From 114b31deb444b2ea080cf56c12dff631603e9512 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Thu, 15 Feb 2024 14:10:31 -0800 Subject: [PATCH 01/67] Initial attempt currently failing on sasl handshake --- mocks/docker-compose.yml | 33 ++++++++++++------- svc-bie-kafka/build.gradle | 1 + .../src/main/resources/application.yaml | 25 +++++++++----- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index cc597789c2..f1d805b0e0 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -74,6 +74,7 @@ services: # Purpose of EXTERNAL port 9094: see https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md#accessing-apache-kafka-with-internal-and-external-clients KAFKA_CFG_LISTENERS: INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:9094 KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL://mock-bie-kafka:9092,EXTERNAL://localhost:9094 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'INTERNAL' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true' KAFKA_MESSAGE_MAX_BYTES: "1000000" @@ -87,24 +88,34 @@ services: # https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md # https://stackoverflow.com/a/72018409 # Caution: Empty string '' and "" don't work; however ' ' or " " does work - KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: " " +# KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: " " # For development without having to deal with connecting via SSL, use port 9092 or do the following. # To disable SSL, uncomment the following and comment out the next SSL section - # ALLOW_PLAINTEXT_LISTENER: 'yes' +# ALLOW_PLAINTEXT_LISTENER: 'yes' # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT + # For SASL for client communications -- see "Security" section of https://hub.docker.com/r/bitnami/kafka/ + KAFKA_CLIENT_USERS: USERNAME + KAFKA_CLIENT_PASSWORDS: PASSWORD + KAFKA_CLIENT_LISTENER_NAME: INTERNAL + KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL: PLAIN + KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN + KAFKA_CFG_SASL_ENABLED_MECHANISMS: PLAIN + KAFKA_CONTROLLER_USER: USERNAME + KAFKA_CONTROLLER_PASSWORD: PASSWORD + KAFKA_INTER_BROKER_USER: USERNAME + KAFKA_INTER_BROKER_PASSWORD: PASSWORD + # Set up SSL -- see "Security" section of https://hub.docker.com/r/bitnami/kafka/ - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:SSL - KAFKA_CFG_SSL_CLIENT_AUTH: required # PEM https://medium.com/analytics-vidhya/how-to-use-pem-certificates-with-apache-kafka-f3b444a00816 - KAFKA_TLS_TYPE: JKS - KAFKA_CERTIFICATE_PASSWORD: mockKafka - - volumes: - # scripts/setenv.sh creates jks files from env variables - - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' - - './mock-bie-kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro' +# KAFKA_TLS_TYPE: JKS +# KAFKA_CERTIFICATE_PASSWORD: mockKafka +# +# volumes: +# # scripts/setenv.sh creates jks files from env variables +# - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' +# - './mock-bie-kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro' networks: - vro_intranet diff --git a/svc-bie-kafka/build.gradle b/svc-bie-kafka/build.gradle index 83b0b18c7d..726af4a7b5 100644 --- a/svc-bie-kafka/build.gradle +++ b/svc-bie-kafka/build.gradle @@ -34,6 +34,7 @@ dependencies { // RabbitMQ implementation 'org.springframework.boot:spring-boot-starter-amqp' implementation 'org.springframework.kafka:spring-kafka' + implementation 'org.apache.kafka:kafka-clients:3.6.1' // Provides endpoint for healthcheck implementation 'org.springframework.boot:spring-boot-starter-web' diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index dac0bd254a..f5280a32fb 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -14,22 +14,29 @@ spring: kafka: bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:localhost:9092}" properties: - schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://schemaregistry.dev.bip.va.gov:443}" + schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://USERNAME:PASSWORD@schemaregistry.dev.bip.va.gov:443}" specific.avro.reader: false + security.protocol: SASL_SSL + security.inter.broker.protocol: SASL_SSL + sasl: + mechanism: PLAIN + username: USERNAME + password: PASSWORD + jaas: + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"USERNAME\" password=\"PASSWORD\";" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-tst-vro}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" auto-offset-reset: earliest # Comment out the following line to connect without SSL - security.protocol: SSL - ssl: - keystore-location: "file:${KEYSTORE_FILE}" - keystore-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - key-store-type: "PKCS12" - truststore-location: "file:${TRUSTSTORE_FILE}" - truststore-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" - trust-store-type: "PKCS12" +# ssl: +# key-store-location: "file:${KEYSTORE_FILE}" +# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" +# key-store-type: "PKCS12" +# trust-store-location: "file:${TRUSTSTORE_FILE}" +# trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" +# trust-store-type: "PKCS12" ## Specify bie properties bie: From 7f53b79177dc33efbb103c9826dae5172e642cdc Mon Sep 17 00:00:00 2001 From: josiahjones Date: Thu, 15 Feb 2024 14:37:16 -0800 Subject: [PATCH 02/67] Enables sasl_ssl for our kafka broker and clients --- mocks/docker-compose.yml | 25 +++++++++---------- .../src/main/resources/application.yaml | 14 +++++------ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index f1d805b0e0..dd98cfe3a5 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -74,7 +74,7 @@ services: # Purpose of EXTERNAL port 9094: see https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md#accessing-apache-kafka-with-internal-and-external-clients KAFKA_CFG_LISTENERS: INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:9094 KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL://mock-bie-kafka:9092,EXTERNAL://localhost:9094 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:SASL_SSL,INTERNAL:SASL_SSL,EXTERNAL:SASL_SSL KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'INTERNAL' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true' KAFKA_MESSAGE_MAX_BYTES: "1000000" @@ -99,23 +99,22 @@ services: KAFKA_CLIENT_USERS: USERNAME KAFKA_CLIENT_PASSWORDS: PASSWORD KAFKA_CLIENT_LISTENER_NAME: INTERNAL - KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL: PLAIN KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN - KAFKA_CFG_SASL_ENABLED_MECHANISMS: PLAIN - KAFKA_CONTROLLER_USER: USERNAME - KAFKA_CONTROLLER_PASSWORD: PASSWORD - KAFKA_INTER_BROKER_USER: USERNAME - KAFKA_INTER_BROKER_PASSWORD: PASSWORD + KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL: PLAIN +# KAFKA_CONTROLLER_USER: USERNAME +# KAFKA_CONTROLLER_PASSWORD: PASSWORD +# KAFKA_INTER_BROKER_USER: USERNAME +# KAFKA_INTER_BROKER_PASSWORD: PASSWORD # Set up SSL -- see "Security" section of https://hub.docker.com/r/bitnami/kafka/ # PEM https://medium.com/analytics-vidhya/how-to-use-pem-certificates-with-apache-kafka-f3b444a00816 -# KAFKA_TLS_TYPE: JKS -# KAFKA_CERTIFICATE_PASSWORD: mockKafka + KAFKA_TLS_TYPE: JKS + KAFKA_CERTIFICATE_PASSWORD: mockKafka # -# volumes: -# # scripts/setenv.sh creates jks files from env variables -# - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' -# - './mock-bie-kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro' + volumes: + # scripts/setenv.sh creates jks files from env variables + - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' + - './mock-bie-kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro' networks: - vro_intranet diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index f5280a32fb..fb26ba58a7 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -30,13 +30,13 @@ spring: value-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" auto-offset-reset: earliest # Comment out the following line to connect without SSL -# ssl: -# key-store-location: "file:${KEYSTORE_FILE}" -# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" -# key-store-type: "PKCS12" -# trust-store-location: "file:${TRUSTSTORE_FILE}" -# trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" -# trust-store-type: "PKCS12" + ssl: + key-store-location: "file:${KEYSTORE_FILE}" + key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + key-store-type: "PKCS12" + trust-store-location: "file:${TRUSTSTORE_FILE}" + trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + trust-store-type: "PKCS12" ## Specify bie properties bie: From a6646b121381ddf9343884e3eef8298b50e126c0 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Thu, 15 Feb 2024 14:42:22 -0800 Subject: [PATCH 03/67] More changes --- mocks/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index dd98cfe3a5..642d166340 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -88,11 +88,11 @@ services: # https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md # https://stackoverflow.com/a/72018409 # Caution: Empty string '' and "" don't work; however ' ' or " " does work -# KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: " " + KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: " " # For development without having to deal with connecting via SSL, use port 9092 or do the following. # To disable SSL, uncomment the following and comment out the next SSL section -# ALLOW_PLAINTEXT_LISTENER: 'yes' + # ALLOW_PLAINTEXT_LISTENER: 'yes' # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT # For SASL for client communications -- see "Security" section of https://hub.docker.com/r/bitnami/kafka/ From 348190b768cf848dab071e2335834c0aaf12db9f Mon Sep 17 00:00:00 2001 From: josiahjones Date: Thu, 15 Feb 2024 15:16:18 -0800 Subject: [PATCH 04/67] Fix for integration test --- .../main/resources/application-integration-test.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-integration-test.yaml b/svc-bie-kafka/src/main/resources/application-integration-test.yaml index 20278871b1..deb084c4d8 100644 --- a/svc-bie-kafka/src/main/resources/application-integration-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-integration-test.yaml @@ -6,13 +6,21 @@ spring: # See https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md#accessing-apache-kafka-with-internal-and-external-clients bootstrap-servers: "localhost:9094" properties: + security.protocol: SASL_SSL + security.inter.broker.protocol: SASL_SSL + sasl: + mechanism: PLAIN + username: USERNAME + password: PASSWORD + jaas: + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"USERNAME\" password=\"PASSWORD\";" ssl: # Disable hostname verification to avoid error for 'localhost' by setting to empty string endpoint.identification.algorithm: producer: key-serializer: "org.apache.kafka.common.serialization.StringSerializer" value-serializer: "org.apache.kafka.common.serialization.StringSerializer" - security.protocol: SSL +# security.protocol: SSL ssl: keystore-location: "file:${KEYSTORE_FILE}" keystore-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" @@ -21,7 +29,7 @@ spring: truststore-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" admin: - security.protocol: SSL +# security.protocol: SSL ssl: keystore-location: "file:${KEYSTORE_FILE}" keystore-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" From 1e3f44b0fd9624c7aa4ed5bd9b59bc5ef1761091 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 20 Feb 2024 10:53:20 -0800 Subject: [PATCH 05/67] Remove comments and unnecessary duplication of ssl --- app/docker-compose.yml | 2 ++ mocks/docker-compose.yml | 6 +--- .../src/main/resources/application-dev.yaml | 30 +++++++++---------- .../application-integration-test.yaml | 27 +++++------------ .../src/main/resources/application.yaml | 6 ++-- 5 files changed, 26 insertions(+), 45 deletions(-) diff --git a/app/docker-compose.yml b/app/docker-compose.yml index e0542660fb..206e1df930 100644 --- a/app/docker-compose.yml +++ b/app/docker-compose.yml @@ -128,5 +128,7 @@ services: BIE_KAFKA_KEYSTORE_PASSWORD: ${BIE_KAFKA_KEYSTORE_PASSWORD} BIE_KAFKA_TRUSTSTORE_INBASE64: ${BIE_KAFKA_TRUSTSTORE_INBASE64} BIE_KAFKA_TRUSTSTORE_PASSWORD: ${BIE_KAFKA_TRUSTSTORE_PASSWORD} + BIE_SASL_SSL_USERNAME: ${BIE_SASL_SSL_USERNAME} + BIE_SASL_SSL_PASSWORD: ${BIE_SASL_SSL_PASSWORD} networks: - vro_intranet diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index 642d166340..ec6764c809 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -101,16 +101,12 @@ services: KAFKA_CLIENT_LISTENER_NAME: INTERNAL KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL: PLAIN -# KAFKA_CONTROLLER_USER: USERNAME -# KAFKA_CONTROLLER_PASSWORD: PASSWORD -# KAFKA_INTER_BROKER_USER: USERNAME -# KAFKA_INTER_BROKER_PASSWORD: PASSWORD # Set up SSL -- see "Security" section of https://hub.docker.com/r/bitnami/kafka/ # PEM https://medium.com/analytics-vidhya/how-to-use-pem-certificates-with-apache-kafka-f3b444a00816 KAFKA_TLS_TYPE: JKS KAFKA_CERTIFICATE_PASSWORD: mockKafka -# + volumes: # scripts/setenv.sh creates jks files from env variables - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index eae38c01b3..f99d69b324 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -2,23 +2,21 @@ spring: kafka: bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:kafka.dev.bip.va.gov:443}" properties: - schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://schemaregistry.dev.bip.va.gov:443}" - schema.registry.ssl.protocol: SSL - schema.registry.ssl.keystore.location: "${KEYSTORE_FILE}" - schema.registry.ssl.keystore.password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - schema.registry.ssl.keystore.type: "PKCS12" - schema.registry.ssl.truststore.location: "${TRUSTSTORE_FILE}" - schema.registry.ssl.truststore.password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" - schema.registry.ssl.truststore.type: "PKCS12" + schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_SASL_USERNAME}:${BIE_SASL_PASSWORD}@schemaregistry.dev.bip.va.gov:443}" + security.protocol: SASL_SSL + security.inter.broker.protocol: SASL_SSL + sasl: + mechanism: PLAIN + jaas: + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_SASL_USERNAME:USERNAME} password=${BIE_SASL_PASSWORD:PASSWORD};" + ssl: + key-store-type: PKCS12 + key-store-location: "${KEYSTORE_FILE}" + key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + trust-store-type: PKCS12 + trust-store-location: "${TRUSTSTORE_FILE}" + trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-tst-vro-1}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" - properties: - security.protocol: SSL - ssl.keystore.type: PKCS12 - ssl.keystore.location: "${KEYSTORE_FILE}" - ssl.keystore.password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - ssl.truststore.type: PKCS12 - ssl.truststore.location: "${TRUSTSTORE_FILE}" - ssl.truststore.password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" diff --git a/svc-bie-kafka/src/main/resources/application-integration-test.yaml b/svc-bie-kafka/src/main/resources/application-integration-test.yaml index deb084c4d8..8fe3944e6a 100644 --- a/svc-bie-kafka/src/main/resources/application-integration-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-integration-test.yaml @@ -10,30 +10,17 @@ spring: security.inter.broker.protocol: SASL_SSL sasl: mechanism: PLAIN - username: USERNAME - password: PASSWORD jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"USERNAME\" password=\"PASSWORD\";" + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_SASL_USERNAME:USERNAME} password=${BIE_SASL_PASSWORD:PASSWORD};" ssl: # Disable hostname verification to avoid error for 'localhost' by setting to empty string endpoint.identification.algorithm: + key-store-location: "file:${KEYSTORE_FILE}" + key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + key-store-type: "PKCS12" + trust-store-location: "file:${TRUSTSTORE_FILE}" + trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + trust-store-type: "PKCS12" producer: key-serializer: "org.apache.kafka.common.serialization.StringSerializer" value-serializer: "org.apache.kafka.common.serialization.StringSerializer" -# security.protocol: SSL - ssl: - keystore-location: "file:${KEYSTORE_FILE}" - keystore-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - key-store-type: "PKCS12" - truststore-location: "file:${TRUSTSTORE_FILE}" - truststore-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" - trust-store-type: "PKCS12" - admin: -# security.protocol: SSL - ssl: - keystore-location: "file:${KEYSTORE_FILE}" - keystore-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - key-store-type: "PKCS12" - truststore-location: "file:${TRUSTSTORE_FILE}" - truststore-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" - trust-store-type: "PKCS12" diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index fb26ba58a7..c24327a7fd 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -14,16 +14,14 @@ spring: kafka: bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:localhost:9092}" properties: - schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://USERNAME:PASSWORD@schemaregistry.dev.bip.va.gov:443}" + schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_SASL_USERNAME}:${BIE_SASL_PASSWORD}@schemaregistry.dev.bip.va.gov:443}" specific.avro.reader: false security.protocol: SASL_SSL security.inter.broker.protocol: SASL_SSL sasl: mechanism: PLAIN - username: USERNAME - password: PASSWORD jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"USERNAME\" password=\"PASSWORD\";" + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_SASL_USERNAME:USERNAME} password=${BIE_SASL_PASSWORD:PASSWORD};" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-tst-vro}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" From 6c99c6fc9b8f7562b7475215064ab96d29158787 Mon Sep 17 00:00:00 2001 From: VRO Machine User Date: Fri, 23 Feb 2024 14:43:10 +0000 Subject: [PATCH 06/67] Automated commit: Unpin versions of updated images --- scripts/image_versions.src | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/scripts/image_versions.src b/scripts/image_versions.src index 00bb214a19..11173059c6 100644 --- a/scripts/image_versions.src +++ b/scripts/image_versions.src @@ -60,15 +60,4 @@ devtools_VER="v3.6.12" # Wed Feb 21 15:29:25 UTC 2024 -- v3.6.13 # Thu Feb 22 20:00:43 UTC 2024 -- v3.6.14 redis_VER="v3.6.14" -rabbitmq_VER="v3.6.14" -apigateway_VER="v3.6.14" -app_VER="v3.6.14" dbinit_VER="v3.6.14" -svcbgsapi_VER="v3.6.14" -svclighthouseapi_VER="v3.6.14" -svcbiekafka_VER="v3.6.14" -svcbipapi_VER="v3.6.14" -xampleworkflows_VER="v3.6.14" -ccapp_VER="v3.6.14" -eemaxcfiapp_VER="v3.6.14" -eeepmergeapp_VER="v3.6.14" From ddce3384b68cd1f07809e01ef57929b3e9b706d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Feb 2024 10:17:27 -0500 Subject: [PATCH 07/67] Bump org.springframework.boot:spring-boot-autoconfigure from 3.2.2 to 3.2.3 (#2659) Bump org.springframework.boot:spring-boot-autoconfigure Bumps [org.springframework.boot:spring-boot-autoconfigure](https://github.com/spring-projects/spring-boot) from 3.2.2 to 3.2.3. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.2.2...v3.2.3) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-autoconfigure dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Cheng <89034346+chengjie8@users.noreply.github.com> --- api-gateway/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-gateway/build.gradle b/api-gateway/build.gradle index b29a1e478f..efbcab0caa 100644 --- a/api-gateway/build.gradle +++ b/api-gateway/build.gradle @@ -25,6 +25,6 @@ dependencies { constraints { // To address Snyk alert on DoS vulnerability - implementation 'org.springframework.boot:spring-boot-autoconfigure:3.2.2' + implementation 'org.springframework.boot:spring-boot-autoconfigure:3.2.3' } } From cda1eb76d5f85db826fe39b254f2f5970c02bdc3 Mon Sep 17 00:00:00 2001 From: Derek Fitchett <135860892+dfitchett@users.noreply.github.com> Date: Fri, 23 Feb 2024 09:01:30 -0800 Subject: [PATCH 08/67] EP Merge: Special Issue Code workaround for testing (#2645) Made special issue code an environment variable, so it can be switched without code change. --- domain-ee/ee-ep-merge-app/pyproject.toml | 3 ++- .../ee-ep-merge-app/src/python_src/config.py | 2 ++ .../python_src/service/ep_merge_machine.py | 20 ++++++++++++++----- ...aim_contentions_increase_tinnitus_200.json | 1 - .../ee-ep-merge-app/tests/service/conftest.py | 5 ++++- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/domain-ee/ee-ep-merge-app/pyproject.toml b/domain-ee/ee-ep-merge-app/pyproject.toml index 3b3140fd44..bec76a0ef8 100644 --- a/domain-ee/ee-ep-merge-app/pyproject.toml +++ b/domain-ee/ee-ep-merge-app/pyproject.toml @@ -7,5 +7,6 @@ testpaths = [ ] env = [ "REQUEST_TIMEOUT = 1", - "REQUEST_RETRIES = 1" + "REQUEST_RETRIES = 1", + "EP_MERGE_SPECIAL_ISSUE_CODE = TEST" ] diff --git a/domain-ee/ee-ep-merge-app/src/python_src/config.py b/domain-ee/ee-ep-merge-app/src/python_src/config.py index b5034e9860..e74b0d0d7a 100644 --- a/domain-ee/ee-ep-merge-app/src/python_src/config.py +++ b/domain-ee/ee-ep-merge-app/src/python_src/config.py @@ -62,6 +62,8 @@ class ClientName(str, Enum): ClientName.BGS_ADD_CLAIM_NOTE: os.environ.get("ADD_CLAIM_NOTE_RESPONSE") or "add-note-response", } +EP_MERGE_SPECIAL_ISSUE_CODE = os.environ.get("EP_MERGE_SPECIAL_ISSUE_CODE") or "EMP" + def create_sqlalchemy_db_uri(): user = quote(os.environ.get("POSTGRES_USER") or "vro_user") diff --git a/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py b/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py index af3d2dbac7..0fbf62632d 100644 --- a/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py +++ b/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py @@ -7,6 +7,7 @@ from hoppy.async_hoppy_client import AsyncHoppyClient from hoppy.exception import ResponseException from pydantic import ValidationError +from config import EP_MERGE_SPECIAL_ISSUE_CODE from schema import ( add_claim_note, cancel_claim, @@ -40,7 +41,6 @@ CANCEL_TRACKING_EP = "60" CANCELLATION_REASON_FORMAT = "Issues moved into or confirmed in pending EP{ep_code} - claim #{claim_id}" -SPECIAL_ISSUE_CODE = "EMP" class Workflow(str, Enum): @@ -168,7 +168,12 @@ def on_get_pending_contentions(self, event): @running_get_ep400_contentions.enter def on_get_ep400_contentions(self, event, pending_contentions_response=None): request = get_contentions.Request(claim_id=self.job.ep400_claim_id) - response = self.make_request(request=request, hoppy_client=HOPPY.get_client(ClientName.GET_CLAIM_CONTENTIONS), response_type=get_contentions.Response, expected_statuses=[200, 204]) + response = self.make_request( + request=request, + hoppy_client=HOPPY.get_client(ClientName.GET_CLAIM_CONTENTIONS), + response_type=get_contentions.Response, + expected_statuses=[200, 204], + ) self.send(event=event, pending_contentions_response=pending_contentions_response, ep400_contentions_response=response) @running_set_temp_station_of_jurisdiction.enter @@ -191,7 +196,10 @@ def on_merge_contentions(self, event, pending_contentions_response=None, ep400_c def on_move_contentions_to_pending_claim(self, event, new_contentions=None, ep400_contentions_response=None): request = create_contentions.Request(claim_id=self.job.pending_claim_id, create_contentions=new_contentions) self.make_request( - request=request, hoppy_client=HOPPY.get_client(ClientName.CREATE_CLAIM_CONTENTIONS), response_type=create_contentions.Response, expected_statuses=201 + request=request, + hoppy_client=HOPPY.get_client(ClientName.CREATE_CLAIM_CONTENTIONS), + response_type=create_contentions.Response, + expected_statuses=201, ) self.send(event=event, ep400_contentions_response=ep400_contentions_response) @@ -230,7 +238,7 @@ def on_pre_cancel_step_failed_remove_special_issue_code(self, event, ep400_conte updates = [] for contention in ContentionsUtil.to_existing_contentions(contentions): contention.special_issue_codes = ( - [code for code in contention.special_issue_codes if code != SPECIAL_ISSUE_CODE] if contention.special_issue_codes else None + [code for code in contention.special_issue_codes if code != EP_MERGE_SPECIAL_ISSUE_CODE] if contention.special_issue_codes else None ) updates.append(contention) @@ -291,7 +299,9 @@ def log_metrics(self, job_duration): if self.skipped_merge: increment(JOB_SKIPPED_MERGE_METRIC) - def make_request(self, request: GeneralRequest, hoppy_client: AsyncHoppyClient, response_type: Type[GeneralResponse], expected_statuses: list[int] | int = 200): + def make_request( + self, request: GeneralRequest, hoppy_client: AsyncHoppyClient, response_type: Type[GeneralResponse], expected_statuses: list[int] | int = 200 + ): if not isinstance(expected_statuses, list): expected_statuses = [expected_statuses] try: diff --git a/domain-ee/ee-ep-merge-app/tests/responses/claim_contentions_increase_tinnitus_200.json b/domain-ee/ee-ep-merge-app/tests/responses/claim_contentions_increase_tinnitus_200.json index 8ab6e4ce1f..3f7ed9d763 100644 --- a/domain-ee/ee-ep-merge-app/tests/responses/claim_contentions_increase_tinnitus_200.json +++ b/domain-ee/ee-ep-merge-app/tests/responses/claim_contentions_increase_tinnitus_200.json @@ -10,7 +10,6 @@ "contentionId": 2, "lastModified": "2023-09-30T00:00:00-06:00", "specialIssueCodes": [ - "EMP" ] } ] diff --git a/domain-ee/ee-ep-merge-app/tests/service/conftest.py b/domain-ee/ee-ep-merge-app/tests/service/conftest.py index 54dfc43a25..9f2c54b57b 100644 --- a/domain-ee/ee-ep-merge-app/tests/service/conftest.py +++ b/domain-ee/ee-ep-merge-app/tests/service/conftest.py @@ -4,6 +4,7 @@ from unittest.mock import ANY, AsyncMock, call import pytest +from src.python_src.config import EP_MERGE_SPECIAL_ISSUE_CODE from schema import ( add_claim_note, cancel_claim, @@ -64,8 +65,10 @@ def load_response(file, response_type): get_pending_contentions_increase_tinnitus_200 = load_response(pending_contentions_increase_tinnitus_200, get_contentions.Response) get_ep400_contentions_req = get_contentions.Request(claim_id=EP400_CLAIM_ID).model_dump(by_alias=True) get_ep400_contentions_200 = load_response(ep400_contentions_increase_tinnitus_200, get_contentions.Response) +# Add special issue code to contention from config +get_ep400_contentions_200.contentions[0].special_issue_codes.append(EP_MERGE_SPECIAL_ISSUE_CODE) get_ep400_contentions_204 = load_response(response_204, get_contentions.Response) -get_ep400_contentions_without_special_issues_200 = load_response(ep400_contentions_increase_tinnitus_without_special_issues_200, get_contentions.Response) +get_ep400_contentions_without_special_issues_200 = load_response(ep400_contentions_increase_tinnitus_200, get_contentions.Response) update_temporary_station_of_jurisdiction_req = tsoj.Request(claim_id=EP400_CLAIM_ID, temp_station_of_jurisdiction="398").model_dump(by_alias=True) revert_temporary_station_of_jurisdiction_req = tsoj.Request(claim_id=EP400_CLAIM_ID, temp_station_of_jurisdiction="111").model_dump(by_alias=True) update_temporary_station_of_jurisdiction_200 = load_response(response_200, tsoj.Response) From 3c61b66f40cda74f903752e56847a8e8599026d0 Mon Sep 17 00:00:00 2001 From: Mason Watson <32622851+msnwatson@users.noreply.github.com> Date: Fri, 23 Feb 2024 09:26:10 -0800 Subject: [PATCH 09/67] Update QA BGS URL (#2622) --- svc-bgs-api/docker-entrypoint.sh | 6 ++++++ svc-bgs-api/src/config/settings.yml | 2 +- svc-bgs-api/src/main_consumer.rb | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/svc-bgs-api/docker-entrypoint.sh b/svc-bgs-api/docker-entrypoint.sh index befb0b3772..e7dd8dc426 100755 --- a/svc-bgs-api/docker-entrypoint.sh +++ b/svc-bgs-api/docker-entrypoint.sh @@ -23,12 +23,18 @@ openssl version # shellcheck disable=SC2086 openssl pkcs12 ${PWD_ARG} -in keystore.p12 -out tls_bip.crt -nokeys -legacy || exit 2 + echo "Emitted tls_bip.crt" + # shellcheck disable=SC2086 openssl pkcs12 ${PWD_ARG} -in keystore.p12 -out tls.key -nocerts -nodes -legacy || exit 3 + echo "Emitted tls.key" + # shellcheck disable=SC2086 openssl pkcs12 ${PWD_ARG} -in truststore.p12 -out va_all.crt -legacy || exit 4 + echo "Emitted va_all.crt" } cd "$CURR_DIR" || exit 11 +echo "Executing Ruby application..." exec bundle exec ruby main_consumer.rb diff --git a/svc-bgs-api/src/config/settings.yml b/svc-bgs-api/src/config/settings.yml index edb571efab..e429dc67e4 100644 --- a/svc-bgs-api/src/config/settings.yml +++ b/svc-bgs-api/src/config/settings.yml @@ -24,7 +24,7 @@ qa: client_station_id: 281 client_username: VROSYSACCT log: true - base_url: http://bepprep.vba.va.gov + base_url: http://bepwebtest.vba.va.gov sandbox: bgs: diff --git a/svc-bgs-api/src/main_consumer.rb b/svc-bgs-api/src/main_consumer.rb index ba79854ce8..fde47448fd 100644 --- a/svc-bgs-api/src/main_consumer.rb +++ b/svc-bgs-api/src/main_consumer.rb @@ -8,6 +8,8 @@ require 'rabbit_subscriber' require 'bgs_client' +$stdout.sync = true + def initialize_subscriber(bgs_client) subscriber = RabbitSubscriber.new(BUNNY_ARGS) From a7892185225c3811958683948bf48f23eb9e5a8d Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 4 Mar 2024 12:38:00 -0800 Subject: [PATCH 10/67] Update to use vault values, added to dev and integration test yamls --- app/docker-compose.yml | 4 ++-- mocks/docker-compose.yml | 3 ++- .../src/main/resources/application-dev.yaml | 13 +++++++------ .../resources/application-integration-test.yaml | 9 +++++---- svc-bie-kafka/src/main/resources/application.yaml | 13 +++++++------ 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/app/docker-compose.yml b/app/docker-compose.yml index 206e1df930..acef22b4e9 100644 --- a/app/docker-compose.yml +++ b/app/docker-compose.yml @@ -128,7 +128,7 @@ services: BIE_KAFKA_KEYSTORE_PASSWORD: ${BIE_KAFKA_KEYSTORE_PASSWORD} BIE_KAFKA_TRUSTSTORE_INBASE64: ${BIE_KAFKA_TRUSTSTORE_INBASE64} BIE_KAFKA_TRUSTSTORE_PASSWORD: ${BIE_KAFKA_TRUSTSTORE_PASSWORD} - BIE_SASL_SSL_USERNAME: ${BIE_SASL_SSL_USERNAME} - BIE_SASL_SSL_PASSWORD: ${BIE_SASL_SSL_PASSWORD} + BIE_KAFKA_RBAC_USERNAME: ${BIE_KAFKA_RBAC_USERNAME} + BIE_KAFKA_RBAC_PASSWORD: ${BIE_KAFKA_RBAC_PASSWORD} networks: - vro_intranet diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index ec6764c809..d35d1ed2bb 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -109,7 +109,8 @@ services: volumes: # scripts/setenv.sh creates jks files from env variables - - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' + # keystore is unneeded only truststore https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 + # - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' - './mock-bie-kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro' networks: - vro_intranet diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index f99d69b324..9da73b36fe 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -1,18 +1,19 @@ spring: kafka: - bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:kafka.dev.bip.va.gov:443}" + bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.tst.bip.va.gov}" properties: - schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_SASL_USERNAME}:${BIE_SASL_PASSWORD}@schemaregistry.dev.bip.va.gov:443}" + schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.tst.bip.va.gov}" security.protocol: SASL_SSL security.inter.broker.protocol: SASL_SSL sasl: mechanism: PLAIN jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_SASL_USERNAME:USERNAME} password=${BIE_SASL_PASSWORD:PASSWORD};" + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" ssl: - key-store-type: PKCS12 - key-store-location: "${KEYSTORE_FILE}" - key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 +# key-store-type: PKCS12 +# key-store-location: "${KEYSTORE_FILE}" +# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" trust-store-type: PKCS12 trust-store-location: "${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" diff --git a/svc-bie-kafka/src/main/resources/application-integration-test.yaml b/svc-bie-kafka/src/main/resources/application-integration-test.yaml index 8fe3944e6a..792359e08e 100644 --- a/svc-bie-kafka/src/main/resources/application-integration-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-integration-test.yaml @@ -11,13 +11,14 @@ spring: sasl: mechanism: PLAIN jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_SASL_USERNAME:USERNAME} password=${BIE_SASL_PASSWORD:PASSWORD};" + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" ssl: # Disable hostname verification to avoid error for 'localhost' by setting to empty string endpoint.identification.algorithm: - key-store-location: "file:${KEYSTORE_FILE}" - key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - key-store-type: "PKCS12" + # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 +# key-store-location: "file:${KEYSTORE_FILE}" +# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" +# key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index c24327a7fd..4b75c844a7 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -14,14 +14,14 @@ spring: kafka: bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:localhost:9092}" properties: - schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_SASL_USERNAME}:${BIE_SASL_PASSWORD}@schemaregistry.dev.bip.va.gov:443}" + schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-kafka.tst.bip.va.gov}" specific.avro.reader: false security.protocol: SASL_SSL security.inter.broker.protocol: SASL_SSL sasl: mechanism: PLAIN jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_SASL_USERNAME:USERNAME} password=${BIE_SASL_PASSWORD:PASSWORD};" + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-tst-vro}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" @@ -29,16 +29,17 @@ spring: auto-offset-reset: earliest # Comment out the following line to connect without SSL ssl: - key-store-location: "file:${KEYSTORE_FILE}" - key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - key-store-type: "PKCS12" + # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 +# key-store-location: "file:${KEYSTORE_FILE}" +# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" +# key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" ## Specify bie properties bie: - kakfa-topic-prefix: "TST_" + kakfa-topic-prefix: "EXT_VRO_TST" ## Actuator for health check, liveness, and readiness management: From 72f6975d7775b75c5f7d249e195fbfdffee1af21 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 4 Mar 2024 12:45:32 -0800 Subject: [PATCH 11/67] Update the remaining environments --- .../src/main/resources/application-dev.yaml | 2 +- .../main/resources/application-prod-test.yaml | 28 ++++++++------ .../src/main/resources/application-qa.yaml | 33 +++++++++------- .../main/resources/application-sandbox.yaml | 38 ++++++++++--------- 4 files changed, 58 insertions(+), 43 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index 9da73b36fe..1504e55df2 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -1,6 +1,6 @@ spring: kafka: - bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.tst.bip.va.gov}" + bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.tst.bip.va.gov:443}" properties: schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.tst.bip.va.gov}" security.protocol: SASL_SSL diff --git a/svc-bie-kafka/src/main/resources/application-prod-test.yaml b/svc-bie-kafka/src/main/resources/application-prod-test.yaml index 3ef94dcdd5..5ec0f29c5f 100644 --- a/svc-bie-kafka/src/main/resources/application-prod-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-prod-test.yaml @@ -1,16 +1,20 @@ spring: kafka: - bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:kafka.preprod.bip.va.gov:443}" + bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.tst.bip.va.gov:443}" properties: schema: registry: - url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://schemaregistry.preprod.bip.va.gov:443}" + url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.tst.bip.va.gov}" + sasl: + mechanism: PLAIN + jaas: + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" ssl: - protocol: SSL - keystore: - type: PKCS12 - location: "${KEYSTORE_FILE}" - password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" +# protocol: SSL +# keystore: +# type: PKCS12 +# location: "${KEYSTORE_FILE}" +# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" truststore: type: PKCS12 location: "${TRUSTSTORE_FILE}" @@ -20,12 +24,12 @@ spring: key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" properties: - security.protocol: SSL + security.protocol: SASL_SSL ssl: - keystore: - type: PKCS12 - location: "${KEYSTORE_FILE}" - password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" +# keystore: +# type: PKCS12 +# location: "${KEYSTORE_FILE}" +# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" truststore: type: PKCS12 location: "${TRUSTSTORE_FILE}" diff --git a/svc-bie-kafka/src/main/resources/application-qa.yaml b/svc-bie-kafka/src/main/resources/application-qa.yaml index bb2daa36e1..509e06495f 100644 --- a/svc-bie-kafka/src/main/resources/application-qa.yaml +++ b/svc-bie-kafka/src/main/resources/application-qa.yaml @@ -1,16 +1,23 @@ spring: kafka: - bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:kafka.stage.bip.va.gov:443}" + bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.tst.bip.va.gov:443}" properties: schema: registry: - url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://schemaregistry.stage.bip.va.gov:443}" + url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.tst.bip.va.gov}" + security.protocol: SASL_SSL + security.inter.broker.protocol: SASL_SSL + sasl: + mechanism: PLAIN + jaas: + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" ssl: - protocol: SSL - keystore: - type: PKCS12 - location: "${KEYSTORE_FILE}" - password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" +# Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 +# protocol: SSL +# keystore: +# type: PKCS12 +# location: "${KEYSTORE_FILE}" +# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" truststore: type: PKCS12 location: "${TRUSTSTORE_FILE}" @@ -20,12 +27,12 @@ spring: key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" properties: - security.protocol: SSL - ssl: - keystore: - type: PKCS12 - location: "${KEYSTORE_FILE}" - password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" +# security.protocol: SSL +# ssl: +# keystore: +# type: PKCS12 +# location: "${KEYSTORE_FILE}" +# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" truststore: type: PKCS12 location: "${TRUSTSTORE_FILE}" diff --git a/svc-bie-kafka/src/main/resources/application-sandbox.yaml b/svc-bie-kafka/src/main/resources/application-sandbox.yaml index 8b69cd5cf3..6a6862c538 100644 --- a/svc-bie-kafka/src/main/resources/application-sandbox.yaml +++ b/svc-bie-kafka/src/main/resources/application-sandbox.yaml @@ -1,31 +1,35 @@ spring: kafka: - bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:kafka.stage.bip.va.gov:443}" + bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.tst.bip.va.gov}" properties: schema: registry: - url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://schemaregistry.stage.bip.va.gov:443}" - ssl: - protocol: SSL - keystore: - type: PKCS12 - location: "${KEYSTORE_FILE}" - password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - truststore: - type: PKCS12 - location: "${TRUSTSTORE_FILE}" - password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.tst.bip.va.gov}" + sasl: + mechanism: PLAIN + jaas: + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" + ssl: +# keystore: +# type: PKCS12 +# location: "${KEYSTORE_FILE}" +# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + truststore: + type: PKCS12 + location: "${TRUSTSTORE_FILE}" + password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-uat-vro}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" properties: - security.protocol: SSL + security.protocol: SASL_SSL ssl: - keystore: - type: PKCS12 - location: "${KEYSTORE_FILE}" - password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 +# keystore: +# type: PKCS12 +# location: "${KEYSTORE_FILE}" +# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" truststore: type: PKCS12 location: "${TRUSTSTORE_FILE}" From 0e473b899fdde7d01b10f99fba0eae4d00e28db7 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Wed, 6 Mar 2024 14:29:02 -0800 Subject: [PATCH 12/67] Updating the names of the topics and add the tls certs needed for communication. --- mocks/docker-compose.yml | 2 +- .../va/vro/model/biekafka/ContentionEvent.java | 16 ++++++++-------- .../src/main/resources/application.yaml | 2 +- .../bie/config/MessageExchangeConfigTest.java | 12 ++++++------ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index d35d1ed2bb..4164bd2ebf 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -110,7 +110,7 @@ services: volumes: # scripts/setenv.sh creates jks files from env variables # keystore is unneeded only truststore https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 - # - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' + - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' - './mock-bie-kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro' networks: - vro_intranet diff --git a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java index 2cb5eb9ecd..71f60fb496 100644 --- a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java +++ b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java @@ -3,11 +3,11 @@ import java.util.Arrays; public enum ContentionEvent { - CONTENTION_ASSOCIATED_TO_CLAIM("CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02"), - CONTENTION_UPDATED("CONTENTION_BIE_CONTENTION_UPDATED_V02"), - CONTENTION_CLASSIFIED("CONTENTION_BIE_CONTENTION_CLASSIFIED_V02"), - CONTENTION_COMPLETED("CONTENTION_BIE_CONTENTION_COMPLETED_V02"), - CONTENTION_DELETED("CONTENTION_BIE_CONTENTION_DELETED_V02"); + CONTENTION_ASSOCIATED_TO_CLAIM("BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM"), + CONTENTION_UPDATED("BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED"), + CONTENTION_CLASSIFIED("BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED"), + CONTENTION_COMPLETED("BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED"), + CONTENTION_DELETED("BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED"); private final String topicName; @@ -20,9 +20,9 @@ public String getTopicName() { } public static ContentionEvent mapTopicToEvent(String topic) { - // remove first word prefix from topic seperated by _ - String noPrefixTopic = topic.substring(topic.indexOf("_") + 1); - + // TODO(3/5/24): Add a better + String prefixPattern = "^EXT_VRO_[A-Z]+_"; + String noPrefixTopic = topic.replaceFirst(prefixPattern, ""); return Arrays.stream(ContentionEvent.values()) .filter(event -> event.getTopicName().equals(noPrefixTopic)) .findFirst() diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index 4b75c844a7..5ea480b037 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -39,7 +39,7 @@ spring: ## Specify bie properties bie: - kakfa-topic-prefix: "EXT_VRO_TST" + kakfa-topic-prefix: "EXT_VRO_TST_" ## Actuator for health check, liveness, and readiness management: diff --git a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java index d1172cf45c..e582d0b35f 100644 --- a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java +++ b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java @@ -17,7 +17,7 @@ class MessageExchangeConfigTest { @BeforeEach void setUp() { bieProperties = new BieProperties(); - bieProperties.kakfaTopicPrefix = "TST_"; + bieProperties.kakfaTopicPrefix = "EXT_VRO_TST_"; } @Test @@ -34,11 +34,11 @@ void topicNames() { final String[] topicNames = bieProperties.topicNames(); assertArrayEquals( new String[] { - "TST_CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02", - "TST_CONTENTION_BIE_CONTENTION_UPDATED_V02", - "TST_CONTENTION_BIE_CONTENTION_CLASSIFIED_V02", - "TST_CONTENTION_BIE_CONTENTION_COMPLETED_V02", - "TST_CONTENTION_BIE_CONTENTION_DELETED_V02" + "EXT_VRO_TST_BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", + "EXT_VRO_TST_BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED", + "EXT_VRO_TST_BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", + "EXT_VRO_TST_BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED", + "EXT_VRO_TST_BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED" }, topicNames); } From f22ff11790a40e5cb9b76dc96ed66bf624e4bda5 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Wed, 6 Mar 2024 14:53:42 -0800 Subject: [PATCH 13/67] Add the var initialization to setenv.sh --- scripts/setenv.sh | 2 ++ svc-bie-kafka/src/main/resources/application.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/setenv.sh b/scripts/setenv.sh index bec4db0df8..2aca1df356 100755 --- a/scripts/setenv.sh +++ b/scripts/setenv.sh @@ -253,3 +253,5 @@ exportSecretIfUnset BIP_KEYSTORE exportSecretIfUnset BIP_PASSWORD exportSecretIfUnset BIP_CLAIM_URL exportSecretIfUnset BIP_EVIDENCE_URL +export BIE_KAFKA_RBAC_USERNAME=USERNAME +export BIE_KAFKA_RBAC_PASSWORD=PASSWORD diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index 5ea480b037..4126ea6966 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -21,7 +21,7 @@ spring: sasl: mechanism: PLAIN jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD};" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-tst-vro}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" @@ -30,9 +30,9 @@ spring: # Comment out the following line to connect without SSL ssl: # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 -# key-store-location: "file:${KEYSTORE_FILE}" -# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" -# key-store-type: "PKCS12" + key-store-location: "file:${KEYSTORE_FILE}" + key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" From c69d5671372a225e60200b630a174f51f4333789 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Fri, 8 Mar 2024 16:12:42 -0800 Subject: [PATCH 14/67] Update urls to dev from tst --- svc-bie-kafka/src/main/resources/application-dev.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index 1504e55df2..3e81548398 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -1,14 +1,14 @@ spring: kafka: - bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.tst.bip.va.gov:443}" + bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.dev.bip.va.gov:443}" properties: - schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.tst.bip.va.gov}" + schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.dev.bip.va.gov}" security.protocol: SASL_SSL security.inter.broker.protocol: SASL_SSL sasl: mechanism: PLAIN jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD};" ssl: # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 # key-store-type: PKCS12 From 00ece5f4484d81cc319110c1d480b994e68cf737 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 11 Mar 2024 10:55:03 -0700 Subject: [PATCH 15/67] Update Contention Event to use new updated topic names --- .../bie/config/ContentionEventTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/ContentionEventTest.java b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/ContentionEventTest.java index f818ff4f45..4e4d882559 100644 --- a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/ContentionEventTest.java +++ b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/ContentionEventTest.java @@ -13,11 +13,11 @@ public class ContentionEventTest { @ParameterizedTest @CsvSource({ - "TST_CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02, CONTENTION_ASSOCIATED_TO_CLAIM", - "TST_CONTENTION_BIE_CONTENTION_UPDATED_V02, CONTENTION_UPDATED", - "TST_CONTENTION_BIE_CONTENTION_CLASSIFIED_V02, CONTENTION_CLASSIFIED", - "TST_CONTENTION_BIE_CONTENTION_COMPLETED_V02, CONTENTION_COMPLETED", - "TST_CONTENTION_BIE_CONTENTION_DELETED_V02, CONTENTION_DELETED" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM, CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED, CONTENTION_UPDATED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED, CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED, CONTENTION_COMPLETED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED, CONTENTION_DELETED" }) public void testMapTopicToEvent_validTopics(String inputTopic, ContentionEvent expectedEvent) { assertEquals(expectedEvent, mapTopicToEvent(inputTopic)); @@ -35,11 +35,11 @@ public void testMapTopicToEvent_unrecognizedTopic() { @ParameterizedTest @CsvSource({ - "TST_CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02, bie-events-contention-associated-to-claim", - "TST_CONTENTION_BIE_CONTENTION_UPDATED_V02, bie-events-contention-updated", - "TST_CONTENTION_BIE_CONTENTION_CLASSIFIED_V02, bie-events-contention-classified", - "TST_CONTENTION_BIE_CONTENTION_COMPLETED_V02, bie-events-contention-completed", - "TST_CONTENTION_BIE_CONTENTION_DELETED_V02, bie-events-contention-deleted" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM, bie-events-contention-associated-to-claim", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED, bie-events-contention-updated", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED, bie-events-contention-classified", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED, bie-events-contention-completed", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED, bie-events-contention-deleted" }) public void testGenerateRabbitMQChannelName_channelNames(String inputTopic, String bieChannel) { assertEquals(bieChannel, ContentionEvent.rabbitMqExchangeName(inputTopic)); From f37a3feaad8f15c2a2c5fa2be704c3582ed10755 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 11 Mar 2024 11:05:11 -0700 Subject: [PATCH 16/67] Update BieMessagePayload to use new updated topic names --- .../vro/model/biekafka/BieMessagePayload.java | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java index da6ab568a7..4cc7192ce7 100644 --- a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java +++ b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java @@ -32,60 +32,60 @@ public class BieMessagePayload { // populated from kafka topic payload @TargetEvents({ - "CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02", - "CONTENTION_BIE_CONTENTION_CLASSIFIED_V02", - "CONTENTION_BIE_CONTENTION_UPDATED_V02" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" }) private String benefitClaimTypeCode; @TargetEvents({ - "CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02", - "CONTENTION_BIE_CONTENTION_CLASSIFIED_V02", - "CONTENTION_BIE_CONTENTION_UPDATED_V02" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" }) private String actorStation; @TargetEvents({ - "CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02", - "CONTENTION_BIE_CONTENTION_CLASSIFIED_V02", - "CONTENTION_BIE_CONTENTION_UPDATED_V02" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" }) private String details; @TargetEvents({ - "CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02", - "CONTENTION_BIE_CONTENTION_CLASSIFIED_V02", - "CONTENTION_BIE_CONTENTION_UPDATED_V02" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" }) private Long veteranParticipantId; @TargetEvents({ - "CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02", - "CONTENTION_BIE_CONTENTION_CLASSIFIED_V02", - "CONTENTION_BIE_CONTENTION_UPDATED_V02" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" }) private String contentionClassificationName; @TargetEvents({ - "CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02", - "CONTENTION_BIE_CONTENTION_CLASSIFIED_V02", - "CONTENTION_BIE_CONTENTION_UPDATED_V02" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" }) private String diagnosticTypeCode; - @TargetEvents({"CONTENTION_BIE_CONTENTION_UPDATED_V02"}) + @TargetEvents({"BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED"}) private String journalStatusTypeCode; @TargetEvents({ - "CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02", - "CONTENTION_BIE_CONTENTION_CLASSIFIED_V02", - "CONTENTION_BIE_CONTENTION_UPDATED_V02" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" }) private Long dateAdded; - @TargetEvents({"CONTENTION_BIE_CONTENTION_UPDATED_V02"}) + @TargetEvents({"BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED"}) private Long dateCompleted; - @TargetEvents({"CONTENTION_BIE_CONTENTION_UPDATED_V02"}) + @TargetEvents({"BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED"}) private Long dateUpdated; } From 0ded0a70f76b454a6b6d514165c9ba8fb9e76fb9 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 11 Mar 2024 11:20:22 -0700 Subject: [PATCH 17/67] Updating Login config to only include Kafka user and no default value --- .../src/main/resources/application-integration-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svc-bie-kafka/src/main/resources/application-integration-test.yaml b/svc-bie-kafka/src/main/resources/application-integration-test.yaml index 792359e08e..2a4b6d1d41 100644 --- a/svc-bie-kafka/src/main/resources/application-integration-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-integration-test.yaml @@ -11,7 +11,7 @@ spring: sasl: mechanism: PLAIN jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD};" ssl: # Disable hostname verification to avoid error for 'localhost' by setting to empty string endpoint.identification.algorithm: From 84f6166152cdb5f0eae565cbee814cbdd9cdf0ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:23:28 -0400 Subject: [PATCH 18/67] Bump org.eclipse.jgit:org.eclipse.jgit from 6.8.0.202311291450-r to 6.9.0.202403050737-r (#2725) Bump org.eclipse.jgit:org.eclipse.jgit Bumps org.eclipse.jgit:org.eclipse.jgit from 6.8.0.202311291450-r to 6.9.0.202403050737-r. --- updated-dependencies: - dependency-name: org.eclipse.jgit:org.eclipse.jgit dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ponnia <151670616+Ponnia-M@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 731c920159..6d427e3789 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ buildscript { // There's a conflict between versions of jgit between spotless and axion release plugin configurations.classpath { resolutionStrategy { - force 'org.eclipse.jgit:org.eclipse.jgit:6.8.0.202311291450-r' + force 'org.eclipse.jgit:org.eclipse.jgit:6.9.0.202403050737-r' } } } From e978c45e8543e6132e8e9688b79fa5b515802adf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:58:56 -0400 Subject: [PATCH 19/67] Bump com.fasterxml.jackson.core:jackson-databind from 2.16.1 to 2.16.2 (#2728) Bumps [com.fasterxml.jackson.core:jackson-databind](https://github.com/FasterXML/jackson) from 2.16.1 to 2.16.2. - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ponnia <151670616+Ponnia-M@users.noreply.github.com> --- db-init/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-init/build.gradle b/db-init/build.gradle index 10db2fb46a..26f1d39f2c 100644 --- a/db-init/build.gradle +++ b/db-init/build.gradle @@ -16,7 +16,7 @@ dependencies { // flywayMigration platform("gov.va.starter:starter-bom:${starter_boot_version}") flywayMigration "org.postgresql:postgresql:${postgresql_version}" constraints { - implementation 'com.fasterxml.jackson.core:jackson-databind:2.16.1' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.16.2' } } From 878f042a88c8d53e004969cb21b032cb9aeb1418 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 18:57:59 -0400 Subject: [PATCH 20/67] Bump org.apache.commons:commons-compress from 1.26.0 to 1.26.1 (#2727) Bumps org.apache.commons:commons-compress from 1.26.0 to 1.26.1. --- updated-dependencies: - dependency-name: org.apache.commons:commons-compress dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ponnia <151670616+Ponnia-M@users.noreply.github.com> --- svc-bie-kafka/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svc-bie-kafka/build.gradle b/svc-bie-kafka/build.gradle index 7291f2d87c..0ed91e249f 100644 --- a/svc-bie-kafka/build.gradle +++ b/svc-bie-kafka/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation 'org.apache.avro:avro:1.11.3' implementation 'io.confluent:kafka-avro-serializer:7.6.0' implementation 'org.apache.kafka:connect-api:7.6.0-ce' - implementation 'org.apache.commons:commons-compress:1.26.0' + implementation 'org.apache.commons:commons-compress:1.26.1' // RabbitMQ implementation 'org.springframework.boot:spring-boot-starter-amqp' From 84633ee012f79e8dfbe46f315932046df50a7645 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 23:54:39 -0400 Subject: [PATCH 21/67] Bump com.fasterxml.jackson.datatype:jackson-datatype-jsr310 from 2.16.1 to 2.16.2 (#2726) Bump com.fasterxml.jackson.datatype:jackson-datatype-jsr310 Bumps com.fasterxml.jackson.datatype:jackson-datatype-jsr310 from 2.16.1 to 2.16.2. --- updated-dependencies: - dependency-name: com.fasterxml.jackson.datatype:jackson-datatype-jsr310 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ponnia <151670616+Ponnia-M@users.noreply.github.com> --- svc-bip-api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svc-bip-api/build.gradle b/svc-bip-api/build.gradle index 30048caabf..905cc4bc3e 100644 --- a/svc-bip-api/build.gradle +++ b/svc-bip-api/build.gradle @@ -18,7 +18,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-core" implementation "com.fasterxml.jackson.core:jackson-databind" - implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.16.1" + implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.16.2" def jjwt_version = "0.12.5" implementation "io.jsonwebtoken:jjwt-api:${jjwt_version}" From be61a823f76cc212612fe6b414cc279f8b44c767 Mon Sep 17 00:00:00 2001 From: Derek Fitchett <135860892+dfitchett@users.noreply.github.com> Date: Tue, 12 Mar 2024 08:24:56 -0700 Subject: [PATCH 22/67] EP Merge: Updated logging of errors and warnings (#2733) * Updated logging of errors and warnings --- .../src/python_src/schema/merge_job.py | 11 ++-- .../src/python_src/schema/response.py | 6 ++- .../python_src/service/ep_merge_machine.py | 52 +++++++++++-------- .../tests/service/test_ep_merge_process.py | 12 ++--- .../test_ep_merge_resume_at_add_note.py | 4 +- .../test_ep_merge_resume_at_cancel_ep400.py | 6 +-- 6 files changed, 51 insertions(+), 40 deletions(-) diff --git a/domain-ee/ee-ep-merge-app/src/python_src/schema/merge_job.py b/domain-ee/ee-ep-merge-app/src/python_src/schema/merge_job.py index ceec934086..e27871c91a 100644 --- a/domain-ee/ee-ep-merge-app/src/python_src/schema/merge_job.py +++ b/domain-ee/ee-ep-merge-app/src/python_src/schema/merge_job.py @@ -57,17 +57,16 @@ def __init__(self, *args, **kwargs) -> None: class Meta: orm_model = model.merge_job.MergeJob - def error(self, messages): + def error(self, message: dict[Any, Any]): self.error_state = self.state self.state = JobState.COMPLETED_ERROR - self.add_message(messages) + self.add_message(message) - def add_message(self, messages): - if messages: + def add_message(self, message: dict[Any, Any]): + if message: if self.messages is None: self.messages = [] - msgs = [str(m) for m in messages] if isinstance(messages, list) else [str(messages)] - self.messages.extend(msgs) + self.messages.append(message) def update(self, new_state: JobState): self.state = new_state diff --git a/domain-ee/ee-ep-merge-app/src/python_src/schema/response.py b/domain-ee/ee-ep-merge-app/src/python_src/schema/response.py index bf7ffdd54b..ee5fc24fa2 100644 --- a/domain-ee/ee-ep-merge-app/src/python_src/schema/response.py +++ b/domain-ee/ee-ep-merge-app/src/python_src/schema/response.py @@ -1,6 +1,6 @@ from datetime import datetime -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, field_serializer from pydantic.alias_generators import to_camel @@ -14,6 +14,10 @@ class Message(BaseModel): timestamp: datetime | None = None http_status: str | None = None + @field_serializer('timestamp') + def serialize_datetime(self, dt: datetime, _info): + return None if dt is None else dt.isoformat() + class GeneralResponse(BaseModel): model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel) diff --git a/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py b/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py index 3498e17ec8..3a345b2289 100644 --- a/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py +++ b/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py @@ -160,7 +160,7 @@ def on_get_pending_claim(self, event): if response is not None and response.status_code == 200: if response.claim is None or response.claim.end_product_code is None: - self.add_error(f"error='Pending claim #{self.job.pending_claim_id} does not have an end product code'") + self.add_job_error(f"Pending claim #{self.job.pending_claim_id} does not have an end product code") else: self.cancellation_reason = CANCELLATION_REASON_FORMAT.format(ep_code=response.claim.end_product_code, claim_id=self.job.pending_claim_id) self.original_tsoj = response.claim.temp_station_of_jurisdiction @@ -187,7 +187,7 @@ def on_get_ep400_contentions(self, event, pending_contentions_response=None): will_retry_condition=ep400_has_no_contentions, ) if response and (response.status_code in expected_responses and not response.contentions): - self.add_error(f"error='EP400 claim #{self.job.ep400_claim_id} does not have any contentions'") + self.add_job_error(f"EP400 claim #{self.job.ep400_claim_id} does not have any contentions") self.send(event=event, pending_contentions_response=pending_contentions_response, ep400_contentions_response=response) @@ -200,11 +200,7 @@ def on_set_temp_station_of_jurisdiction(self, event, pending_contentions_respons @running_merge_contentions.enter def on_merge_contentions(self, event, pending_contentions_response=None, ep400_contentions_response=None): self.skipped_merge = False - new_contentions = None - try: - new_contentions = ContentionsUtil.new_contentions(pending_contentions_response.contentions, ep400_contentions_response.contentions) - except Exception as e: - self.add_error(e.message) + new_contentions = ContentionsUtil.new_contentions(pending_contentions_response.contentions, ep400_contentions_response.contentions) self.send(event=event, new_contentions=new_contentions, ep400_contentions_response=ep400_contentions_response) @running_move_contentions_to_pending_claim.enter @@ -259,6 +255,8 @@ def on_pre_cancel_step_failed_remove_special_issue_code(self, event, ep400_conte request = update_contentions.Request(claim_id=self.job.ep400_claim_id, update_contentions=updates) self.make_request(request=request, hoppy_client=HOPPY.get_client(ClientName.UPDATE_CLAIM_CONTENTIONS), response_type=update_contentions.Response) + else: + self.add_warning_message('Could not remove special issues since EP400 has no contentions') self.send(event=event) @@ -266,7 +264,10 @@ def on_pre_cancel_step_failed_remove_special_issue_code(self, event, ep400_conte @running_cancel_claim_failed_revert_temp_station_of_jurisdiction.enter def on_move_contentions_or_cancel_claim_failed_revert_temp_station_of_jurisdiction(self, event): request = tsoj.Request(temp_station_of_jurisdiction=self.original_tsoj, claim_id=self.job.ep400_claim_id) - self.make_request(request=request, hoppy_client=HOPPY.get_client(ClientName.PUT_TSOJ), response_type=tsoj.Response) + response = self.make_request(request=request, hoppy_client=HOPPY.get_client(ClientName.PUT_TSOJ), response_type=tsoj.Response) + if not response or response.status_code != 200: + self.add_warning_message(f'Could not revert temporary station of jurisdiction back to original: {self.original_tsoj}') + self.send(event=event) @completed_success.enter @@ -282,10 +283,10 @@ def on_completed(self, event): f"job_id={self.job.job_id} " f"pending_claim_id={self.job.pending_claim_id} " f"ep400_claim_id={self.job.ep400_claim_id} " + f"job_duration_seconds={job_duration} " f"state={self.job.state} " f"errorState={self.job.error_state} " - f"error=\"{jsonable_encoder(self.job.messages)}\" " - f"job_duration_seconds={job_duration}" + f"errors={jsonable_encoder(self.job.messages)}" ) else: logging.info( @@ -294,8 +295,8 @@ def on_completed(self, event): f"job_id={self.job.job_id} " f"pending_claim_id={self.job.pending_claim_id} " f"ep400_claim_id={self.job.ep400_claim_id} " - f"state={self.job.state} " - f"job_duration_seconds={job_duration}" + f"job_duration_seconds={job_duration} " + f"state={self.job.state}" ) def log_metrics(self, job_duration): @@ -330,10 +331,8 @@ async def make_hoppy_request( response = await hoppy_client.make_request(request_id, request_body) model = response_type.model_validate(response) if model.status_code not in expected_statuses: - self.add_error( - model.messages - if model.messages - else f"client={hoppy_client.name} error='Unknown Downstream Error' status={model.status_code} status_message={model.status_message}" + self.add_client_error( + hoppy_client.name, GeneralResponse(status_code=model.status_code, status_message=model.status_message, messages=model.messages) ) break @@ -369,11 +368,11 @@ def make_request( ) return loop.run_until_complete(req) except ValidationError as e: - self.add_error(f"client={hoppy_client.name} error={e.errors(include_url=False, include_input=False)}") + self.add_client_error(hoppy_client.name, e.errors(include_url=False, include_input=False)) except ResponseException as e: - self.add_error(f"client={hoppy_client.name} error={e.message}") + self.add_client_error(hoppy_client.name, e.message) except Exception as e: - self.add_error(f"client={hoppy_client.name} error='Unknown Exception Caught {e}'") + self.add_client_error(hoppy_client.name, f"Unknown Exception Caught {e}") return None def has_error(self): @@ -384,6 +383,15 @@ def has_new_contentions(self, pending_contentions_response: get_contentions.Resp self.num_new_contentions = len(contentions) return contentions - def add_error(self, error): - logging.warning(f"event=jobError job_id={self.job.job_id} state={self.job.state} {error}") - self.job.error(error) + def add_job_error(self, message): + errors = {'state': self.job.state, 'error': message} + logging.warning(f"event=jobError job_id={self.job.job_id} error={jsonable_encoder(errors)}") + self.job.error(errors) + + def add_client_error(self, client_name, message): + errors = {'state': self.job.state, 'client': client_name, 'error': message} + logging.warning(f"event=jobError job_id={self.job.job_id} error={jsonable_encoder(errors)}") + self.job.error(errors) + + def add_warning_message(self, message): + self.job.add_message({'warning': message}) diff --git a/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_process.py b/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_process.py index 060146b35e..d50233ece0 100644 --- a/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_process.py +++ b/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_process.py @@ -101,7 +101,7 @@ def test_no_contentions_on_ep400_after_get_pending_claim_failure( self, machine, mock_hoppy_async_client, metric_logger_distribution, metric_logger_increment, no_contentions_response ): mock_async_responses(mock_hoppy_async_client, [ResponseException("Oops"), no_contentions_response]) - process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM, 1) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM, 2) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), @@ -129,7 +129,7 @@ def test_invalid_request_at_get_ep400_contentions_after_get_pending_claim_failur invalid_request, ], ) - process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE, 2) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE, 3) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), @@ -207,7 +207,7 @@ def test_no_contentions_on_ep400_after_get_pending_contentions_failure( self, machine, mock_hoppy_async_client, metric_logger_distribution, metric_logger_increment, no_contentions_response ): mock_async_responses(mock_hoppy_async_client, [get_pending_claim_200, ResponseException("Oops"), no_contentions_response]) - process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_CONTENTIONS, 1) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_CONTENTIONS, 2) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), @@ -237,7 +237,7 @@ def test_invalid_request_at_get_ep400_contentions_after_get_pending_contentions_ invalid_request, ], ) - process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_CONTENTIONS_FAILED_REMOVE_SPECIAL_ISSUE, 2) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_CONTENTIONS_FAILED_REMOVE_SPECIAL_ISSUE, 3) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), @@ -488,7 +488,7 @@ def test_fail_to_revert_tsoj_after_failure_to_move_contentions_to_pending_claim( invalid_request, ], ) - process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.MOVE_CONTENTIONS_FAILED_REVERT_TEMP_STATION_OF_JURISDICTION, 2) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.MOVE_CONTENTIONS_FAILED_REVERT_TEMP_STATION_OF_JURISDICTION, 3) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), @@ -574,7 +574,7 @@ def test_invalid_request_at_revert_tsoj_due_to_failure_to_cancel_claim( invalid_request, ], ) - process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.CANCEL_CLAIM_FAILED_REVERT_TEMP_STATION_OF_JURISDICTION, 2) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.CANCEL_CLAIM_FAILED_REVERT_TEMP_STATION_OF_JURISDICTION, 3) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_contentions_req), diff --git a/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_resume_at_add_note.py b/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_resume_at_add_note.py index 64329afeba..bace944522 100644 --- a/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_resume_at_add_note.py +++ b/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_resume_at_add_note.py @@ -68,7 +68,7 @@ def test_invalid_request(self, machine, mock_hoppy_async_client, invalid_request ) def test_no_contentions_on_ep400_after_get_pending_claim_failure(self, machine, mock_hoppy_async_client, no_contentions_response): mock_async_responses(mock_hoppy_async_client, [ResponseException("Oops"), no_contentions_response]) - process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM, 1) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM, 2) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), @@ -94,7 +94,7 @@ def test_invalid_request_at_get_ep400_contentions_after_get_pending_claim_failur invalid_request, ], ) - process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE, 2) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE, 3) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), diff --git a/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_resume_at_cancel_ep400.py b/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_resume_at_cancel_ep400.py index aecb052931..6d02255fae 100644 --- a/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_resume_at_cancel_ep400.py +++ b/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_resume_at_cancel_ep400.py @@ -79,7 +79,7 @@ def test_invalid_request(self, machine, mock_hoppy_async_client, invalid_request ) def test_no_contentions_on_ep400_after_get_pending_claim_failure(self, machine, mock_hoppy_async_client, no_contentions_response): mock_async_responses(mock_hoppy_async_client, [ResponseException("Oops"), no_contentions_response]) - process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM, 1) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM, 2) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), @@ -105,7 +105,7 @@ def test_invalid_request_at_get_ep400_contentions_after_get_pending_claim_failur invalid_request, ], ) - process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE, 2) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE, 3) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), @@ -182,7 +182,7 @@ def test_invalid_request_at_cancel_claim_due_to_exception(self, machine, mock_ho ) def test_invalid_request_at_revert_tsoj_due_to_failure_to_cancel_claim(self, machine, mock_hoppy_async_client, invalid_request): mock_async_responses(mock_hoppy_async_client, [get_pending_claim_200, ResponseException("Oops"), invalid_request]) - process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.CANCEL_CLAIM_FAILED_REVERT_TEMP_STATION_OF_JURISDICTION, 2) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.CANCEL_CLAIM_FAILED_REVERT_TEMP_STATION_OF_JURISDICTION, 3) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), From 2699f4021ddb40b69f1ba623d3006ad5cc9bf3e1 Mon Sep 17 00:00:00 2001 From: YANG YANG Date: Tue, 12 Mar 2024 12:05:11 -0400 Subject: [PATCH 23/67] Simplify EP Merge integration test sorter (#2683) Co-authored-by: Erik Nelsestuen Co-authored-by: Derek Fitchett <135860892+dfitchett@users.noreply.github.com> --- domain-ee/ee-ep-merge-app/integration/conftest.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/domain-ee/ee-ep-merge-app/integration/conftest.py b/domain-ee/ee-ep-merge-app/integration/conftest.py index 288304014f..f0469ce2ab 100644 --- a/domain-ee/ee-ep-merge-app/integration/conftest.py +++ b/domain-ee/ee-ep-merge-app/integration/conftest.py @@ -12,14 +12,9 @@ def pytest_collection_modifyitems(items): 1. All test from modules not specified in the `module_order` below 2. Test from each module in module_order (in order) """ - module_order = ["integration.test_get_endpoints", "integration.test_merge_request"] - module_mapping = {item: item.module.__name__ for item in items} - - sorted_items = items.copy() - # Iteratively move tests of each module to the end of the test queue - for module in module_order: - sorted_items = [it for it in sorted_items if module_mapping[it] != module] + [it for it in sorted_items if module_mapping[it] == module] - items[:] = sorted_items + module_order = {"integration.test_get_endpoints": 1, "integration.test_merge_request": 2} + item_order = {item: module_order.get(item.module.__name__, 0) for item in items} + items.sort(key=item_order.get) @pytest.fixture(autouse=True, scope="session") From 8b6679664d8939fe84cc39479951cae8fe93cccd Mon Sep 17 00:00:00 2001 From: Ponnia <151670616+Ponnia-M@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:07:53 -0400 Subject: [PATCH 24/67] Github action version rollback (#2737) * rollback version for yoomlam/delete-older-releases * Revert "rollback version for yoomlam/delete-older-releases" This reverts commit 19ad81ecd940dac950f477c57021ffb445f3d8cd. * rollback github-action version --- .github/workflows/delete-old-releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/delete-old-releases.yml b/.github/workflows/delete-old-releases.yml index 3a62a65410..cd05997ec9 100644 --- a/.github/workflows/delete-old-releases.yml +++ b/.github/workflows/delete-old-releases.yml @@ -65,7 +65,7 @@ jobs: runs-on: ubuntu-latest steps: - name: "Delete old releases in GHCR" - uses: yoomlam/delete-older-releases@v0.3.2 + uses: yoomlam/delete-older-releases@v0.3.0 with: dry_run: ${{ inputs.dry_run || false }} delete_tags: true From d039e800a2df31dd4b091da12a756d4afb6cf663 Mon Sep 17 00:00:00 2001 From: Derek Fitchett <135860892+dfitchett@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:34:57 -0700 Subject: [PATCH 25/67] EPMerge: Fix serialization issue resulting in bad request response from BIP API (#2732) Changed rest template to use configured objectmapper --- .../gov/va/vro/bip/config/BipApiConfig.java | 19 +- .../gov/va/vro/bip/config/JacksonConfig.java | 12 + .../gov/va/vro/bip/service/BipApiService.java | 24 +- .../va/vro/bip/config/BipApiConfigTest.java | 20 +- .../va/vro/bip/service/BipApiServiceTest.java | 811 ++++++++---------- .../bip-test-data/claim_response_404.json | 2 +- .../contention_response_201.json | 5 + .../contention_response_412.json | 2 +- .../resources/bip-test-data/response_500.json | 2 +- 9 files changed, 410 insertions(+), 487 deletions(-) create mode 100644 svc-bip-api/src/test/resources/bip-test-data/contention_response_201.json diff --git a/svc-bip-api/src/main/java/gov/va/vro/bip/config/BipApiConfig.java b/svc-bip-api/src/main/java/gov/va/vro/bip/config/BipApiConfig.java index ec4db8092a..541639fbb9 100644 --- a/svc-bip-api/src/main/java/gov/va/vro/bip/config/BipApiConfig.java +++ b/svc-bip-api/src/main/java/gov/va/vro/bip/config/BipApiConfig.java @@ -11,11 +11,11 @@ import org.apache.hc.client5.http.io.HttpClientConnectionManager; import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.http.client.ClientHttpRequestFactory; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.http.converter.HttpMessageConverter; import org.springframework.web.client.RestTemplate; import java.io.ByteArrayInputStream; @@ -28,6 +28,7 @@ import java.security.SecureRandom; import java.security.cert.CertificateException; import java.util.Base64; +import java.util.List; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManagerFactory; @@ -65,7 +66,7 @@ private KeyStore getKeyStore(String base64, String password) throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException { KeyStore keyStore = KeyStore.getInstance("PKCS12"); String noSpaceBase64 = base64.replaceAll("\\s+", ""); - byte[] decodedBytes = new byte[] {}; + byte[] decodedBytes; try { decodedBytes = Base64.getDecoder().decode(noSpaceBase64); } catch (IllegalArgumentException e) { @@ -77,15 +78,9 @@ private KeyStore getKeyStore(String base64, String password) return keyStore; } - /** - * Get Rest template for BIP API connection. - * - * @param builder RestTemplateBuilder - * @return Rest template, request factory - * @throws BipException failure to create connection - */ @Bean(name = "bipCERestTemplate") - public RestTemplate getHttpsRestTemplate(RestTemplateBuilder builder) throws BipException { + public RestTemplate getHttpsRestTemplate(List> messageConverters) + throws BipException { try { if (trustStore.isEmpty() && password.isEmpty()) { log.info("No valid BIP mTLS setup. Skip related setup."); @@ -122,7 +117,9 @@ public RestTemplate getHttpsRestTemplate(RestTemplateBuilder builder) throws Bip HttpClients.custom().setConnectionManager(connectionManager).build(); ClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(httpClient); - return new RestTemplate(requestFactory); + RestTemplate restTemplate = new RestTemplate(requestFactory); + restTemplate.setMessageConverters(messageConverters); + return restTemplate; } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { log.error("Failed to create SSL context for VA certificate. {}", e.getMessage(), e); throw new BipException("Failed to create SSL context.", e); diff --git a/svc-bip-api/src/main/java/gov/va/vro/bip/config/JacksonConfig.java b/svc-bip-api/src/main/java/gov/va/vro/bip/config/JacksonConfig.java index 6c8440046d..53a7bfc359 100644 --- a/svc-bip-api/src/main/java/gov/va/vro/bip/config/JacksonConfig.java +++ b/svc-bip-api/src/main/java/gov/va/vro/bip/config/JacksonConfig.java @@ -1,5 +1,6 @@ package gov.va.vro.bip.config; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; @@ -9,6 +10,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; @Configuration public class JacksonConfig { @@ -19,12 +21,22 @@ public ObjectMapper objectMapper() { objectMapper.registerModule(new JavaTimeModule()); objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); objectMapper.enable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + objectMapper.enable(JsonParser.Feature.INCLUDE_SOURCE_IN_LOCATION); return objectMapper; } + // Used for AMQP message conversion @Bean @Primary public MessageConverter jackson2MessageConverter(ObjectMapper objectMapper) { return new Jackson2JsonMessageConverter(objectMapper); } + + // Used for RestTemplate message conversion + @Bean + @Primary + public MappingJackson2HttpMessageConverter mappingJackson2HttpMessageConverter( + ObjectMapper objectMapper) { + return new MappingJackson2HttpMessageConverter(objectMapper); + } } diff --git a/svc-bip-api/src/main/java/gov/va/vro/bip/service/BipApiService.java b/svc-bip-api/src/main/java/gov/va/vro/bip/service/BipApiService.java index d66d0b6081..ef02bf53a9 100644 --- a/svc-bip-api/src/main/java/gov/va/vro/bip/service/BipApiService.java +++ b/svc-bip-api/src/main/java/gov/va/vro/bip/service/BipApiService.java @@ -33,7 +33,9 @@ import java.nio.charset.StandardCharsets; import java.security.Key; import java.util.Date; +import java.util.HashMap; import java.util.Map; +import java.util.Objects; import javax.crypto.spec.SecretKeySpec; /** @@ -52,8 +54,6 @@ public class BipApiService implements IBipApiService { static final String CONTENTION = "/claims/%s/contentions"; static final String SPECIAL_ISSUE_TYPES = "/contentions/special_issue_types"; - static final String JWT_TYPE = "JWT"; - @Qualifier("bipCERestTemplate") @NonNull final RestTemplate restTemplate; @@ -66,7 +66,7 @@ public class BipApiService implements IBipApiService { public GetClaimResponse getClaimDetails(long claimId) { String url = bipApiProps.getClaimRequestUrl(String.format(CLAIM_DETAILS, claimId)); - return makeRequest(url, HttpMethod.GET, GetClaimResponse.class); + return makeRequest(url, HttpMethod.GET, null, GetClaimResponse.class); } @Override @@ -82,7 +82,7 @@ public PutClaimLifecycleResponse putClaimLifecycleStatus(PutClaimLifecycleReques public GetClaimContentionsResponse getClaimContentions(long claimId) { String url = bipApiProps.getClaimRequestUrl(String.format(CONTENTION, claimId)); - return makeRequest(url, HttpMethod.GET, GetClaimContentionsResponse.class); + return makeRequest(url, HttpMethod.GET, null, GetClaimContentionsResponse.class); } @Override @@ -125,8 +125,8 @@ public PutTempStationOfJurisdictionResponse putTempStationOfJurisdiction( String url = bipApiProps.getClaimRequestUrl(String.format(TEMP_STATION_OF_JURISDICTION, claimId)); - String tsoj = request.getTempStationOfJurisdiction(); - Map requestBody = Map.of("tempStationOfJurisdiction", tsoj); + Map requestBody = new HashMap<>(); + requestBody.put("tempStationOfJurisdiction", request.getTempStationOfJurisdiction()); return makeRequest( url, HttpMethod.PUT, requestBody, PutTempStationOfJurisdictionResponse.class); @@ -138,16 +138,17 @@ private T makeRequest( try { HttpEntity httpEntity = new HttpEntity<>(requestBody, getBipHeader()); log.info("event=requestSent url={} method={}", url, method); - ResponseEntity bipResponse = - restTemplate.exchange(url, method, httpEntity, String.class); + ResponseEntity bipResponse = + restTemplate.exchange(url, method, httpEntity, expectedResponse); log.info( "event=responseReceived url={} method={} status={}", url, method, bipResponse.getStatusCode().value()); + BipPayloadResponse.BipPayloadResponseBuilder responseBuilder; if (bipResponse.hasBody()) { - responseBuilder = mapper.readValue(bipResponse.getBody(), expectedResponse).toBuilder(); + responseBuilder = Objects.requireNonNull(bipResponse.getBody()).toBuilder(); } else { responseBuilder = mapper.readValue("{}", expectedResponse).toBuilder(); } @@ -175,11 +176,6 @@ private T makeRequest( } } - private T makeRequest( - String url, HttpMethod method, Class expectedResponse) { - return makeRequest(url, method, null, expectedResponse); - } - /** * Verifies that the BIP Api responds to a request. Calls the special_issue_types URL and confirms * the response status is OK and body is not empty diff --git a/svc-bip-api/src/test/java/gov/va/vro/bip/config/BipApiConfigTest.java b/svc-bip-api/src/test/java/gov/va/vro/bip/config/BipApiConfigTest.java index 85bddbb21f..d64278c8c3 100644 --- a/svc-bip-api/src/test/java/gov/va/vro/bip/config/BipApiConfigTest.java +++ b/svc-bip-api/src/test/java/gov/va/vro/bip/config/BipApiConfigTest.java @@ -4,14 +4,15 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import com.fasterxml.jackson.databind.ObjectMapper; import gov.va.vro.bip.service.BipException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.web.client.RestTemplate; import java.io.InputStream; +import java.util.List; /** * BIP API configuration tests. @@ -21,16 +22,19 @@ class BipApiConfigTest { private BipApiConfig config; + private JacksonConfig jacksonConfig; @BeforeEach public void setUp() { config = new BipApiConfig(); + jacksonConfig = new JacksonConfig(); } @Test public void testGetHttpsRestTemplate_WithoutConfiguringCerts() { try { - config.getHttpsRestTemplate(new RestTemplateBuilder()); + config.getHttpsRestTemplate( + List.of(jacksonConfig.mappingJackson2HttpMessageConverter(new ObjectMapper()))); fail(); } catch (Exception e) { assertTrue(e.getCause() instanceof NullPointerException); @@ -45,7 +49,8 @@ public void testGetHttpsRestTemplate_WithBadCerts() { config.setTrustStore("biptruststore.jks"); config.setKeystore("biptruststore.jks"); config.setPassword("bad"); - config.getHttpsRestTemplate(new RestTemplateBuilder()); + config.getHttpsRestTemplate( + List.of(jacksonConfig.mappingJackson2HttpMessageConverter(new ObjectMapper()))); }); } @@ -53,7 +58,9 @@ public void testGetHttpsRestTemplate_WithBadCerts() { public void testGetHttpsRestTemplate_WithoutTrustStore() { config.setTrustStore(""); config.setPassword(""); - RestTemplate temp = config.getHttpsRestTemplate(new RestTemplateBuilder()); + RestTemplate temp = + config.getHttpsRestTemplate( + List.of(jacksonConfig.mappingJackson2HttpMessageConverter(new ObjectMapper()))); assertNotNull(temp); } @@ -66,7 +73,10 @@ public void testGetHttpsRestTemplate_WithValidCerts() { config.setTrustStore(store); config.setKeystore(store); config.setPassword("vropassword"); - RestTemplate template = config.getHttpsRestTemplate(new RestTemplateBuilder()); + RestTemplate template = + config.getHttpsRestTemplate( + List.of(jacksonConfig.mappingJackson2HttpMessageConverter(new ObjectMapper()))); + assertNotNull(template); } catch (Exception e) { fail(); diff --git a/svc-bip-api/src/test/java/gov/va/vro/bip/service/BipApiServiceTest.java b/svc-bip-api/src/test/java/gov/va/vro/bip/service/BipApiServiceTest.java index 1b024179f4..bd4b2243bd 100644 --- a/svc-bip-api/src/test/java/gov/va/vro/bip/service/BipApiServiceTest.java +++ b/svc-bip-api/src/test/java/gov/va/vro/bip/service/BipApiServiceTest.java @@ -31,10 +31,13 @@ import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.NullAndEmptySource; import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Mockito; @@ -61,12 +64,13 @@ public class BipApiServiceTest { private static final ObjectMapper MAPPER = new ObjectMapper(); private static final long GOOD_CLAIM_ID = 9666959L; - private static final long BAD_CLAIM_ID = 9666958L; private static final long NOT_FOUND_CLAIM_ID = 9234567L; private static final long INTERNAL_SERVER_ERROR_CLAIM_ID = 9345678L; private static final String RESPONSE_500 = "bip-test-data/response_500.json"; private static final String CONTENTION_RESPONSE_200 = "bip-test-data/contention_response_200.json"; + private static final String CONTENTION_RESPONSE_201 = + "bip-test-data/contention_response_201.json"; private static final String CLAIM_RESPONSE_404 = "bip-test-data/claim_response_404.json"; private static final String CLAIM_RESPONSE_200 = "bip-test-data/claim_response_200.json"; private static final String CLAIM_DETAILS = "/claims/%s"; @@ -88,6 +92,8 @@ public class BipApiServiceTest { private BipApiService service; + private final ObjectMapper mapper = new JacksonConfig().objectMapper(); + @Mock private RestTemplate restTemplate; @BeforeEach @@ -100,7 +106,7 @@ public void setUp() { bipApiProps.setStationId(STATION_ID); bipApiProps.setClaimClientId(CLAIM_USERID); - service = new BipApiService(restTemplate, bipApiProps, new JacksonConfig().objectMapper()); + service = new BipApiService(restTemplate, bipApiProps, mapper); } private String formatClaimUrl(String format, Long claimId) { @@ -116,482 +122,354 @@ private static String getTestData(String dataFile) throws Exception { return Files.readString(filePath); } - private void mockResponseForUrl(Stubber response, String claimUrl, HttpMethod httpMethod) { + private void mockResponseForUrl( + Stubber response, + String claimUrl, + HttpMethod httpMethod, + Class clazz) { response .when(restTemplate) .exchange( ArgumentMatchers.eq(claimUrl), ArgumentMatchers.eq(httpMethod), ArgumentMatchers.any(HttpEntity.class), - ArgumentMatchers.eq(String.class)); - } - - @Test - public void testGetClaimDetails_200() throws Exception { - String resp200Body = getTestData(CLAIM_RESPONSE_200); - - ResponseEntity resp200 = ResponseEntity.ok(resp200Body); - mockResponseForUrl( - Mockito.doReturn(resp200), formatClaimUrl(CLAIM_DETAILS, GOOD_CLAIM_ID), HttpMethod.GET); - - GetClaimResponse result = service.getClaimDetails(GOOD_CLAIM_ID); - assertResponseIsSuccess(result, HttpStatus.OK); - } - - @Test - public void testGetClaimDetails_404() throws Exception { - String resp404Body = getTestData(CLAIM_RESPONSE_404); - mockResponseForUrl( - Mockito.doThrow( - new HttpClientErrorException( - HttpStatus.NOT_FOUND, - HttpStatus.NOT_FOUND.name(), - resp404Body.getBytes(), - Charset.defaultCharset())), - formatClaimUrl(CLAIM_DETAILS, NOT_FOUND_CLAIM_ID), - HttpMethod.GET); - - HttpStatusCodeException ex = - Assertions.assertThrows( - HttpStatusCodeException.class, () -> service.getClaimDetails(NOT_FOUND_CLAIM_ID)); - assertResponseHasMessageWithStatus(ex.getResponseBodyAsString(), HttpStatus.NOT_FOUND); - } - - @Test - public void testGetClaimDetailsDownstreamServerError_500() throws Exception { - String resp500Body = getTestData(RESPONSE_500); - mockResponseForUrl( - Mockito.doThrow( - new HttpServerErrorException( - HttpStatus.INTERNAL_SERVER_ERROR, - HttpStatus.INTERNAL_SERVER_ERROR.name(), - resp500Body.getBytes(), - Charset.defaultCharset())), - formatClaimUrl(CLAIM_DETAILS, INTERNAL_SERVER_ERROR_CLAIM_ID), - HttpMethod.GET); - - HttpStatusCodeException ex = - Assertions.assertThrows( - HttpStatusCodeException.class, - () -> service.getClaimDetails(INTERNAL_SERVER_ERROR_CLAIM_ID)); - assertResponseHasMessageWithStatus( - ex.getResponseBodyAsString(), HttpStatus.INTERNAL_SERVER_ERROR); - } - - @Test - public void testGetClaimDetailsInternalServerError_500() { - mockResponseForUrl( - Mockito.doThrow(new RuntimeException("nope")), - formatClaimUrl(CLAIM_DETAILS, INTERNAL_SERVER_ERROR_CLAIM_ID), - HttpMethod.GET); - - BipException ex = - Assertions.assertThrows( - BipException.class, () -> service.getClaimDetails(INTERNAL_SERVER_ERROR_CLAIM_ID)); - assertSame(HttpStatus.INTERNAL_SERVER_ERROR, ex.getStatus()); - } - - @Test - public void testGetClaimContention_200() throws Exception { - String resp200Body = getTestData(CONTENTION_RESPONSE_200); - ResponseEntity resp200 = ResponseEntity.ok(resp200Body); - - mockResponseForUrl( - Mockito.doReturn(resp200), formatClaimUrl(CONTENTION, GOOD_CLAIM_ID), HttpMethod.GET); - - GetClaimContentionsResponse result = service.getClaimContentions(GOOD_CLAIM_ID); - assertResponseIsSuccess(result, HttpStatus.OK); - assertEquals(1, result.getContentions().size()); - } - - @Test - public void testGetClaimContention_204() throws Exception { - ResponseEntity resp204 = ResponseEntity.noContent().build(); - - mockResponseForUrl( - Mockito.doReturn(resp204), formatClaimUrl(CONTENTION, GOOD_CLAIM_ID), HttpMethod.GET); - - GetClaimContentionsResponse result = service.getClaimContentions(GOOD_CLAIM_ID); - assertResponseIsSuccess(result, HttpStatus.NO_CONTENT); - assertNull(result.getContentions()); - } - - @Test - public void testGetClaimContentions_404() throws Exception { - String resp404Body = getTestData(CLAIM_RESPONSE_404); - - mockResponseForUrl( - Mockito.doThrow( - new HttpClientErrorException( - HttpStatus.NOT_FOUND, - HttpStatus.NOT_FOUND.name(), - resp404Body.getBytes(), - Charset.defaultCharset())), - formatClaimUrl(CONTENTION, BAD_CLAIM_ID), - HttpMethod.GET); - - HttpStatusCodeException ex = - Assertions.assertThrows( - HttpStatusCodeException.class, () -> service.getClaimContentions(BAD_CLAIM_ID)); - assertResponseHasMessageWithStatus(ex.getResponseBodyAsString(), HttpStatus.NOT_FOUND); - } - - @Test - public void testGetClaimContentionsDownstreamServerError_500() throws Exception { - String resp500Body = getTestData(RESPONSE_500); - mockResponseForUrl( - Mockito.doThrow( - new HttpServerErrorException( - HttpStatus.INTERNAL_SERVER_ERROR, - HttpStatus.INTERNAL_SERVER_ERROR.name(), - resp500Body.getBytes(), - Charset.defaultCharset())), - formatClaimUrl(CONTENTION, INTERNAL_SERVER_ERROR_CLAIM_ID), - HttpMethod.GET); - - HttpStatusCodeException ex = - Assertions.assertThrows( - HttpStatusCodeException.class, - () -> service.getClaimContentions(INTERNAL_SERVER_ERROR_CLAIM_ID)); - assertResponseHasMessageWithStatus( - ex.getResponseBodyAsString(), HttpStatus.INTERNAL_SERVER_ERROR); + ArgumentMatchers.eq(clazz)); } - @Test - public void testGetClaimContentionsInternalServerError_500() { - mockResponseForUrl( - Mockito.doThrow(new RuntimeException("nope")), - formatClaimUrl(CONTENTION, INTERNAL_SERVER_ERROR_CLAIM_ID), - HttpMethod.GET); - - BipException ex = - Assertions.assertThrows( - BipException.class, () -> service.getClaimContentions(INTERNAL_SERVER_ERROR_CLAIM_ID)); - assertSame(HttpStatus.INTERNAL_SERVER_ERROR, ex.getStatus()); + private void mockResponseForUrl(Stubber response, String claimUrl, HttpMethod httpMethod) { + response + .when(restTemplate) + .exchange( + ArgumentMatchers.eq(claimUrl), + ArgumentMatchers.eq(httpMethod), + ArgumentMatchers.any(HttpEntity.class), + ArgumentMatchers.eq(String.class)); } - @Test - public void testCreateClaimContentions_201() { - String resp200Body = "{\"contentionIds\":[1]}"; - ResponseEntity resp200 = new ResponseEntity<>(resp200Body, HttpStatus.CREATED); + private void mock2xxResponse( + HttpMethod httpMethod, + String urlFormat, + HttpStatus httpStatus, + Class responseType, + String responseFile) + throws Exception { + ResponseEntity responseEntity; + if (Objects.isNull(responseFile)) { + responseEntity = new ResponseEntity<>(httpStatus); + } else { + T body = mapper.readValue(getTestData(responseFile), responseType); + responseEntity = new ResponseEntity<>(body, httpStatus); + } mockResponseForUrl( - Mockito.doReturn(resp200), formatClaimUrl(CONTENTION, GOOD_CLAIM_ID), HttpMethod.POST); - CreateClaimContentionsRequest request = - CreateClaimContentionsRequest.builder() - .claimId(GOOD_CLAIM_ID) - .createContentions( - List.of( - Contention.builder() - .medicalInd(true) - .beginDate(OffsetDateTime.parse("2023-09-27T00:00:00-06:00")) - .contentionTypeCode("NEW") - .claimantText("tendinitis/bilateral") - .build())) - .build(); - CreateClaimContentionsResponse response = service.createClaimContentions(request); - assertResponseIsSuccess(response, HttpStatus.CREATED); - assertNotNull(response.getContentionIds()); - assertEquals(1, response.getContentionIds().size()); - assertEquals(1, response.getContentionIds().get(0)); + Mockito.doReturn(responseEntity), + formatClaimUrl(urlFormat, BipApiServiceTest.GOOD_CLAIM_ID), + httpMethod, + responseType); } - @ParameterizedTest(name = "testCreateClaimContentions_{0}") - @EnumSource(TestCase.class) - public void testCreateClaimContentions_Non2xx(TestCase test) throws JsonProcessingException { - mockResponseForUrl( - Mockito.doThrow(test.ex), formatClaimUrl(CONTENTION, test.claimId), HttpMethod.POST); - CreateClaimContentionsRequest request = - CreateClaimContentionsRequest.builder() - .claimId(test.claimId) - .createContentions( - List.of( - Contention.builder() - .medicalInd(true) - .beginDate(OffsetDateTime.parse("2023-09-27T00:00:00-06:00")) - .contentionTypeCode("NEW") - .claimantText("tendinitis/bilateral") - .build())) - .build(); - HttpStatusCodeException ex = - Assertions.assertThrows(test.ex.getClass(), () -> service.createClaimContentions(request)); - assertResponseHasMessageWithStatus(ex.getResponseBodyAsString(), test.status); + private void mockExceptionResponse( + Exception exception, String url, HttpMethod httpMethod, Class responseType) { + mockResponseForUrl(Mockito.doThrow(exception), url, httpMethod, responseType); } - @Test - public void testCreateClaimContentionsInternalServerError_500() { - mockResponseForUrl( - Mockito.doThrow(new RuntimeException("nope")), - formatClaimUrl(CONTENTION, INTERNAL_SERVER_ERROR_CLAIM_ID), - HttpMethod.POST); - CreateClaimContentionsRequest request = - CreateClaimContentionsRequest.builder() - .claimId(INTERNAL_SERVER_ERROR_CLAIM_ID) - .createContentions( - List.of( - Contention.builder() - .medicalInd(true) - .beginDate(OffsetDateTime.parse("2023-09-27T00:00:00-06:00")) - .contentionTypeCode("NEW") - .claimantText("tendinitis/bilateral") - .build())) - .build(); - BipException ex = - Assertions.assertThrows(BipException.class, () -> service.createClaimContentions(request)); - assertSame(HttpStatus.INTERNAL_SERVER_ERROR, ex.getStatus()); - } + @Nested + public class GetClaimDetails { + @Test + public void testGetClaimDetails_200() throws Exception { + mock2xxResponse( + HttpMethod.GET, CLAIM_DETAILS, HttpStatus.OK, GetClaimResponse.class, CLAIM_RESPONSE_200); - @Test - public void testUpdateClaimContentions_200() throws Exception { - String resp200Body = getTestData(CONTENTION_RESPONSE_200); - ResponseEntity resp200 = ResponseEntity.ok(resp200Body); - mockResponseForUrl( - Mockito.doReturn(resp200), formatClaimUrl(CONTENTION, GOOD_CLAIM_ID), HttpMethod.PUT); - - UpdateClaimContentionsRequest request = - UpdateClaimContentionsRequest.builder() - .claimId(GOOD_CLAIM_ID) - .updateContentions( - List.of( - ExistingContention.builder() - .medicalInd(true) - .beginDate(OffsetDateTime.parse("2023-09-27T00:00:00-06:00")) - .contentionTypeCode("NEW") - .claimantText("tendinitis/bilateral") - .build())) - .build(); - - UpdateClaimContentionsResponse response = service.updateClaimContentions(request); - assertResponseIsSuccess(response, HttpStatus.OK); - } + GetClaimResponse result = service.getClaimDetails(GOOD_CLAIM_ID); + assertResponseIsSuccess(result, HttpStatus.OK); + } - @ParameterizedTest(name = "testUpdateClaimContentions_{0}") - @EnumSource(TestCase.class) - public void testUpdateClaimContentions_Non2xx(TestCase test) throws JsonProcessingException { - mockResponseForUrl( - Mockito.doThrow(test.ex), formatClaimUrl(CONTENTION, test.claimId), HttpMethod.PUT); - UpdateClaimContentionsRequest request = - UpdateClaimContentionsRequest.builder() - .claimId(test.claimId) - .updateContentions( - List.of( - ExistingContention.builder() - .medicalInd(true) - .beginDate(OffsetDateTime.parse("2023-09-27T00:00:00-06:00")) - .contentionTypeCode("NEW") - .claimantText("tendinitis/bilateral") - .build())) - .build(); - HttpStatusCodeException ex = - Assertions.assertThrows(test.ex.getClass(), () -> service.updateClaimContentions(request)); - assertResponseHasMessageWithStatus(ex.getResponseBodyAsString(), test.status); + @ParameterizedTest(name = "testGetClaimDetails_{0}") + @EnumSource(TestCase.class) + public void testGetClaimDetails_non2xx(TestCase test) throws Exception { + mockExceptionResponse( + test.ex, + formatClaimUrl(CLAIM_DETAILS, test.claimId), + HttpMethod.GET, + GetClaimResponse.class); + + Exception ex = + Assertions.assertThrows(test.ex.getClass(), () -> service.getClaimDetails(test.claimId)); + assertResponseExceptionWithStatus(ex, test.status); + } } - @Test - public void testUpdateClaimContentionsInternalServerError_500() { - mockResponseForUrl( - Mockito.doThrow(new RuntimeException("nope")), - formatClaimUrl(CONTENTION, INTERNAL_SERVER_ERROR_CLAIM_ID), - HttpMethod.PUT); - UpdateClaimContentionsRequest request = - UpdateClaimContentionsRequest.builder() - .claimId(INTERNAL_SERVER_ERROR_CLAIM_ID) - .updateContentions( - List.of( - ExistingContention.builder() - .medicalInd(true) - .beginDate(OffsetDateTime.parse("2023-09-27T00:00:00-06:00")) - .contentionTypeCode("NEW") - .claimantText("tendinitis/bilateral") - .build())) - .build(); - BipException ex = - Assertions.assertThrows(BipException.class, () -> service.updateClaimContentions(request)); - assertSame(HttpStatus.INTERNAL_SERVER_ERROR, ex.getStatus()); - } + @Nested + public class GetClaimContentions { + @Test + public void testGetClaimContention_200() throws Exception { + mock2xxResponse( + HttpMethod.GET, + CONTENTION, + HttpStatus.OK, + GetClaimContentionsResponse.class, + CONTENTION_RESPONSE_200); + + GetClaimContentionsResponse result = service.getClaimContentions(GOOD_CLAIM_ID); + assertResponseIsSuccess(result, HttpStatus.OK); + assertEquals(1, result.getContentions().size()); + } - @Test - public void testCancelClaim_200() { - ResponseEntity resp200 = ResponseEntity.ok("{}"); - mockResponseForUrl( - Mockito.doReturn(resp200), formatClaimUrl(CANCEL_CLAIM, GOOD_CLAIM_ID), HttpMethod.PUT); - CancelClaimRequest request = - CancelClaimRequest.builder() - .claimId(GOOD_CLAIM_ID) - .closeReasonText("because we are testing") - .lifecycleStatusReasonCode("60") - .build(); - CancelClaimResponse response = service.cancelClaim(request); - assertResponseIsSuccess(response, HttpStatus.OK); - } + @Test + public void testGetClaimContention_204() throws Exception { + mock2xxResponse( + HttpMethod.GET, + CONTENTION, + HttpStatus.NO_CONTENT, + GetClaimContentionsResponse.class, + null); + + GetClaimContentionsResponse result = service.getClaimContentions(GOOD_CLAIM_ID); + assertResponseIsSuccess(result, HttpStatus.NO_CONTENT); + assertNull(result.getContentions()); + } - @ParameterizedTest(name = "testCancelClaim_{0}") - @EnumSource(TestCase.class) - public void testCancelClaim_Non2xx(TestCase test) throws JsonProcessingException { - mockResponseForUrl( - Mockito.doThrow(test.ex), formatClaimUrl(CANCEL_CLAIM, test.claimId), HttpMethod.PUT); - CancelClaimRequest request = - CancelClaimRequest.builder() - .claimId(test.claimId) - .closeReasonText("because we are testing") - .lifecycleStatusReasonCode("60") - .build(); - HttpStatusCodeException ex = - Assertions.assertThrows(test.ex.getClass(), () -> service.cancelClaim(request)); - assertResponseHasMessageWithStatus(ex.getResponseBodyAsString(), test.status); + @ParameterizedTest(name = "testGetClaimContentions_{0}") + @EnumSource(TestCase.class) + public void testGetClaimContentions_Non2xx(TestCase test) throws Exception { + mockExceptionResponse( + test.ex, + formatClaimUrl(CONTENTION, test.claimId), + HttpMethod.GET, + GetClaimContentionsResponse.class); + + Exception ex = + Assertions.assertThrows( + test.ex.getClass(), () -> service.getClaimContentions(test.claimId)); + assertResponseExceptionWithStatus(ex, test.status); + } } - @Test - public void testCancelClaimInternalServerError_500() { - mockResponseForUrl( - Mockito.doThrow(new RuntimeException("nope")), - formatClaimUrl(CANCEL_CLAIM, INTERNAL_SERVER_ERROR_CLAIM_ID), - HttpMethod.PUT); - - CancelClaimRequest request = - CancelClaimRequest.builder() - .claimId(INTERNAL_SERVER_ERROR_CLAIM_ID) - .closeReasonText("because we are testing") - .lifecycleStatusReasonCode("60") - .build(); - BipException ex = - Assertions.assertThrows(BipException.class, () -> service.cancelClaim(request)); - assertSame(HttpStatus.INTERNAL_SERVER_ERROR, ex.getStatus()); - } + @Nested + public class CreateClaimContentions { + @Test + public void testCreateClaimContentions_201() throws Exception { + mock2xxResponse( + HttpMethod.POST, + CONTENTION, + HttpStatus.CREATED, + CreateClaimContentionsResponse.class, + CONTENTION_RESPONSE_201); + CreateClaimContentionsRequest request = + CreateClaimContentionsRequest.builder() + .claimId(GOOD_CLAIM_ID) + .createContentions( + List.of( + Contention.builder() + .medicalInd(true) + .beginDate(OffsetDateTime.parse("2023-09-27T00:00:00-06:00")) + .contentionTypeCode("NEW") + .claimantText("tendinitis/bilateral") + .build())) + .build(); + CreateClaimContentionsResponse response = service.createClaimContentions(request); + assertResponseIsSuccess(response, HttpStatus.CREATED); + assertNotNull(response.getContentionIds()); + assertEquals(1, response.getContentionIds().size()); + assertEquals(1, response.getContentionIds().get(0)); + } - @Test - public void testPutLifecycleStatus_200() { - ResponseEntity resp200 = ResponseEntity.ok("{}"); - mockResponseForUrl( - Mockito.doReturn(resp200), - formatClaimUrl(CLAIM_LIFECYCLE_STATUS, GOOD_CLAIM_ID), - HttpMethod.PUT); - PutClaimLifecycleRequest request = - PutClaimLifecycleRequest.builder() - .claimId(GOOD_CLAIM_ID) - .claimLifecycleStatus("Just a test") - .build(); - PutClaimLifecycleResponse response = service.putClaimLifecycleStatus(request); - assertResponseIsSuccess(response, HttpStatus.OK); + @ParameterizedTest(name = "testCreateClaimContentions_{0}") + @EnumSource(TestCase.class) + public void testCreateClaimContentions_Non2xx(TestCase test) throws Exception { + mockExceptionResponse( + test.ex, + formatClaimUrl(CONTENTION, test.claimId), + HttpMethod.POST, + CreateClaimContentionsResponse.class); + CreateClaimContentionsRequest request = + CreateClaimContentionsRequest.builder() + .claimId(test.claimId) + .createContentions( + List.of( + Contention.builder() + .medicalInd(true) + .beginDate(OffsetDateTime.parse("2023-09-27T00:00:00-06:00")) + .contentionTypeCode("NEW") + .claimantText("tendinitis/bilateral") + .build())) + .build(); + Exception ex = + Assertions.assertThrows( + test.ex.getClass(), () -> service.createClaimContentions(request)); + assertResponseExceptionWithStatus(ex, test.status); + } } - @ParameterizedTest(name = "testPutLifecycleStatus_{0}") - @EnumSource(TestCase.class) - public void testPutLifecycleStatus_Non2xx(TestCase test) throws JsonProcessingException { - mockResponseForUrl( - Mockito.doThrow(test.ex), - formatClaimUrl(CLAIM_LIFECYCLE_STATUS, test.claimId), - HttpMethod.PUT); - PutClaimLifecycleRequest request = - PutClaimLifecycleRequest.builder() - .claimId(test.claimId) - .claimLifecycleStatus("Just a test") - .build(); - HttpStatusCodeException ex = - Assertions.assertThrows(test.ex.getClass(), () -> service.putClaimLifecycleStatus(request)); - assertResponseHasMessageWithStatus(ex.getResponseBodyAsString(), test.status); - } + @Nested + public class UpdateClaimContentions { + @Test + public void testUpdateClaimContentions_200() throws Exception { + mock2xxResponse( + HttpMethod.PUT, + CONTENTION, + HttpStatus.OK, + UpdateClaimContentionsResponse.class, + CONTENTION_RESPONSE_200); + UpdateClaimContentionsRequest request = + UpdateClaimContentionsRequest.builder() + .claimId(GOOD_CLAIM_ID) + .updateContentions( + List.of( + ExistingContention.builder() + .medicalInd(true) + .beginDate(OffsetDateTime.parse("2023-09-27T00:00:00-06:00")) + .contentionTypeCode("NEW") + .claimantText("tendinitis/bilateral") + .build())) + .build(); + + UpdateClaimContentionsResponse response = service.updateClaimContentions(request); + assertResponseIsSuccess(response, HttpStatus.OK); + } - @Test - public void testPutLifecycleStatusInternalServerError_500() { - mockResponseForUrl( - Mockito.doThrow(new RuntimeException("nope")), - formatClaimUrl(CLAIM_LIFECYCLE_STATUS, INTERNAL_SERVER_ERROR_CLAIM_ID), - HttpMethod.PUT); - - PutClaimLifecycleRequest request = - PutClaimLifecycleRequest.builder() - .claimId(INTERNAL_SERVER_ERROR_CLAIM_ID) - .claimLifecycleStatus("Just a test") - .build(); - BipException ex = - Assertions.assertThrows(BipException.class, () -> service.putClaimLifecycleStatus(request)); - assertSame(HttpStatus.INTERNAL_SERVER_ERROR, ex.getStatus()); + @ParameterizedTest(name = "testUpdateClaimContentions_{0}") + @EnumSource(TestCase.class) + public void testUpdateClaimContentions_Non2xx(TestCase test) throws Exception { + mockExceptionResponse( + test.ex, + formatClaimUrl(CONTENTION, test.claimId), + HttpMethod.PUT, + UpdateClaimContentionsResponse.class); + UpdateClaimContentionsRequest request = + UpdateClaimContentionsRequest.builder() + .claimId(test.claimId) + .updateContentions( + List.of( + ExistingContention.builder() + .medicalInd(true) + .beginDate(OffsetDateTime.parse("2023-09-27T00:00:00-06:00")) + .contentionTypeCode("NEW") + .claimantText("tendinitis/bilateral") + .build())) + .build(); + Exception ex = + Assertions.assertThrows( + test.ex.getClass(), () -> service.updateClaimContentions(request)); + assertResponseExceptionWithStatus(ex, test.status); + } } - @Test - public void testPutTemporaryStationOfJurisdiction_200() { - ResponseEntity resp200 = ResponseEntity.ok("{}"); + @Nested + public class CancelClaim { + @Test + public void testCancelClaim_200() throws Exception { + mock2xxResponse(HttpMethod.PUT, CANCEL_CLAIM, HttpStatus.OK, CancelClaimResponse.class, null); + CancelClaimRequest request = + CancelClaimRequest.builder() + .claimId(GOOD_CLAIM_ID) + .closeReasonText("because we are testing") + .lifecycleStatusReasonCode("60") + .build(); + CancelClaimResponse response = service.cancelClaim(request); + assertResponseIsSuccess(response, HttpStatus.OK); + } - mockResponseForUrl( - Mockito.doReturn(resp200), - formatClaimUrl(TEMP_STATION_OF_JURISDICTION, GOOD_CLAIM_ID), - HttpMethod.PUT); - - PutTempStationOfJurisdictionRequest request = - PutTempStationOfJurisdictionRequest.builder() - .claimId(GOOD_CLAIM_ID) - .tempStationOfJurisdiction("398") - .build(); - PutTempStationOfJurisdictionResponse result = service.putTempStationOfJurisdiction(request); - assertResponseIsSuccess(result, HttpStatus.OK); + @ParameterizedTest(name = "testCancelClaim_{0}") + @EnumSource(TestCase.class) + public void testCancelClaim_Non2xx(TestCase test) throws Exception { + mockExceptionResponse( + test.ex, + formatClaimUrl(CANCEL_CLAIM, test.claimId), + HttpMethod.PUT, + CancelClaimResponse.class); + CancelClaimRequest request = + CancelClaimRequest.builder() + .claimId(test.claimId) + .closeReasonText("because we are testing") + .lifecycleStatusReasonCode("60") + .build(); + Exception ex = + Assertions.assertThrows(test.ex.getClass(), () -> service.cancelClaim(request)); + assertResponseExceptionWithStatus(ex, test.status); + } } - @Test - public void testPutTemporaryStationOfJurisdiction_404() throws Exception { - String resp404Body = getTestData(CLAIM_RESPONSE_404); + @Nested + public class PutLifecycleStatus { + @Test + public void testPutLifecycleStatus_200() throws Exception { + mock2xxResponse( + HttpMethod.PUT, + CLAIM_LIFECYCLE_STATUS, + HttpStatus.OK, + PutClaimLifecycleResponse.class, + null); + + PutClaimLifecycleRequest request = + PutClaimLifecycleRequest.builder() + .claimId(GOOD_CLAIM_ID) + .claimLifecycleStatus("Just a test") + .build(); + PutClaimLifecycleResponse response = service.putClaimLifecycleStatus(request); + assertResponseIsSuccess(response, HttpStatus.OK); + } - mockResponseForUrl( - Mockito.doThrow( - new HttpClientErrorException( - HttpStatus.NOT_FOUND, - HttpStatus.NOT_FOUND.name(), - resp404Body.getBytes(), - Charset.defaultCharset())), - formatClaimUrl(TEMP_STATION_OF_JURISDICTION, BAD_CLAIM_ID), - HttpMethod.PUT); - - PutTempStationOfJurisdictionRequest request = - PutTempStationOfJurisdictionRequest.builder() - .claimId(BAD_CLAIM_ID) - .tempStationOfJurisdiction("398") - .build(); - HttpStatusCodeException ex = - Assertions.assertThrows( - HttpStatusCodeException.class, () -> service.putTempStationOfJurisdiction(request)); - assertResponseHasMessageWithStatus(ex.getResponseBodyAsString(), HttpStatus.NOT_FOUND); + @ParameterizedTest(name = "testPutLifecycleStatus_{0}") + @EnumSource(TestCase.class) + public void testPutLifecycleStatus_Non2xx(TestCase test) throws Exception { + mockExceptionResponse( + test.ex, + formatClaimUrl(CLAIM_LIFECYCLE_STATUS, test.claimId), + HttpMethod.PUT, + PutClaimLifecycleResponse.class); + PutClaimLifecycleRequest request = + PutClaimLifecycleRequest.builder() + .claimId(test.claimId) + .claimLifecycleStatus("Just a test") + .build(); + Exception ex = + Assertions.assertThrows( + test.ex.getClass(), () -> service.putClaimLifecycleStatus(request)); + assertResponseExceptionWithStatus(ex, test.status); + } } - @Test - public void testPutTemporaryStationOfJurisdictionDownstreamServerError_500() throws Exception { - String resp500Body = getTestData(RESPONSE_500); - mockResponseForUrl( - Mockito.doThrow( - new HttpServerErrorException( - HttpStatus.INTERNAL_SERVER_ERROR, - HttpStatus.INTERNAL_SERVER_ERROR.name(), - resp500Body.getBytes(), - Charset.defaultCharset())), - formatClaimUrl(TEMP_STATION_OF_JURISDICTION, INTERNAL_SERVER_ERROR_CLAIM_ID), - HttpMethod.PUT); - - PutTempStationOfJurisdictionRequest request = - PutTempStationOfJurisdictionRequest.builder() - .claimId(INTERNAL_SERVER_ERROR_CLAIM_ID) - .tempStationOfJurisdiction("398") - .build(); - HttpStatusCodeException ex = - Assertions.assertThrows( - HttpStatusCodeException.class, () -> service.putTempStationOfJurisdiction(request)); - assertResponseHasMessageWithStatus( - ex.getResponseBodyAsString(), HttpStatus.INTERNAL_SERVER_ERROR); - } + @Nested + public class PutTemporaryStationOfJursidiction { + @ParameterizedTest + @NullAndEmptySource + @CsvSource(value = {"398"}) + public void testPutTemporaryStationOfJurisdiction_200(String tsoj) throws Exception { + mock2xxResponse( + HttpMethod.PUT, + TEMP_STATION_OF_JURISDICTION, + HttpStatus.OK, + PutTempStationOfJurisdictionResponse.class, + null); + + PutTempStationOfJurisdictionRequest request = + PutTempStationOfJurisdictionRequest.builder() + .claimId(GOOD_CLAIM_ID) + .tempStationOfJurisdiction(tsoj) + .build(); + PutTempStationOfJurisdictionResponse result = service.putTempStationOfJurisdiction(request); + assertResponseIsSuccess(result, HttpStatus.OK); + } - @Test - public void testPutTemporaryStationOfJurisdictionInternalServerError_500() { - mockResponseForUrl( - Mockito.doThrow(new RuntimeException("nope")), - formatClaimUrl(TEMP_STATION_OF_JURISDICTION, INTERNAL_SERVER_ERROR_CLAIM_ID), - HttpMethod.PUT); - - PutTempStationOfJurisdictionRequest request = - PutTempStationOfJurisdictionRequest.builder() - .claimId(INTERNAL_SERVER_ERROR_CLAIM_ID) - .tempStationOfJurisdiction("398") - .build(); - BipException ex = - Assertions.assertThrows( - BipException.class, () -> service.putTempStationOfJurisdiction(request)); - assertSame(HttpStatus.INTERNAL_SERVER_ERROR, ex.getStatus()); + @ParameterizedTest(name = "testPutTemporaryStationOfJurisdiction_{0}") + @EnumSource(TestCase.class) + public void testPutTemporaryStationOfJurisdiction_Non2xx(TestCase test) throws Exception { + mockExceptionResponse( + test.ex, + formatClaimUrl(TEMP_STATION_OF_JURISDICTION, test.claimId), + HttpMethod.PUT, + PutTempStationOfJurisdictionResponse.class); + + PutTempStationOfJurisdictionRequest request = + PutTempStationOfJurisdictionRequest.builder() + .claimId(test.claimId) + .tempStationOfJurisdiction("398") + .build(); + Exception ex = + Assertions.assertThrows( + test.ex.getClass(), () -> service.putTempStationOfJurisdiction(request)); + assertResponseExceptionWithStatus(ex, test.status); + } } @Test @@ -635,6 +513,18 @@ private void assertResponseIsSuccess(BipPayloadResponse response, HttpStatus sta assertNull(response.getMessages()); } + private void assertResponseExceptionWithStatus(Exception ex, HttpStatus expected) + throws JsonProcessingException { + if (ex instanceof HttpStatusCodeException httpStatusCodeException) { + assertResponseHasMessageWithStatus( + httpStatusCodeException.getResponseBodyAsString(), expected); + } else if (ex instanceof BipException bipException) { + assertSame(expected, bipException.getStatus()); + } else { + fail("Unsupported Error Type"); + } + } + private void assertResponseHasMessageWithStatus(String response, HttpStatus expected) throws JsonProcessingException { BipPayloadResponse bipResponse = MAPPER.readValue(response, BipPayloadResponse.class); @@ -646,28 +536,41 @@ private void assertResponseHasMessageWithStatus(String response, HttpStatus expe assertEquals(expected.name(), message.getHttpStatus()); } + private enum ErrType { + CLIENT, + SERVER, + INTERNAL + } + public enum TestCase { - NOT_FOUND(NOT_FOUND_CLAIM_ID, HttpStatus.NOT_FOUND, CLAIM_RESPONSE_404), + NOT_FOUND(ErrType.CLIENT, NOT_FOUND_CLAIM_ID, HttpStatus.NOT_FOUND, CLAIM_RESPONSE_404), DOWNSTREAM_ERROR( - INTERNAL_SERVER_ERROR_CLAIM_ID, HttpStatus.INTERNAL_SERVER_ERROR, RESPONSE_500); + ErrType.SERVER, + INTERNAL_SERVER_ERROR_CLAIM_ID, + HttpStatus.INTERNAL_SERVER_ERROR, + RESPONSE_500), + BIP_INTERNAL( + ErrType.INTERNAL, + INTERNAL_SERVER_ERROR_CLAIM_ID, + HttpStatus.INTERNAL_SERVER_ERROR, + RESPONSE_500); final long claimId; final HttpStatus status; - final HttpStatusCodeException ex; + final Exception ex; @SneakyThrows - TestCase(long claimId, HttpStatus status, String dataFile) { + TestCase(ErrType type, long claimId, HttpStatus status, String dataFile) { this.claimId = claimId; this.status = status; this.ex = - status == HttpStatus.INTERNAL_SERVER_ERROR - ? new HttpServerErrorException( - status, status.name(), getTestData(dataFile).getBytes(), Charset.defaultCharset()) - : new HttpClientErrorException( - status, - status.name(), - getTestData(dataFile).getBytes(), - Charset.defaultCharset()); + switch (type) { + case CLIENT -> new HttpClientErrorException( + status, status.name(), getTestData(dataFile).getBytes(), Charset.defaultCharset()); + case SERVER -> new HttpServerErrorException( + status, status.name(), getTestData(dataFile).getBytes(), Charset.defaultCharset()); + case INTERNAL -> new RuntimeException("nope"); + }; } } } diff --git a/svc-bip-api/src/test/resources/bip-test-data/claim_response_404.json b/svc-bip-api/src/test/resources/bip-test-data/claim_response_404.json index a841b5db1f..9279d8ce4a 100644 --- a/svc-bip-api/src/test/resources/bip-test-data/claim_response_404.json +++ b/svc-bip-api/src/test/resources/bip-test-data/claim_response_404.json @@ -1,7 +1,7 @@ { "messages": [ { - "timestamp": "2023-02-06T18:10:59.288", + "timestamp": "2023-02-06T18:10:59.288Z", "key": "bip.vetservices.claims.contentions.claimidnotfound", "severity": "FATAL", "status": "404", diff --git a/svc-bip-api/src/test/resources/bip-test-data/contention_response_201.json b/svc-bip-api/src/test/resources/bip-test-data/contention_response_201.json new file mode 100644 index 0000000000..2a1a7dff90 --- /dev/null +++ b/svc-bip-api/src/test/resources/bip-test-data/contention_response_201.json @@ -0,0 +1,5 @@ +{ + "contentionIds": [ + 1 + ] +} diff --git a/svc-bip-api/src/test/resources/bip-test-data/contention_response_412.json b/svc-bip-api/src/test/resources/bip-test-data/contention_response_412.json index 3a4625a03a..1aaef709de 100644 --- a/svc-bip-api/src/test/resources/bip-test-data/contention_response_412.json +++ b/svc-bip-api/src/test/resources/bip-test-data/contention_response_412.json @@ -1,7 +1,7 @@ { "messages": [ { - "timestamp": "2023-02-06T18:27:32.316", + "timestamp": "2023-02-06T18:27:32.316Z", "key": "bip.vetservices.lastmodified.not.matched", "severity": "ERROR", "status": "412", diff --git a/svc-bip-api/src/test/resources/bip-test-data/response_500.json b/svc-bip-api/src/test/resources/bip-test-data/response_500.json index 83b80fbd19..fa4f75d48d 100644 --- a/svc-bip-api/src/test/resources/bip-test-data/response_500.json +++ b/svc-bip-api/src/test/resources/bip-test-data/response_500.json @@ -1,7 +1,7 @@ { "messages": [ { - "timestamp": "2023-02-06T18:10:59.288", + "timestamp": "2023-02-06T18:10:59.288Z", "key": "bip.vetservices.internalServerError", "severity": "FATAL", "status": "500", From 39dfc58c46f0c55c31783d953dfb7a25c91075ec Mon Sep 17 00:00:00 2001 From: Derek Fitchett <135860892+dfitchett@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:01:13 -0700 Subject: [PATCH 26/67] mock-bip-claims-api: Updated mocks for new EE EP Merge End2End tests. (#2746) Updated mocks for new EE EP Merge End2End tests. --- .../config/ClaimIdConstants.java | 3 +- .../controller/ContentionsController.java | 6 +- .../src/main/resources/mock-claims.json | 86 ++++++++++++++++--- 3 files changed, 82 insertions(+), 13 deletions(-) diff --git a/mocks/mock-bip-claims-api/src/main/java/gov/va/vro/mockbipclaims/config/ClaimIdConstants.java b/mocks/mock-bip-claims-api/src/main/java/gov/va/vro/mockbipclaims/config/ClaimIdConstants.java index 3b903effd2..14eccc56c3 100644 --- a/mocks/mock-bip-claims-api/src/main/java/gov/va/vro/mockbipclaims/config/ClaimIdConstants.java +++ b/mocks/mock-bip-claims-api/src/main/java/gov/va/vro/mockbipclaims/config/ClaimIdConstants.java @@ -12,7 +12,8 @@ public class ClaimIdConstants { public static final int CLAIM_ID_SET_TSOJ_YIELDS_500 = 5003; // GET /claims/{claimId}/contentions - public static final int CLAIM_ID_GET_CONTENTIONS_YIELDS_500 = 5004; + public static final int CLAIM_ID_GET_PENDING_EP_CONTENTIONS_YIELDS_500 = 5004; + public static final int CLAIM_ID_GET_SUPP_EP_CONTENTIONS_YIELDS_500 = 50041; // PUT /claims/{claimId}/contentions public static final int CLAIM_ID_UPDATE_CONTENTIONS_YIELDS_500 = 5005; // POST /claims/{claimId}/contentions diff --git a/mocks/mock-bip-claims-api/src/main/java/gov/va/vro/mockbipclaims/controller/ContentionsController.java b/mocks/mock-bip-claims-api/src/main/java/gov/va/vro/mockbipclaims/controller/ContentionsController.java index 2dd6ca0072..d9da363bd3 100644 --- a/mocks/mock-bip-claims-api/src/main/java/gov/va/vro/mockbipclaims/controller/ContentionsController.java +++ b/mocks/mock-bip-claims-api/src/main/java/gov/va/vro/mockbipclaims/controller/ContentionsController.java @@ -2,7 +2,8 @@ import static gov.va.vro.mockbipclaims.config.ClaimIdConstants.CLAIM_ID_ALL_ENDPOINTS_YIELDS_500; import static gov.va.vro.mockbipclaims.config.ClaimIdConstants.CLAIM_ID_CREATE_CONTENTIONS_YIELDS_500; -import static gov.va.vro.mockbipclaims.config.ClaimIdConstants.CLAIM_ID_GET_CONTENTIONS_YIELDS_500; +import static gov.va.vro.mockbipclaims.config.ClaimIdConstants.CLAIM_ID_GET_PENDING_EP_CONTENTIONS_YIELDS_500; +import static gov.va.vro.mockbipclaims.config.ClaimIdConstants.CLAIM_ID_GET_SUPP_EP_CONTENTIONS_YIELDS_500; import static gov.va.vro.mockbipclaims.config.ClaimIdConstants.CLAIM_ID_UPDATE_CONTENTIONS_YIELDS_500; import gov.va.vro.mockbipclaims.api.ContentionsApi; @@ -79,7 +80,8 @@ public ResponseEntity getContentionsForClaim(Long c return createClaim404(response, claimId); } if (claimId == CLAIM_ID_ALL_ENDPOINTS_YIELDS_500 - || claimId == CLAIM_ID_GET_CONTENTIONS_YIELDS_500) { + || claimId == CLAIM_ID_GET_PENDING_EP_CONTENTIONS_YIELDS_500 + || claimId == CLAIM_ID_GET_SUPP_EP_CONTENTIONS_YIELDS_500) { return create500(response); } diff --git a/mocks/mock-bip-claims-api/src/main/resources/mock-claims.json b/mocks/mock-bip-claims-api/src/main/resources/mock-claims.json index ecba205c3a..db86281ae6 100644 --- a/mocks/mock-bip-claims-api/src/main/resources/mock-claims.json +++ b/mocks/mock-bip-claims-api/src/main/resources/mock-claims.json @@ -814,7 +814,11 @@ "claimId": 10001, "phase": "Claim Received", "endProductCode": "400", - "claimLifecycleStatus": "Open" + "claimLifecycleStatus": "Open", + "benefitClaimType": { + "name": "eBenefits 526EZ-Supplemental (400)", + "code": "400SUPP" + } }, "contentions": [ { @@ -840,7 +844,11 @@ "claimId": 10002, "phase": "Claim Received", "endProductCode": "400", - "claimLifecycleStatus": "Open" + "claimLifecycleStatus": "Open", + "benefitClaimType": { + "name": "eBenefits 526EZ-Supplemental (400)", + "code": "400SUPP" + } }, "contentions": [ { @@ -866,7 +874,11 @@ "claimId": 10003, "phase": "Claim Received", "endProductCode": "400", - "claimLifecycleStatus": "Open" + "claimLifecycleStatus": "Open", + "benefitClaimType": { + "name": "eBenefits 526EZ-Supplemental (400)", + "code": "400SUPP" + } }, "contentions": [ { @@ -892,7 +904,11 @@ "claimId": 10004, "phase": "Claim Received", "endProductCode": "400", - "claimLifecycleStatus": "Open" + "claimLifecycleStatus": "Open", + "benefitClaimType": { + "name": "eBenefits 526EZ-Supplemental (400)", + "code": "400SUPP" + } }, "contentions": [ { @@ -931,7 +947,11 @@ "claimId": 10005, "phase": "Claim Received", "endProductCode": "400", - "claimLifecycleStatus": "Open" + "claimLifecycleStatus": "Open", + "benefitClaimType": { + "name": "eBenefits 526EZ-Supplemental (400)", + "code": "400SUPP" + } }, "contentions": [ { @@ -970,7 +990,11 @@ "claimId": 10006, "phase": "Claim Received", "endProductCode": "400", - "claimLifecycleStatus": "Open" + "claimLifecycleStatus": "Open", + "benefitClaimType": { + "name": "eBenefits 526EZ-Supplemental (400)", + "code": "400SUPP" + } }, "contentions": [] }, @@ -1005,7 +1029,12 @@ "claimId": 5002, "phase": "Claim Received", "tempStationOfJurisdiction": "398", - "endProductCode": "400" + "endProductCode": "400", + "claimLifecycleStatus": "Open", + "benefitClaimType": { + "name": "eBenefits 526EZ-Supplemental (400)", + "code": "400SUPP" + } }, "contentions": [ { @@ -1028,7 +1057,12 @@ "claimId": 5003, "phase": "Claim Received", "tempStationOfJurisdiction": "398", - "endProductCode": "400" + "endProductCode": "400", + "claimLifecycleStatus": "Open", + "benefitClaimType": { + "name": "eBenefits 526EZ-Supplemental (400)", + "code": "400SUPP" + } }, "contentions": [ { @@ -1045,7 +1079,7 @@ }, { "description": [ - "Employee Experience EP Merge End 2 End Testing - Fails at get claim contentions" + "Employee Experience EP Merge End 2 End Testing - Fails at get pending claim contentions" ], "claimDetail": { "claimId": 5004, @@ -1066,6 +1100,33 @@ } ] }, + { + "description": [ + "Employee Experience EP Merge End 2 End Testing - Fails at get ep400 claim contentions" + ], + "claimDetail": { + "claimId": 50041, + "phase": "Claim Received", + "endProductCode": "400", + "claimLifecycleStatus": "Open", + "benefitClaimType": { + "name": "eBenefits 526EZ-Supplemental (400)", + "code": "400SUPP" + } + }, + "contentions": [ + { + "medicalInd": true, + "beginDate": "2023-01-01T00:00:00Z", + "contentionTypeCode": "NEW", + "classificationType": 1250, + "diagnosticTypeCode": "6100", + "claimantText": "tendinitis/bilateral", + "contentionId": "1", + "lastModified": "2023-01-01T00:00:01Z" + } + ] + }, { "description": [ "Employee Experience EP Merge End 2 End Testing - Fails at update claim contentions" @@ -1074,7 +1135,12 @@ "claimId": 5005, "phase": "Claim Received", "tempStationOfJurisdiction": "398", - "endProductCode": "400" + "endProductCode": "400", + "claimLifecycleStatus": "Open", + "benefitClaimType": { + "name": "eBenefits 526EZ-Supplemental (400)", + "code": "400SUPP" + } }, "contentions": [ { From 66cca254284806b525b3e340f6bda1377fbd7fc9 Mon Sep 17 00:00:00 2001 From: Derek Fitchett <135860892+dfitchett@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:18:09 -0700 Subject: [PATCH 27/67] EP Merge: add checks for get EP400 claim details (#2747) * Added checks for EP 400 claim details to only allow claims with EP codes in [400, 409] with a benefit claim type code of '400SUPP' --- .../test_process_new_merge_request.py | 16 +- .../end_to_end/test_restart.py | 16 +- .../integration/test_merge_request.py | 77 ++++++-- .../src/python_src/graph_export.py | 2 +- .../src/python_src/schema/claim.py | 10 + .../src/python_src/schema/merge_job.py | 2 + .../python_src/service/ep_merge_machine.py | 29 ++- .../tests/responses/get_ep400_claim_200.json | 13 ++ .../ee-ep-merge-app/tests/service/conftest.py | 3 + .../tests/service/test_ep_merge_process.py | 177 +++++++++++++++++- 10 files changed, 312 insertions(+), 33 deletions(-) create mode 100644 domain-ee/ee-ep-merge-app/tests/responses/get_ep400_claim_200.json diff --git a/domain-ee/ee-ep-merge-app/end_to_end/test_process_new_merge_request.py b/domain-ee/ee-ep-merge-app/end_to_end/test_process_new_merge_request.py index 896d3f3be5..0d1a66a3f8 100644 --- a/domain-ee/ee-ep-merge-app/end_to_end/test_process_new_merge_request.py +++ b/domain-ee/ee-ep-merge-app/end_to_end/test_process_new_merge_request.py @@ -19,7 +19,8 @@ CLAIM_ID_ERROR_AT_GET_CLAIM_DETAILS = 5001 CLAIM_ID_ERROR_AT_CANCEL_CLAIM = 5002 CLAIM_ID_ERROR_AT_SET_TSOJ = 5003 -CLAIM_ID_ERROR_AT_GET_CONTENTIONS = 5004 +CLAIM_ID_ERROR_AT_GET_PENDING_CONTENTIONS = 5004 +CLAIM_ID_ERROR_AT_GET_EP400_CONTENTIONS = 50041 CLAIM_ID_ERROR_AT_UPDATE_CONTENTIONS = 5005 CLAIM_ID_ERROR_AT_CREATE_CONTENTIONS = 5006 @@ -125,14 +126,21 @@ class TestError: id="fail to get pending claim details", ), pytest.param( - CLAIM_ID_ERROR_AT_GET_CONTENTIONS, + PENDING_CLAIM_ID, + CLAIM_ID_ERROR_AT_GET_CLAIM_DETAILS, + JobState.GET_EP400_CLAIM, + 1, + id="fail to get ep400 claim details", + ), + pytest.param( + CLAIM_ID_ERROR_AT_GET_PENDING_CONTENTIONS, EP400_WITH_MULTI_CONTENTION_NO_DUPLICATES, JobState.GET_PENDING_CLAIM_CONTENTIONS, 1, id="fail to get pending claim contentions", ), pytest.param( - PENDING_CLAIM_ID, CLAIM_ID_ERROR_AT_GET_CONTENTIONS, JobState.GET_EP400_CLAIM_CONTENTIONS, 1, id="fail to get ep400 claim contentions" + PENDING_CLAIM_ID, CLAIM_ID_ERROR_AT_GET_EP400_CONTENTIONS, JobState.GET_EP400_CLAIM_CONTENTIONS, 1, id="fail to get ep400 claim contentions" ), pytest.param(PENDING_CLAIM_ID, EP400_WITH_NO_CONTENTIONS, JobState.GET_EP400_CLAIM_CONTENTIONS, 1, id="ep400 claim has zero contentions"), pytest.param(PENDING_CLAIM_ID, CLAIM_ID_ERROR_AT_SET_TSOJ, JobState.SET_TEMP_STATION_OF_JURISDICTION, 1, id="fail to set tsoj on ep400"), @@ -152,7 +160,7 @@ class TestError: id="fail to remove special issues from ep400 claim after failing to get pending claim", ), pytest.param( - CLAIM_ID_ERROR_AT_GET_CONTENTIONS, + CLAIM_ID_ERROR_AT_GET_EP400_CONTENTIONS, CLAIM_ID_ERROR_AT_UPDATE_CONTENTIONS, JobState.GET_PENDING_CLAIM_CONTENTIONS_FAILED_REMOVE_SPECIAL_ISSUE, 2, diff --git a/domain-ee/ee-ep-merge-app/end_to_end/test_restart.py b/domain-ee/ee-ep-merge-app/end_to_end/test_restart.py index 30a6121386..227b752a36 100644 --- a/domain-ee/ee-ep-merge-app/end_to_end/test_restart.py +++ b/domain-ee/ee-ep-merge-app/end_to_end/test_restart.py @@ -18,7 +18,8 @@ CLAIM_ID_ERROR_AT_GET_CLAIM_DETAILS = 5001 CLAIM_ID_ERROR_AT_CANCEL_CLAIM = 5002 CLAIM_ID_ERROR_AT_SET_TSOJ = 5003 -CLAIM_ID_ERROR_AT_GET_CONTENTIONS = 5004 +CLAIM_ID_ERROR_AT_GET_PENDING_CONTENTIONS = 5004 +CLAIM_ID_ERROR_AT_GET_EP400_CONTENTIONS = 50041 CLAIM_ID_ERROR_AT_UPDATE_CONTENTIONS = 5005 CLAIM_ID_ERROR_AT_CREATE_CONTENTIONS = 5006 @@ -80,7 +81,14 @@ class TestError: id="fail to get pending claim details", ), pytest.param( - CLAIM_ID_ERROR_AT_GET_CONTENTIONS, + PENDING_CLAIM_ID, + CLAIM_ID_ERROR_AT_GET_CLAIM_DETAILS, + JobState.GET_EP400_CLAIM, + 1, + id="fail to get ep400 claim details", + ), + pytest.param( + CLAIM_ID_ERROR_AT_GET_PENDING_CONTENTIONS, EP400_WITH_MULTI_CONTENTION_NO_DUPLICATES, JobState.GET_PENDING_CLAIM_CONTENTIONS, 1, @@ -88,7 +96,7 @@ class TestError: ), pytest.param( PENDING_CLAIM_ID, - CLAIM_ID_ERROR_AT_GET_CONTENTIONS, + CLAIM_ID_ERROR_AT_GET_EP400_CONTENTIONS, JobState.GET_EP400_CLAIM_CONTENTIONS, 1, id="fail to get ep400 claim contentions", @@ -111,7 +119,7 @@ class TestError: id="fail to remove special issues from ep400 claim after failing to get pending claim", ), pytest.param( - CLAIM_ID_ERROR_AT_GET_CONTENTIONS, + CLAIM_ID_ERROR_AT_GET_EP400_CONTENTIONS, CLAIM_ID_ERROR_AT_UPDATE_CONTENTIONS, JobState.GET_PENDING_CLAIM_CONTENTIONS_FAILED_REMOVE_SPECIAL_ISSUE, 2, diff --git a/domain-ee/ee-ep-merge-app/integration/test_merge_request.py b/domain-ee/ee-ep-merge-app/integration/test_merge_request.py index 29ee3d5d8e..ca0037b027 100644 --- a/domain-ee/ee-ep-merge-app/integration/test_merge_request.py +++ b/domain-ee/ee-ep-merge-app/integration/test_merge_request.py @@ -13,6 +13,7 @@ response_500 = f'{RESPONSE_DIR}/500_response.json' pending_claim_200 = f'{RESPONSE_DIR}/get_pending_claim_200.json' pending_contentions_200 = f'{RESPONSE_DIR}/claim_contentions_increase_tendinitis_200.json' +ep400_claim_200 = f'{RESPONSE_DIR}/get_ep400_claim_200.json' ep400_contentions_200 = f'{RESPONSE_DIR}/claim_contentions_increase_tinnitus_200.json' ep400_duplicate_contentions_200 = f'{RESPONSE_DIR}/claim_contentions_increase_tendinitis_200.json' @@ -68,7 +69,7 @@ async def test_completed_success( cancel_claim_endpoint: MqEndpoint, add_claim_note_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, ep400_contentions_200]) put_tsoj_endpoint.set_responses([response_200]) create_claim_contentions_endpoint.set_responses([response_201]) @@ -89,7 +90,7 @@ async def test_completed_success_with_duplicate_contention( cancel_claim_endpoint: MqEndpoint, add_claim_note_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, ep400_duplicate_contentions_200]) put_tsoj_endpoint.set_responses([response_200]) cancel_claim_endpoint.set_responses([response_200]) @@ -109,7 +110,7 @@ async def test_completed_no_ep400_contentions_on_first_attempt( cancel_claim_endpoint: MqEndpoint, add_claim_note_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, response_204, ep400_contentions_200]) put_tsoj_endpoint.set_responses([response_200]) create_claim_contentions_endpoint.set_responses([response_201]) @@ -163,11 +164,53 @@ async def test_error_at_remove_special_issue_fail_to_update_ep400_contentions( assert_error_response(response, JobState.GET_PENDING_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE) +class TestErrorAtGetEP400Claim(TestMergeRequestBase): + + @pytest.mark.asyncio(scope="session") + async def test(self, get_claim_endpoint: MqEndpoint, get_claim_contentions_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint): + get_claim_endpoint.set_responses([pending_claim_200, response_500]) + + # Needed after get claim failure + get_claim_contentions_endpoint.set_responses([ep400_contentions_200]) + update_claim_contentions_endpoint.set_responses([response_200]) + + async with AsyncClient(app=app, base_url="http://test") as client: + response = await submit_request_and_process(client) + assert_error_response(response, JobState.GET_EP400_CLAIM) + + @pytest.mark.asyncio(scope="session") + async def test_error_at_remove_special_issue_fail_to_get_ep400_contentions( + self, get_claim_endpoint: MqEndpoint, get_claim_contentions_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint + ): + get_claim_endpoint.set_responses([pending_claim_200, response_500]) + + # Needed after get claim failure + get_claim_contentions_endpoint.set_responses([response_500]) + + async with AsyncClient(app=app, base_url="http://test") as client: + response = await submit_request_and_process(client) + assert_error_response(response, JobState.GET_EP400_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE) + + @pytest.mark.asyncio(scope="session") + async def test_error_at_remove_special_issue_fail_to_update_ep400_contentions( + self, get_claim_endpoint: MqEndpoint, get_claim_contentions_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint + ): + get_claim_endpoint.set_responses([pending_claim_200, response_500]) + + # Needed after get claim failure + get_claim_contentions_endpoint.set_responses([ep400_contentions_200]) + update_claim_contentions_endpoint.set_responses([response_500]) + + async with AsyncClient(app=app, base_url="http://test") as client: + response = await submit_request_and_process(client) + assert_error_response(response, JobState.GET_EP400_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE) + + class TestErrorAtGetPendingClaimContentions(TestMergeRequestBase): @pytest.mark.asyncio(scope="session") async def test(self, get_claim_endpoint: MqEndpoint, get_claim_contentions_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([response_500, ep400_contentions_200]) # Needed after get claim failure, note second response from get_claim_contentions_endpoint above @@ -181,7 +224,7 @@ async def test(self, get_claim_endpoint: MqEndpoint, get_claim_contentions_endpo async def test_error_at_remove_special_issue_fail_to_get_ep400_contentions( self, get_claim_endpoint: MqEndpoint, get_claim_contentions_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([response_500, response_500]) # Note second response from get_claim_contentions_endpoint above @@ -193,7 +236,7 @@ async def test_error_at_remove_special_issue_fail_to_get_ep400_contentions( async def test_error_at_remove_special_issue_fail_to_update_ep400_contentions( self, get_claim_endpoint: MqEndpoint, get_claim_contentions_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([response_500, ep400_contentions_200]) # Needed after get claim failure, note second response from get_claim_contentions_endpoint above @@ -208,7 +251,7 @@ class TestErrorAtGetEp400ClaimContentions(TestMergeRequestBase): @pytest.mark.asyncio(scope="session") async def test(self, get_claim_endpoint: MqEndpoint, get_claim_contentions_endpoint: MqEndpoint): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, response_500]) async with AsyncClient(app=app, base_url="http://test") as client: @@ -217,7 +260,7 @@ async def test(self, get_claim_endpoint: MqEndpoint, get_claim_contentions_endpo @pytest.mark.asyncio(scope="session") async def test_no_contentions_found(self, get_claim_endpoint: MqEndpoint, get_claim_contentions_endpoint: MqEndpoint): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, response_204, response_204]) # Note the second 204 is because the tests are set up to try to get the ep400 contentions twice in pyproject.toml @@ -236,7 +279,7 @@ async def test( put_tsoj_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, ep400_contentions_200]) put_tsoj_endpoint.set_responses([response_500]) update_claim_contentions_endpoint.set_responses([response_200]) @@ -253,7 +296,7 @@ async def test_error_at_remove_special_issue_fail_to_update_ep400_contentions( put_tsoj_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, ep400_contentions_200]) put_tsoj_endpoint.set_responses([response_500]) update_claim_contentions_endpoint.set_responses([response_500]) @@ -274,7 +317,7 @@ async def test( create_claim_contentions_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, ep400_contentions_200]) put_tsoj_endpoint.set_responses([response_200, response_200]) # Second response is to revert the tsoj create_claim_contentions_endpoint.set_responses([response_500]) @@ -293,7 +336,7 @@ async def test_error_at_remove_special_issue_fail_to_update_ep400_contentions( create_claim_contentions_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, ep400_contentions_200]) put_tsoj_endpoint.set_responses([response_200, response_200]) create_claim_contentions_endpoint.set_responses([response_500]) @@ -312,7 +355,7 @@ async def test_error_at_revert_tsoj( create_claim_contentions_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, ep400_contentions_200]) put_tsoj_endpoint.set_responses([response_200, response_500]) # Note the 500 on second response create_claim_contentions_endpoint.set_responses([response_500]) @@ -334,7 +377,7 @@ async def test( create_claim_contentions_endpoint: MqEndpoint, cancel_claim_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, ep400_contentions_200]) put_tsoj_endpoint.set_responses([response_200, response_200]) # Note the 200 to revert tsoj create_claim_contentions_endpoint.set_responses([response_201]) @@ -353,7 +396,7 @@ async def test_error_at_revert_tsoj( create_claim_contentions_endpoint: MqEndpoint, cancel_claim_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, ep400_contentions_200]) put_tsoj_endpoint.set_responses([response_200, response_500]) # Note the 500 on second response create_claim_contentions_endpoint.set_responses([response_201]) @@ -377,7 +420,7 @@ async def test( add_claim_note_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, ep400_contentions_200]) put_tsoj_endpoint.set_responses([response_200]) create_claim_contentions_endpoint.set_responses([response_201]) @@ -400,7 +443,7 @@ async def test_error_at_remove_special_issue_fail_to_update_ep400_contentions( add_claim_note_endpoint: MqEndpoint, update_claim_contentions_endpoint: MqEndpoint, ): - get_claim_endpoint.set_responses([pending_claim_200]) + get_claim_endpoint.set_responses([pending_claim_200, ep400_claim_200]) get_claim_contentions_endpoint.set_responses([pending_contentions_200, ep400_contentions_200]) put_tsoj_endpoint.set_responses([response_200]) create_claim_contentions_endpoint.set_responses([response_201]) diff --git a/domain-ee/ee-ep-merge-app/src/python_src/graph_export.py b/domain-ee/ee-ep-merge-app/src/python_src/graph_export.py index a932afd661..06660564b1 100644 --- a/domain-ee/ee-ep-merge-app/src/python_src/graph_export.py +++ b/domain-ee/ee-ep-merge-app/src/python_src/graph_export.py @@ -25,7 +25,7 @@ def _get_graph(self): label=f"{machine.name} {machine.main_event}", fontname=self.font_name, fontsize=self.state_font_size, - rankdir="LR", + rankdir="TB", ) def _initial_node(self): diff --git a/domain-ee/ee-ep-merge-app/src/python_src/schema/claim.py b/domain-ee/ee-ep-merge-app/src/python_src/schema/claim.py index 1da044973f..66f6c5e82e 100644 --- a/domain-ee/ee-ep-merge-app/src/python_src/schema/claim.py +++ b/domain-ee/ee-ep-merge-app/src/python_src/schema/claim.py @@ -2,11 +2,21 @@ from pydantic.alias_generators import to_camel +class BenefitClaimType(BaseModel): + """Benefit Claim Type with extra fields ignored.""" + + model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel) + + name: str | None = None + code: str | None = None + + class ClaimDetail(BaseModel): """Contention Summary with extra fields ignored.""" model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel) claim_id: int + benefit_claim_type: BenefitClaimType | None = None end_product_code: str | None = None temp_station_of_jurisdiction: str | None = None diff --git a/domain-ee/ee-ep-merge-app/src/python_src/schema/merge_job.py b/domain-ee/ee-ep-merge-app/src/python_src/schema/merge_job.py index e27871c91a..911b3e4ac9 100644 --- a/domain-ee/ee-ep-merge-app/src/python_src/schema/merge_job.py +++ b/domain-ee/ee-ep-merge-app/src/python_src/schema/merge_job.py @@ -12,6 +12,7 @@ class JobState(StrEnum): PENDING = auto() GET_PENDING_CLAIM = auto() GET_PENDING_CLAIM_CONTENTIONS = auto() + GET_EP400_CLAIM = auto() GET_EP400_CLAIM_CONTENTIONS = auto() SET_TEMP_STATION_OF_JURISDICTION = auto() MERGE_CONTENTIONS = auto() @@ -21,6 +22,7 @@ class JobState(StrEnum): COMPLETED_SUCCESS = auto() GET_PENDING_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE = auto() + GET_EP400_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE = auto() GET_PENDING_CLAIM_CONTENTIONS_FAILED_REMOVE_SPECIAL_ISSUE = auto() SET_TEMP_STATION_OF_JURISDICTION_FAILED_REMOVE_SPECIAL_ISSUE = auto() MOVE_CONTENTIONS_FAILED_REMOVE_SPECIAL_ISSUE = auto() diff --git a/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py b/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py index 3a345b2289..875d84153c 100644 --- a/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py +++ b/domain-ee/ee-ep-merge-app/src/python_src/service/ep_merge_machine.py @@ -40,6 +40,9 @@ JOB_SKIPPED_MERGE_METRIC = 'job.skipped_merge' JOB_NEW_CONTENTIONS_METRIC = 'job.new_contentions' +EP400_PRODUCT_CODES = frozenset([str(i) for i in range(400, 410)]) +EP400_BENEFIT_CLAIM_TYPE_CODES = frozenset(['400SUPP']) + CANCEL_TRACKING_EP = "60" CANCELLATION_REASON_FORMAT = "Issues moved into or confirmed in pending EP{ep_code} - claim #{claim_id}" @@ -73,6 +76,8 @@ class EpMergeMachine(StateMachine): running_get_pending_claim_failed_remove_special_issue = State(value=JobState.GET_PENDING_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE) running_get_pending_contentions = State(value=JobState.GET_PENDING_CLAIM_CONTENTIONS) running_get_pending_contentions_failed_remove_special_issue = State(value=JobState.GET_PENDING_CLAIM_CONTENTIONS_FAILED_REMOVE_SPECIAL_ISSUE) + running_get_ep400_claim = State(value=JobState.GET_EP400_CLAIM) + running_get_ep400_claim_failed_remove_special_issue = State(value=JobState.GET_EP400_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE) running_get_ep400_contentions = State(value=JobState.GET_EP400_CLAIM_CONTENTIONS) running_set_temp_station_of_jurisdiction = State(value=JobState.SET_TEMP_STATION_OF_JURISDICTION) running_set_temp_station_of_jurisdiction_failed_remove_special_issue = State(value=JobState.SET_TEMP_STATION_OF_JURISDICTION_FAILED_REMOVE_SPECIAL_ISSUE) @@ -88,9 +93,12 @@ class EpMergeMachine(StateMachine): process = ( pending.to(running_get_pending_claim) - | running_get_pending_claim.to(running_get_pending_contentions, unless="has_error") + | running_get_pending_claim.to(running_get_ep400_claim, unless="has_error") | running_get_pending_claim.to(running_get_pending_claim_failed_remove_special_issue, cond="has_error") | running_get_pending_claim_failed_remove_special_issue.to(completed_error) + | running_get_ep400_claim.to(running_get_pending_contentions, unless="has_error") + | running_get_ep400_claim.to(running_get_ep400_claim_failed_remove_special_issue, cond="has_error") + | running_get_ep400_claim_failed_remove_special_issue.to(completed_error) | running_get_pending_contentions.to(running_get_ep400_contentions, unless="has_error") | running_get_pending_contentions.to(running_get_pending_contentions_failed_remove_special_issue, cond="has_error") | running_get_pending_contentions_failed_remove_special_issue.to(completed_error) @@ -167,6 +175,24 @@ def on_get_pending_claim(self, event): self.send(event=event) + @running_get_ep400_claim.enter + def on_get_ep400_claim(self, event): + request = get_claim.Request(claim_id=self.job.ep400_claim_id) + response = self.make_request(request=request, hoppy_client=HOPPY.get_client(ClientName.GET_CLAIM), response_type=get_claim.Response) + + if response is not None and response.status_code == 200: + claim = response.claim + if claim is None or claim.end_product_code is None: + self.add_job_error(f"EP400 claim #{self.job.ep400_claim_id} does not have an end product code") + elif claim.end_product_code not in EP400_PRODUCT_CODES: + self.add_job_error(f"EP400 claim #{self.job.ep400_claim_id} end product code of '{claim.end_product_code}' is not supported") + elif claim.benefit_claim_type is None or claim.benefit_claim_type.code is None: + self.add_job_error(f"EP400 claim #{self.job.ep400_claim_id} does not have a benefit claim type code") + elif claim.benefit_claim_type.code not in EP400_BENEFIT_CLAIM_TYPE_CODES: + self.add_job_error(f"EP400 claim #{self.job.ep400_claim_id} benefit claim type code of '{claim.benefit_claim_type.code}' is not supported") + + self.send(event=event) + @running_get_pending_contentions.enter def on_get_pending_contentions(self, event): request = get_contentions.Request(claim_id=self.job.pending_claim_id) @@ -230,6 +256,7 @@ def on_add_claim_note_to_ep400(self, event): @running_get_pending_claim_failed_remove_special_issue.enter @running_get_pending_contentions_failed_remove_special_issue.enter + @running_get_ep400_claim_failed_remove_special_issue.enter @running_set_temp_station_of_jurisdiction_failed_remove_special_issue.enter @running_move_contentions_failed_remove_special_issue.enter def on_pre_cancel_step_failed_remove_special_issue_code(self, event, ep400_contentions_response=None): diff --git a/domain-ee/ee-ep-merge-app/tests/responses/get_ep400_claim_200.json b/domain-ee/ee-ep-merge-app/tests/responses/get_ep400_claim_200.json new file mode 100644 index 0000000000..ce03e6997c --- /dev/null +++ b/domain-ee/ee-ep-merge-app/tests/responses/get_ep400_claim_200.json @@ -0,0 +1,13 @@ +{ + "statusCode": 200, + "statusMessage": "OK", + "claim": { + "claimId": 2, + "endProductCode": "400", + "tempStationOfJurisdiction": "111", + "benefitClaimType": { + "name": "eBenefits 526EZ-Supplemental (400)", + "code": "400SUPP" + } + } +} diff --git a/domain-ee/ee-ep-merge-app/tests/service/conftest.py b/domain-ee/ee-ep-merge-app/tests/service/conftest.py index 9f2c54b57b..1c6a6840e6 100644 --- a/domain-ee/ee-ep-merge-app/tests/service/conftest.py +++ b/domain-ee/ee-ep-merge-app/tests/service/conftest.py @@ -43,6 +43,7 @@ response_500 = f'{RESPONSE_DIR}/500_response.json' pending_claim_200 = f'{RESPONSE_DIR}/get_pending_claim_200.json' pending_contentions_increase_tendinitis_200 = f'{RESPONSE_DIR}/claim_contentions_increase_tendinitis_200.json' +ep400_claim_200 = f'{RESPONSE_DIR}/get_ep400_claim_200.json' ep400_contentions_increase_tinnitus_200 = f'{RESPONSE_DIR}/claim_contentions_increase_tinnitus_200.json' ep400_contentions_increase_tinnitus_without_special_issues_200 = f'{RESPONSE_DIR}/claim_contentions_increase_tinnitus_without_special_issues_200.json' pending_contentions_increase_tinnitus_200 = f'{RESPONSE_DIR}/claim_contentions_increase_tinnitus_200.json' @@ -63,6 +64,8 @@ def load_response(file, response_type): get_pending_contentions_req = get_contentions.Request(claim_id=PENDING_CLAIM_ID).model_dump(by_alias=True) get_pending_contentions_200 = load_response(pending_contentions_increase_tendinitis_200, get_contentions.Response) get_pending_contentions_increase_tinnitus_200 = load_response(pending_contentions_increase_tinnitus_200, get_contentions.Response) +get_ep400_claim_req = get_claim.Request(claim_id=EP400_CLAIM_ID).model_dump(by_alias=True) +get_ep400_claim_200 = load_response(ep400_claim_200, get_claim.Response) get_ep400_contentions_req = get_contentions.Request(claim_id=EP400_CLAIM_ID).model_dump(by_alias=True) get_ep400_contentions_200 = load_response(ep400_contentions_increase_tinnitus_200, get_contentions.Response) # Add special issue code to contention from config diff --git a/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_process.py b/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_process.py index d50233ece0..6d48f3daf3 100644 --- a/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_process.py +++ b/domain-ee/ee-ep-merge-app/tests/service/test_ep_merge_process.py @@ -15,6 +15,8 @@ ep400_contentions_increase_multicontention_200, ep400_contentions_increase_tinnitus_200, ep400_contentions_new_tinnitus_200, + get_ep400_claim_200, + get_ep400_claim_req, get_ep400_contentions_200, get_ep400_contentions_204, get_ep400_contentions_req, @@ -48,7 +50,7 @@ update_contentions, ) from schema import update_temp_station_of_jurisdiction as tsoj -from schema.claim import ClaimDetail +from schema.claim import BenefitClaimType, ClaimDetail from schema.merge_job import JobState, MergeJob from service.ep_merge_machine import EpMergeMachine from util.contentions_util import ContentionsUtil @@ -172,6 +174,140 @@ def test_invalid_request_at_update_ep400_contentions_after_get_pending_claim_fai ) +class TestUpToGetEP400Claim: + @pytest.mark.parametrize( + "invalid_request", + [ + pytest.param(ResponseException("Oops"), id="Caught Exception"), + pytest.param(load_response(response_400, get_claim.Response), id="400"), + pytest.param(load_response(response_404, get_claim.Response), id="404"), + pytest.param(load_response(response_500, get_claim.Response), id="500"), + pytest.param(get_claim.Response(statusCode=200, statusMessage="OK", claim=ClaimDetail(claimId=3)).model_dump(), id="claim has no endProductCode"), + pytest.param( + get_claim.Response(statusCode=200, statusMessage="OK", claim=ClaimDetail(claimId=3, endProductCode="399")).model_dump(), + id="claim endProductCode not supported", + ), + pytest.param( + get_claim.Response(statusCode=200, statusMessage="OK", claim=ClaimDetail(claimId=3, endProductCode="400")).model_dump(), + id="claim no benefitClaimType", + ), + pytest.param( + get_claim.Response( + statusCode=200, statusMessage="OK", claim=ClaimDetail(claimId=3, endProductCode="400", benefitClaimType=BenefitClaimType()) + ).model_dump(), + id="claim no benefitClaimType.code", + ), + pytest.param( + get_claim.Response( + statusCode=200, + statusMessage="OK", + claim=ClaimDetail(claimId=3, endProductCode="400", benefitClaimType=BenefitClaimType(code="NOTTHISCODE")), + ).model_dump(), + id="claim benefitClaimType.code not supported", + ), + ], + ) + def test_invalid_request(self, machine, mock_hoppy_async_client, metric_logger_distribution, metric_logger_increment, invalid_request): + mock_async_responses(mock_hoppy_async_client, [get_pending_claim_200, invalid_request, get_ep400_contentions_200, update_contentions_on_ep400_200]) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_EP400_CLAIM, 1) + mock_hoppy_async_client.make_request.assert_has_calls( + [ + call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), + call(machine.job.job_id, get_ep400_contentions_req), + call(machine.job.job_id, update_contentions_on_ep400_req), + ] + ) + assert_metrics_called(metric_logger_distribution, metric_logger_increment, JobState.COMPLETED_ERROR, JobState.GET_EP400_CLAIM) + + @pytest.mark.parametrize( + "no_contentions_response", + [ + pytest.param(get_contentions.Response(status_code=200, status_message="OK"), id="Implicit None"), + pytest.param(get_contentions.Response(status_code=200, status_message="OK", contentions=None), id="Explicit None"), + pytest.param(get_contentions.Response(status_code=200, status_message="OK", contentions=[]), id="Empty"), + ], + ) + def test_no_contentions_on_ep400_after_get_ep400_claim_failure( + self, machine, mock_hoppy_async_client, metric_logger_distribution, metric_logger_increment, no_contentions_response + ): + mock_async_responses(mock_hoppy_async_client, [get_pending_claim_200, ResponseException("Oops"), no_contentions_response]) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_EP400_CLAIM, 2) + mock_hoppy_async_client.make_request.assert_has_calls( + [ + call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), + call(machine.job.job_id, get_ep400_contentions_req), + ] + ) + assert_metrics_called(metric_logger_distribution, metric_logger_increment, JobState.COMPLETED_ERROR, JobState.GET_EP400_CLAIM) + + @pytest.mark.parametrize( + "invalid_request", + [ + pytest.param(ResponseException("Oops"), id="Caught Exception"), + pytest.param(load_response(response_400, get_contentions.Response), id="400"), + pytest.param(load_response(response_404, get_contentions.Response), id="404"), + pytest.param(load_response(response_500, get_contentions.Response), id="500"), + ], + ) + def test_invalid_request_at_get_ep400_contentions_after_get_ep400_claim_failure( + self, machine, mock_hoppy_async_client, metric_logger_distribution, metric_logger_increment, invalid_request + ): + mock_async_responses( + mock_hoppy_async_client, + [ + get_pending_claim_200, + ResponseException("Oops"), + invalid_request, + ], + ) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_EP400_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE, 3) + mock_hoppy_async_client.make_request.assert_has_calls( + [ + call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), + call(machine.job.job_id, get_ep400_contentions_req), + ] + ) + assert_metrics_called( + metric_logger_distribution, metric_logger_increment, JobState.COMPLETED_ERROR, JobState.GET_EP400_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE + ) + + @pytest.mark.parametrize( + "invalid_request", + [ + pytest.param(ResponseException("Oops"), id="Caught Exception"), + pytest.param(load_response(response_400, update_contentions.Response), id="400"), + pytest.param(load_response(response_404, update_contentions.Response), id="404"), + pytest.param(load_response(response_500, update_contentions.Response), id="500"), + ], + ) + def test_invalid_request_at_update_ep400_contentions_after_get_ep400_claim_failure( + self, machine, mock_hoppy_async_client, metric_logger_distribution, metric_logger_increment, invalid_request + ): + mock_async_responses( + mock_hoppy_async_client, + [ + get_pending_claim_200, + ResponseException("Oops"), + get_ep400_contentions_200, + invalid_request, + ], + ) + process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_EP400_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE, 2) + mock_hoppy_async_client.make_request.assert_has_calls( + [ + call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), + call(machine.job.job_id, get_ep400_contentions_req), + ] + ) + assert_metrics_called( + metric_logger_distribution, metric_logger_increment, JobState.COMPLETED_ERROR, JobState.GET_EP400_CLAIM_FAILED_REMOVE_SPECIAL_ISSUE + ) + + class TestUpToGetPendingContentions: @pytest.mark.parametrize( "invalid_request", @@ -183,11 +319,14 @@ class TestUpToGetPendingContentions: ], ) def test_invalid_request(self, machine, mock_hoppy_async_client, metric_logger_distribution, metric_logger_increment, invalid_request): - mock_async_responses(mock_hoppy_async_client, [get_pending_claim_200, invalid_request, get_ep400_contentions_200, update_contentions_on_ep400_200]) + mock_async_responses( + mock_hoppy_async_client, [get_pending_claim_200, get_ep400_claim_200, invalid_request, get_ep400_contentions_200, update_contentions_on_ep400_200] + ) process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_CONTENTIONS, 1) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), call(machine.job.job_id, get_pending_contentions_req), call(machine.job.job_id, get_ep400_contentions_req), call(machine.job.job_id, update_contentions_on_ep400_req), @@ -206,11 +345,12 @@ def test_invalid_request(self, machine, mock_hoppy_async_client, metric_logger_d def test_no_contentions_on_ep400_after_get_pending_contentions_failure( self, machine, mock_hoppy_async_client, metric_logger_distribution, metric_logger_increment, no_contentions_response ): - mock_async_responses(mock_hoppy_async_client, [get_pending_claim_200, ResponseException("Oops"), no_contentions_response]) + mock_async_responses(mock_hoppy_async_client, [get_pending_claim_200, get_ep400_claim_200, ResponseException("Oops"), no_contentions_response]) process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_PENDING_CLAIM_CONTENTIONS, 2) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), call(machine.job.job_id, get_pending_contentions_req), call(machine.job.job_id, get_ep400_contentions_req), ] @@ -233,6 +373,7 @@ def test_invalid_request_at_get_ep400_contentions_after_get_pending_contentions_ mock_hoppy_async_client, [ get_pending_claim_200, + get_ep400_claim_200, ResponseException("Oops"), invalid_request, ], @@ -241,6 +382,7 @@ def test_invalid_request_at_get_ep400_contentions_after_get_pending_contentions_ mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), call(machine.job.job_id, get_pending_contentions_req), call(machine.job.job_id, get_ep400_contentions_req), ] @@ -261,13 +403,14 @@ def test_invalid_request_at_get_ep400_contentions_after_get_pending_contentions_ pytest.param(load_response(response_500, update_contentions.Response), id="500"), ], ) - def test_invalid_request_at_update_ep400_contentions_after_get_pending_claim_failure( + def test_invalid_request_at_update_ep400_contentions_after_get_pending_contentions_failure( self, machine, mock_hoppy_async_client, metric_logger_distribution, metric_logger_increment, invalid_request ): mock_async_responses( mock_hoppy_async_client, [ get_pending_claim_200, + get_ep400_claim_200, ResponseException("Oops"), get_ep400_contentions_200, invalid_request, @@ -277,6 +420,7 @@ def test_invalid_request_at_update_ep400_contentions_after_get_pending_claim_fai mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), call(machine.job.job_id, get_pending_contentions_req), call(machine.job.job_id, get_ep400_contentions_req), ] @@ -305,7 +449,7 @@ class TestUpToGetEp400Contentions: def test_invalid_request_at_get_ep400_contentions( self, machine, mock_hoppy_async_client, metric_logger_distribution, metric_logger_increment, invalid_request ): - mock_async_responses(mock_hoppy_async_client, [get_pending_claim_200, get_pending_contentions_200, invalid_request]) + mock_async_responses(mock_hoppy_async_client, [get_pending_claim_200, get_ep400_claim_200, get_pending_contentions_200, invalid_request]) process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.GET_EP400_CLAIM_CONTENTIONS, 1) mock_hoppy_async_client.make_request.assert_has_calls( [call(machine.job.job_id, get_pending_contentions_req), call(machine.job.job_id, get_ep400_contentions_req)] @@ -326,12 +470,20 @@ class TestUpToSetTemporaryStationOfJurisdiction: def test_invalid_request(self, machine, mock_hoppy_async_client, metric_logger_distribution, metric_logger_increment, invalid_request): mock_async_responses( mock_hoppy_async_client, - [get_pending_claim_200, get_pending_contentions_200, get_ep400_contentions_200, invalid_request, update_contentions_on_ep400_200], + [ + get_pending_claim_200, + get_ep400_claim_200, + get_pending_contentions_200, + get_ep400_contentions_200, + invalid_request, + update_contentions_on_ep400_200, + ], ) process_and_assert(machine, JobState.COMPLETED_ERROR, JobState.SET_TEMP_STATION_OF_JURISDICTION, 1) mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), call(machine.job.job_id, get_pending_contentions_req), call(machine.job.job_id, get_ep400_contentions_req), call(machine.job.job_id, update_temporary_station_of_jurisdiction_req), @@ -356,6 +508,7 @@ def test_invalid_request_on_update_contentions_after_set_tsoj_failure( mock_hoppy_async_client, [ get_pending_claim_200, + get_ep400_claim_200, get_pending_contentions_200, get_ep400_contentions_200, ResponseException("Oops"), @@ -366,6 +519,7 @@ def test_invalid_request_on_update_contentions_after_set_tsoj_failure( mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), call(machine.job.job_id, get_pending_contentions_req), call(machine.job.job_id, get_ep400_contentions_req), call(machine.job.job_id, update_temporary_station_of_jurisdiction_req), @@ -402,6 +556,7 @@ def test_fail(self, machine, mock_hoppy_async_client, metric_logger_distribution mock_hoppy_async_client, [ get_pending_claim_200, + get_ep400_claim_200, get_pending_contentions_200, get_ep400_contentions_200, update_temporary_station_of_jurisdiction_200, @@ -414,6 +569,7 @@ def test_fail(self, machine, mock_hoppy_async_client, metric_logger_distribution mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), call(machine.job.job_id, get_pending_contentions_req), call(machine.job.job_id, get_ep400_contentions_req), call(machine.job.job_id, update_temporary_station_of_jurisdiction_req), @@ -440,6 +596,7 @@ def test_fail_to_remove_special_issues_after_move_contentions_to_pending_claim_f mock_hoppy_async_client, [ get_pending_claim_200, + get_ep400_claim_200, get_pending_contentions_200, get_ep400_contentions_200, update_temporary_station_of_jurisdiction_200, @@ -452,6 +609,7 @@ def test_fail_to_remove_special_issues_after_move_contentions_to_pending_claim_f mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), call(machine.job.job_id, get_pending_contentions_req), call(machine.job.job_id, get_ep400_contentions_req), call(machine.job.job_id, update_temporary_station_of_jurisdiction_req), @@ -480,6 +638,7 @@ def test_fail_to_revert_tsoj_after_failure_to_move_contentions_to_pending_claim( mock_hoppy_async_client, [ get_pending_claim_200, + get_ep400_claim_200, get_pending_contentions_200, get_ep400_contentions_200, update_temporary_station_of_jurisdiction_200, @@ -492,6 +651,7 @@ def test_fail_to_revert_tsoj_after_failure_to_move_contentions_to_pending_claim( mock_hoppy_async_client.make_request.assert_has_calls( [ call(machine.job.job_id, get_pending_claim_req), + call(machine.job.job_id, get_ep400_claim_req), call(machine.job.job_id, get_pending_contentions_req), call(machine.job.job_id, get_ep400_contentions_req), call(machine.job.job_id, update_temporary_station_of_jurisdiction_req), @@ -530,6 +690,7 @@ def test_invalid_request_at_cancel_claim_due_to_exception( mock_hoppy_async_client, [ get_pending_claim_200, + get_ep400_claim_200, get_pending_contentions_200, get_ep400_contentions_200, update_temporary_station_of_jurisdiction_200, @@ -566,6 +727,7 @@ def test_invalid_request_at_revert_tsoj_due_to_failure_to_cancel_claim( mock_hoppy_async_client, [ get_pending_claim_200, + get_ep400_claim_200, get_pending_contentions_200, get_ep400_contentions_200, update_temporary_station_of_jurisdiction_200, @@ -612,6 +774,7 @@ def test_invalid_request_at_add_claim_note_due_to_exception( mock_hoppy_async_client, [ get_pending_claim_200, + get_ep400_claim_200, get_pending_contentions_200, get_ep400_contentions_200, update_temporary_station_of_jurisdiction_200, @@ -672,6 +835,7 @@ def test_process_succeeds_with_different_contention( mock_hoppy_async_client, [ get_pending_claim_200, + get_ep400_claim_200, pending_contentions, ep400_contentions, update_temporary_station_of_jurisdiction_200, @@ -697,6 +861,7 @@ def test_process_succeeds_with_duplicate_contention(self, machine, mock_hoppy_as mock_hoppy_async_client, [ get_pending_claim_200, + get_ep400_claim_200, get_pending_contentions_increase_tinnitus_200, get_ep400_contentions_200, update_temporary_station_of_jurisdiction_200, From d6c0cccd69b11bac63f236178c8283a353993d3d Mon Sep 17 00:00:00 2001 From: Luke Short Date: Wed, 13 Mar 2024 17:21:01 -0700 Subject: [PATCH 28/67] bump pydantic version for Contention Classification App (#2674) bump pydantic version, use model validator w/ mode="before" for validating CfI Co-authored-by: Erik Nelsestuen --- domain-cc/cc-app/src/dev-requirements.txt | 1 + domain-cc/cc-app/src/python_src/api.py | 4 +++- domain-cc/cc-app/src/python_src/pydantic_models.py | 13 +++++++++---- domain-cc/cc-app/src/requirements.txt | 2 +- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/domain-cc/cc-app/src/dev-requirements.txt b/domain-cc/cc-app/src/dev-requirements.txt index 729ac43ef1..dfbeb6f40f 100644 --- a/domain-cc/cc-app/src/dev-requirements.txt +++ b/domain-cc/cc-app/src/dev-requirements.txt @@ -1,4 +1,5 @@ black==23.1.0 +bump-pydantic==0.8.0 flake8==6.0.0 isort==5.12.0 pre-commit==3.1.1 diff --git a/domain-cc/cc-app/src/python_src/api.py b/domain-cc/cc-app/src/python_src/api.py index 9c976fbda4..0c8b43b6dc 100644 --- a/domain-cc/cc-app/src/python_src/api.py +++ b/domain-cc/cc-app/src/python_src/api.py @@ -93,7 +93,9 @@ def log_claim_stats(claim: Claim, classification: Optional[PredictedClassificati contention_text = claim.contention_text or "" is_in_dropdown = contention_text.strip().lower() in dropdown_values is_mapped_text = dropdown_lookup_table.get(contention_text, None) is not None - log_contention_text = contention_text if is_mapped_text else "unmapped contention text" + log_contention_text = ( + contention_text if is_mapped_text else "unmapped contention text" + ) log_as_json( { diff --git a/domain-cc/cc-app/src/python_src/pydantic_models.py b/domain-cc/cc-app/src/python_src/pydantic_models.py index f8f689c42d..343de0cc3e 100644 --- a/domain-cc/cc-app/src/python_src/pydantic_models.py +++ b/domain-cc/cc-app/src/python_src/pydantic_models.py @@ -1,16 +1,21 @@ from typing import Optional -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, model_validator class Claim(BaseModel): claim_id: int form526_submission_id: int - diagnostic_code: Optional[int] # only required for claim_type: "claim_for_increase" + diagnostic_code: Optional[ + int + ] = None # only required for claim_type: "claim_for_increase" claim_type: str = "claim_for_increase" - contention_text: Optional[str] # marked optional to retain compatibility with v1 + contention_text: Optional[ + str + ] = None # marked optional to retain compatibility with v1 - @root_validator(pre="before") + @model_validator(mode="before") + @classmethod def check_dc_for_cfi(cls, values): claim_type = values.get("claim_type") diagnostic_code = values.get("diagnostic_code") diff --git a/domain-cc/cc-app/src/requirements.txt b/domain-cc/cc-app/src/requirements.txt index 7351f198ba..8bcba9b396 100644 --- a/domain-cc/cc-app/src/requirements.txt +++ b/domain-cc/cc-app/src/requirements.txt @@ -1,7 +1,7 @@ fastapi==0.109.* httpx==0.24.* pika==1.3.* -pydantic==1.10.13 +pydantic==2.6.1 pytest==7.3.1 starlette>=0.27.0 uvicorn[standard]==0.20.* From 84f58ff2c0cc9ad7bc9384164fc3c277ebbc9688 Mon Sep 17 00:00:00 2001 From: Josiah Jones Date: Thu, 14 Mar 2024 09:56:05 -0700 Subject: [PATCH 29/67] #2749: PROD FIX, Increase rabbitmq container memory (#2750) Production FIX: Increase the rabbitmq container memory --- helm/platform/charts/rabbitmq/values.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/helm/platform/charts/rabbitmq/values.yaml b/helm/platform/charts/rabbitmq/values.yaml index 2b0d3537db..7c31401389 100644 --- a/helm/platform/charts/rabbitmq/values.yaml +++ b/helm/platform/charts/rabbitmq/values.yaml @@ -13,3 +13,11 @@ service: mq: sourcePort: 5672 targetPort: 5672 + +resources: + requests: + cpu: 150m + memory: 1024Mi + limits: + cpu: 2000m + memory: 4096Mi From 3b630fd6f8005a57afa661892ad0f441038f037a Mon Sep 17 00:00:00 2001 From: Josiah Jones Date: Thu, 14 Mar 2024 10:44:34 -0700 Subject: [PATCH 30/67] Update SpringBootStarterWeb (#2752) Snyk fix, update spring-boot-web to 3.2.3 --- domain-xample/xample-workflows/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-xample/xample-workflows/build.gradle b/domain-xample/xample-workflows/build.gradle index 6e8a81c68a..af9957dd1c 100644 --- a/domain-xample/xample-workflows/build.gradle +++ b/domain-xample/xample-workflows/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation "org.apache.camel:camel-spring-rabbitmq:${camel_version}" // Actuator for health check - implementation 'org.springframework.boot:spring-boot-starter-web' + implementation 'org.springframework.boot:spring-boot-starter-web:3.2.3' implementation 'org.springframework.boot:spring-boot-starter-actuator' testImplementation "org.apache.camel:camel-test-junit5:${camel_version}" From b75b81c0d8701516b8431414c6381ba3915a98aa Mon Sep 17 00:00:00 2001 From: Cheng <89034346+chengjie8@users.noreply.github.com> Date: Thu, 14 Mar 2024 14:10:32 -0400 Subject: [PATCH 31/67] Upgrade spring-boot-starter-web package version (#2754) Upgrade spring-boot-starter-web package version --- domain-xample/xample-workflows/build.gradle | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/domain-xample/xample-workflows/build.gradle b/domain-xample/xample-workflows/build.gradle index af9957dd1c..6e8a81c68a 100644 --- a/domain-xample/xample-workflows/build.gradle +++ b/domain-xample/xample-workflows/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation "org.apache.camel:camel-spring-rabbitmq:${camel_version}" // Actuator for health check - implementation 'org.springframework.boot:spring-boot-starter-web:3.2.3' + implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-actuator' testImplementation "org.apache.camel:camel-test-junit5:${camel_version}" diff --git a/gradle.properties b/gradle.properties index cf3cab00cd..33c94e2c87 100644 --- a/gradle.properties +++ b/gradle.properties @@ -16,7 +16,7 @@ hibernate_types_version=3.7.3 postgresql_version=42.7.1 spring_security_version=6.0.3 -spring_boot_version=3.2.2 +spring_boot_version=3.2.3 spring_doc_version=2.2.0 swagger_version=2.2.16 From 230db9bf46888cd8f8208ac19e5e145d8413c15f Mon Sep 17 00:00:00 2001 From: Mason Watson <32622851+msnwatson@users.noreply.github.com> Date: Thu, 14 Mar 2024 11:39:55 -0700 Subject: [PATCH 32/67] remove all versioning (#2741) * remove all versioning * undo some changes * run image-names script * add back sourcing file --- .github/actions/publish-images/action.yml | 68 +---------- .github/workflows/continuous-delivery.yml | 2 +- .github/workflows/create-gh-release.yml | 41 ------- .github/workflows/create-release-tag.yml | 71 ----------- .github/workflows/publish-3rd-party-image.yml | 1 - .github/workflows/secrel.yml | 44 +------ .github/workflows/update-deployment-cc.yml | 2 +- .github/workflows/update-deployment-ee.yml | 2 +- .github/workflows/update-deployment.yml | 11 +- build.gradle | 74 ------------ helm/api-gateway/Chart.yaml | 5 - helm/api-gateway/templates/deployment.yaml | 2 - helm/deploy.sh | 4 - helm/dev-tools/Chart.yaml | 6 - helm/domain-cc/Chart.yaml | 5 - helm/domain-ee-ep-merge-app/Chart.yaml | 5 - helm/domain-ee-max-cfi-app/Chart.yaml | 5 - helm/domain-xample/Chart.yaml | 5 - helm/platform/Chart.yaml | 6 - helm/platform/charts/rabbitmq/Chart.yaml | 5 - helm/platform/charts/redis/Chart.yaml | 5 - helm/svc-bgs-api/Chart.yaml | 5 - helm/svc-bie-kafka/Chart.yaml | 5 - helm/svc-bip-api/Chart.yaml | 5 - helm/svc-lighthouse-api/Chart.yaml | 5 - helm/vro-app/Chart.yaml | 5 - scripts/image-names.sh | 21 ---- scripts/image-version.sh | 113 ------------------ scripts/image_vars.src | 34 ------ scripts/image_versions.src | 64 ---------- versions.txt | 55 --------- 31 files changed, 9 insertions(+), 672 deletions(-) delete mode 100644 .github/workflows/create-gh-release.yml delete mode 100644 .github/workflows/create-release-tag.yml delete mode 100755 scripts/image-version.sh delete mode 100644 scripts/image_versions.src delete mode 100644 versions.txt diff --git a/.github/actions/publish-images/action.yml b/.github/actions/publish-images/action.yml index e08b650301..38f5d71970 100644 --- a/.github/actions/publish-images/action.yml +++ b/.github/actions/publish-images/action.yml @@ -7,7 +7,7 @@ inputs: type: string default: 'dev_' image_tag: - description: 'The tag/version for the image, e.g., first 7 of the commit hash' + description: 'The tag for the image, e.g., first 7 of the commit hash' type: string ghcr_username: description: 'GHCR username' @@ -21,10 +21,7 @@ inputs: description: 'Run test before publishing images' type: string default: true - publish_mode: - description: 'Publish certain images to reduce SecRel' - type: string - default: 'only_changed' + outputs: images_list: @@ -50,41 +47,6 @@ runs: username: ${{ inputs.ghcr_username }} password: ${{ inputs.ghcr_password }} - - name: "Unpin changed pinned images" - id: unpin-changed-pinned-images - shell: bash - run: | - if [ "$(git rev-parse --abbrev-ref --symbolic-full-name HEAD)" == "HEAD" ]; then - echo "Presuming this GitHub Action workflow was triggered by a GitHub release, which means no unpinning needs to happen." - elif [ "${{ inputs.publish_mode }}" == "all" ]; then - echo "Since publish_mode = 'all', skipping unnecessary comparing (and unpinning) of image versions." - else - # Install container-diff needed by image-version.sh - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && \ - mkdir -p "/tmp/bin" && \ - export PATH="/tmp/bin:$PATH" && \ - install container-diff-linux-amd64 /tmp/bin/container-diff - - # Unpin pinned images that have changed so that new image versions can be published - scripts/image-version.sh unpinIfDiff - - # Commit updated files -- https://stackoverflow.com/a/3879077 - # The '|| echo' prevents this step from reporting a failure - git update-index --refresh || echo "Files changed" - if git diff-index --quiet HEAD --; then - echo "No files changed" - else - git config user.name "VRO Machine User" - git config user.email "abd-vro-machine@users.noreply.github.com" - git add scripts/image_versions.src - git commit -m "Automated commit: Unpin versions of updated images" - - # Push to public repo, which will mirror to internal repo - git remote set-url origin https://github.com/department-of-veterans-affairs/abd-vro.git - git push - fi - fi - - name: "Tag and push images using commit hash and `latest`" id: push-images shell: bash @@ -97,35 +59,13 @@ runs: echo -n "[" >> "$GITHUB_OUTPUT" source scripts/image_vars.src for PREFIX in "${VAR_PREFIXES_ARR[@]}"; do - # Set IMG_TAG, which will determine if a new image will get published - case "${{ inputs.publish_mode }}" in - all) - # This IMG_TAG value will cause a new image to be publish (and scanned by SecRel) - IMG_TAG="${{ inputs.image_tag }}" - ;; - pinned_only) - # For release events (see secrel.yml), all images are pinned - IMG_TAG=$(getVarValue "${PREFIX}" _VER) - ;; - only_changed) - # If image is pinned, then use it to avoid publishing unchanged image (and avoids SecRel scan) - if grep -w "${PREFIX}_VER" scripts/image_versions.src; then - # Use the pinned version - IMG_TAG=$(getVarValue "${PREFIX}" _VER) - else - IMG_TAG="${{ inputs.image_tag }}" - fi - ;; - *) echo "Unknown publish_mode: ${{ inputs.publish_mode }}" - exit 30 - ;; - esac + # This IMG_TAG value will cause a new image to be publish (and scanned by SecRel) + IMG_TAG="${{ inputs.image_tag }}" IMG_NAME="${{ inputs.image_prefix }}$(getVarValue "${PREFIX}" _IMG)" echo "::group::Push image $IMG_NAME $IMG_TAG" GHCR_PATH="ghcr.io/${{ github.repository }}/${IMG_NAME}" if [ "$(imageTagExists "$IMG_NAME" "$IMG_TAG")" == "200" ]; then - # Pinned versions of images will already exist, so they don't need to be published echo "Image already exists: $IMG_NAME:$IMG_TAG -- not overwriting" echo "* ($IMG_NAME:$IMG_TAG -- already exists, not overwriting)" >> "$GITHUB_STEP_SUMMARY" else diff --git a/.github/workflows/continuous-delivery.yml b/.github/workflows/continuous-delivery.yml index 9d02c874bf..e5d1d5ca24 100644 --- a/.github/workflows/continuous-delivery.yml +++ b/.github/workflows/continuous-delivery.yml @@ -23,7 +23,7 @@ on: - prod image_tag: - description: 'Image tag: first 7 of commit hash, "latest", "v1.2.3", or "release-1.2.3"' + description: 'Image tag: first 7 of commit hash, "latest"' required: true default: 'latest' type: string diff --git a/.github/workflows/create-gh-release.yml b/.github/workflows/create-gh-release.yml deleted file mode 100644 index ea987acce6..0000000000 --- a/.github/workflows/create-gh-release.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: "6. Auto-create GitHub release" - -on: - push: - tags: - - "v*.*.*" - -jobs: - create-gh-release: - runs-on: ubuntu-latest - steps: - - name: "Checkout source code" - uses: actions/checkout@v4 - - - name: "Check if release already exists" - id: check-release - env: - GH_TOKEN: ${{ github.token }} - run: | - echo "Tag name from github.ref_name: ${{ github.ref_name }}" - if gh release view ${{ github.ref_name }}; then - # This workflow is triggered more than once in the internal repo. - # Subsequent runs are undesirable, so don't continue. - # See notes at https://github.com/department-of-veterans-affairs/abd-vro/issues/1955 - echo "continue=false" >> "$GITHUB_OUTPUT" - echo "Skipping b/c GitHub release ${{ github.ref_name }} already exists." | tee -a "$GITHUB_STEP_SUMMARY" - else - echo "continue=true" >> "$GITHUB_OUTPUT" - fi - - - name: "Create GitHub Release" - if: steps.check-release.outputs.continue == 'true' - uses: softprops/action-gh-release@v1 - with: - name: "${{ github.ref_name }}-auto" - # Use a token so that auto-creating a GH Release will trigger the SecRel workflow - token: ${{ secrets.ACCESS_TOKEN_PUSH_TO_DEVELOP }} - prerelease: true - generate_release_notes: true - append_body: true - body: "Automatically created by create-gh-release GitHub Action (run #${{github.run_number}})" diff --git a/.github/workflows/create-release-tag.yml b/.github/workflows/create-release-tag.yml deleted file mode 100644 index 2296461b45..0000000000 --- a/.github/workflows/create-release-tag.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: "4. Create release tag" -run-name: "Create release: ${{inputs.version}}" - -on: - workflow_dispatch: - inputs: - version: - description: 'The new semantic version number using the form X.Y.Z. \ - Using the default (next patch version) will increment Z.' - required: true - default: "(next patch version)" - type: string - -jobs: - update-code-version: - # only run in public repo - if: github.repository == 'department-of-veterans-affairs/abd-vro' - runs-on: ubuntu-latest - steps: - - name: "Install Java" - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: '17' - # "When using gradle-build-action we recommend that you - # not use actions/cache or actions/setup-java@v4 to explicitly - # cache the Gradle User Home" - # cache: 'gradle' - - name: "Setup Gradle" - uses: gradle/actions/setup-gradle@v3 - with: - # Only write to the cache for builds on the 'main' and 'develop' branches. (Default is 'main' only.) - # Builds on other branches will only read existing entries from the cache. - cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/qa' && github.ref != 'refs/heads/develop' }} - # To avoid a growing cache over time, attempt to delete any files in the Gradle User Home - # that were not used by Gradle during the workflow, prior to saving the cache. - # https://github.com/gradle/actions/setup-gradle@v3#removing-unused-files-from-gradle-user-home-before-saving-to-cache - gradle-home-cache-cleanup: true - - name: "Checkout source code" - uses: actions/checkout@v4 - with: - # Checkout using a PAT so that we can do `git push` later - token: ${{ secrets.ACCESS_TOKEN_PUSH_TO_DEVELOP }} - - - name: "Run gradle release" - run: | - if [ "${{ inputs.version }}" == "(next patch version)" ]; then - # Fetch prior tags to determine last version: https://axion-release-plugin.readthedocs.io/en/latest/configuration/ci_servers/#github-actions - # Fetch only tags, not branches: https://stackoverflow.com/questions/1204190/does-git-fetch-tags-include-git-fetch - git fetch --tags --unshallow origin 'refs/tags/*:refs/tags/*' - ./gradlew release - else - ./gradlew release -Prelease.forceVersion="${{ inputs.version }}" - fi - - # Pin version numbers AFTER creating the new version - - name: "Pin unpinned image versions" - run: | - scripts/image-version.sh pin - - git config --worktree user.name "VRO Machine User" - git config --worktree user.email "abd-vro-machine@users.noreply.github.com" - git add scripts/image_versions.src - git commit -m "Automated commit: Pin versions of unpinned images" - - - name: "Push updated image versions and new tag" - run: | - git log -5 - git push - # Push the release tag after a successful `git push` - git push --tags diff --git a/.github/workflows/publish-3rd-party-image.yml b/.github/workflows/publish-3rd-party-image.yml index 531801bd40..aed70c3eb6 100644 --- a/.github/workflows/publish-3rd-party-image.yml +++ b/.github/workflows/publish-3rd-party-image.yml @@ -64,7 +64,6 @@ jobs: secrel: name: SecRel Pipeline needs: publish-image - # https://psychic-disco-c1251ea1.pages.github.io/pipeline/release_notes/v4.1.0/ uses: department-of-veterans-affairs/lighthouse-tornado-secrel-pipeline/.github/workflows/pipeline.yml@v4 with: config-file: .github/secrel/config.yml diff --git a/.github/workflows/secrel.yml b/.github/workflows/secrel.yml index 69c160b998..9fa82d47c3 100644 --- a/.github/workflows/secrel.yml +++ b/.github/workflows/secrel.yml @@ -6,10 +6,6 @@ on: push: branches: [ main, qa, develop, domain-* ] - # Trigger on all published GitHub Releases - release: - types: [ "created" ] - pull_request: branches: [ develop ] @@ -34,14 +30,6 @@ on: required: true type: boolean default: true - publish_mode: - description: "Which images to publish" - required: true - type: choice - options: - - "only_changed" - - "all" - - "pinned_only" env: # Id for the #benefits-vro-devops Slack channel @@ -60,23 +48,6 @@ jobs: id: check-state run: | # Gate Check - - # Use triggering_actor instead of actor to allow manually re-running this workflow as another user - if [ "${{github.triggering_actor}}" == "abd-vro-machine" ]; then - if echo "$HEAD_COMMIT_MSG" | grep '^Automated commit: Pin versions'; then - CONTINUE=false - echo "Skipping this workflow b/c it was triggered by a commit that pins image versions \ - immediately after a release. Otherwise, the workflow will fail. - Until the release images are published to GHCR (so that they're available for comparison), \ - the 'Publish to GHCR' step of this workflow will fail." | tee -a "$GITHUB_STEP_SUMMARY" - - elif echo "$HEAD_COMMIT_MSG" | grep '^Automated commit:'; then - CONTINUE=false - echo "Skipping this workflow b/c it was triggered by an automated commit that \ - doesn't change container images." | tee -a "$GITHUB_STEP_SUMMARY" - fi - fi - if [ "${{ github.event_name }}" == "pull_request" ]; then if echo "$GITHUB_HEAD_REF" | grep '^dependabot/'; then CONTINUE=true @@ -106,24 +77,16 @@ jobs: IMG_TAG=${GITHUB_SHA:0:7} IMG_PREFIX="dev_" RUN_GRADLE_TESTS="true" - PUBLISH_MODE="all" RUN_SECREL="true" # Override some defaults depending on the branch/ref_name echo "ref_name: ${{ github.ref_name }}" case "${{ github.ref_name }}" in main) IMG_PREFIX="" - PUBLISH_MODE="all" - ;; - release-*|v*.*.*) # if git tag reflects a release, then publish non-dev images - IMG_PREFIX="" - # For release events, all images are pinned so use the pinned version - PUBLISH_MODE="pinned_only" ;; develop) # To reduce workflow runtime, don't run tests RUN_GRADLE_TESTS=false # To reduce SecRel scans, only published changed images - PUBLISH_MODE="only_changed" ;; esac @@ -132,7 +95,6 @@ jobs: [ "${{ inputs.image_prefix }}" == "(no prefix)" ] && IMG_PREFIX="" RUN_GRADLE_TESTS=${{ inputs.run_tests }} RUN_SECREL=${{ inputs.run_secrel }} - PUBLISH_MODE=${{ inputs.publish_mode }} fi { @@ -140,7 +102,6 @@ jobs: echo "image_prefix=${IMG_PREFIX}" echo "run_tests=${RUN_GRADLE_TESTS}" echo "run_secrel=${RUN_SECREL}" - echo "publish_mode=${PUBLISH_MODE}" } >> "$GITHUB_OUTPUT" - name: "DEBUG" @@ -148,7 +109,6 @@ jobs: echo "image_prefix: ${{ steps.image-props.outputs.image_prefix }}" echo "image_tag: ${{ steps.image-props.outputs.image_tag }}" echo "run_tests: ${{ steps.image-props.outputs.run_tests }}" - echo "publish_mode: ${{ steps.image-props.outputs.publish_mode }}" - name: "Checkout source code" uses: actions/checkout@v4 @@ -180,7 +140,6 @@ jobs: ghcr_username: ${{ github.actor }} ghcr_password: ${{ secrets.GITHUB_TOKEN }} run_tests: ${{ steps.image-props.outputs.run_tests }} - publish_mode: ${{ steps.image-props.outputs.publish_mode }} - name: "Slack thread: Images published" uses: archive/github-actions-slack@v2.9.0 @@ -203,8 +162,7 @@ jobs: name: SecRel Pipeline needs: publish-to-ghcr if: needs.publish-to-ghcr.outputs.run-secrel == 'true' - # https://psychic-disco-c1251ea1.pages.github.io/pipeline/release_notes/v4.2.0/ - uses: department-of-veterans-affairs/lighthouse-tornado-secrel-pipeline/.github/workflows/pipeline.yml@v5.0.19 + uses: department-of-veterans-affairs/lighthouse-tornado-secrel-pipeline/.github/workflows/pipeline.yml@v5 with: config-file: .github/secrel/config.yml images: ${{ needs.publish-to-ghcr.outputs.vro-images }} diff --git a/.github/workflows/update-deployment-cc.yml b/.github/workflows/update-deployment-cc.yml index 261a6665fb..ccf80846e1 100644 --- a/.github/workflows/update-deployment-cc.yml +++ b/.github/workflows/update-deployment-cc.yml @@ -18,7 +18,7 @@ on: - prod-test image_tag: - description: 'Image tag: first 7 of commit hash, "latest", "v1.2.3", or "release-1.2.3"' + description: 'Image tag: first 7 of commit hash or "latest"' required: true default: 'latest' type: string diff --git a/.github/workflows/update-deployment-ee.yml b/.github/workflows/update-deployment-ee.yml index d0995d5f20..cbbbe9bed4 100644 --- a/.github/workflows/update-deployment-ee.yml +++ b/.github/workflows/update-deployment-ee.yml @@ -18,7 +18,7 @@ on: - prod-test image_tag: - description: 'Image tag: first 7 of commit hash, "latest", "v1.2.3", or "release-1.2.3"' + description: 'Image tag: first 7 of commit hash or "latest"' required: true default: 'latest' type: string diff --git a/.github/workflows/update-deployment.yml b/.github/workflows/update-deployment.yml index 39c437eca8..765f0efdfb 100644 --- a/.github/workflows/update-deployment.yml +++ b/.github/workflows/update-deployment.yml @@ -33,7 +33,7 @@ on: - dev-tools image_tag: - description: 'Image tag: first 7 of commit hash, "latest", "v1.2.3", or "release-1.2.3"' + description: 'Image tag: first 7 of commit hash or "latest"' required: true default: 'latest' type: string @@ -179,17 +179,8 @@ jobs: export WAIT_TIMEOUT=10m export TRIGGERING_ACTOR="${{github.triggering_actor}}" - # Load default values for *_VER environment variables, which pin image versions source scripts/image_vars.src - { - echo '## Default image versions' - echo '```' - imageVersions | sort - echo '```' - } | tee -a "$GITHUB_STEP_SUMMARY" - - # To override pinned image version default, export env variable, e.g., 'apigateway_VER' for Helm chart 'api-gateway' VAR_PREFIX=$(bashVarPrefix "${{inputs.helm_chart}}") eval export "${VAR_PREFIX}_VER=${{inputs.image_tag}}" diff --git a/build.gradle b/build.gradle index 6d427e3789..5436b889df 100644 --- a/build.gradle +++ b/build.gradle @@ -2,18 +2,10 @@ buildscript { dependencies { classpath "org.yaml:snakeyaml:2.2" } - - // There's a conflict between versions of jgit between spotless and axion release plugin - configurations.classpath { - resolutionStrategy { - force 'org.eclipse.jgit:org.eclipse.jgit:6.9.0.202403050737-r' - } - } } plugins { id 'local.std.base-project-conventions' - id 'pl.allegro.tech.build.axion-release' version '1.17.0' // Set versions for these plugins (but don't apply) so we don't have to repeat the versions in subprojects id "org.springframework.boot" version "${spring_boot_version}" apply false @@ -95,72 +87,6 @@ ext { jacoco_minimum_coverage = 0.8 } -// Used by `release` task to create a new version -scmVersion { - // never connect to remote - localOnly = true - useHighestVersion = true - - def helmChartAppVersions = [ - - // These change often (i.e., with each release) - // A null value will result in the currentVersion being used - '..' : null, // '..' will resolve to helm/Charts.yaml - 'api-gateway': null, // Update the version presented in SwaggerUI - 'vro-app' : null, - 'svc-bgs-api': null, - 'svc-lighthouse-api': null, - 'redis' : null, - 'rabbitmq' : null, - // Once stable, set a version and move to previous section - 'domain-cc' : null, - 'domain-ee-max-cfi-app' : null, - 'domain-ee-ep-merge-app' : null, - ] - - def setHelmChartAppVersions = { currentVersion -> - helmChartAppVersions.collect { projectName, version -> - if (version == null) version = currentVersion - - def helmChartFilePaths = [ - project.findProperty("helm.${projectName}.chart_file"), - "/helm/${projectName}/Chart.yaml", - "/helm/platform/charts/${projectName}/Chart.yaml", - "/helm/vro/charts/${projectName}/Chart.yaml" - ] - def helmChartFilePath = helmChartFilePaths.find{ it != null && file("${project.rootDir}/$it").exists() } - println String.format( "%-14s %-12s => %s", projectName, version, helmChartFilePath) - if (version != null && helmChartFilePath != null) { - def helmChartFile = file("${project.rootDir}/$helmChartFilePath") - String s = helmChartFile.text.replaceFirst(/(?m)^appVersion: ".*"/, "appVersion: \"$version\"") - helmChartFile.setText(s) - helmChartFile - } - } - } - - hooks { - pre("fileUpdate", [file: "scripts/image_versions.src", - pattern : { v, p -> /CURRENT_RELEASE_VER="v.*"/ }, - replacement: { v, p -> "CURRENT_RELEASE_VER=\"v$v\"" }]) - pre({ context -> - def versionsFile = file("${project.rootDir}/versions.txt") - versionsFile.append("v${context.currentVersion}\n") - context.addCommitPattern(versionsFile.canonicalPath) - }) - pre({ context -> - setHelmChartAppVersions(context.currentVersion) - .findAll{ it !=null }.each { file -> context.addCommitPattern(file.canonicalPath) } - }) - pre("commit") - } -} - -allprojects { - // https://axion-release-plugin.readthedocs.io/en/latest/configuration/basic_usage - project.version = scmVersion.version -} - tasks.named("dockerComposeDown").configure { // Shut down subprojects' containers before Platform containers, volumes, and network mustRunAfter ':app:dockerComposeDown' diff --git a/helm/api-gateway/Chart.yaml b/helm/api-gateway/Chart.yaml index e889e954e1..2f7ee394b0 100644 --- a/helm/api-gateway/Chart.yaml +++ b/helm/api-gateway/Chart.yaml @@ -12,8 +12,3 @@ description: ABD VRO (Automated Benefits Delivery - Virtual Regional Office) # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.6.15" diff --git a/helm/api-gateway/templates/deployment.yaml b/helm/api-gateway/templates/deployment.yaml index 107cc3b193..87452e5229 100644 --- a/helm/api-gateway/templates/deployment.yaml +++ b/helm/api-gateway/templates/deployment.yaml @@ -7,7 +7,6 @@ metadata: annotations: app.kubernetes.io/owner: {{ .Values.owner }} app.kubernetes.io/name: {{ .Values.name }} - app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} app.kubernetes.io/env: {{ .Values.global.environment }} spec: replicas: {{ .Values.replicaCount }} @@ -21,7 +20,6 @@ spec: {{- include "vro.annotations.reloadLatest" . | nindent 8 }} app.kubernetes.io/podowner: {{ .Values.owner }} app.kubernetes.io/podname: {{ .Values.name }} - app.kubernetes.io/podversion: {{ .Chart.AppVersion | quote }} app.kubernetes.io/podenv: {{ .Values.global.environment }} spec: {{- include "vro.imagePullSecrets" . | nindent 6 }} diff --git a/helm/deploy.sh b/helm/deploy.sh index 6ae0a16782..7ed0db834d 100755 --- a/helm/deploy.sh +++ b/helm/deploy.sh @@ -23,9 +23,6 @@ NAMESPACE=va-abd-rrd-${TARGET_ENV} : ${GITHUB_SHA:=$(git rev-parse HEAD)} : ${TRIGGERING_ACTOR:=$USER} -#echo -e "TARGET_ENV=$TARGET_ENV \t HELM_CHART=HELM_CHART \t IMAGE_TAG=$IMAGE_TAG" -#echo -e "RELEASE_NAME=$RELEASE_NAME \t NAMESPACE=$NAMESPACE \t GITHUB_SHA=$GITHUB_SHA" - if [ "${SHUTDOWN_FIRST}" == "true" ]; then helm del "$RELEASE_NAME" --wait -n "$NAMESPACE" || exit 5 fi @@ -91,7 +88,6 @@ case "$HELM_CHART" in HELM_ARGS="$HELM_ARGS --set-string imageTag=$postgres_VER";; esac -#echo "HELM_ARGS: $HELM_ARGS" set -x # Exit with error code when command fails so that GH Action fails set -e diff --git a/helm/dev-tools/Chart.yaml b/helm/dev-tools/Chart.yaml index c375265a7d..7da95fc1c4 100644 --- a/helm/dev-tools/Chart.yaml +++ b/helm/dev-tools/Chart.yaml @@ -16,9 +16,3 @@ type: application # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "3.6.1" diff --git a/helm/domain-cc/Chart.yaml b/helm/domain-cc/Chart.yaml index f7fe1c204d..a8fba3bd88 100644 --- a/helm/domain-cc/Chart.yaml +++ b/helm/domain-cc/Chart.yaml @@ -12,8 +12,3 @@ description: Workflows and microservices for the Contention Classification domai # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.6.15" diff --git a/helm/domain-ee-ep-merge-app/Chart.yaml b/helm/domain-ee-ep-merge-app/Chart.yaml index 339db3da94..ec47f5e641 100644 --- a/helm/domain-ee-ep-merge-app/Chart.yaml +++ b/helm/domain-ee-ep-merge-app/Chart.yaml @@ -12,8 +12,3 @@ description: EP400 Merge App for the Employee Experience domain # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.6.15" diff --git a/helm/domain-ee-max-cfi-app/Chart.yaml b/helm/domain-ee-max-cfi-app/Chart.yaml index 10bc61982e..aca89e3470 100644 --- a/helm/domain-ee-max-cfi-app/Chart.yaml +++ b/helm/domain-ee-max-cfi-app/Chart.yaml @@ -12,8 +12,3 @@ description: Max Claim for Increase (CFI) API for the Employee Experience domain # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.6.15" diff --git a/helm/domain-xample/Chart.yaml b/helm/domain-xample/Chart.yaml index 78d8ab4afc..ad6c0bab79 100644 --- a/helm/domain-xample/Chart.yaml +++ b/helm/domain-xample/Chart.yaml @@ -12,8 +12,3 @@ description: Workflows and microservices for the Xample domain # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.2.1 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.3.1" diff --git a/helm/platform/Chart.yaml b/helm/platform/Chart.yaml index 866be31fec..10703893f3 100644 --- a/helm/platform/Chart.yaml +++ b/helm/platform/Chart.yaml @@ -17,12 +17,6 @@ type: application # Versions are expected to follow Semantic Versioning (https://semver.org/) version: 0.2.0 -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "3.6.15" - # https://levelup.gitconnected.com/helm-data-sharing-between-parent-and-child-chart-c4487a452d4e dependencies: # name must match the subchart's Chart.yaml's name diff --git a/helm/platform/charts/rabbitmq/Chart.yaml b/helm/platform/charts/rabbitmq/Chart.yaml index 026414814f..a2b89cbd55 100644 --- a/helm/platform/charts/rabbitmq/Chart.yaml +++ b/helm/platform/charts/rabbitmq/Chart.yaml @@ -12,8 +12,3 @@ description: Message Queue service used by VRO and its microservices # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.6.15" diff --git a/helm/platform/charts/redis/Chart.yaml b/helm/platform/charts/redis/Chart.yaml index 40e7bd3e3a..388ae7b5ff 100644 --- a/helm/platform/charts/redis/Chart.yaml +++ b/helm/platform/charts/redis/Chart.yaml @@ -12,8 +12,3 @@ description: Redis cache service used by VRO and its microservices # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.6.15" diff --git a/helm/svc-bgs-api/Chart.yaml b/helm/svc-bgs-api/Chart.yaml index b844dfdfa4..bb8b4abd79 100644 --- a/helm/svc-bgs-api/Chart.yaml +++ b/helm/svc-bgs-api/Chart.yaml @@ -12,8 +12,3 @@ description: VRO microservice - client for BGS API # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.1.5 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.6.15" diff --git a/helm/svc-bie-kafka/Chart.yaml b/helm/svc-bie-kafka/Chart.yaml index 680c312b8c..337364b8e1 100644 --- a/helm/svc-bie-kafka/Chart.yaml +++ b/helm/svc-bie-kafka/Chart.yaml @@ -12,8 +12,3 @@ description: VRO microservice - client for BIE Kafka # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.1.4 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.3.0" diff --git a/helm/svc-bip-api/Chart.yaml b/helm/svc-bip-api/Chart.yaml index 85099fc2fe..e45ef82897 100644 --- a/helm/svc-bip-api/Chart.yaml +++ b/helm/svc-bip-api/Chart.yaml @@ -12,8 +12,3 @@ description: VRO microservice - client for BIP API # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.1.4 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.3.0" diff --git a/helm/svc-lighthouse-api/Chart.yaml b/helm/svc-lighthouse-api/Chart.yaml index 1445672d09..457eb03317 100644 --- a/helm/svc-lighthouse-api/Chart.yaml +++ b/helm/svc-lighthouse-api/Chart.yaml @@ -12,8 +12,3 @@ description: VRO microservice - client for Lighthouse API # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.1.4 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.6.15" diff --git a/helm/vro-app/Chart.yaml b/helm/vro-app/Chart.yaml index 8db0b86c13..18aee7c3e0 100644 --- a/helm/vro-app/Chart.yaml +++ b/helm/vro-app/Chart.yaml @@ -12,8 +12,3 @@ description: ABD VRO (Automated Benefits Delivery - Virtual Regional Office) # to the chart and its templates, including the app version. # TODO discuss: How can we enforce or automate incrementing this? version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -# This field is informational, and has no impact on chart version calculations. -appVersion: "3.6.15" diff --git a/scripts/image-names.sh b/scripts/image-names.sh index 6a1b1d0524..12aa71d9c8 100755 --- a/scripts/image-names.sh +++ b/scripts/image-names.sh @@ -64,8 +64,6 @@ overwriteSrcFile(){ echo '# shellcheck disable=SC2034' echo "VAR_PREFIXES_ARR=( ${VAR_PREFIXES[@]} )" echo "export VAR_PREFIXES=\"${VAR_PREFIXES[@]}\"" - echo - echo 'LAST_RELEASE_VERSION=$(tail -1 versions.txt)' echo ' ## Helper functions # Usage example to get the variable value for app_GRADLE_IMG: GRADLE_IMG_TAG=`getVarValue app _GRADLE_IMG` @@ -97,19 +95,10 @@ imageTagExists(){ # echo "The value of ${PREFIX}_GRADLE_IMG is $(getVarValue ${PREFIX} _GRADLE_IMG)" # echo # done - -imageVersions(){ - # shellcheck disable=SC2068 - for PREFIX in ${VAR_PREFIXES_ARR[@]}; do - echo "$(getVarValue ${PREFIX} _IMG) $(getVarValue ${PREFIX} _VER)" - done -} - ###################################### ' # Load current image versions by setting *_VER variables - >&2 source scripts/image_versions.src getVarValue(){ local VARNAME=${1}${2} echo "${!VARNAME}" @@ -118,17 +107,7 @@ imageVersions(){ local PREFIX=$(bashVarPrefix "$IMG") echo "export ${PREFIX}_GRADLE_IMG=\"$(gradleImageName "$IMG")\"" echo "export ${PREFIX}_IMG=\"$(prodImageName "$IMG")\"" - echo "export ${PREFIX}_VER=\"\$LAST_RELEASE_VERSION\"" echo done - - echo '######################################## -# Override default *_VER variables above -source scripts/image_versions.src - -if [ "$1" ]; then - eval "$@" -fi -# End of file' } overwriteSrcFile > "$SRC_FILE" diff --git a/scripts/image-version.sh b/scripts/image-version.sh deleted file mode 100755 index e97d39f683..0000000000 --- a/scripts/image-version.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/bash - -source scripts/image_vars.src - -# Pins image versions to the latest release version -# if the image is not already pinned. -pinImageVersions(){ - echo "# $(date) -- $LAST_RELEASE_VERSION" - for PREFIX in ${VAR_PREFIXES_ARR[@]}; do - local IMG_VAR="${PREFIX}_VER" - # If not (automatically or manually) pinned, then pin to latest release version - if ! grep -q -w "${IMG_VAR}" scripts/image_versions.src; then - local IMG_VER=$(getVarValue "${PREFIX}" _VER) - >&2 echo "Pinning ${IMG_VAR}=\"$IMG_VER\"" - echo "${IMG_VAR}=\"$IMG_VER\"" - fi - done -} - -# Unpins auto-pinned image versions -unpinImageVersion(){ - local PREFIX=$1 - local IMG_VAR="${PREFIX}_VER" - >&2 echo "Unpinning ${IMG_VAR}" - sed "/^${IMG_VAR}=/d" scripts/image_versions.src -} - -# Returns only versions that have been automatically pinned (by this script) -autoPinnedImages(){ - for PREFIX in ${VAR_PREFIXES_ARR[@]}; do - if grep -q "^${PREFIX}_VER=" scripts/image_versions.src; then - echo "${PREFIX}" - fi - done -} - -# Returns only versions that have been automatically pinned (by this script) -# and have changed -changedAutoPinnedImages(){ - for PREFIX in $(autoPinnedImages); do - >&2 echo "Found pinned image: ${PREFIX}" - local IMG_DIFFS=$(comparePinnedImages) - >&2 echo "$IMG_DIFFS" | jq - if [ "$IMG_DIFFS" = " Error" ]; then - return 4 - elif ! isImageSame "$IMG_DIFFS"; then - echo "${PREFIX}" - fi - done -} - -# Returns JSON of image differences between locally created image -# and the pinned image version (which has a release tag) -comparePinnedImages(){ - local IMG_VER=$(getVarValue "${PREFIX}" _VER) - # Release versions are tagged on non-dev images only (see secrel.yml) so no image `dev_` image prefix is needed - local IMG_NAME=$(getVarValue "${PREFIX}" _IMG) - local GHCR_PATH="ghcr.io/department-of-veterans-affairs/abd-vro-internal/${IMG_NAME}" - local GRADLE_IMG_NAME=$(getVarValue "${PREFIX}" _GRADLE_IMG) - >&2 echo " Comparing local ${GRADLE_IMG_NAME} vs GHCR's ${IMG_NAME}:$IMG_VER" - container-diff diff --type=history --type=size --json \ - "daemon://${GRADLE_IMG_NAME}" \ - "remote://${GHCR_PATH}:${IMG_VER}" || echo " Error" -} - -# Using container-diff, every tiny difference is detected. -# There are docker-build-time differences (e.g., installing packages and file timestamps) -# that cause two images that are practically the same to be different. -# This function tries to account for some trivial differences but -# an image size difference cannot be ignored. -isImageSame(){ - local IMG_DIFFS=$1 - local SIZE_DIFF_LEN=$(echo "${IMG_DIFFS}" | jq '.[] | select(.DiffType == "Size") | .Diff | length') - # The 'grep -v " is used to ignore difference in the file identifier - local HIST_DIFF_LEN=$(echo "${IMG_DIFFS}" | jq '.[] | select(.DiffType == "History")' | \ - grep -v "/bin/sh -c #(nop) COPY file:.* in fat.jar" | \ - jq '.Diff.Adds + .Diff.Dels | length') - - if [ "$SIZE_DIFF_LEN" = 0 ] && [ "$HIST_DIFF_LEN" = 0 ]; then - >&2 echo " Same" - return 0 - else - >&2 echo " Different" - return 1 - fi -} - -## The functions above perform only read operations. -## File modifications are done below. - -case "$1" in - pin) pinImageVersions >> scripts/image_versions.src - ;; - unpinIfDiff) - CHANGED_PINNED_IMAGES=$(changedAutoPinnedImages) - if [ "$?" = 4 ]; then - >&2 echo "Error comparing images, probably due to missing image.\ - Retry after secrel.yml workflow publishes release versions." - exit 44 - fi - for PREFIX in $CHANGED_PINNED_IMAGES; do - unpinImageVersion "${PREFIX}" > unpinned_versions.src && \ - mv unpinned_versions.src scripts/image_versions.src - done - ;; - "") echo "Usage: - To pin versions of unpinned images: - $0 pin - To unpin versions of pinned images that have changed: - $0 unpinIfDiff -" - ;; -esac diff --git a/scripts/image_vars.src b/scripts/image_vars.src index 6f7339fd96..ba3e52bdca 100755 --- a/scripts/image_vars.src +++ b/scripts/image_vars.src @@ -14,8 +14,6 @@ export VRO_IMAGES="postgres redis rabbitmq api-gateway app db-init dev-tools svc VAR_PREFIXES_ARR=( postgres redis rabbitmq apigateway app dbinit devtools svcbgsapi svclighthouseapi svcbiekafka svcbipapi xampleworkflows ccapp eemaxcfiapp eeepmergeapp ) export VAR_PREFIXES="postgres redis rabbitmq apigateway app dbinit devtools svcbgsapi svclighthouseapi svcbiekafka svcbipapi xampleworkflows ccapp eemaxcfiapp eeepmergeapp" -LAST_RELEASE_VERSION=$(tail -1 versions.txt) - ## Helper functions # Usage example to get the variable value for app_GRADLE_IMG: GRADLE_IMG_TAG=`getVarValue app _GRADLE_IMG` getVarValue(){ @@ -46,81 +44,49 @@ imageTagExists(){ # echo "The value of ${PREFIX}_GRADLE_IMG is $(getVarValue ${PREFIX} _GRADLE_IMG)" # echo # done - -imageVersions(){ - # shellcheck disable=SC2068 - for PREFIX in ${VAR_PREFIXES_ARR[@]}; do - echo "$(getVarValue ${PREFIX} _IMG) $(getVarValue ${PREFIX} _VER)" - done -} - ###################################### export postgres_GRADLE_IMG="va/abd_vro-postgres" export postgres_IMG="vro-postgres" -export postgres_VER="$LAST_RELEASE_VERSION" export redis_GRADLE_IMG="va/abd_vro-redis" export redis_IMG="vro-redis" -export redis_VER="$LAST_RELEASE_VERSION" export rabbitmq_GRADLE_IMG="va/abd_vro-rabbitmq" export rabbitmq_IMG="vro-rabbitmq" -export rabbitmq_VER="$LAST_RELEASE_VERSION" export apigateway_GRADLE_IMG="va/abd_vro-api-gateway" export apigateway_IMG="vro-api-gateway" -export apigateway_VER="$LAST_RELEASE_VERSION" export app_GRADLE_IMG="va/abd_vro-app" export app_IMG="vro-app" -export app_VER="$LAST_RELEASE_VERSION" export dbinit_GRADLE_IMG="va/abd_vro-db-init" export dbinit_IMG="vro-db-init" -export dbinit_VER="$LAST_RELEASE_VERSION" export devtools_GRADLE_IMG="va/abd_vro-dev-tools" export devtools_IMG="vro-dev-tools" -export devtools_VER="$LAST_RELEASE_VERSION" export svcbgsapi_GRADLE_IMG="va/abd_vro-svc-bgs-api" export svcbgsapi_IMG="vro-svc-bgs-api" -export svcbgsapi_VER="$LAST_RELEASE_VERSION" export svclighthouseapi_GRADLE_IMG="va/abd_vro-svc-lighthouse-api" export svclighthouseapi_IMG="vro-svc-lighthouse-api" -export svclighthouseapi_VER="$LAST_RELEASE_VERSION" export svcbiekafka_GRADLE_IMG="va/abd_vro-svc-bie-kafka" export svcbiekafka_IMG="vro-svc-bie-kafka" -export svcbiekafka_VER="$LAST_RELEASE_VERSION" export svcbipapi_GRADLE_IMG="va/abd_vro-svc-bip-api" export svcbipapi_IMG="vro-svc-bip-api" -export svcbipapi_VER="$LAST_RELEASE_VERSION" export xampleworkflows_GRADLE_IMG="va/abd_vro-xample-workflows" export xampleworkflows_IMG="vro-xample-workflows" -export xampleworkflows_VER="$LAST_RELEASE_VERSION" export ccapp_GRADLE_IMG="va/abd_vro-cc-app" export ccapp_IMG="vro-cc-app" -export ccapp_VER="$LAST_RELEASE_VERSION" export eemaxcfiapp_GRADLE_IMG="va/abd_vro-ee-max-cfi-app" export eemaxcfiapp_IMG="vro-ee-max-cfi-app" -export eemaxcfiapp_VER="$LAST_RELEASE_VERSION" export eeepmergeapp_GRADLE_IMG="va/abd_vro-ee-ep-merge-app" export eeepmergeapp_IMG="vro-ee-ep-merge-app" -export eeepmergeapp_VER="$LAST_RELEASE_VERSION" - -######################################## -# Override default *_VER variables above -source scripts/image_versions.src - -if [ "$1" ]; then - eval "$@" -fi -# End of file diff --git a/scripts/image_versions.src b/scripts/image_versions.src deleted file mode 100644 index de7952775b..0000000000 --- a/scripts/image_versions.src +++ /dev/null @@ -1,64 +0,0 @@ -# This file sets environment variables that enables pinning a version for specific containers -# overriding the variables set in image_vars.src. -# -# These versions are used as image tags for: -# - image publishing to GHCR (see .github/actions/publish-images/action.yml) -# - Helm chart deployments to LHDI (see .github/workflows/update-deployment.yml) - -# Manually pin these images; they don't need to be automatically updated -# Use the `export myimage_VER="v1.2.3"` syntax so that they will be ignored by `image-version.sh` -# export console_VER="v3.2.5" - -# The following are pinned versions of images; -# they are automatically pinned and unpinned by scripts/image-version.sh. -# See https://github.com/department-of-veterans-affairs/abd-vro/wiki/Container-Image-Versions -# Mon Jul 31 17:42:17 UTC 2023 -- v3.4.0 -# Tue Aug 15 18:14:47 UTC 2023 -- v3.4.1 -# Fri Aug 18 18:35:18 UTC 2023 -- v3.4.3 -# Tue Aug 22 15:10:09 UTC 2023 -- v3.4.4 -# Tue Aug 22 15:48:51 UTC 2023 -- v3.4.5 -# Thu Aug 24 13:10:53 UTC 2023 -- v3.4.6 -# Fri Aug 25 16:19:51 UTC 2023 -- v3.4.7 -# Fri Aug 25 16:36:30 UTC 2023 -- v3.4.8 -# Wed Aug 30 18:36:53 UTC 2023 -- v3.4.9 -# Fri Sep 1 21:08:58 UTC 2023 -- v3.4.12 -# Wed Sep 6 17:58:41 UTC 2023 -- v3.4.13 -# Thu Sep 7 20:28:30 UTC 2023 -- v3.4.14 -# Tue Sep 12 20:23:04 UTC 2023 -- v3.4.15 -# Wed Sep 13 14:29:58 UTC 2023 -- v3.4.16 -# Fri Sep 15 15:14:20 UTC 2023 -- v3.4.17 -# Wed Sep 20 17:22:23 UTC 2023 -- v3.4.18 -# Thu Sep 28 18:53:10 UTC 2023 -- v3.4.19 -# Fri Sep 29 16:52:44 UTC 2023 -- v3.4.20 -# Fri Sep 29 17:50:35 UTC 2023 -- v3.4.21 -# Mon Oct 2 20:38:20 UTC 2023 -- v3.5.0 -# Wed Oct 11 19:06:17 UTC 2023 -- v3.5.1 -# Thu Oct 12 16:21:59 UTC 2023 -- v3.5.2 -# Tue Oct 24 11:02:54 PDT 2023 -- v3.5.3 -# Thu Oct 26 12:23:43 PDT 2023 -- v3.5.4 -# Thu Oct 26 13:03:08 PDT 2023 -- v3.5.6 -postgres_VER="v3.5.6" -# Thu Nov 2 18:18:12 UTC 2023 -- v3.5.7 -# Thu Nov 9 00:56:04 UTC 2023 -- v3.5.8 -# Fri Nov 17 21:33:22 UTC 2023 -- v3.5.9 -# Tue Nov 28 20:49:39 UTC 2023 -- v3.5.10 -# Tue Nov 28 22:32:12 UTC 2023 -- v3.5.11 -# Wed Dec 13 22:20:53 UTC 2023 -- v3.6.0 -# Thu Dec 14 02:59:08 UTC 2023 -- v3.6.1 -# Mon Jan 22 18:32:01 UTC 2024 -- v3.6.2 -# Thu Jan 25 22:01:17 UTC 2024 -- v3.6.3 -# Tue Jan 30 02:15:11 UTC 2024 -- v3.6.4 -# Wed Jan 31 22:06:50 UTC 2024 -- v3.6.5 -# Wed Jan 31 22:15:37 UTC 2024 -- v3.6.6 -# Thu Feb 1 19:41:58 UTC 2024 -- v3.6.7 -# Thu Feb 8 19:13:49 UTC 2024 -- v3.6.8 -# Thu Feb 15 00:26:56 UTC 2024 -- v3.6.9 -# Fri Feb 16 23:58:06 UTC 2024 -- v3.6.10 -# Tue Feb 20 14:32:01 UTC 2024 -- v3.6.11 -# Tue Feb 20 22:15:30 UTC 2024 -- v3.6.12 -devtools_VER="v3.6.12" -# Wed Feb 21 15:29:25 UTC 2024 -- v3.6.13 -# Thu Feb 22 20:00:43 UTC 2024 -- v3.6.14 -redis_VER="v3.6.14" -# Thu Feb 29 21:50:04 UTC 2024 -- v3.6.15 -rabbitmq_VER="v3.6.15" diff --git a/versions.txt b/versions.txt deleted file mode 100644 index 0a5387b34e..0000000000 --- a/versions.txt +++ /dev/null @@ -1,55 +0,0 @@ -# This file contains significant versions of VRO and its components. -# The versions are used to present options for manual runs of GitHub Action workflows. -# This file is automatically updated when a new release is created. -# Manually delete versions that are not significant milestones or will not be deployed. - -v3.0.7-domaincc1 -v3.1.0 -v3.2.0 -v3.3.0 -v3.4.0 -v3.4.5 -v3.4.6 -v3.4.7 -v3.4.8 -v3.4.9 -v3.4.10 -v3.4.11 -v3.4.12 -v3.4.13 -v3.4.14 -v3.4.15 -v3.4.16 -v3.4.17 -v3.4.18 -v3.4.19 -v3.4.20 -v3.4.21 -v3.5.0 -v3.5.1 -v3.5.2 -v3.5.3 -v3.5.4 -v3.5.5 -v3.5.6 -v3.5.7 -v3.5.8 -v3.5.9 -v3.5.10 -v3.5.11 -v3.6.0 -v3.6.1 -v3.6.2 -v3.6.3 -v3.6.4 -v3.6.5 -v3.6.6 -v3.6.7 -v3.6.8 -v3.6.9 -v3.6.10 -v3.6.11 -v3.6.12 -v3.6.13 -v3.6.14 -v3.6.15 From 9b6b0a01c3e3cc7238e0d342ecb4d53a0ed4e467 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 19 Mar 2024 15:18:35 -0700 Subject: [PATCH 33/67] Update the substring env instead of a prefix for bia kafka topics. --- .../vro/model/biekafka/BieMessagePayload.java | 48 +++++++++---------- .../vro/model/biekafka/ContentionEvent.java | 20 ++++---- .../services/bie/config/BieProperties.java | 8 +++- .../src/main/resources/application-dev.yaml | 4 ++ .../main/resources/application-prod-test.yaml | 2 +- .../src/main/resources/application-prod.yaml | 2 +- .../src/main/resources/application-qa.yaml | 2 +- .../main/resources/application-sandbox.yaml | 2 +- .../src/main/resources/application.yaml | 2 +- .../bie/config/MessageExchangeConfigTest.java | 12 ++--- 10 files changed, 55 insertions(+), 47 deletions(-) diff --git a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java index 7c2f1faafc..3ceaa60ac4 100644 --- a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java +++ b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java @@ -34,60 +34,60 @@ public class BieMessagePayload { // populated from kafka topic payload @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" }) private String benefitClaimTypeCode; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" }) private String actorStation; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" }) private String details; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" }) private Long veteranParticipantId; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" }) private String contentionClassificationName; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" }) private String diagnosticTypeCode; - @TargetEvents({"BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED"}) + @TargetEvents({"BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED"}) private String journalStatusTypeCode; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" }) private Long dateAdded; - @TargetEvents({"BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED"}) + @TargetEvents({"BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED"}) private Long dateCompleted; - @TargetEvents({"BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED"}) + @TargetEvents({"BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED"}) private Long dateUpdated; } diff --git a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java index 71f60fb496..28b783ee8b 100644 --- a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java +++ b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java @@ -3,11 +3,11 @@ import java.util.Arrays; public enum ContentionEvent { - CONTENTION_ASSOCIATED_TO_CLAIM("BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM"), - CONTENTION_UPDATED("BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED"), - CONTENTION_CLASSIFIED("BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED"), - CONTENTION_COMPLETED("BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED"), - CONTENTION_DELETED("BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED"); + CONTENTION_ASSOCIATED_TO_CLAIM("BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM"), + CONTENTION_UPDATED("BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED"), + CONTENTION_CLASSIFIED("BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED"), + CONTENTION_COMPLETED("BIA_SERVICES_BIE_CATALOG_CONTENTION_COMPLETED"), + CONTENTION_DELETED("BIA_SERVICES_BIE_CATALOG_CONTENTION_DELETED"); private final String topicName; @@ -20,13 +20,13 @@ public String getTopicName() { } public static ContentionEvent mapTopicToEvent(String topic) { - // TODO(3/5/24): Add a better - String prefixPattern = "^EXT_VRO_[A-Z]+_"; - String noPrefixTopic = topic.replaceFirst(prefixPattern, ""); + String subString = "CATALOG_.*?_CONTENTION"; + String noSubStringTopic = topic.replaceAll(subString, "CATALOG_CONTENTION"); + return Arrays.stream(ContentionEvent.values()) - .filter(event -> event.getTopicName().equals(noPrefixTopic)) + .filter(event -> event.getTopicName().equals(noSubStringTopic)) .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Unrecognized topic: " + noPrefixTopic)); + .orElseThrow(() -> new IllegalArgumentException("Unrecognized topic: " + noSubStringTopic)); } public static String rabbitMqExchangeName(String topic) { diff --git a/svc-bie-kafka/src/main/java/gov/va/vro/services/bie/config/BieProperties.java b/svc-bie-kafka/src/main/java/gov/va/vro/services/bie/config/BieProperties.java index 5e87624a8b..bd042f625f 100644 --- a/svc-bie-kafka/src/main/java/gov/va/vro/services/bie/config/BieProperties.java +++ b/svc-bie-kafka/src/main/java/gov/va/vro/services/bie/config/BieProperties.java @@ -13,11 +13,15 @@ @Setter public class BieProperties { - @Getter String kakfaTopicPrefix; + @Getter String kafkaTopicEnv; public String[] topicNames() { return Arrays.stream(ContentionEvent.values()) - .map(contention -> kakfaTopicPrefix + contention.getTopicName()) + .map( + contention -> { + String subString = "CATALOG_" + kafkaTopicEnv + "_CONTENTION"; + return contention.getTopicName().replaceAll("CATALOG_CONTENTION", subString); + }) .toArray(String[]::new); } } diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index 3e81548398..253924d63a 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -21,3 +21,7 @@ spring: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-tst-vro-1}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" + + +bie: + kafka-topic-env: "DEV" \ No newline at end of file diff --git a/svc-bie-kafka/src/main/resources/application-prod-test.yaml b/svc-bie-kafka/src/main/resources/application-prod-test.yaml index 5ec0f29c5f..494962ff5c 100644 --- a/svc-bie-kafka/src/main/resources/application-prod-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-prod-test.yaml @@ -36,4 +36,4 @@ spring: password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" bie: - kakfa-topic-prefix: "PRE_" + kafka-topic-env: "PRE_" diff --git a/svc-bie-kafka/src/main/resources/application-prod.yaml b/svc-bie-kafka/src/main/resources/application-prod.yaml index 8810a8b0f1..39498e84f3 100644 --- a/svc-bie-kafka/src/main/resources/application-prod.yaml +++ b/svc-bie-kafka/src/main/resources/application-prod.yaml @@ -32,4 +32,4 @@ spring: password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" bie: - kakfa-topic-prefix: "PROD_" + kafka-topic-env: "PROD_" diff --git a/svc-bie-kafka/src/main/resources/application-qa.yaml b/svc-bie-kafka/src/main/resources/application-qa.yaml index 509e06495f..12d8049468 100644 --- a/svc-bie-kafka/src/main/resources/application-qa.yaml +++ b/svc-bie-kafka/src/main/resources/application-qa.yaml @@ -39,4 +39,4 @@ spring: password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" bie: - kakfa-topic-prefix: "IVV_" + kafka-topic-env: "IVV_" diff --git a/svc-bie-kafka/src/main/resources/application-sandbox.yaml b/svc-bie-kafka/src/main/resources/application-sandbox.yaml index 6a6862c538..5ea2d40dcd 100644 --- a/svc-bie-kafka/src/main/resources/application-sandbox.yaml +++ b/svc-bie-kafka/src/main/resources/application-sandbox.yaml @@ -36,4 +36,4 @@ spring: password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" bie: - kakfa-topic-prefix: "UAT_" + kafka-topic-env: "UAT_" diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index 4126ea6966..f1b42d8fc1 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -39,7 +39,7 @@ spring: ## Specify bie properties bie: - kakfa-topic-prefix: "EXT_VRO_TST_" + kafka-topic-env: "TST" ## Actuator for health check, liveness, and readiness management: diff --git a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java index e582d0b35f..43873c1f7f 100644 --- a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java +++ b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java @@ -17,7 +17,7 @@ class MessageExchangeConfigTest { @BeforeEach void setUp() { bieProperties = new BieProperties(); - bieProperties.kakfaTopicPrefix = "EXT_VRO_TST_"; + bieProperties.kafkaTopicEnv = "TST"; } @Test @@ -34,11 +34,11 @@ void topicNames() { final String[] topicNames = bieProperties.topicNames(); assertArrayEquals( new String[] { - "EXT_VRO_TST_BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", - "EXT_VRO_TST_BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED", - "EXT_VRO_TST_BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", - "EXT_VRO_TST_BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED", - "EXT_VRO_TST_BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED" }, topicNames); } From 3837add10d4dee55c256ad754c387b2df6379f49 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 25 Mar 2024 11:18:56 -0700 Subject: [PATCH 34/67] Experiment with bie-kafka-end2end-test.yml --- .github/workflows/bie-kafka-end2end-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bie-kafka-end2end-test.yml b/.github/workflows/bie-kafka-end2end-test.yml index 160ea979fd..ce25deb2b5 100644 --- a/.github/workflows/bie-kafka-end2end-test.yml +++ b/.github/workflows/bie-kafka-end2end-test.yml @@ -77,7 +77,10 @@ jobs: - name: 'Wait for Kafka to be ready' run: | # Verify broker is up with 30 second timeout - BROKER_QUERY=$(kafkacat -m 30 -b localhost:9092 -L) + BROKER_QUERY=$(kafkacat -m 30 -b localhost:9092 \ + -X security.protocol=sasl_ssl -X sasl.mechanisms=PLAIN \ + -X sasl.username=USERNAME -X sasl.password=PASSWORD \ + -L ) if echo "$BROKER_QUERY" | grep -q '1 brokers' then echo "Found Broker: $BROKER_QUERY" From 3b25beafe1acf81fe23c83f979b4b0655767460c Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 25 Mar 2024 11:49:47 -0700 Subject: [PATCH 35/67] Experiment with bie-kafka-end2end-test.yml --- .github/workflows/bie-kafka-end2end-test.yml | 1 + mocks/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bie-kafka-end2end-test.yml b/.github/workflows/bie-kafka-end2end-test.yml index ce25deb2b5..8858381827 100644 --- a/.github/workflows/bie-kafka-end2end-test.yml +++ b/.github/workflows/bie-kafka-end2end-test.yml @@ -78,6 +78,7 @@ jobs: run: | # Verify broker is up with 30 second timeout BROKER_QUERY=$(kafkacat -m 30 -b localhost:9092 \ + -X debug='security,broker' -X security.protocol=sasl_ssl -X sasl.mechanisms=PLAIN \ -X sasl.username=USERNAME -X sasl.password=PASSWORD \ -L ) diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index 4164bd2ebf..b262020e1b 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -88,7 +88,7 @@ services: # https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md # https://stackoverflow.com/a/72018409 # Caution: Empty string '' and "" don't work; however ' ' or " " does work - KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: " " +# KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: " " # For development without having to deal with connecting via SSL, use port 9092 or do the following. # To disable SSL, uncomment the following and comment out the next SSL section From 746d1df70c6ed45843ac0a2d86a4e3f49bada8d2 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 26 Mar 2024 10:39:48 -0700 Subject: [PATCH 36/67] Experiment with bie-kafka-end2end-test.yml --- .github/workflows/bie-kafka-end2end-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bie-kafka-end2end-test.yml b/.github/workflows/bie-kafka-end2end-test.yml index 8858381827..a0d0ff2cb5 100644 --- a/.github/workflows/bie-kafka-end2end-test.yml +++ b/.github/workflows/bie-kafka-end2end-test.yml @@ -78,10 +78,10 @@ jobs: run: | # Verify broker is up with 30 second timeout BROKER_QUERY=$(kafkacat -m 30 -b localhost:9092 \ - -X debug='security,broker' + -X debug='security,broker' \ -X security.protocol=sasl_ssl -X sasl.mechanisms=PLAIN \ -X sasl.username=USERNAME -X sasl.password=PASSWORD \ - -L ) + -L) if echo "$BROKER_QUERY" | grep -q '1 brokers' then echo "Found Broker: $BROKER_QUERY" From 3c35aad56beb5490b8d5c69c7f8f01e7e3d08dd8 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Thu, 28 Mar 2024 10:34:44 -0700 Subject: [PATCH 37/67] Update group-id --- svc-bie-kafka/src/main/resources/application-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index 253924d63a..a8c71177d4 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -18,7 +18,7 @@ spring: trust-store-location: "${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" consumer: - group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-tst-vro-1}" + group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:EXT_VRO_DEV}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" From 4c20a5968eaff80480d8fc6ede24251fa5131cd4 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Thu, 28 Mar 2024 12:54:08 -0700 Subject: [PATCH 38/67] Add V02 to the end of topic names --- .../vro/model/biekafka/BieMessagePayload.java | 48 +++++++++---------- .../vro/model/biekafka/ContentionEvent.java | 10 ++-- .../bie/config/ContentionEventTest.java | 20 ++++---- .../bie/config/MessageExchangeConfigTest.java | 10 ++-- 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java index 3ceaa60ac4..743dc4cc85 100644 --- a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java +++ b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/BieMessagePayload.java @@ -34,60 +34,60 @@ public class BieMessagePayload { // populated from kafka topic payload @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED_V02" }) private String benefitClaimTypeCode; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED_V02" }) private String actorStation; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED_V02" }) private String details; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED_V02" }) private Long veteranParticipantId; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED_V02" }) private String contentionClassificationName; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED_V02" }) private String diagnosticTypeCode; - @TargetEvents({"BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED"}) + @TargetEvents({"BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED_V02"}) private String journalStatusTypeCode; @TargetEvents({ - "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED" + "BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED_V02", + "BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED_V02" }) private Long dateAdded; - @TargetEvents({"BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED"}) + @TargetEvents({"BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED_V02"}) private Long dateCompleted; - @TargetEvents({"BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED"}) + @TargetEvents({"BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED_V02"}) private Long dateUpdated; } diff --git a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java index 28b783ee8b..b16a60268f 100644 --- a/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java +++ b/shared/lib-bie-kafka/src/main/java/gov/va/vro/model/biekafka/ContentionEvent.java @@ -3,11 +3,11 @@ import java.util.Arrays; public enum ContentionEvent { - CONTENTION_ASSOCIATED_TO_CLAIM("BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM"), - CONTENTION_UPDATED("BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED"), - CONTENTION_CLASSIFIED("BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED"), - CONTENTION_COMPLETED("BIA_SERVICES_BIE_CATALOG_CONTENTION_COMPLETED"), - CONTENTION_DELETED("BIA_SERVICES_BIE_CATALOG_CONTENTION_DELETED"); + CONTENTION_ASSOCIATED_TO_CLAIM("BIA_SERVICES_BIE_CATALOG_CONTENTION_ASSOCIATED_TO_CLAIM_V02"), + CONTENTION_UPDATED("BIA_SERVICES_BIE_CATALOG_CONTENTION_UPDATED_V02"), + CONTENTION_CLASSIFIED("BIA_SERVICES_BIE_CATALOG_CONTENTION_CLASSIFIED_V02"), + CONTENTION_COMPLETED("BIA_SERVICES_BIE_CATALOG_CONTENTION_COMPLETED_V02"), + CONTENTION_DELETED("BIA_SERVICES_BIE_CATALOG_CONTENTION_DELETED_V02"); private final String topicName; diff --git a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/ContentionEventTest.java b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/ContentionEventTest.java index 4e4d882559..63813d45a6 100644 --- a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/ContentionEventTest.java +++ b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/ContentionEventTest.java @@ -13,11 +13,11 @@ public class ContentionEventTest { @ParameterizedTest @CsvSource({ - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM, CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED, CONTENTION_UPDATED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED, CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED, CONTENTION_COMPLETED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED, CONTENTION_DELETED" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM_V02, CONTENTION_ASSOCIATED_TO_CLAIM", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED_V02, CONTENTION_UPDATED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED_V02, CONTENTION_CLASSIFIED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED_V02, CONTENTION_COMPLETED", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED_V02, CONTENTION_DELETED" }) public void testMapTopicToEvent_validTopics(String inputTopic, ContentionEvent expectedEvent) { assertEquals(expectedEvent, mapTopicToEvent(inputTopic)); @@ -35,11 +35,11 @@ public void testMapTopicToEvent_unrecognizedTopic() { @ParameterizedTest @CsvSource({ - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM, bie-events-contention-associated-to-claim", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED, bie-events-contention-updated", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED, bie-events-contention-classified", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED, bie-events-contention-completed", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED, bie-events-contention-deleted" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM_V02, bie-events-contention-associated-to-claim", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED_V02, bie-events-contention-updated", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED_V02, bie-events-contention-classified", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED_V02, bie-events-contention-completed", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED_V02, bie-events-contention-deleted" }) public void testGenerateRabbitMQChannelName_channelNames(String inputTopic, String bieChannel) { assertEquals(bieChannel, ContentionEvent.rabbitMqExchangeName(inputTopic)); diff --git a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java index 43873c1f7f..819288a612 100644 --- a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java +++ b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java @@ -34,11 +34,11 @@ void topicNames() { final String[] topicNames = bieProperties.topicNames(); assertArrayEquals( new String[] { - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED", - "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED" + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_ASSOCIATED_TO_CLAIM_V02", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_UPDATED_V02", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_CLASSIFIED_V02", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_COMPLETED_V02", + "BIA_SERVICES_BIE_CATALOG_TST_CONTENTION_DELETED_V02" }, topicNames); } From 30e1a1acec32826e0e8ab8e428002b2af60623fa Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 1 Apr 2024 11:38:39 -0700 Subject: [PATCH 39/67] Update the escaped "" --- svc-bie-kafka/src/main/resources/application-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index a8c71177d4..06d1c36e93 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -8,7 +8,7 @@ spring: sasl: mechanism: PLAIN jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD};" + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${BIE_KAFKA_RBAC_USERNAME}\" password=\"${BIE_KAFKA_RBAC_PASSWORD}\";" ssl: # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 # key-store-type: PKCS12 From 4b86b6c39909c71dcad8838956cffeeaff390604 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 1 Apr 2024 12:11:47 -0700 Subject: [PATCH 40/67] Changes to the key and value -deserializer --- svc-bie-kafka/src/main/resources/application-dev.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index 06d1c36e93..d34a858369 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -19,9 +19,8 @@ spring: trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:EXT_VRO_DEV}" - key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" - value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" - + key-deserializer: "org.springframework.kafka.support.serializer.JsonDeserializer" + value-deserializer: "org.springframework.kafka.support.serializer.ErrorHandlingDeserializer" bie: kafka-topic-env: "DEV" \ No newline at end of file From eb800c61a4dd253cb74c99ccbf4f45da9cf20c63 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 2 Apr 2024 09:45:51 -0700 Subject: [PATCH 41/67] Changes to the key and value -deserializer --- svc-bie-kafka/src/main/resources/application-dev.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index d34a858369..d2196ee528 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -19,8 +19,8 @@ spring: trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:EXT_VRO_DEV}" - key-deserializer: "org.springframework.kafka.support.serializer.JsonDeserializer" - value-deserializer: "org.springframework.kafka.support.serializer.ErrorHandlingDeserializer" + key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" + value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" bie: kafka-topic-env: "DEV" \ No newline at end of file From bf4f10e37543d16836a68ea6d90ca3487ff8fddd Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 2 Apr 2024 10:16:11 -0700 Subject: [PATCH 42/67] Add deserializer delegate class --- svc-bie-kafka/src/main/resources/application-dev.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index d2196ee528..f185b324a9 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -21,6 +21,8 @@ spring: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:EXT_VRO_DEV}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" + properties: + spring-deserializer-value-delegate-class: "org.springframework.kafka.support.serializer.JsonDeserializer" bie: kafka-topic-env: "DEV" \ No newline at end of file From 44ec917b56c35a8b06a65dc7c37cf5145ca42f4a Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 2 Apr 2024 10:55:07 -0700 Subject: [PATCH 43/67] Add deserializer delegate class --- .../src/main/resources/application-dev.yaml | 14 +++++--------- svc-bie-kafka/src/main/resources/application.yaml | 6 +++--- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index f185b324a9..2bb33ee64f 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -4,25 +4,21 @@ spring: properties: schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.dev.bip.va.gov}" security.protocol: SASL_SSL - security.inter.broker.protocol: SASL_SSL sasl: mechanism: PLAIN jaas: config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${BIE_KAFKA_RBAC_USERNAME}\" password=\"${BIE_KAFKA_RBAC_PASSWORD}\";" - ssl: - # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 -# key-store-type: PKCS12 -# key-store-location: "${KEYSTORE_FILE}" -# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - trust-store-type: PKCS12 - trust-store-location: "${TRUSTSTORE_FILE}" - trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:EXT_VRO_DEV}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" properties: spring-deserializer-value-delegate-class: "org.springframework.kafka.support.serializer.JsonDeserializer" + ssl: + # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 + trust-store-location: "file:${TRUSTSTORE_FILE}" + trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + trust-store-type: "PKCS12" bie: kafka-topic-env: "DEV" \ No newline at end of file diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index f1b42d8fc1..192e1db9db 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -30,9 +30,9 @@ spring: # Comment out the following line to connect without SSL ssl: # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 - key-store-location: "file:${KEYSTORE_FILE}" - key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - key-store-type: "PKCS12" +# key-store-location: "file:${KEYSTORE_FILE}" +# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" +# key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" From a19f753ce154297fe0e04e174d27842367d9d3ed Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 2 Apr 2024 11:07:00 -0700 Subject: [PATCH 44/67] Fix integration test attempt #1 --- .../resources/application-integration-test.yaml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-integration-test.yaml b/svc-bie-kafka/src/main/resources/application-integration-test.yaml index 2a4b6d1d41..58c41a4735 100644 --- a/svc-bie-kafka/src/main/resources/application-integration-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-integration-test.yaml @@ -11,17 +11,12 @@ spring: sasl: mechanism: PLAIN jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD};" + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${BIE_KAFKA_RBAC_USERNAME}\" password=\"${BIE_KAFKA_RBAC_PASSWORD}\";" + producer: + key-serializer: "org.apache.kafka.common.serialization.StringSerializer" + value-serializer: "org.apache.kafka.common.serialization.StringSerializer" ssl: - # Disable hostname verification to avoid error for 'localhost' by setting to empty string - endpoint.identification.algorithm: # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 -# key-store-location: "file:${KEYSTORE_FILE}" -# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" -# key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" - producer: - key-serializer: "org.apache.kafka.common.serialization.StringSerializer" - value-serializer: "org.apache.kafka.common.serialization.StringSerializer" From 5c48b9df5d184033af4dce901c8523977d5cfcf6 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 2 Apr 2024 11:34:45 -0700 Subject: [PATCH 45/67] Update kafka cert generation script to only generate the truststore. --- scripts/svc-bie-kafka-certgen.sh | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/scripts/svc-bie-kafka-certgen.sh b/scripts/svc-bie-kafka-certgen.sh index 8d26c65de8..ff729c4448 100755 --- a/scripts/svc-bie-kafka-certgen.sh +++ b/scripts/svc-bie-kafka-certgen.sh @@ -39,29 +39,22 @@ generate_password() { openssl rand -base64 20 | tr -dc "$all_chars" | head -c 16 } -export KEYSTORE_PWD=$(generate_password) export TRUSTSTORE_PWD=$(generate_password) # Delete temporary files (if they exist) to avoid issues when re-running script. # Keeping them will result in password mismatch error. They are safe to delete as they are recreated. rm -f bip.truststore.jks keystore.p12 bip.truststore.p12 passwd output.json -openssl pkcs12 -export -in "$2" -inkey "$3" -out keystore.p12 -name kafka-keystore-$4-$1 -CAfile VACACerts.pem -caname root -passout env:KEYSTORE_PWD - -keytool -importkeystore -srckeystore keystore.p12 -srcstoretype pkcs12 -destkeystore bip.truststore.jks -deststoretype JKS -srcstorepass "$KEYSTORE_PWD" -deststorepass "$TRUSTSTORE_PWD" - -echo "yes" | keytool -import -alias AllVA1 -file VA-Internal-S2-ICA4.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" +keytool -import -alias AllVA1 -file VA-Internal-S2-ICA4.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" # shellcheck disable=SC2086 -echo "yes" | keytool -import -alias AllVA2 -file VA-Internal-S2-ICA19.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" -echo "yes" | keytool -import -alias AllVA3 -file VA-Internal-S2-ICA11.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" -echo "yes" | keytool -import -alias AllVA4 -file VA-Internal-S2-RCA2.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" - -echo "$KEYSTORE_PWD" | keytool -importkeystore -srckeystore bip.truststore.jks -srcstoretype jks -srcstorepass "$TRUSTSTORE_PWD" -destkeystore bip.truststore.p12 -deststoretype pkcs12 -deststorepass "$TRUSTSTORE_PWD" +keytool -import -alias AllVA2 -file VA-Internal-S2-ICA19.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" +keytool -import -alias AllVA3 -file VA-Internal-S2-ICA11.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" +keytool -import -alias AllVA4 -file VA-Internal-S2-RCA2.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" +echo "$TRUSTSTORE_PWD" | keytool -importkeystore -srckeystore bip.truststore.jks -srcstoretype jks -srcstorepass "$TRUSTSTORE_PWD" -destkeystore bip.truststore.p12 -deststoretype pkcs12 -deststorepass "$TRUSTSTORE_PWD" # Encode the files -keystore=$(cat keystore.p12 | base64 | tr -d '\n') bip_truststore=$(cat bip.truststore.p12 | base64 | tr -d '\n') # Create the JSON file -echo -e "{\n\"BIE_KAFKA_KEYSTORE_INBASE64\": \"$keystore\", \n\"BIE_KAFKA_KEYSTORE_PASSWORD\": \"$KEYSTORE_PWD\", \n\"BIE_KAFKA_TRUSTSTORE_INBASE64\": \"$bip_truststore\", \n\"BIE_KAFKA_TRUSTSTORE_PASSWORD\": \"$TRUSTSTORE_PWD\"\n}" > output.json +echo -e "{\n\"BIE_KAFKA_TRUSTSTORE_INBASE64\": \"$bip_truststore\", \n\"BIE_KAFKA_TRUSTSTORE_PASSWORD\": \"$TRUSTSTORE_PWD\"\n}" > output.json From 60bfef700e6015613ea04c4440909881cd71f2e0 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 2 Apr 2024 11:41:58 -0700 Subject: [PATCH 46/67] Add yes back --- scripts/svc-bie-kafka-certgen.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/svc-bie-kafka-certgen.sh b/scripts/svc-bie-kafka-certgen.sh index ff729c4448..ddfd99e62e 100755 --- a/scripts/svc-bie-kafka-certgen.sh +++ b/scripts/svc-bie-kafka-certgen.sh @@ -45,11 +45,11 @@ export TRUSTSTORE_PWD=$(generate_password) # Keeping them will result in password mismatch error. They are safe to delete as they are recreated. rm -f bip.truststore.jks keystore.p12 bip.truststore.p12 passwd output.json -keytool -import -alias AllVA1 -file VA-Internal-S2-ICA4.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" +echo "yes" | keytool -import -alias AllVA1 -file VA-Internal-S2-ICA4.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" # shellcheck disable=SC2086 -keytool -import -alias AllVA2 -file VA-Internal-S2-ICA19.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" -keytool -import -alias AllVA3 -file VA-Internal-S2-ICA11.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" -keytool -import -alias AllVA4 -file VA-Internal-S2-RCA2.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" +echo "yes" | keytool -import -alias AllVA2 -file VA-Internal-S2-ICA19.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" +echo "yes" | keytool -import -alias AllVA3 -file VA-Internal-S2-ICA11.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" +echo "yes" | keytool -import -alias AllVA4 -file VA-Internal-S2-RCA2.cer -storetype JKS -keystore bip.truststore.jks -storepass "$TRUSTSTORE_PWD" echo "$TRUSTSTORE_PWD" | keytool -importkeystore -srckeystore bip.truststore.jks -srcstoretype jks -srcstorepass "$TRUSTSTORE_PWD" -destkeystore bip.truststore.p12 -deststoretype pkcs12 -deststorepass "$TRUSTSTORE_PWD" From 51561239d6dae3aabe534ccb297a0cc51810388d Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 2 Apr 2024 11:52:13 -0700 Subject: [PATCH 47/67] Revert config --- .../main/resources/application-integration-test.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-integration-test.yaml b/svc-bie-kafka/src/main/resources/application-integration-test.yaml index 58c41a4735..18ca44008b 100644 --- a/svc-bie-kafka/src/main/resources/application-integration-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-integration-test.yaml @@ -12,11 +12,16 @@ spring: mechanism: PLAIN jaas: config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${BIE_KAFKA_RBAC_USERNAME}\" password=\"${BIE_KAFKA_RBAC_PASSWORD}\";" - producer: - key-serializer: "org.apache.kafka.common.serialization.StringSerializer" - value-serializer: "org.apache.kafka.common.serialization.StringSerializer" ssl: + # Disable hostname verification to avoid error for 'localhost' by setting to empty string + endpoint.identification.algorithm: # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 +# key-store-location: "file:${KEYSTORE_FILE}" +# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" +# key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" + producer: + key-serializer: "org.apache.kafka.common.serialization.StringSerializer" + value-serializer: "org.apache.kafka.common.serialization.StringSerializer" From f46dad43720a974c8de570b491c88c0433a6b29e Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 2 Apr 2024 12:47:20 -0700 Subject: [PATCH 48/67] Update application yamls --- .../application-integration-test.yaml | 24 +++++++++---------- .../src/main/resources/application.yaml | 3 --- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-integration-test.yaml b/svc-bie-kafka/src/main/resources/application-integration-test.yaml index 18ca44008b..0fe368d4d2 100644 --- a/svc-bie-kafka/src/main/resources/application-integration-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-integration-test.yaml @@ -6,22 +6,20 @@ spring: # See https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md#accessing-apache-kafka-with-internal-and-external-clients bootstrap-servers: "localhost:9094" properties: - security.protocol: SASL_SSL - security.inter.broker.protocol: SASL_SSL - sasl: - mechanism: PLAIN - jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${BIE_KAFKA_RBAC_USERNAME}\" password=\"${BIE_KAFKA_RBAC_PASSWORD}\";" ssl: # Disable hostname verification to avoid error for 'localhost' by setting to empty string endpoint.identification.algorithm: - # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 -# key-store-location: "file:${KEYSTORE_FILE}" -# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" -# key-store-type: "PKCS12" - trust-store-location: "file:${TRUSTSTORE_FILE}" - trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" - trust-store-type: "PKCS12" producer: key-serializer: "org.apache.kafka.common.serialization.StringSerializer" value-serializer: "org.apache.kafka.common.serialization.StringSerializer" + security.protocol: SASL_SSL + ssl: + trust-store-location: "file:${TRUSTSTORE_FILE}" + trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + trust-store-type: "PKCS12" + admin: + security.protocol: SASL_SSL + ssl: + trust-store-location: "file:${TRUSTSTORE_FILE}" + trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + trust-store-type: "PKCS12" \ No newline at end of file diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index 192e1db9db..9408c19892 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -30,9 +30,6 @@ spring: # Comment out the following line to connect without SSL ssl: # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 -# key-store-location: "file:${KEYSTORE_FILE}" -# key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" -# key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" From 58dccebfc3f99dec6c493517dd75d831293c4867 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Wed, 3 Apr 2024 09:52:44 -0700 Subject: [PATCH 49/67] Debug application integration test yaml --- .../src/main/resources/application-integration-test.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/svc-bie-kafka/src/main/resources/application-integration-test.yaml b/svc-bie-kafka/src/main/resources/application-integration-test.yaml index 0fe368d4d2..b2ef4fca92 100644 --- a/svc-bie-kafka/src/main/resources/application-integration-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-integration-test.yaml @@ -18,8 +18,11 @@ spring: trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" admin: - security.protocol: SASL_SSL + security.protocol: SSL ssl: + key-store-location: "file:${KEYSTORE_FILE}" + key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" \ No newline at end of file From 39cde8a78dde1f3c47814cdd12f1464498156513 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Wed, 3 Apr 2024 10:13:56 -0700 Subject: [PATCH 50/67] Changes to bie kafka docker vars --- mocks/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index b262020e1b..e2f80b4def 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -88,7 +88,7 @@ services: # https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md # https://stackoverflow.com/a/72018409 # Caution: Empty string '' and "" don't work; however ' ' or " " does work -# KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: " " + KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: " " # For development without having to deal with connecting via SSL, use port 9092 or do the following. # To disable SSL, uncomment the following and comment out the next SSL section @@ -110,7 +110,7 @@ services: volumes: # scripts/setenv.sh creates jks files from env variables # keystore is unneeded only truststore https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 - - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' +# - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' - './mock-bie-kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro' networks: - vro_intranet From c698664216e44c11c6fa136c871a3ef3db81c74f Mon Sep 17 00:00:00 2001 From: josiahjones Date: Wed, 3 Apr 2024 10:29:56 -0700 Subject: [PATCH 51/67] Changes to bie kafka docker vars --- .github/workflows/bie-kafka-end2end-test.yml | 6 +----- mocks/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bie-kafka-end2end-test.yml b/.github/workflows/bie-kafka-end2end-test.yml index a0d0ff2cb5..160ea979fd 100644 --- a/.github/workflows/bie-kafka-end2end-test.yml +++ b/.github/workflows/bie-kafka-end2end-test.yml @@ -77,11 +77,7 @@ jobs: - name: 'Wait for Kafka to be ready' run: | # Verify broker is up with 30 second timeout - BROKER_QUERY=$(kafkacat -m 30 -b localhost:9092 \ - -X debug='security,broker' \ - -X security.protocol=sasl_ssl -X sasl.mechanisms=PLAIN \ - -X sasl.username=USERNAME -X sasl.password=PASSWORD \ - -L) + BROKER_QUERY=$(kafkacat -m 30 -b localhost:9092 -L) if echo "$BROKER_QUERY" | grep -q '1 brokers' then echo "Found Broker: $BROKER_QUERY" diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index e2f80b4def..e00f17d8b8 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -74,7 +74,7 @@ services: # Purpose of EXTERNAL port 9094: see https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md#accessing-apache-kafka-with-internal-and-external-clients KAFKA_CFG_LISTENERS: INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:9094 KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL://mock-bie-kafka:9092,EXTERNAL://localhost:9094 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:SASL_SSL,INTERNAL:SASL_SSL,EXTERNAL:SASL_SSL + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:SASL_SSL KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'INTERNAL' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true' KAFKA_MESSAGE_MAX_BYTES: "1000000" From 0a8a6716b1431d34f87a63676dbe73a22ebf3f4f Mon Sep 17 00:00:00 2001 From: josiahjones Date: Wed, 3 Apr 2024 10:46:41 -0700 Subject: [PATCH 52/67] Changes to bie kafka docker vars --- .github/workflows/bie-kafka-end2end-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bie-kafka-end2end-test.yml b/.github/workflows/bie-kafka-end2end-test.yml index 160ea979fd..b9f815e7d9 100644 --- a/.github/workflows/bie-kafka-end2end-test.yml +++ b/.github/workflows/bie-kafka-end2end-test.yml @@ -77,7 +77,11 @@ jobs: - name: 'Wait for Kafka to be ready' run: | # Verify broker is up with 30 second timeout - BROKER_QUERY=$(kafkacat -m 30 -b localhost:9092 -L) + BROKER_QUERY=$(kafkacat -m 30 -b localhost:9092 \ + -v -X debug=generic,broker,security \ + -X security.protocol=sasl_ssl -X sasl.mechanisms=PLAIN \ + -X sasl.username=USERNAME -X sasl.password=PASSWORD \ + -L) if echo "$BROKER_QUERY" | grep -q '1 brokers' then echo "Found Broker: $BROKER_QUERY" From 23e76ad00c24f84d06333471899d473493cdcf39 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Wed, 3 Apr 2024 11:12:30 -0700 Subject: [PATCH 53/67] Add truststore back --- .github/workflows/bie-kafka-end2end-test.yml | 6 +----- mocks/docker-compose.yml | 2 +- svc-bie-kafka/src/main/resources/application-dev.yaml | 4 +++- .../src/main/resources/application-integration-test.yaml | 3 +++ 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/bie-kafka-end2end-test.yml b/.github/workflows/bie-kafka-end2end-test.yml index b9f815e7d9..160ea979fd 100644 --- a/.github/workflows/bie-kafka-end2end-test.yml +++ b/.github/workflows/bie-kafka-end2end-test.yml @@ -77,11 +77,7 @@ jobs: - name: 'Wait for Kafka to be ready' run: | # Verify broker is up with 30 second timeout - BROKER_QUERY=$(kafkacat -m 30 -b localhost:9092 \ - -v -X debug=generic,broker,security \ - -X security.protocol=sasl_ssl -X sasl.mechanisms=PLAIN \ - -X sasl.username=USERNAME -X sasl.password=PASSWORD \ - -L) + BROKER_QUERY=$(kafkacat -m 30 -b localhost:9092 -L) if echo "$BROKER_QUERY" | grep -q '1 brokers' then echo "Found Broker: $BROKER_QUERY" diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index e00f17d8b8..63d43476fb 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -110,7 +110,7 @@ services: volumes: # scripts/setenv.sh creates jks files from env variables # keystore is unneeded only truststore https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 -# - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' + - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' - './mock-bie-kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro' networks: - vro_intranet diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index 2bb33ee64f..c7e111f663 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -15,7 +15,9 @@ spring: properties: spring-deserializer-value-delegate-class: "org.springframework.kafka.support.serializer.JsonDeserializer" ssl: - # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 + key-store-location: "file:${KEYSTORE_FILE}" + key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" diff --git a/svc-bie-kafka/src/main/resources/application-integration-test.yaml b/svc-bie-kafka/src/main/resources/application-integration-test.yaml index b2ef4fca92..84ca4124ae 100644 --- a/svc-bie-kafka/src/main/resources/application-integration-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-integration-test.yaml @@ -14,6 +14,9 @@ spring: value-serializer: "org.apache.kafka.common.serialization.StringSerializer" security.protocol: SASL_SSL ssl: + key-store-location: "file:${KEYSTORE_FILE}" + key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" From 58d8003ac16b888d19a0e80b40d7cc71279048d0 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Wed, 3 Apr 2024 11:45:38 -0700 Subject: [PATCH 54/67] Edits to ssl/sasl connection --- mocks/docker-compose.yml | 2 +- svc-bie-kafka/src/main/resources/application.yaml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index 63d43476fb..34cecb73c2 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -74,7 +74,7 @@ services: # Purpose of EXTERNAL port 9094: see https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md#accessing-apache-kafka-with-internal-and-external-clients KAFKA_CFG_LISTENERS: INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:9094 KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL://mock-bie-kafka:9092,EXTERNAL://localhost:9094 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:SASL_SSL + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:SASL_SSL,EXTERNAL:SASL_SSL KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'INTERNAL' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true' KAFKA_MESSAGE_MAX_BYTES: "1000000" diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index 9408c19892..495e1bf5a5 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -29,7 +29,9 @@ spring: auto-offset-reset: earliest # Comment out the following line to connect without SSL ssl: - # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 + key-store-location: "file:${KEYSTORE_FILE}" + key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" From 4a2d12654f010a66d32171b815f6b86fe5146dea Mon Sep 17 00:00:00 2001 From: josiahjones Date: Fri, 5 Apr 2024 12:40:05 -0700 Subject: [PATCH 55/67] Updates to docker-entryprep.sh to ignore the keystore creation/verification --- scripts/kafka-truststore-gen.sh | 39 +++++++++++++++++++++++++++++++ svc-bie-kafka/docker-entryprep.sh | 8 ------- 2 files changed, 39 insertions(+), 8 deletions(-) create mode 100755 scripts/kafka-truststore-gen.sh diff --git a/scripts/kafka-truststore-gen.sh b/scripts/kafka-truststore-gen.sh new file mode 100755 index 0000000000..4932fd2bdb --- /dev/null +++ b/scripts/kafka-truststore-gen.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +generate_password() { + # Define characters for password generation + upper="ABCDEFGHIJKLMNOPQRSTUVWXYZ" + lower="abcdefghijklmnopqrstuvwxyz" + numbers="0123456789" + special="@#\$%^&*()_-+=" + + # Combine all characters into one set + all_chars="${upper}${lower}${numbers}${special}" + + # Generate a random password of length 16 using openssl + openssl rand -base64 20 | tr -dc "$all_chars" | head -c 16 +} + +# Generate a password for the keystore +STOREPASS=$(generate_password) +echo "Generated keystore password: $STOREPASS" + +# Define the keystore file name +KEYSTORE="vro-keystore.p12" + +# Create or clear the existing keystore +rm -f "$KEYSTORE" + +# Loop through the .cer files and import each into the keystore +for certfile in *.cer; do + # Extract alias name by removing the 'VA-Internal-' prefix from the filename + alias=$(echo "$certfile" | sed 's/VA-Internal-//; s/.cer$//') + + # Import the certificate into the keystore + keytool -import -noprompt -alias "$alias" -file "$certfile" -keystore "$KEYSTORE" -storepass "$STOREPASS" -storetype PKCS12 +done + + +# Encode the keystore file to Base64 and print it +echo "Base64 Encoded Keystore:" +base64 -i "$KEYSTORE" diff --git a/svc-bie-kafka/docker-entryprep.sh b/svc-bie-kafka/docker-entryprep.sh index 1c2bcddacd..592ce059a6 100755 --- a/svc-bie-kafka/docker-entryprep.sh +++ b/svc-bie-kafka/docker-entryprep.sh @@ -6,20 +6,12 @@ getEnvVarValue(){ } for ENV_VAR in \ - BIE_KAFKA_KEYSTORE_INBASE64 BIE_KAFKA_KEYSTORE_PASSWORD \ BIE_KAFKA_TRUSTSTORE_INBASE64 BIE_KAFKA_TRUSTSTORE_PASSWORD; do if [ "$(getEnvVarValue "$ENV_VAR")" = "" ]; then >&2 echo "ERROR: Missing expected environment variable: $ENV_VAR" fi done -export KEYSTORE_FILE="$PWD/keystore.p12" -echo "$BIE_KAFKA_KEYSTORE_INBASE64" | base64 -d > "$KEYSTORE_FILE" -echo -e "\nVerifying keystore ($KEYSTORE_FILE) and its password..." -if ! keytool -list -v -keystore "$KEYSTORE_FILE" -storepass "$BIE_KAFKA_KEYSTORE_PASSWORD" | grep "Alias name:"; then - >&2 echo "ERROR: with keystore" -fi - export TRUSTSTORE_FILE="$PWD/truststore.p12" echo "$BIE_KAFKA_TRUSTSTORE_INBASE64" | base64 -d > "$TRUSTSTORE_FILE" echo -e "\nVerifying truststore ($TRUSTSTORE_FILE) and its password..." From 0be686600bae59e8f233e67e77f91bddc1c12664 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Fri, 5 Apr 2024 13:13:43 -0700 Subject: [PATCH 56/67] Remove references to keystore file --- svc-bie-kafka/src/main/resources/application-dev.yaml | 7 ++----- svc-bie-kafka/src/main/resources/application.yaml | 3 --- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index c7e111f663..d9dd6e4121 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -9,18 +9,15 @@ spring: jaas: config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${BIE_KAFKA_RBAC_USERNAME}\" password=\"${BIE_KAFKA_RBAC_PASSWORD}\";" consumer: - group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:EXT_VRO_DEV}" + group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:EXT_VRO_TST}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" properties: spring-deserializer-value-delegate-class: "org.springframework.kafka.support.serializer.JsonDeserializer" ssl: - key-store-location: "file:${KEYSTORE_FILE}" - key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" bie: - kafka-topic-env: "DEV" \ No newline at end of file + kafka-topic-env: "TST" \ No newline at end of file diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index 495e1bf5a5..46aef9ec40 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -29,9 +29,6 @@ spring: auto-offset-reset: earliest # Comment out the following line to connect without SSL ssl: - key-store-location: "file:${KEYSTORE_FILE}" - key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - key-store-type: "PKCS12" trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" From 0ae57918e0711a663bd47934553f83ad0639a999 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 8 Apr 2024 12:44:26 -0700 Subject: [PATCH 57/67] Add a ssl piece to the properties file --- svc-bie-kafka/src/main/resources/application-dev.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index d9dd6e4121..cf70cf24d1 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -8,12 +8,14 @@ spring: mechanism: PLAIN jaas: config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${BIE_KAFKA_RBAC_USERNAME}\" password=\"${BIE_KAFKA_RBAC_PASSWORD}\";" + ssl: + trust-store-location: "file:${TRUSTSTORE_FILE}" + trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + trust-store-type: "PKCS12" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:EXT_VRO_TST}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" - properties: - spring-deserializer-value-delegate-class: "org.springframework.kafka.support.serializer.JsonDeserializer" ssl: trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" From 38ab6c63433a7bc3deb6d81de9b4ef232834f953 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 8 Apr 2024 12:51:08 -0700 Subject: [PATCH 58/67] Add ssl truststore everywhere. --- .../src/main/resources/application-dev.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index cf70cf24d1..cd8ce06619 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -2,20 +2,23 @@ spring: kafka: bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.dev.bip.va.gov:443}" properties: - schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.dev.bip.va.gov}" + schema.registry: + url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.dev.bip.va.gov}" + ssl: + trust-store-location: "${TRUSTSTORE_FILE}" + trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + trust-store-type: "PKCS12" security.protocol: SASL_SSL sasl: mechanism: PLAIN jaas: config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${BIE_KAFKA_RBAC_USERNAME}\" password=\"${BIE_KAFKA_RBAC_PASSWORD}\";" - ssl: - trust-store-location: "file:${TRUSTSTORE_FILE}" - trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" - trust-store-type: "PKCS12" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:EXT_VRO_TST}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" + security: + protocol: SASL_SSL ssl: trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" From 14a29897d0373c2bd5ced658179c0551c98c5e4f Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 8 Apr 2024 13:09:49 -0700 Subject: [PATCH 59/67] Change shape of schema.registry.truststore.* --- svc-bie-kafka/src/main/resources/application-dev.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index cd8ce06619..189d93cf8c 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -5,9 +5,9 @@ spring: schema.registry: url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.dev.bip.va.gov}" ssl: - trust-store-location: "${TRUSTSTORE_FILE}" - trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" - trust-store-type: "PKCS12" + truststore.location: "${TRUSTSTORE_FILE}" + truststore.password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + truststore.type: "PKCS12" security.protocol: SASL_SSL sasl: mechanism: PLAIN From 4065cfc673ace44754772a03479e03060dcc7c6b Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 8 Apr 2024 13:37:06 -0700 Subject: [PATCH 60/67] Add in USER_INFO basic auth for schema registry --- svc-bie-kafka/src/main/resources/application-dev.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index 189d93cf8c..835c9df036 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -2,8 +2,12 @@ spring: kafka: bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.dev.bip.va.gov:443}" properties: + basic.auth.credentials.source: USER_INFO + basic.auth.user.info: "${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}" schema.registry: - url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.dev.bip.va.gov}" + basic.auth.credentials.source: USER_INFO + basic.auth.user.info: "${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}" + url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://bip-schemaregistry.dev.bip.va.gov}" ssl: truststore.location: "${TRUSTSTORE_FILE}" truststore.password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" From c408ab7651447d4075e7f7cc064ea47efb6f39e1 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Mon, 8 Apr 2024 14:01:52 -0700 Subject: [PATCH 61/67] Revert higher environments for RBAC to match develop --- .../main/resources/application-prod-test.yaml | 30 ++++++-------- .../src/main/resources/application-prod.yaml | 2 +- .../src/main/resources/application-qa.yaml | 35 +++++++--------- .../main/resources/application-sandbox.yaml | 40 +++++++++---------- .../src/main/resources/application.yaml | 14 ++++++- 5 files changed, 58 insertions(+), 63 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-prod-test.yaml b/svc-bie-kafka/src/main/resources/application-prod-test.yaml index 494962ff5c..3ef94dcdd5 100644 --- a/svc-bie-kafka/src/main/resources/application-prod-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-prod-test.yaml @@ -1,20 +1,16 @@ spring: kafka: - bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.tst.bip.va.gov:443}" + bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:kafka.preprod.bip.va.gov:443}" properties: schema: registry: - url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.tst.bip.va.gov}" - sasl: - mechanism: PLAIN - jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" + url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://schemaregistry.preprod.bip.va.gov:443}" ssl: -# protocol: SSL -# keystore: -# type: PKCS12 -# location: "${KEYSTORE_FILE}" -# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + protocol: SSL + keystore: + type: PKCS12 + location: "${KEYSTORE_FILE}" + password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" truststore: type: PKCS12 location: "${TRUSTSTORE_FILE}" @@ -24,16 +20,16 @@ spring: key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" properties: - security.protocol: SASL_SSL + security.protocol: SSL ssl: -# keystore: -# type: PKCS12 -# location: "${KEYSTORE_FILE}" -# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + keystore: + type: PKCS12 + location: "${KEYSTORE_FILE}" + password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" truststore: type: PKCS12 location: "${TRUSTSTORE_FILE}" password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" bie: - kafka-topic-env: "PRE_" + kakfa-topic-prefix: "PRE_" diff --git a/svc-bie-kafka/src/main/resources/application-prod.yaml b/svc-bie-kafka/src/main/resources/application-prod.yaml index 39498e84f3..8810a8b0f1 100644 --- a/svc-bie-kafka/src/main/resources/application-prod.yaml +++ b/svc-bie-kafka/src/main/resources/application-prod.yaml @@ -32,4 +32,4 @@ spring: password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" bie: - kafka-topic-env: "PROD_" + kakfa-topic-prefix: "PROD_" diff --git a/svc-bie-kafka/src/main/resources/application-qa.yaml b/svc-bie-kafka/src/main/resources/application-qa.yaml index 12d8049468..bb2daa36e1 100644 --- a/svc-bie-kafka/src/main/resources/application-qa.yaml +++ b/svc-bie-kafka/src/main/resources/application-qa.yaml @@ -1,23 +1,16 @@ spring: kafka: - bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.tst.bip.va.gov:443}" + bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:kafka.stage.bip.va.gov:443}" properties: schema: registry: - url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.tst.bip.va.gov}" - security.protocol: SASL_SSL - security.inter.broker.protocol: SASL_SSL - sasl: - mechanism: PLAIN - jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" + url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://schemaregistry.stage.bip.va.gov:443}" ssl: -# Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 -# protocol: SSL -# keystore: -# type: PKCS12 -# location: "${KEYSTORE_FILE}" -# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + protocol: SSL + keystore: + type: PKCS12 + location: "${KEYSTORE_FILE}" + password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" truststore: type: PKCS12 location: "${TRUSTSTORE_FILE}" @@ -27,16 +20,16 @@ spring: key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" properties: -# security.protocol: SSL -# ssl: -# keystore: -# type: PKCS12 -# location: "${KEYSTORE_FILE}" -# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + security.protocol: SSL + ssl: + keystore: + type: PKCS12 + location: "${KEYSTORE_FILE}" + password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" truststore: type: PKCS12 location: "${TRUSTSTORE_FILE}" password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" bie: - kafka-topic-env: "IVV_" + kakfa-topic-prefix: "IVV_" diff --git a/svc-bie-kafka/src/main/resources/application-sandbox.yaml b/svc-bie-kafka/src/main/resources/application-sandbox.yaml index 5ea2d40dcd..8b69cd5cf3 100644 --- a/svc-bie-kafka/src/main/resources/application-sandbox.yaml +++ b/svc-bie-kafka/src/main/resources/application-sandbox.yaml @@ -1,39 +1,35 @@ spring: kafka: - bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.tst.bip.va.gov}" + bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:kafka.stage.bip.va.gov:443}" properties: schema: registry: - url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-schemaregistry.tst.bip.va.gov}" - sasl: - mechanism: PLAIN - jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME:USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD:PASSWORD};" - ssl: -# keystore: -# type: PKCS12 -# location: "${KEYSTORE_FILE}" -# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - truststore: - type: PKCS12 - location: "${TRUSTSTORE_FILE}" - password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://schemaregistry.stage.bip.va.gov:443}" + ssl: + protocol: SSL + keystore: + type: PKCS12 + location: "${KEYSTORE_FILE}" + password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + truststore: + type: PKCS12 + location: "${TRUSTSTORE_FILE}" + password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-uat-vro}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" properties: - security.protocol: SASL_SSL + security.protocol: SSL ssl: - # Keystore not needed https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 -# keystore: -# type: PKCS12 -# location: "${KEYSTORE_FILE}" -# password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + keystore: + type: PKCS12 + location: "${KEYSTORE_FILE}" + password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" truststore: type: PKCS12 location: "${TRUSTSTORE_FILE}" password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" bie: - kafka-topic-env: "UAT_" + kakfa-topic-prefix: "UAT_" diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index 46aef9ec40..528586b80a 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -14,7 +14,16 @@ spring: kafka: bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:localhost:9092}" properties: - schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}@bip-kafka.tst.bip.va.gov}" + basic.auth.credentials.source: USER_INFO + basic.auth.user.info: "${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}" + schema.registry: + basic.auth.credentials.source: USER_INFO + basic.auth.user.info: "${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}" + url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://bip-kafka.tst.bip.va.gov}" + ssl: + truststore.location: "${TRUSTSTORE_FILE}" + truststore.password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + truststore.type: "PKCS12" specific.avro.reader: false security.protocol: SASL_SSL security.inter.broker.protocol: SASL_SSL @@ -27,7 +36,8 @@ spring: key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" auto-offset-reset: earliest - # Comment out the following line to connect without SSL + security: + protocol: SASL_SSL ssl: trust-store-location: "file:${TRUSTSTORE_FILE}" trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" From fed6802e2dca7711c1bcf0747a11adc92c81f94c Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 9 Apr 2024 08:58:17 -0700 Subject: [PATCH 62/67] Small nits and update to infix from env --- mocks/docker-compose.yml | 2 -- .../java/gov/va/vro/services/bie/config/BieProperties.java | 4 ++-- svc-bie-kafka/src/main/resources/application-dev.yaml | 2 +- .../va/vro/services/bie/config/MessageExchangeConfigTest.java | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index 34cecb73c2..a370937654 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -109,8 +109,6 @@ services: volumes: # scripts/setenv.sh creates jks files from env variables - # keystore is unneeded only truststore https://dsva.slack.com/archives/C04U8GESRT7/p1708456986385019?thread_ts=1706878417.546749&cid=C04U8GESRT7 - - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' - './mock-bie-kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro' networks: - vro_intranet diff --git a/svc-bie-kafka/src/main/java/gov/va/vro/services/bie/config/BieProperties.java b/svc-bie-kafka/src/main/java/gov/va/vro/services/bie/config/BieProperties.java index bd042f625f..49edcb2437 100644 --- a/svc-bie-kafka/src/main/java/gov/va/vro/services/bie/config/BieProperties.java +++ b/svc-bie-kafka/src/main/java/gov/va/vro/services/bie/config/BieProperties.java @@ -13,13 +13,13 @@ @Setter public class BieProperties { - @Getter String kafkaTopicEnv; + @Getter String kafkaTopicInfix; public String[] topicNames() { return Arrays.stream(ContentionEvent.values()) .map( contention -> { - String subString = "CATALOG_" + kafkaTopicEnv + "_CONTENTION"; + String subString = "CATALOG_" + kafkaTopicInfix + "_CONTENTION"; return contention.getTopicName().replaceAll("CATALOG_CONTENTION", subString); }) .toArray(String[]::new); diff --git a/svc-bie-kafka/src/main/resources/application-dev.yaml b/svc-bie-kafka/src/main/resources/application-dev.yaml index 835c9df036..ef39a4c0fa 100644 --- a/svc-bie-kafka/src/main/resources/application-dev.yaml +++ b/svc-bie-kafka/src/main/resources/application-dev.yaml @@ -29,4 +29,4 @@ spring: trust-store-type: "PKCS12" bie: - kafka-topic-env: "TST" \ No newline at end of file + kafka-topic-infix: "TST" diff --git a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java index 819288a612..f40581176c 100644 --- a/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java +++ b/svc-bie-kafka/src/test/java/gov/va/vro/services/bie/config/MessageExchangeConfigTest.java @@ -17,7 +17,7 @@ class MessageExchangeConfigTest { @BeforeEach void setUp() { bieProperties = new BieProperties(); - bieProperties.kafkaTopicEnv = "TST"; + bieProperties.kafkaTopicInfix = "TST"; } @Test From 781f8c0ac152e0e87bf5b4399569c6a9bd5a8965 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 9 Apr 2024 08:59:56 -0700 Subject: [PATCH 63/67] rm unnecessary Kafka cert gen --- scripts/kafka-truststore-gen.sh | 39 --------------------------------- 1 file changed, 39 deletions(-) delete mode 100755 scripts/kafka-truststore-gen.sh diff --git a/scripts/kafka-truststore-gen.sh b/scripts/kafka-truststore-gen.sh deleted file mode 100755 index 4932fd2bdb..0000000000 --- a/scripts/kafka-truststore-gen.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -generate_password() { - # Define characters for password generation - upper="ABCDEFGHIJKLMNOPQRSTUVWXYZ" - lower="abcdefghijklmnopqrstuvwxyz" - numbers="0123456789" - special="@#\$%^&*()_-+=" - - # Combine all characters into one set - all_chars="${upper}${lower}${numbers}${special}" - - # Generate a random password of length 16 using openssl - openssl rand -base64 20 | tr -dc "$all_chars" | head -c 16 -} - -# Generate a password for the keystore -STOREPASS=$(generate_password) -echo "Generated keystore password: $STOREPASS" - -# Define the keystore file name -KEYSTORE="vro-keystore.p12" - -# Create or clear the existing keystore -rm -f "$KEYSTORE" - -# Loop through the .cer files and import each into the keystore -for certfile in *.cer; do - # Extract alias name by removing the 'VA-Internal-' prefix from the filename - alias=$(echo "$certfile" | sed 's/VA-Internal-//; s/.cer$//') - - # Import the certificate into the keystore - keytool -import -noprompt -alias "$alias" -file "$certfile" -keystore "$KEYSTORE" -storepass "$STOREPASS" -storetype PKCS12 -done - - -# Encode the keystore file to Base64 and print it -echo "Base64 Encoded Keystore:" -base64 -i "$KEYSTORE" From 3b4e83108cdea9c3c77d4e76c4aecdda7a736547 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 9 Apr 2024 16:18:21 -0700 Subject: [PATCH 64/67] Add qa and sandbox as well --- .../src/main/resources/application-qa.yaml | 51 +++++++++---------- .../main/resources/application-sandbox.yaml | 51 +++++++++---------- 2 files changed, 48 insertions(+), 54 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application-qa.yaml b/svc-bie-kafka/src/main/resources/application-qa.yaml index bb2daa36e1..331f520e60 100644 --- a/svc-bie-kafka/src/main/resources/application-qa.yaml +++ b/svc-bie-kafka/src/main/resources/application-qa.yaml @@ -1,35 +1,32 @@ spring: kafka: - bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:kafka.stage.bip.va.gov:443}" + bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.qa.bip.va.gov:443}" properties: - schema: - registry: - url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://schemaregistry.stage.bip.va.gov:443}" - ssl: - protocol: SSL - keystore: - type: PKCS12 - location: "${KEYSTORE_FILE}" - password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - truststore: - type: PKCS12 - location: "${TRUSTSTORE_FILE}" - password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + basic.auth.credentials.source: USER_INFO + basic.auth.user.info: "${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}" + schema.registry: + basic.auth.credentials.source: USER_INFO + basic.auth.user.info: "${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}" + url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://bip-schemaregistry.dev.bip.va.gov}" + ssl: + truststore.location: "${TRUSTSTORE_FILE}" + truststore.password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + truststore.type: "PKCS12" + security.protocol: SASL_SSL + sasl: + mechanism: PLAIN + jaas: + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${BIE_KAFKA_RBAC_USERNAME}\" password=\"${BIE_KAFKA_RBAC_PASSWORD}\";" consumer: - group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-ivv-vro}" + group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:EXT_VRO_QA}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" - properties: - security.protocol: SSL - ssl: - keystore: - type: PKCS12 - location: "${KEYSTORE_FILE}" - password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - truststore: - type: PKCS12 - location: "${TRUSTSTORE_FILE}" - password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + security: + protocol: SASL_SSL + ssl: + trust-store-location: "file:${TRUSTSTORE_FILE}" + trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + trust-store-type: "PKCS12" bie: - kakfa-topic-prefix: "IVV_" + kafka-topic-infix: "QA" diff --git a/svc-bie-kafka/src/main/resources/application-sandbox.yaml b/svc-bie-kafka/src/main/resources/application-sandbox.yaml index 8b69cd5cf3..03b25e7e76 100644 --- a/svc-bie-kafka/src/main/resources/application-sandbox.yaml +++ b/svc-bie-kafka/src/main/resources/application-sandbox.yaml @@ -1,35 +1,32 @@ spring: kafka: - bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:kafka.stage.bip.va.gov:443}" + bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-kafka.stage.bip.va.gov:443}" properties: - schema: - registry: - url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://schemaregistry.stage.bip.va.gov:443}" - ssl: - protocol: SSL - keystore: - type: PKCS12 - location: "${KEYSTORE_FILE}" - password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - truststore: - type: PKCS12 - location: "${TRUSTSTORE_FILE}" - password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + basic.auth.credentials.source: USER_INFO + basic.auth.user.info: "${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}" + schema.registry: + basic.auth.credentials.source: USER_INFO + basic.auth.user.info: "${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}" + url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:bip-schemaregistry.stage.bip.va.go}" + ssl: + truststore.location: "${TRUSTSTORE_FILE}" + truststore.password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + truststore.type: "PKCS12" + security.protocol: SASL_SSL + sasl: + mechanism: PLAIN + jaas: + config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${BIE_KAFKA_RBAC_USERNAME}\" password=\"${BIE_KAFKA_RBAC_PASSWORD}\";" consumer: - group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-uat-vro}" + group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:EXT_VRO_IVV}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "io.confluent.kafka.serializers.KafkaAvroDeserializer" - properties: - security.protocol: SSL - ssl: - keystore: - type: PKCS12 - location: "${KEYSTORE_FILE}" - password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" - truststore: - type: PKCS12 - location: "${TRUSTSTORE_FILE}" - password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + security: + protocol: SASL_SSL + ssl: + trust-store-location: "file:${TRUSTSTORE_FILE}" + trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + trust-store-type: "PKCS12" bie: - kakfa-topic-prefix: "UAT_" + kafka-topic-infix: "IVV" From 3c60d98d31e251ea0500a2f7623e17db2da21fb0 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 9 Apr 2024 16:21:38 -0700 Subject: [PATCH 65/67] Revert mock BIE kafka to existing state --- mocks/docker-compose.yml | 11 +++------- .../application-integration-test.yaml | 21 +++++++++---------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml index a370937654..cc597789c2 100644 --- a/mocks/docker-compose.yml +++ b/mocks/docker-compose.yml @@ -74,7 +74,6 @@ services: # Purpose of EXTERNAL port 9094: see https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md#accessing-apache-kafka-with-internal-and-external-clients KAFKA_CFG_LISTENERS: INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:9094 KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL://mock-bie-kafka:9092,EXTERNAL://localhost:9094 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:SASL_SSL,EXTERNAL:SASL_SSL KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'INTERNAL' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true' KAFKA_MESSAGE_MAX_BYTES: "1000000" @@ -95,20 +94,16 @@ services: # ALLOW_PLAINTEXT_LISTENER: 'yes' # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT - # For SASL for client communications -- see "Security" section of https://hub.docker.com/r/bitnami/kafka/ - KAFKA_CLIENT_USERS: USERNAME - KAFKA_CLIENT_PASSWORDS: PASSWORD - KAFKA_CLIENT_LISTENER_NAME: INTERNAL - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN - KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL: PLAIN - # Set up SSL -- see "Security" section of https://hub.docker.com/r/bitnami/kafka/ + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:SSL + KAFKA_CFG_SSL_CLIENT_AUTH: required # PEM https://medium.com/analytics-vidhya/how-to-use-pem-certificates-with-apache-kafka-f3b444a00816 KAFKA_TLS_TYPE: JKS KAFKA_CERTIFICATE_PASSWORD: mockKafka volumes: # scripts/setenv.sh creates jks files from env variables + - './mock-bie-kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro' - './mock-bie-kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro' networks: - vro_intranet diff --git a/svc-bie-kafka/src/main/resources/application-integration-test.yaml b/svc-bie-kafka/src/main/resources/application-integration-test.yaml index 84ca4124ae..80e20e9a34 100644 --- a/svc-bie-kafka/src/main/resources/application-integration-test.yaml +++ b/svc-bie-kafka/src/main/resources/application-integration-test.yaml @@ -1,4 +1,3 @@ - spring: kafka: # To prevent java.net.UnknownHostException mock-bie-kafka, use mock Kafka's 'EXTERNAL' port 9094 @@ -12,20 +11,20 @@ spring: producer: key-serializer: "org.apache.kafka.common.serialization.StringSerializer" value-serializer: "org.apache.kafka.common.serialization.StringSerializer" - security.protocol: SASL_SSL + security.protocol: SSL ssl: - key-store-location: "file:${KEYSTORE_FILE}" - key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + keystore-location: "file:${KEYSTORE_FILE}" + keystore-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" key-store-type: "PKCS12" - trust-store-location: "file:${TRUSTSTORE_FILE}" - trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + truststore-location: "file:${TRUSTSTORE_FILE}" + truststore-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" admin: security.protocol: SSL ssl: - key-store-location: "file:${KEYSTORE_FILE}" - key-store-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + keystore-location: "file:${KEYSTORE_FILE}" + keystore-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" key-store-type: "PKCS12" - trust-store-location: "file:${TRUSTSTORE_FILE}" - trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" - trust-store-type: "PKCS12" \ No newline at end of file + truststore-location: "file:${TRUSTSTORE_FILE}" + truststore-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + trust-store-type: "PKCS12" From 268577e55be19b5dfb31c3a4abf6fd09419591d1 Mon Sep 17 00:00:00 2001 From: josiahjones Date: Tue, 9 Apr 2024 16:47:36 -0700 Subject: [PATCH 66/67] Revert local config --- .../src/main/resources/application.yaml | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/svc-bie-kafka/src/main/resources/application.yaml b/svc-bie-kafka/src/main/resources/application.yaml index 528586b80a..755316a52f 100644 --- a/svc-bie-kafka/src/main/resources/application.yaml +++ b/svc-bie-kafka/src/main/resources/application.yaml @@ -14,38 +14,26 @@ spring: kafka: bootstrap-servers: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:localhost:9092}" properties: - basic.auth.credentials.source: USER_INFO - basic.auth.user.info: "${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}" - schema.registry: - basic.auth.credentials.source: USER_INFO - basic.auth.user.info: "${BIE_KAFKA_RBAC_USERNAME}:${BIE_KAFKA_RBAC_PASSWORD}" - url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://bip-kafka.tst.bip.va.gov}" - ssl: - truststore.location: "${TRUSTSTORE_FILE}" - truststore.password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" - truststore.type: "PKCS12" + schema.registry.url: "${BIE_KAFKA_PLACEHOLDERS_BROKERS:https://schemaregistry.dev.bip.va.gov:443}" specific.avro.reader: false - security.protocol: SASL_SSL - security.inter.broker.protocol: SASL_SSL - sasl: - mechanism: PLAIN - jaas: - config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=${BIE_KAFKA_RBAC_USERNAME} password=${BIE_KAFKA_RBAC_PASSWORD};" consumer: group-id: "${BIE_KAFKA_PLACEHOLDERS_GROUP_ID:vro-bie-tst-vro}" key-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" value-deserializer: "org.apache.kafka.common.serialization.StringDeserializer" auto-offset-reset: earliest - security: - protocol: SASL_SSL + # Comment out the following line to connect without SSL + security.protocol: SSL ssl: - trust-store-location: "file:${TRUSTSTORE_FILE}" - trust-store-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" + keystore-location: "file:${KEYSTORE_FILE}" + keystore-password: "${BIE_KAFKA_KEYSTORE_PASSWORD}" + key-store-type: "PKCS12" + truststore-location: "file:${TRUSTSTORE_FILE}" + truststore-password: "${BIE_KAFKA_TRUSTSTORE_PASSWORD}" trust-store-type: "PKCS12" ## Specify bie properties bie: - kafka-topic-env: "TST" + kafka-topic-infix: "TST" ## Actuator for health check, liveness, and readiness management: From fa020b8c9f58cab67802374bc4a87914a210cd5a Mon Sep 17 00:00:00 2001 From: Chengjie Yin <89034346+chengjie8@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:49:18 -0400 Subject: [PATCH 67/67] RBAC user id password are not needed for local test --- app/docker-compose.yml | 2 -- scripts/setenv.sh | 2 -- 2 files changed, 4 deletions(-) diff --git a/app/docker-compose.yml b/app/docker-compose.yml index acef22b4e9..e0542660fb 100644 --- a/app/docker-compose.yml +++ b/app/docker-compose.yml @@ -128,7 +128,5 @@ services: BIE_KAFKA_KEYSTORE_PASSWORD: ${BIE_KAFKA_KEYSTORE_PASSWORD} BIE_KAFKA_TRUSTSTORE_INBASE64: ${BIE_KAFKA_TRUSTSTORE_INBASE64} BIE_KAFKA_TRUSTSTORE_PASSWORD: ${BIE_KAFKA_TRUSTSTORE_PASSWORD} - BIE_KAFKA_RBAC_USERNAME: ${BIE_KAFKA_RBAC_USERNAME} - BIE_KAFKA_RBAC_PASSWORD: ${BIE_KAFKA_RBAC_PASSWORD} networks: - vro_intranet diff --git a/scripts/setenv.sh b/scripts/setenv.sh index 2aca1df356..bec4db0df8 100755 --- a/scripts/setenv.sh +++ b/scripts/setenv.sh @@ -253,5 +253,3 @@ exportSecretIfUnset BIP_KEYSTORE exportSecretIfUnset BIP_PASSWORD exportSecretIfUnset BIP_CLAIM_URL exportSecretIfUnset BIP_EVIDENCE_URL -export BIE_KAFKA_RBAC_USERNAME=USERNAME -export BIE_KAFKA_RBAC_PASSWORD=PASSWORD