From 0bb9aefc3665a2381cf49a57a5ade09db9b436ae Mon Sep 17 00:00:00 2001 From: Clay Woods Date: Mon, 23 Sep 2024 10:50:14 -0700 Subject: [PATCH] #356 Spark working with java17 356 Update mavenUtils to be compatable with jdk17 356 Create aissemble compatible quarkus bom 356 Create migration to update spark configs with version update --- bom/aissemble-quarkus-bom/LICENSE.txt | 49 +++++++++++++++++++ bom/aissemble-quarkus-bom/pom.xml | 36 ++++++++++++++ bom/bom-component/pom.xml | 7 --- bom/pom.xml | 1 + build-parent/pom.xml | 13 ++++- .../extensions-alerting-teams/pom.xml | 15 ++---- extensions/extensions-alerting/pom.xml | 8 +-- .../pyproject.toml | 2 +- .../messaging/PipelineMessagingConfig.java | 2 +- .../base/pipeline-messaging.properties | 1 + .../messaging-configuration.feature | 2 +- .../object-store-validation.feature | 6 +-- .../src/main/resources/docker/Dockerfile | 7 ++- extensions/extensions-docker/pom.xml | 2 +- .../templates/rbac.yaml | 3 +- .../aissemble-spark-history-chart/values.yaml | 4 +- .../aissemble-thrift-server-chart/values.yaml | 4 +- extensions/extensions-messaging/pom.xml | 8 +-- .../extensions-metadata-service/pom.xml | 8 +-- .../pom.xml | 6 +-- .../extensions-authzforce/pom.xml | 5 ++ .../pyproject.toml | 2 +- .../pyproject.toml | 2 + .../foundation-configuration-store/pom.xml | 6 +-- foundation/foundation-data-access/pom.xml | 7 +++ .../pom.xml | 6 +-- .../foundation-data-lineage-consumer/pom.xml | 10 +++- .../aiops/mda/generator/util/MavenUtil.java | 8 ++- .../data-access/data.access.pom.xml.vm | 6 +-- .../foundation-metadata-producer/pom.xml | 17 ++++--- .../foundation-pdp-client-java/pom.xml | 5 ++ .../foundation-service-discovery/pom.xml | 6 +-- .../v1_10_0/SparkVersionUpgradeMigration.java | 48 ++++++++++++++++++ .../src/main/resources/migrations.json | 12 +++++ .../SparkVersionUpgradeMigrationSteps.java | 47 ++++++++++++++++++ .../spark-version-upgrade-migration.feature | 11 +++++ .../migration/outdatedApps/base-values.yaml | 27 ++++++++++ .../migration/updatedApps/base-values.yaml | 27 ++++++++++ .../validation/outdatedApps/base-values.yaml | 27 ++++++++++ .../validation/updatedApps/base-values.yaml | 27 ++++++++++ .../pyproject.toml | 2 +- .../pyproject.toml | 2 +- 42 files changed, 418 insertions(+), 76 deletions(-) create mode 100644 bom/aissemble-quarkus-bom/LICENSE.txt create mode 100644 bom/aissemble-quarkus-bom/pom.xml create mode 100644 foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/SparkVersionUpgradeMigration.java create mode 100644 foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/SparkVersionUpgradeMigrationSteps.java create mode 100644 foundation/foundation-upgrade/src/test/resources/specifications/spark-version-upgrade-migration.feature create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/migration/outdatedApps/base-values.yaml create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/migration/updatedApps/base-values.yaml create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/validation/outdatedApps/base-values.yaml create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/validation/updatedApps/base-values.yaml diff --git a/bom/aissemble-quarkus-bom/LICENSE.txt b/bom/aissemble-quarkus-bom/LICENSE.txt new file mode 100644 index 000000000..a9bd80cda --- /dev/null +++ b/bom/aissemble-quarkus-bom/LICENSE.txt @@ -0,0 +1,49 @@ +Booz Allen Public License v1.0 +------------------------------ + +INTRODUCTION +The Booz Allen Public License allows government, non-profit academic, other non-profit, and commercial entities access to distinctive, disruptive, and robust code with the goal of Empowering People to Change the World(SM). Products licensed under the Booz Allen Public License are founded on the basis that collective ingenuity can make the largest impact in the community. + +DEFINITIONS +* **Commercial Entity.** “Commercial Entity” means any individual or entity other than a government, non-profit academic, or other non-profit entity. +* **Derivative.** “Derivative” means any work of authorship in Source Code or Object Code form that results from an addition to, deletion from, or modification of the Source Code of the Product. +* **License.** “License” means this Booz Allen Public License. +* **Object Code.** “Object Code” means the form resulting from transformation or translation of Source Code into machine readable code, including but not limited to, compiled object code. +* **Originator.** “Originator” means each individual or legal entity that creates, contributes to the creation of, or owns the Product. +* **Patent Claims.** “Patent Claims” means any patent claim(s) in any patent to which Originator has a right to grant a license that would be infringed by Your making, using, selling, offering for sale, having made, or importing of the Product, but for the grant of this License. +* **Product.** “Product” means the Source Code of the software which the initial Originator made available under this License, and any Derivative of such Source Code. +* **Source Code.** “Source Code” means software in human-readable form. +* **You.** “You” means either an individual or an entity (if you are taking this license on behalf of an entity) that exercises the rights granted under this License. + +LICENSE +**Government/Non-Profit Academic/Other Non-Profit.** +This Section applies if You are not a Commercial Entity. + +* **License.** Subject to the terms and conditions of this License, each Originator hereby grants You a perpetual, worldwide, non-exclusive, royalty-free license to reproduce, display, perform, modify, distribute and otherwise use the Product and Derivatives, in Source Code and Object Code form, in accordance with the terms and conditions of this License in order to support the general public good and for your internal business purposes. +* **Distribution.** You may distribute to third parties copies of the Product, including any Derivative that You create, in Source Code or Object Code form. If You distribute copies of the Product, including any Derivative that You create, in Source Code form, such distribution must be under the terms of this License and You must inform recipients of the Source Code that the Product is governed under this License and how they can obtain a copy of this License. You may distribute to third parties copies of the Product, including any Derivative that You create, in Object Code form, or allow third parties to access or use the Product, including any Derivative that You create, under a license of Your choice. +* **Commercial Sales.** You may not distribute, or allow third parties to access or use, the Product or any Derivative for a fee, unless You first obtain permission from the Originator. If Booz Allen Hamilton is the Originator, please contact Booz Allen Hamilton at . + +**Commercial Entities**. +This Section applies if You are a Commercial Entity. + +* **License.** Subject to the terms and conditions of this License, each Originator hereby grants You a perpetual, worldwide, non-exclusive, royalty-free license to reproduce, display, perform, modify, distribute and otherwise use the Product and Derivatives, in Source Code and Object Code form, in accordance with the terms and conditions of this License for the sole purpose of Your internal business purposes and the provision of services to government, non-profit academic, and other non-profit entities. +* **Distribution and Derivatives.** You may distribute to third parties copies of the Product, including any Derivative that You create, in Source Code or Object Code form. If You distribute copies of the Product, including any Derivative that You create, in Source Code form, such distribution must be under the terms of this License and You must inform recipients of the Source Code that the Product is governed under this License and how they can obtain a copy of this License. You may distribute to third parties copies of the Product, including any Derivative that You create, in Object Code form, or allow third parties to access or use the Product, including any Derivative that You create, under a license of Your choice, provided that You make available, and inform the recipient of such distribution how they can obtain, a copy of the Source Code thereof, at no charge, and inform the recipient of the Source Code that the Product is governed under this License and how they can obtain a copy of this License. +* **Commercial Sales.** You may not distribute, or allow third parties to access or use, the Product or any Derivative for a fee, unless You first obtain permission from the Originator. If Booz Allen Hamilton, please contact Booz Allen Hamilton at . + + +**Patent Claim(s)**. +This Section applies regardless of whether You are a government, non-profit academic, or other non-profit entity or a Commercial Entity. + +* **Patent License.** Subject to the limitations in the Sections above, each Originator hereby grants You a perpetual, worldwide, non-exclusive, royalty-free license under Patent Claims of such Originator to make, use, sell, offer for sale, have made, and import the Product. The foregoing patent license does not apply (a) to any code that an Originator has removed from the Product, or (b) for infringement caused by Your modifications of the Product or the combination of any Derivative created by You or on Your behalf with other software. + +GENERAL TERMS +This Section applies regardless of whether You are a government, non-profit academic, or other non-profit entity or a Commercial Entity. + +* **Required Notices.** If You distribute the Product or a Derivative, in Object Code or Source Code form, You shall not remove or otherwise modify any proprietary markings or notices contained within or placed upon the Product or any Derivative. Any distribution of the Product or a Derivative, in Object Code or Source Code form, shall contain a clear and conspicuous Originator copyright and license reference in accordance with the below: + * *Unmodified Product Notice*: “This software package is licensed under the Booz Allen Public License. Copyright © 20__ [Copyright Holder Name]. All Rights Reserved.” + * *Derivative Notice*: “This software package is licensed under the Booz Allen Public License. Portions of this code are Copyright © 20__ [Copyright Holder Name]. All Rights Reserved.” +* **Compliance with Laws.** You agree that You shall not reproduce, display, perform, modify, distribute and otherwise use the Product in any way that violates applicable law or regulation or infringes or violates the rights of others, including, but not limited to, third party intellectual property, privacy, and publicity rights. +* **Disclaimer.** You understand that the Product is licensed to You, and not sold. The Product is provided on an “As Is” basis, without any warranties, representations, and guarantees, whether oral or written, express, implied or statutory, with regard to the Product, including without limitation, warranties of merchantability, fitness for a particular purpose, title, non-infringement, non-interference, and warranties arising from course of dealing or usage of trade, to the maximum extent permitted by applicable law. Originator does not warrant that (i) the Product will meet your needs; (ii) the Product will be error-free or accessible at all times; or (iii) the use or the results of the use of the Product will be correct, accurate, timely, or otherwise reliable. You acknowledge that the Product has not been prepared to meet Your individual requirements, whether or not such requirements have been communicated to Originator. You assume all responsibility for use of the Product. +* **Limitation of Liability.** Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Originator, or anyone who distributes the Product in accordance with this License, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if informed of the possibility of such damages. +* **Export Control.** The Product is subject to U.S. export control laws and may be subject to export or import regulations in other countries. You agree to strictly comply with all such laws and regulations and acknowledges that You are responsible for obtaining such licenses to export, re-export, or import as may be required. +* **Severability.** If the application of any provision of this License to any particular facts or circumstances shall be held to be invalid or unenforceable, then the validity and enforceability of other provisions of this License shall not in any way be affected or impaired thereby. diff --git a/bom/aissemble-quarkus-bom/pom.xml b/bom/aissemble-quarkus-bom/pom.xml new file mode 100644 index 000000000..e6f030ea8 --- /dev/null +++ b/bom/aissemble-quarkus-bom/pom.xml @@ -0,0 +1,36 @@ + + + 4.0.0 + + + com.boozallen.aissemble + bom + 1.10.0-SNAPSHOT + + + aissemble-quarkus-bom + pom + + aiSSEMBLE::BOM::Quarkus + An aiSSEMBLE compabible version of the quarkus-universe-bom + + + + + + io.quarkus + quarkus-universe-bom + ${version.quarkus.plugin} + pom + import + + + io.smallrye.reactive + * + + + + + + + diff --git a/bom/bom-component/pom.xml b/bom/bom-component/pom.xml index 3b175cf84..f5a43eb5d 100644 --- a/bom/bom-component/pom.xml +++ b/bom/bom-component/pom.xml @@ -24,13 +24,6 @@ pom import - - io.quarkus - quarkus-universe-bom - ${version.quarkus.plugin} - pom - import - org.technologybrewery.krausening krausening diff --git a/bom/pom.xml b/bom/pom.xml index d3f8b7fe2..154837f10 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -16,6 +16,7 @@ bom-component bom-instantiation + aissemble-quarkus-bom diff --git a/build-parent/pom.xml b/build-parent/pom.xml index 8992cd33d..85689a684 100644 --- a/build-parent/pom.xml +++ b/build-parent/pom.xml @@ -55,7 +55,7 @@ 2.1.1 42.5.1 4.7.7.Final - 4.2.5 + 4.4.6 ^1.26.18 >=0.95.0 >=0.18.0 @@ -71,11 +71,14 @@ 4.0.1 2.0.1 3.0.0 + 3.1.0 + 2.1.2 3.1.0 - 3.4.0 + 3.5.2 2.12.13 + 2.12 2.4.0 8.9.0 1.4.0 @@ -370,6 +373,7 @@ ${TESTCONTAINERS_RYUK_DISABLED} + --add-exports java.base/sun.nio.ch=ALL-UNNAMED @@ -544,6 +548,7 @@ ${docker.baseline.repo.id}/ ${version.aissemble} ${version.spark} + ${version.scala.minor} @@ -589,6 +594,7 @@ ${docker.baseline.repo.id}/ ${version.aissemble} ${version.spark} + ${version.scala.minor} default @@ -863,6 +869,9 @@ ${TESTCONTAINERS_RYUK_DISABLED} + + --add-exports java.base/sun.nio.ch=ALL-UNNAMED diff --git a/extensions/extensions-alerting/extensions-alerting-teams/pom.xml b/extensions/extensions-alerting/extensions-alerting-teams/pom.xml index 68c8ce16c..496f5bbc8 100644 --- a/extensions/extensions-alerting/extensions-alerting-teams/pom.xml +++ b/extensions/extensions-alerting/extensions-alerting-teams/pom.xml @@ -16,18 +16,11 @@ - io.quarkus - quarkus-universe-bom - ${version.quarkus.plugin} + ${project.groupId} + aissemble-quarkus-bom + ${project.version} pom import - - - - io.smallrye.reactive - * - - io.smallrye.reactive @@ -147,4 +140,4 @@ - \ No newline at end of file + diff --git a/extensions/extensions-alerting/pom.xml b/extensions/extensions-alerting/pom.xml index 6f6acbf80..c1a6897e9 100644 --- a/extensions/extensions-alerting/pom.xml +++ b/extensions/extensions-alerting/pom.xml @@ -38,9 +38,9 @@ ${project.version} - io.quarkus - quarkus-universe-bom - ${version.quarkus.plugin} + ${project.groupId} + aissemble-quarkus-bom + ${project.version} pom import @@ -119,4 +119,4 @@ - \ No newline at end of file + diff --git a/extensions/extensions-data-delivery/aissemble-extensions-data-delivery-spark-py/pyproject.toml b/extensions/extensions-data-delivery/aissemble-extensions-data-delivery-spark-py/pyproject.toml index 5103fa6f4..330d5aa8d 100644 --- a/extensions/extensions-data-delivery/aissemble-extensions-data-delivery-spark-py/pyproject.toml +++ b/extensions/extensions-data-delivery/aissemble-extensions-data-delivery-spark-py/pyproject.toml @@ -11,7 +11,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.8" krausening = ">=20" -pyspark = "3.4.0" +pyspark = "3.5.2" pyyaml = "^6.0" [tool.poetry.group.dev.dependencies] diff --git a/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/main/java/com/boozallen/aiops/data/delivery/messaging/PipelineMessagingConfig.java b/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/main/java/com/boozallen/aiops/data/delivery/messaging/PipelineMessagingConfig.java index 8bc4829dc..42872e6b6 100644 --- a/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/main/java/com/boozallen/aiops/data/delivery/messaging/PipelineMessagingConfig.java +++ b/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/main/java/com/boozallen/aiops/data/delivery/messaging/PipelineMessagingConfig.java @@ -47,7 +47,7 @@ public Map getProperties() { Map map = new HashMap<>(); for (String name : pipelineProperties.stringPropertyNames()) { String reactiveName = mapToReactiveProperty(name); - String value = pipelineProperties.getProperty(reactiveName); + String value = pipelineProperties.getProperty(name); map.put(reactiveName, value); } return map; diff --git a/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/krausening/base/pipeline-messaging.properties b/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/krausening/base/pipeline-messaging.properties index a7666fcfb..4dbc846f1 100644 --- a/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/krausening/base/pipeline-messaging.properties +++ b/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/krausening/base/pipeline-messaging.properties @@ -18,3 +18,4 @@ NoChannelTestStep.in.connector=smallrye-kafka # these are overridden to in-memory in test/pipeline-messaging.properties OverrideTestStep.in.connector=smallrye-kafka OverrideTestStep.out.connector=smallrye-kafka + diff --git a/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/specifications/messaging-configuration.feature b/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/specifications/messaging-configuration.feature index d7872f0a3..78708944d 100644 --- a/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/specifications/messaging-configuration.feature +++ b/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/specifications/messaging-configuration.feature @@ -35,4 +35,4 @@ Feature: Pipeline messaging steps can be configured And a pipeline configuration for the step When messages flow from the external system Then the configuration directs the messaging to the step - And the configuration directs the step result to the external system \ No newline at end of file + And the configuration directs the step result to the external system diff --git a/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/specifications/object-store-validation.feature b/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/specifications/object-store-validation.feature index c354e24b9..999e4471c 100644 --- a/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/specifications/object-store-validation.feature +++ b/extensions/extensions-data-delivery/extensions-data-delivery-spark/src/test/resources/specifications/object-store-validation.feature @@ -1,8 +1,8 @@ @data-delivery @object-store-validation -Feature: Object store credentials can be verified for read and write access - I want to use configured credentials to connect to an object store and +Feature: Object store credentials can be verified for read and write access + I want to use configured credentials to connect to an object store and verify read and write access Scenario: I can retrieve credentials from a Krausening file Given a properties file exists - Then the credentials are used to verify object store connectivity \ No newline at end of file + Then the credentials are used to verify object store connectivity diff --git a/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile index 7973dfc09..77c6ca4ea 100644 --- a/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile @@ -1,7 +1,8 @@ # Script for creating base Spark Docker image # ARG SPARK_VERSION -FROM apache/spark-py:v${SPARK_VERSION} +ARG SCALA_VERSION +FROM apache/spark:${SPARK_VERSION}-scala${SCALA_VERSION}-java17-python3-ubuntu LABEL org.opencontainers.image.source="https://github.com/boozallen/aissemble" @@ -14,9 +15,7 @@ RUN mkdir $SPARK_HOME/checkpoint && \ mkdir $SPARK_HOME/krausening && \ mkdir $SPARK_HOME/warehouse && \ mkdir $SPARK_HOME/policies && \ - mkdir $SPARK_HOME/logs && \ - useradd --home /opt/spark --group 0 --shell /usr/sbin/nologin --uid 185 spark && \ - chown -R spark:spark /opt/spark + mkdir $SPARK_HOME/logs # Update repositories and add necessary ones RUN apt-get -y update \ diff --git a/extensions/extensions-docker/pom.xml b/extensions/extensions-docker/pom.xml index 7464d0fc7..48aac4fe2 100644 --- a/extensions/extensions-docker/pom.xml +++ b/extensions/extensions-docker/pom.xml @@ -78,7 +78,7 @@ - com.boozallen.aissemble + ${project.groupId} bom-component ${project.version} pom diff --git a/extensions/extensions-helm/aissemble-spark-operator-chart/templates/rbac.yaml b/extensions/extensions-helm/aissemble-spark-operator-chart/templates/rbac.yaml index 5d5938b26..bd49c01eb 100644 --- a/extensions/extensions-helm/aissemble-spark-operator-chart/templates/rbac.yaml +++ b/extensions/extensions-helm/aissemble-spark-operator-chart/templates/rbac.yaml @@ -38,6 +38,7 @@ rules: - delete - update - list + - patch - apiGroups: - extensions - networking.k8s.io @@ -137,4 +138,4 @@ roleRef: kind: ClusterRole name: {{ include "spark-operator.fullname" (index .Subcharts "spark-operator") }} apiGroup: rbac.authorization.k8s.io -{{- end }} \ No newline at end of file +{{- end }} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/values.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/values.yaml index f1d7dcf33..3f335b0a7 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/values.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-spark-history-chart/values.yaml @@ -7,9 +7,9 @@ deployment: labels: {} replicas: 1 image: - repository: "apache/spark" + repository: "ghcr.io/boozallen/aissemble-spark" imagePullPolicy: IfNotPresent - tag: "3.5.1" + tag: "1.10.0-SNAPSHOT" command: ["/opt/spark/sbin/start-history-server.sh"] env: SPARK_NO_DAEMONIZE: "true" diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/values.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/values.yaml index 772f96ec1..ea8d2eb58 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/values.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-thrift-server-chart/values.yaml @@ -7,9 +7,9 @@ deployment: labels: {} replicas: 1 image: - repository: "apache/spark" + repository: "ghcr.io/boozallen/aissemble-spark" imagePullPolicy: IfNotPresent - tag: "3.5.1" + tag: "1.10.0-SNAPSHOT" command: ["/opt/spark/sbin/start-thriftserver.sh"] env: SPARK_NO_DAEMONIZE: "true" diff --git a/extensions/extensions-messaging/pom.xml b/extensions/extensions-messaging/pom.xml index 807449f52..f00d123cc 100644 --- a/extensions/extensions-messaging/pom.xml +++ b/extensions/extensions-messaging/pom.xml @@ -42,12 +42,12 @@ ${version.mutiny} - io.quarkus - quarkus-universe-bom - ${version.quarkus.plugin} + ${project.groupId} + aissemble-quarkus-bom + ${project.version} pom import - \ No newline at end of file + diff --git a/extensions/extensions-metadata-service/pom.xml b/extensions/extensions-metadata-service/pom.xml index 2a55f602a..26e8c2a79 100644 --- a/extensions/extensions-metadata-service/pom.xml +++ b/extensions/extensions-metadata-service/pom.xml @@ -75,9 +75,9 @@ - io.quarkus - quarkus-universe-bom - ${version.quarkus.plugin} + ${project.groupId} + aissemble-quarkus-bom + ${project.version} pom import @@ -205,4 +205,4 @@ - \ No newline at end of file + diff --git a/extensions/extensions-pipeline-invocation-service/pom.xml b/extensions/extensions-pipeline-invocation-service/pom.xml index cf6ac7e07..b4d84ddc8 100644 --- a/extensions/extensions-pipeline-invocation-service/pom.xml +++ b/extensions/extensions-pipeline-invocation-service/pom.xml @@ -23,9 +23,9 @@ import - io.quarkus - quarkus-universe-bom - ${version.quarkus.plugin} + ${project.groupId} + aissemble-quarkus-bom + ${project.version} pom import diff --git a/extensions/extensions-security/extensions-authzforce/pom.xml b/extensions/extensions-security/extensions-authzforce/pom.xml index b119dff87..32e5ce3d3 100644 --- a/extensions/extensions-security/extensions-authzforce/pom.xml +++ b/extensions/extensions-security/extensions-authzforce/pom.xml @@ -90,6 +90,11 @@ slf4j-api ${version.slf4j} + + jakarta.activation + jakarta.activation-api + ${version.jakarta.activation.api} + io.cucumber diff --git a/extensions/extensions-transform/aissemble-extensions-transform-spark-python/pyproject.toml b/extensions/extensions-transform/aissemble-extensions-transform-spark-python/pyproject.toml index cb4f3c2d0..5dc4d4151 100644 --- a/extensions/extensions-transform/aissemble-extensions-transform-spark-python/pyproject.toml +++ b/extensions/extensions-transform/aissemble-extensions-transform-spark-python/pyproject.toml @@ -13,7 +13,7 @@ python = ">=3.8" krausening = ">=20" pydantic = ">=2.8.0" -pyspark = "3.4.0" +pyspark = "3.5.2" cryptography = ">=42.0.4" urllib3 = "^1.26.18" diff --git a/foundation/aissemble-foundation-core-python/pyproject.toml b/foundation/aissemble-foundation-core-python/pyproject.toml index 5e9d772f7..9b583d3dc 100644 --- a/foundation/aissemble-foundation-core-python/pyproject.toml +++ b/foundation/aissemble-foundation-core-python/pyproject.toml @@ -21,6 +21,8 @@ krausening = ">=20" cryptography = ">=42.0.4" pyjwt = ">=2.3.0" pyjks = ">=20.0.0" +#pyasn1 = ">=0.3.5,<0.6.1" +#pyasn1-modules = ">=0.0.8,<0.4.1" jsonpickle = ">=2.1.0" aiohttp = ">=3.8.0" apache-libcloud = ">=3.6.0" diff --git a/foundation/foundation-configuration-store/pom.xml b/foundation/foundation-configuration-store/pom.xml index a1f27c5df..f2df24471 100644 --- a/foundation/foundation-configuration-store/pom.xml +++ b/foundation/foundation-configuration-store/pom.xml @@ -100,9 +100,9 @@ - io.quarkus - quarkus-universe-bom - ${version.quarkus.plugin} + ${project.groupId} + aissemble-quarkus-bom + ${project.version} pom import diff --git a/foundation/foundation-data-access/pom.xml b/foundation/foundation-data-access/pom.xml index 3f60f43cc..dee6b4811 100644 --- a/foundation/foundation-data-access/pom.xml +++ b/foundation/foundation-data-access/pom.xml @@ -22,6 +22,13 @@ pom import + + com.boozallen.aissemble + aissemble-quarkus-bom + ${project.version} + pom + import + diff --git a/foundation/foundation-drift-detection/foundation-drift-detection-service/pom.xml b/foundation/foundation-drift-detection/foundation-drift-detection-service/pom.xml index a2f4386e5..f42560ed6 100644 --- a/foundation/foundation-drift-detection/foundation-drift-detection-service/pom.xml +++ b/foundation/foundation-drift-detection/foundation-drift-detection-service/pom.xml @@ -14,9 +14,9 @@ - io.quarkus - quarkus-universe-bom - ${version.quarkus} + ${project.groupId} + aissemble-quarkus-bom + ${project.version} pom import diff --git a/foundation/foundation-lineage/foundation-data-lineage/foundation-data-lineage-consumer/pom.xml b/foundation/foundation-lineage/foundation-data-lineage/foundation-data-lineage-consumer/pom.xml index fcc8caec4..4da934678 100644 --- a/foundation/foundation-lineage/foundation-data-lineage/foundation-data-lineage-consumer/pom.xml +++ b/foundation/foundation-lineage/foundation-data-lineage/foundation-data-lineage-consumer/pom.xml @@ -33,12 +33,19 @@ - com.boozallen.aissemble + ${project.groupId} bom-component ${project.version} pom import + + ${project.groupId} + aissemble-quarkus-bom + ${project.version} + pom + import + @@ -82,6 +89,7 @@ io.smallrye.reactive smallrye-reactive-messaging-api + ${version.smallrye.reactive.messaging} net.masterthought diff --git a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/util/MavenUtil.java b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/util/MavenUtil.java index 2d08923f4..36b095698 100644 --- a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/util/MavenUtil.java +++ b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/util/MavenUtil.java @@ -120,7 +120,13 @@ public static String getDataRecordModuleName(GenerationContext context, String m String projectPrefix = context.getRootArtifactId(); String defaultDataRecords = projectPrefix + "-" + dataModule.getBaseName(); Path projectRoot = context.getExecutionRootDirectory().toPath(); - Path defaultModulePath = projectRoot.resolve(Path.of(getSharedModuleName(context.getExecutionRootDirectory()), defaultDataRecords)); + Path defaultModulePath; + if (getSharedModuleName(context.getExecutionRootDirectory()) != null) { + defaultModulePath = projectRoot.resolve(Path.of(getSharedModuleName(context.getExecutionRootDirectory()), + defaultDataRecords)); + } else { + defaultModulePath = projectRoot.resolve(Path.of(defaultDataRecords)); + } Language defaultLanguage = getModuleLanguage(defaultModulePath.resolve("pom.xml")); if (SemanticDataUtil.arePythonDataRecordsNeeded(metadataContext) && SemanticDataUtil.areJavaDataRecordsNeeded(metadataContext) diff --git a/foundation/foundation-mda/src/main/resources/templates/data-access/data.access.pom.xml.vm b/foundation/foundation-mda/src/main/resources/templates/data-access/data.access.pom.xml.vm index 5dfc49f53..52cac8685 100644 --- a/foundation/foundation-mda/src/main/resources/templates/data-access/data.access.pom.xml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/data-access/data.access.pom.xml.vm @@ -19,9 +19,9 @@ - io.quarkus - quarkus-universe-bom - ${version.quarkus.plugin} + com.boozallen.aissemble + aissemble-quarkus-bom + ${project.version} pom import diff --git a/foundation/foundation-metadata/foundation-metadata-producer/pom.xml b/foundation/foundation-metadata/foundation-metadata-producer/pom.xml index a6eb7d7b5..8b9c9b34b 100644 --- a/foundation/foundation-metadata/foundation-metadata-producer/pom.xml +++ b/foundation/foundation-metadata/foundation-metadata-producer/pom.xml @@ -19,13 +19,13 @@ ${project.version} pom import - - - - io.smallrye.reactive - * - - + + + com.boozallen.aissemble + aissemble-quarkus-bom + ${project.version} + pom + import io.smallrye.reactive @@ -68,6 +68,7 @@ io.smallrye.reactive smallrye-reactive-messaging-api + ${version.smallrye.reactive.messaging} @@ -80,4 +81,4 @@ - \ No newline at end of file + diff --git a/foundation/foundation-security/foundation-pdp-client-java/pom.xml b/foundation/foundation-security/foundation-pdp-client-java/pom.xml index 864900c18..f10102c4b 100644 --- a/foundation/foundation-security/foundation-pdp-client-java/pom.xml +++ b/foundation/foundation-security/foundation-pdp-client-java/pom.xml @@ -47,6 +47,11 @@ resteasy-jaxb-provider ${version.resteasy} + + jakarta.ws.rs + jakarta.ws.rs-api + ${version.jakarta.wr.rs} + diff --git a/foundation/foundation-service-discovery/pom.xml b/foundation/foundation-service-discovery/pom.xml index 1a4038d18..b1e6bd18e 100644 --- a/foundation/foundation-service-discovery/pom.xml +++ b/foundation/foundation-service-discovery/pom.xml @@ -14,9 +14,9 @@ - io.quarkus - quarkus-universe-bom - ${version.quarkus.plugin} + ${project.groupId} + aissemble-quarkus-bom + ${project.version} pom import diff --git a/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/SparkVersionUpgradeMigration.java b/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/SparkVersionUpgradeMigration.java new file mode 100644 index 000000000..13f6ca911 --- /dev/null +++ b/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/SparkVersionUpgradeMigration.java @@ -0,0 +1,48 @@ +package com.boozallen.aissemble.upgrade.migration.v1_10_0; + +/*- + * #%L + * aiSSEMBLE::Foundation::Upgrade + * %% + * Copyright (C) 2021 Booz Allen + * %% + * This software package is licensed under the Booz Allen Public License. All Rights Reserved. + * #L% + */ + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +import com.boozallen.aissemble.upgrade.migration.AbstractAissembleMigration; + +public class SparkVersionUpgradeMigration extends AbstractAissembleMigration { + private static final String OLD_FAILURE_VALIDITY_INTERVAL = "spark.yarn.executor.failuresValidityInterval"; + private static final String NEW_FAILURE_VALIDITY_INTERVAL = "spark.executor.failuresValidityInterval"; + private static final String OLD_MAX_FAILURES = "spark.yarn.max.executor.failures"; + private static final String NEW_MAX_FAILURES = "spark.executor.maxNumFailures"; + + @Override + protected boolean shouldExecuteOnFile(File file) { + try { + System.out.println("Clay: " + file.getName()); + return (Files.readString(file.toPath()).contains(OLD_FAILURE_VALIDITY_INTERVAL) || + Files.readString(file.toPath()).contains(OLD_MAX_FAILURES)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + protected boolean performMigration(File file) { + try { + Files.writeString(file.toPath(), Files.readString(file.toPath()) + .replace(OLD_FAILURE_VALIDITY_INTERVAL, NEW_FAILURE_VALIDITY_INTERVAL)); + Files.writeString(file.toPath(), Files.readString(file.toPath()) + .replace(OLD_MAX_FAILURES, NEW_MAX_FAILURES)); + return true; + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/foundation/foundation-upgrade/src/main/resources/migrations.json b/foundation/foundation-upgrade/src/main/resources/migrations.json index ec41fe384..fcbd3b3d3 100644 --- a/foundation/foundation-upgrade/src/main/resources/migrations.json +++ b/foundation/foundation-upgrade/src/main/resources/migrations.json @@ -3,6 +3,18 @@ "group": "v1-10-0-migrations", "type": "ordered", "migrations": [ + { + "name": "spark-version-upgrade-migration", + "implementation": "com.boozallen.aissemble.upgrade.migration.v1_10_0.SparkVersionUpgradeMigration", + "fileSets": [ + { + "includes": [ + "*-pipelines/*/src/*/resources/apps/*-values.yaml", + "*-pipelines/*/src/main/resources/apps/*-values.yaml" + ] + } + ] + } ] }, { diff --git a/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/SparkVersionUpgradeMigrationSteps.java b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/SparkVersionUpgradeMigrationSteps.java new file mode 100644 index 000000000..4ac47d1d7 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/SparkVersionUpgradeMigrationSteps.java @@ -0,0 +1,47 @@ +package com.boozallen.aissemble.upgrade.migration.v1_10_0; + +/*- + * #%L + * aiSSEMBLE::Foundation::Upgrade + * %% + * Copyright (C) 2021 Booz Allen + * %% + * This software package is licensed under the Booz Allen Public License. All Rights Reserved. + * #L% + */ + +import com.boozallen.aissemble.upgrade.migration.AbstractMigrationTest; + +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; + +public class SparkVersionUpgradeMigrationSteps extends AbstractMigrationTest { + + @Given("a project with outdated spark application configuration") + public void aProjectWithOutdatedSparkApplicationConfiguration() { + testFile = getTestFile("v1_10_0/SparkVersionUpgrade/migration/outdatedApps/base-values.yaml"); + } + + @Given("a project with up to date spark application configuration") + public void aProjectWithUpToDateSparkApplicationConfiguration() { + testFile = getTestFile("v1_10_0/SparkVersionUpgrade/migration/updatedApps/base-values.yaml"); + } + + @When("the spark version upgrade migration executes") + public void theSparkVersionUpgradeMigrationExecutes() { + performMigration(new SparkVersionUpgradeMigration()); + } + + @Then("the spark application configs are updated") + public void theSparkApplicationConfigsAreUpdated() { + assertMigrationSuccess(); + assertTestFileMatchesExpectedFile("Spark configs were not updated correctly following version migration"); + } + + @Then("the spark application configs are not updated") + public void theSparkApplicationConfigsAreNotUpdated() { + assertMigrationSkipped(); + assertTestFileMatchesExpectedFile("Spark configs were not updated correctly following version migration"); + } +} diff --git a/foundation/foundation-upgrade/src/test/resources/specifications/spark-version-upgrade-migration.feature b/foundation/foundation-upgrade/src/test/resources/specifications/spark-version-upgrade-migration.feature new file mode 100644 index 000000000..4a6e5b0c6 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/specifications/spark-version-upgrade-migration.feature @@ -0,0 +1,11 @@ +Feature: Spark is updated to use appropriate v3.5.0 configs + + Scenario: Update a project with outdated spark configuration + Given a project with outdated spark application configuration + When the spark version upgrade migration executes + Then the spark application configs are updated + + Scenario: A project with up to date spark configuration is not migrated + Given a project with up to date spark application configuration + When the spark version upgrade migration executes + Then the spark application configs are not updated diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/migration/outdatedApps/base-values.yaml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/migration/outdatedApps/base-values.yaml new file mode 100644 index 000000000..6db7445f8 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/migration/outdatedApps/base-values.yaml @@ -0,0 +1,27 @@ +metadata: + name: spark-persist +sparkApp: + spec: + sparkConf: + spark.yarn.executor.failuresValidityInterval: "2h" + spark.yarn.max.executor.failures: 10 + volumes: [] + type: Java + deps: + packages: + - mysql:mysql-connector-java:8.0.30 + - org.apache.hadoop:hadoop-aws:3.3.4 + - com.amazonaws:aws-java-sdk-bundle:1.12.262 + excludePackages: [] + hadoopConf: + driver: + volumeMounts: [] + cores: 1 + coreLimit: "1200m" + memory: "2048m" + javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base" + executor: + volumeMounts: [] + cores: 1 + memory: "4096m" + javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base" diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/migration/updatedApps/base-values.yaml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/migration/updatedApps/base-values.yaml new file mode 100644 index 000000000..f260531d4 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/migration/updatedApps/base-values.yaml @@ -0,0 +1,27 @@ +metadata: + name: spark-persist +sparkApp: + spec: + sparkConf: + spark.executor.failuresValidityInterval: "2h" + spark.executor.maxNumFailures: 10 + volumes: [] + type: Java + deps: + packages: + - mysql:mysql-connector-java:8.0.30 + - org.apache.hadoop:hadoop-aws:3.3.4 + - com.amazonaws:aws-java-sdk-bundle:1.12.262 + excludePackages: [] + hadoopConf: + driver: + volumeMounts: [] + cores: 1 + coreLimit: "1200m" + memory: "2048m" + javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base" + executor: + volumeMounts: [] + cores: 1 + memory: "4096m" + javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base" diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/validation/outdatedApps/base-values.yaml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/validation/outdatedApps/base-values.yaml new file mode 100644 index 000000000..f260531d4 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/validation/outdatedApps/base-values.yaml @@ -0,0 +1,27 @@ +metadata: + name: spark-persist +sparkApp: + spec: + sparkConf: + spark.executor.failuresValidityInterval: "2h" + spark.executor.maxNumFailures: 10 + volumes: [] + type: Java + deps: + packages: + - mysql:mysql-connector-java:8.0.30 + - org.apache.hadoop:hadoop-aws:3.3.4 + - com.amazonaws:aws-java-sdk-bundle:1.12.262 + excludePackages: [] + hadoopConf: + driver: + volumeMounts: [] + cores: 1 + coreLimit: "1200m" + memory: "2048m" + javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base" + executor: + volumeMounts: [] + cores: 1 + memory: "4096m" + javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base" diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/validation/updatedApps/base-values.yaml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/validation/updatedApps/base-values.yaml new file mode 100644 index 000000000..f260531d4 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/validation/updatedApps/base-values.yaml @@ -0,0 +1,27 @@ +metadata: + name: spark-persist +sparkApp: + spec: + sparkConf: + spark.executor.failuresValidityInterval: "2h" + spark.executor.maxNumFailures: 10 + volumes: [] + type: Java + deps: + packages: + - mysql:mysql-connector-java:8.0.30 + - org.apache.hadoop:hadoop-aws:3.3.4 + - com.amazonaws:aws-java-sdk-bundle:1.12.262 + excludePackages: [] + hadoopConf: + driver: + volumeMounts: [] + cores: 1 + coreLimit: "1200m" + memory: "2048m" + javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base" + executor: + volumeMounts: [] + cores: 1 + memory: "4096m" + javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base" diff --git a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model-basic/pyproject.toml b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model-basic/pyproject.toml index 88613cfc4..6ce6eea3e 100644 --- a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model-basic/pyproject.toml +++ b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model-basic/pyproject.toml @@ -22,7 +22,7 @@ aissemble-extensions-encryption-vault-python = {path = "../../../extensions/exte kafka-python = "^2.0.2" python = ">=3.8" krausening = ">=20" -pyspark = "3.4.0" +pyspark = "3.5.2" jsonpickle = "^2.1.0" pyyaml = "^6.0" cryptography = ">=42.0.4" diff --git a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pyproject.toml b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pyproject.toml index 33acdc3ff..94e2f8212 100644 --- a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pyproject.toml +++ b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pyproject.toml @@ -21,7 +21,7 @@ aissemble-extensions-encryption-vault-python = {path = "../../../extensions/exte kafka-python = "^2.0.2" python = ">=3.8" krausening = ">=20" -pyspark = "3.4.0" +pyspark = "3.5.2" jsonpickle = "^2.1.0" pyyaml = "^6.0" cryptography = ">=42.0.4"