diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..ca46685ab --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,95 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Build aissemble + +on: + workflow_dispatch: + inputs: + buildBranch: + description: "Branch you want to build" + required: true + type: string + default: 'dev' + push: + branches: [ "dev" ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.buildBranch }} + - name: Install Python # use direct install rather than pyenv for CI for large speed improvement + uses: actions/setup-python@v5 + with: + python-version: '3.11.4' + - name: Load cached Poetry installation + id: cached-poetry + uses: actions/cache@v4 + with: + path: ~/.local + key: poetry-0 # increment to reset cache + - name: Load m2 repository cache # Manually caching .m2 repo as the setup-java caching isn't falling back to older caches + id: cached-m2-repo + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven- + - name: Load m2 build cache + id: cached-m2-build + uses: actions/cache@v4 + with: + path: ~/.m2/build-cache + key: maven-build-cache-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-build-cache- + - name: Install Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' + uses: snok/install-poetry@v1 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + - name: Install Docker + run: | + sudo apt-get update + sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install docker-ce docker-ce-cli containerd.io + - name: Install Helm + run: | + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh + - name: Build aiSSEMBLE + run: ./mvnw -B clean install --file pom.xml -Dhabushu.usePyenv=false -Pci + #NB: The following two explicit cache saves are necessary to ensure caches are saved on build failure, until + # https://github.com/actions/cache/issues/1315 is resolved + - name: Save m2 repository cache + id: save-m2-repo + uses: actions/cache/save@v4 + if: always() + with: + path: ~/.m2/repository + key: maven-${{ hashFiles('**/pom.xml') }} + - name: Save m2 build cache + id: save-m2-build + uses: actions/cache/save@v4 + if: always() + with: + path: ~/.m2/build-cache + key: maven-build-cache-${{ hashFiles('**/pom.xml') }} diff --git a/build-parent/pom.xml b/build-parent/pom.xml index a1052b5bc..b6be0cb85 100644 --- a/build-parent/pom.xml +++ b/build-parent/pom.xml @@ -58,6 +58,7 @@ >=42.0.4 >=${version.krausening} 2.0.7 + 0.6.0 3.4.0 @@ -963,51 +964,6 @@ To suppress enforce-helm-version rule, you must add following plugin to the root - - - aiops-release - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - attach-javadocs - - jar - - - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-sources - - jar-no-fork - - - - - - - doclint-java8-disable diff --git a/extensions/extensions-docker/aissemble-airflow/pom.xml b/extensions/extensions-docker/aissemble-airflow/pom.xml index 163eb2479..ebafd25dc 100644 --- a/extensions/extensions-docker/aissemble-airflow/pom.xml +++ b/extensions/extensions-docker/aissemble-airflow/pom.xml @@ -14,33 +14,4 @@ orphedomos - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack - prepare-package - - unpack - - - - - ${project.groupId} - extensions-docker-cacerts - ${project.version} - true - ${project.build.directory}/cacerts - zip - - - - - - - - \ No newline at end of file diff --git a/extensions/extensions-docker/aissemble-airflow/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-airflow/src/main/resources/docker/Dockerfile index e2d9a4414..396ed18e5 100644 --- a/extensions/extensions-docker/aissemble-airflow/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-airflow/src/main/resources/docker/Dockerfile @@ -5,9 +5,7 @@ LABEL org.opencontainers.image.source = "https://github.com/boozallen/aissemble" USER root -COPY ./target/cacerts/* /usr/local/share/ca-certificates/ - -RUN update-ca-certificates && apt-get update && apt-get install -y --no-install-recommends apt-utils +RUN apt-get update && apt-get install -y --no-install-recommends apt-utils RUN apt-get update && apt-get install -y \ ant \ fontconfig \ diff --git a/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/pom.xml b/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/pom.xml index 2e766cdba..ac2deac8e 100644 --- a/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/pom.xml +++ b/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/pom.xml @@ -13,34 +13,4 @@ A base image for a Jenkins Controller orphedomos - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack - prepare-package - - unpack - - - - - ${project.groupId} - extensions-docker-cacerts - ${project.version} - true - ${project.build.directory}/cacerts - zip - - - - - - - - diff --git a/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/src/main/resources/docker/Dockerfile index d51112211..38d59dca4 100644 --- a/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/src/main/resources/docker/Dockerfile @@ -3,12 +3,6 @@ FROM jenkins/jenkins:lts-jdk11 LABEL org.opencontainers.image.source = "https://github.com/boozallen/aissemble" USER root -COPY ./target/cacerts/* /usr/local/share/ca-certificates/ -# Update CA certificates and import each certificate in the directory -RUN update-ca-certificates && \ - for cert in /usr/local/share/ca-certificates/*.crt; do \ - keytool -import -trustcacerts -keystore "$JAVA_HOME/lib/security/cacerts" -storepass changeit -noprompt -file "$cert"; \ - done USER jenkins RUN jenkins-plugin-cli --plugins git:4.11.2 diff --git a/extensions/extensions-docker/aissemble-nvidia/pom.xml b/extensions/extensions-docker/aissemble-nvidia/pom.xml index 62fed1e7e..3b91252cb 100644 --- a/extensions/extensions-docker/aissemble-nvidia/pom.xml +++ b/extensions/extensions-docker/aissemble-nvidia/pom.xml @@ -13,34 +13,4 @@ A base image for a Nvidia Cuda implementation orphedomos - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack - prepare-package - - unpack - - - - - ${project.groupId} - extensions-docker-cacerts - ${project.version} - true - ${project.build.directory}/cacerts - zip - - - - - - - - diff --git a/extensions/extensions-docker/aissemble-nvidia/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-nvidia/src/main/resources/docker/Dockerfile index df92b87cb..6eb88369a 100644 --- a/extensions/extensions-docker/aissemble-nvidia/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-nvidia/src/main/resources/docker/Dockerfile @@ -9,9 +9,7 @@ RUN printf '#!/bin/bash\n\nif [[ "$1" == "-r" ]] ;then\n echo '4.9.250'\n exit\n RUN chmod 755 /bin/uname # Workaround for ubuntu fix end -COPY ./target/cacerts/* /usr/local/share/ca-certificates/ - -RUN update-ca-certificates && apt-get update \ +RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y openjdk-11-jdk \ && update-ca-certificates \ diff --git a/extensions/extensions-docker/aissemble-spark-infrastructure/pom.xml b/extensions/extensions-docker/aissemble-spark-infrastructure/pom.xml index 1b48d9bbc..0ac965ae2 100644 --- a/extensions/extensions-docker/aissemble-spark-infrastructure/pom.xml +++ b/extensions/extensions-docker/aissemble-spark-infrastructure/pom.xml @@ -51,11 +51,6 @@ delta-storage ${version.delta} - - io.delta - delta-hive-assembly_2.12 - ${version.delta.hive.connector} - diff --git a/extensions/extensions-docker/aissemble-spark-infrastructure/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-spark-infrastructure/src/main/resources/docker/Dockerfile index 92f33466e..75189d063 100644 --- a/extensions/extensions-docker/aissemble-spark-infrastructure/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-spark-infrastructure/src/main/resources/docker/Dockerfile @@ -1,11 +1,13 @@ ARG VERSION_AISSEMBLE ARG DOCKER_BASELINE_REPO_ID FROM ${DOCKER_BASELINE_REPO_ID}boozallen/aissemble-spark:${VERSION_AISSEMBLE} +ARG DELTA_HIVE_CONNECTOR_VERSION LABEL org.opencontainers.image.source = "https://github.com/boozallen/aissemble" +RUN curl -L https://github.com/delta-io/connectors/releases/download/v${DELTA_HIVE_CONNECTOR_VERSION}/delta-hive-assembly_2.12-${DELTA_HIVE_CONNECTOR_VERSION}.jar \ + -o "${SPARK_HOME}"/jars/delta-hive-assembly_2.12-${DELTA_HIVE_CONNECTOR_VERSION}.jar ARG JARS_DIR - ADD ${JARS_DIR}/* ${SPARK_HOME}/jars/ ENV SPARK_NO_DAEMONIZE=true diff --git a/extensions/extensions-docker/aissemble-spark/pom.xml b/extensions/extensions-docker/aissemble-spark/pom.xml index a2a976386..f4ad72edd 100644 --- a/extensions/extensions-docker/aissemble-spark/pom.xml +++ b/extensions/extensions-docker/aissemble-spark/pom.xml @@ -61,31 +61,6 @@ - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack - prepare-package - - unpack - - - - - ${project.groupId} - extensions-docker-cacerts - ${project.version} - true - ${project.build.directory}/cacerts - zip - - - - - - diff --git a/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile index 63021da95..8bfae16f9 100644 --- a/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile @@ -7,11 +7,8 @@ LABEL org.opencontainers.image.source = "https://github.com/boozallen/aissemble" USER root -COPY ./target/cacerts/* /usr/local/share/ca-certificates/ - # Configures the desired version of Python to install -ARG PYTHON_VERSION=3.11.4 - +ARG PYTHON_VERSION=3.11 # Setup Spark home directory RUN mkdir ${SPARK_HOME}/checkpoint && \ mkdir ${SPARK_HOME}/krausening && \ @@ -19,38 +16,26 @@ RUN mkdir ${SPARK_HOME}/checkpoint && \ mkdir ${SPARK_HOME}/policies && \ mkdir ${SPARK_HOME}/logs && \ useradd --home /opt/spark --group 0 --shell /usr/sbin/nologin --uid 185 spark && \ - chown -R spark:spark /opt/spark && \ - ln -s /usr/bin/python3 /usr/bin/python + chown -R spark:spark /opt/spark + +# Update repositories and add necessary ones +RUN apt-get -y update \ + && apt-get install -y software-properties-common \ + && add-apt-repository ppa:deadsnakes/ppa # git and build-essential are used during pip install of some dependencies (e.g. python-deequ) -# software-properties-common and *-dev dependencies are needed to build an updated version of Python -# as Python 3.11.4 is packaged by default with the base spark-py image -RUN update-ca-certificates && apt-get update -y && apt-get install --assume-yes \ +RUN apt-get update -y && apt-get install --assume-yes \ build-essential \ curl \ - software-properties-common \ - libnss3-dev \ - zlib1g-dev \ - libgdbm-dev \ - libncurses5-dev \ - libssl-dev \ - libffi-dev \ - libreadline-dev \ - libsqlite3-dev \ - libbz2-dev \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ +#TODO is distutils needed? + python${PYTHON_VERSION}-distutils \ #Patch for CVE-2023-4863: upgrade libwebp7 to latest && apt-get upgrade -y libwebp7 \ && rm -rf /var/lib/apt/lists/* \ - && apt-get clean - -# Install newer version of python -RUN curl -L https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \ - -o /tmp/Python-${PYTHON_VERSION}.tgz && \ - tar xvf /tmp/Python-${PYTHON_VERSION}.tgz -C /tmp && \ - cd /tmp/Python-${PYTHON_VERSION} && \ - ./configure --enable-optimizations && \ - make install && \ - ln -sf /usr/local/bin/python3 /usr/bin/python + && apt-get clean \ + && ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python ## Add spark configurations COPY ./src/main/resources/conf/ ${SPARK_HOME}/conf/ diff --git a/extensions/extensions-docker/pom.xml b/extensions/extensions-docker/pom.xml index ae4a36793..8b4c38538 100644 --- a/extensions/extensions-docker/pom.xml +++ b/extensions/extensions-docker/pom.xml @@ -42,6 +42,32 @@ ${project.version}-${target.architecture} + + + + org.codehaus.mojo + exec-maven-plugin + + + clear-buildkit-cache + clean + + exec + + + docker + + buildx + prune + --all + --force + + + + + + + ensure-docker-dependencies @@ -105,6 +131,7 @@ ${dockerImageVersion} + ${version.delta.hive.connector} diff --git a/extensions/extensions-helm/pom.xml b/extensions/extensions-helm/pom.xml index cfa19d738..ddd548e97 100644 --- a/extensions/extensions-helm/pom.xml +++ b/extensions/extensions-helm/pom.xml @@ -61,11 +61,33 @@ - ci + release true + + integration-test + + + + + ${group.helm.plugin} + helm-maven-plugin + + + simulates-an-install + integration-test + + dry-run + + + + + + + + @@ -83,13 +105,6 @@ dependency-update - - simulates-an-install - integration-test - - dry-run - - true diff --git a/foundation/foundation-archetype/src/main/resources/archetype-resources/pom.xml b/foundation/foundation-archetype/src/main/resources/archetype-resources/pom.xml index 9a4d7c8bb..e7995da08 100644 --- a/foundation/foundation-archetype/src/main/resources/archetype-resources/pom.xml +++ b/foundation/foundation-archetype/src/main/resources/archetype-resources/pom.xml @@ -28,10 +28,6 @@ ${projectName} ${projectDescription} - - nexus - - ${artifactId}-pipeline-models ${artifactId}-pipelines diff --git a/pom.xml b/pom.xml index a4453d494..849ff734c 100644 --- a/pom.xml +++ b/pom.xml @@ -284,6 +284,7 @@ the habushu-managed updates to pyproject.toml files lags behind. To prevent the first committer after a release from needing to commit these files, we force the update here. --> org.technologybrewery.habushu:habushu-maven-plugin:initialize-habushu + release