diff --git a/.github/workflows/dockerhub_ci.yml b/.github/workflows/dockerhub_ci.yml index 042af99..a1faef8 100644 --- a/.github/workflows/dockerhub_ci.yml +++ b/.github/workflows/dockerhub_ci.yml @@ -21,92 +21,94 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Specify which tags to build - DOCKER_TAG: - - "3.5.3,3.5,latest" - - "3.4.4,3.4" - - "3.3.4,3.3" - - "3.1.3,3.1" - - "3.0.3,3.0" - - "2.4.8,2.4" + image: + - + SPARK_VERSION: "3.5.3" + OPENJDK_VERSION: "17" + HADOOP_VERSION: "3" + LATEST: "true" + - + SPARK_VERSION: "3.4.4" + OPENJDK_VERSION: "11" + HADOOP_VERSION: "3" + LATEST: "false" + - + SPARK_VERSION: "3.3.4" + OPENJDK_VERSION: "11" + HADOOP_VERSION: "3" + LATEST: "false" + - + SPARK_VERSION: "3.1.3" + OPENJDK_VERSION: "11" + HADOOP_VERSION: "3.2" + LATEST: "false" + - + SPARK_VERSION: "3.0.3" + OPENJDK_VERSION: "11" + HADOOP_VERSION: "3.2" + LATEST: "false" + - + SPARK_VERSION: "2.4.8" + OPENJDK_VERSION: "8" + HADOOP_VERSION: "2.7" + LATEST: "false" timeout-minutes: 30 steps: - name: Checkout code uses: actions/checkout@v4 - - name: Parse args - id: args - run: | - DOCKER_TAG="${{ matrix.DOCKER_TAG }}" - - # Manipulate DOCKER_TAG to create build args - SPARK_MAJOR_VERSION=${DOCKER_TAG:0:1} - SPARK_MAJOR_MINOR_VERSION=${DOCKER_TAG:0:3} - if [[ ${SPARK_MAJOR_MINOR_VERSION} = "3.5" || ${DOCKER_TAG} == "latest" ]]; then - OPENJDK_VERSION=17 - HADOOP_VERSION=3 - elif [[ ${SPARK_MAJOR_MINOR_VERSION} = "3.3" || ${SPARK_MAJOR_MINOR_VERSION} = "3.4" ]]; then - OPENJDK_VERSION=11 - HADOOP_VERSION=3 - elif [ ${SPARK_MAJOR_VERSION} = "3" ]; then - OPENJDK_VERSION=11 - HADOOP_VERSION=3.2 - else - OPENJDK_VERSION=8 - HADOOP_VERSION=2.7 - fi - - BUILD_ARGS="OPENJDK_VERSION=${OPENJDK_VERSION} - HADOOP_VERSION=${HADOOP_VERSION} - SPARK_VERSION=${DOCKER_TAG%%,*}" - - # No modification needed beyond this point - BUILD_ARGS="${BUILD_ARGS//'%'/'%25'}" - BUILD_ARGS="${BUILD_ARGS//$'\n'/'%0A'}" - BUILD_ARGS="${BUILD_ARGS//$'\r'/'%0D'}" - - echo "::set-output name=build_args::$BUILD_ARGS" - - - name: Prepare - id: prep - run: | - DOCKER_TAG="${{ matrix.DOCKER_TAG }}" - DOCKER_IMAGE_NAME="$DOCKER_REPO:${DOCKER_TAG%%,*}" - echo ::set-output name=image_name::${DOCKER_IMAGE_NAME} - - TAGS="$DOCKER_REPO:${DOCKER_TAG//,/,$DOCKER_REPO:}" - echo ::set-output name=tags::${TAGS} - - echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') - - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx + id: setup-buildx uses: docker/setup-buildx-action@v3 + - name: Generate image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: "${{ env.DOCKER_REPO }}" + flavor: | + latest=${{ matrix.image.LATEST }} + prefix= + suffix= + tags: | + type=semver,pattern={{version}},value=${{ matrix.image.SPARK_VERSION }} + type=semver,pattern={{major}}.{{minor}},value=${{ matrix.image.SPARK_VERSION }} + labels: | + org.opencontainers.image.source=${{ github.event.repository.clone_url }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.vendor=Xebia Data, https://xebia.com/ + org.opencontainers.image.description=An Apache Spark Docker image. + - name: Build image + id: docker-build uses: docker/build-push-action@v6 with: context: . file: ./Dockerfile push: false load: true - tags: ${{ steps.prep.outputs.tags }} + tags: ${{ steps.meta.outputs.tags }} build-args: | - ${{ steps.args.outputs.build_args }} - labels: | - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} + HADOOP_VERSION=${{ matrix.image.HADOOP_VERSION }} + OPENJDK_VERSION=${{ matrix.image.OPENJDK_VERSION }} + SPARK_VERSION=${{ matrix.image.SPARK_VERSION }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + # https://docs.docker.com/build/ci/github-actions/share-image-jobs/ + outputs: type=docker,dest=/tmp/gdd-spark-${{ matrix.image.SPARK_VERSION }}.tar - name: Test image env: - IMAGE_NAME: ${{ steps.prep.outputs.image_name }} + IMAGE_NAME: "godatadriven/spark:${{ matrix.image.SPARK_VERSION }}" run: | + docker load --input /tmp/gdd-spark-${{ matrix.image.SPARK_VERSION }}.tar + docker image ls -a if [[ -f "docker-compose.test.yml" ]]; then - docker compose --file docker-compose.test.yml build + docker compose --file docker-compose.test.yml build --builder ${{ steps.setup-buildx.outputs.name }} docker compose --file docker-compose.test.yml run sut fi @@ -127,13 +129,13 @@ jobs: push: true provenance: false sbom: false - tags: ${{ steps.prep.outputs.tags }} + tags: ${{ steps.meta.outputs.tags }} build-args: | - ${{ steps.args.outputs.build_args }} - labels: | - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} + HADOOP_VERSION=${{ matrix.image.HADOOP_VERSION }} + OPENJDK_VERSION=${{ matrix.image.OPENJDK_VERSION }} + SPARK_VERSION=${{ matrix.image.SPARK_VERSION }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} platforms: linux/amd64,linux/arm64 - name: Report Status