Reduces workflow footprint and updates pom for new cache key

XComp · Nov 13, 2023 · b13b722 · b13b722
1 parent 165d1c0
commit b13b722
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 272 deletions.
diff --git a/.github/workflows/flink-ci-template.yml b/.github/workflows/flink-ci-template.yml
@@ -135,274 +135,3 @@ jobs:
           name: build-artifacts-${{ steps.workflow-prep-step.outputs.stringified-workflow-name }}-${{ github.run_number }}
           path: ${{ runner.temp }}/${{ env.FLINK_ARTIFACT_FILENAME }}
           if-no-files-found: error
-
-  packaging:
-    name: "Test packaging/licensing"
-    needs: compile
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: "Flink Checkout"
-        uses: actions/checkout@v3
-        with:
-          persist-credentials: false
-
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "temurin"
-          java-version: ${{ inputs.jdk-version }}
-
-      - name: "Setup Maven package cache"
-        uses: actions/cache@v3
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: ${{ runner.os }}-maven-
-
-      - name: "Download build artifacts from compile job"
-        uses: actions/download-artifact@v3
-        with:
-          name: build-artifacts-${{ needs.compile.outputs.stringified-workflow-name }}-${{ github.run_number }}
-          path: ${{ runner.temp }}
-
-      - name: "Unpack build artifact"
-        run: |
-          ./tools/azure-pipelines/unpack_build_artifact.sh -f ${{ runner.temp }}/${{ env.FLINK_ARTIFACT_FILENAME }} -t .
-
-      - name: "Test"
-        run: |
-          ${{ inputs.environment }} ./tools/ci/compile_ci.sh || exit $?
-
-  test:
-    name: "Test (module: ${{ matrix.module }})"
-    needs: compile
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - module: core
-            stringified-module-name: core
-          - module: python
-            stringified-module-name: python
-          - module: table
-            stringified-module-name: table
-          - module: connect_1
-            stringified-module-name: connect_1
-          - module: connect_2
-            stringified-module-name: connect_2
-          - module: tests
-            stringified-module-name: tests
-          - module: misc
-            stringified-module-name: misc
-
-    steps:
-      - name: "Flink Checkout"
-        uses: actions/checkout@v3
-        with:
-          persist-credentials: false
-
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "temurin"
-          java-version: ${{ inputs.jdk-version }}
-
-      - name: "Setup Maven package cache"
-        uses: actions/cache@v3
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: ${{ runner.os }}-maven-
-
-      - name: "Set coredump pattern"
-        run: sudo sysctl -w kernel.core_pattern=core.%p
-
-      - name: "Download build artifacts from compile job"
-        uses: actions/download-artifact@v3
-        with:
-          name: build-artifacts-${{ needs.compile.outputs.stringified-workflow-name }}-${{ github.run_number }}
-          path: ${{ runner.temp }}
-
-      - name: "Unpack build artifact"
-        run: |
-          ./tools/azure-pipelines/unpack_build_artifact.sh -f ${{ runner.temp }}/${{ env.FLINK_ARTIFACT_FILENAME }} -t .
-
-      - name: "Try loading Docker images from Cache"
-        id: docker-cache
-        uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/${{ env.DOCKER_IMAGES_CACHE_FOLDER }}
-          key: ${{ matrix.module }}-docker-${{ runner.os }}-${{ hashFiles('**/cache_docker_images.sh', '**/flink-test-utils-parent/**/DockerImageVersions.java') }}
-          restore-keys: ${{ matrix.module }}-docker-${{ runner.os }}
-
-      - name: "Load Docker images if not present in cache, yet"
-        if: ${{ !cancelled() && !steps.docker-cache.cache.hit }}
-        run: ./tools/azure-pipelines/cache_docker_images.sh -f ${{ github.workspace }}/${{ env.DOCKER_IMAGES_CACHE_FOLDER }} load
-
-      - name: "Test - ${{ matrix.module }}"
-        id: test-run
-        env:
-          IT_CASE_S3_BUCKET: ${{ secrets.s3_bucket }}
-          IT_CASE_S3_ACCESS_KEY: ${{ secrets.s3_access_key }}
-          IT_CASE_S3_SECRET_KEY: ${{ secrets.s3_secret_key }}
-          IT_CASE_GLUE_SCHEMA_ACCESS_KEY: ${{ secrets.glue_schema_access_key }}
-          IT_CASE_GLUE_SCHEMA_SECRET_KEY: ${{ secrets.glue_schema_secret_key }}
-        # timeout-minutes: 240
-        run: |
-          ${{ inputs.environment }} ./tools/azure-pipelines/uploading_watchdog.sh \
-              -a ${{ github.job }} \
-              -d ${{ github.workspace }} \
-              -t 480 \
-              ./tools/ci/test_controller.sh ${{ matrix.module }}
-
-      - name: "Post-process build artifacts"
-        run: find ${{ steps.test-run.outputs.debug-files-output-dir }} -type f -exec rename 's/[:<>|*?]/-/' {} \;
-
-      - name: "Upload build artifacts"
-        uses: actions/upload-artifact@v3
-        if: ${{ failure() && steps.test-run.outputs.debug-files-output-dir }} != ''
-        with:
-          name: logs-test-${{ needs.compile.outputs.stringified-workflow-name }}-${{ github.run_number }}-${{ matrix.stringified-module-name }}-${{ steps.test-run.outputs.debug-files-name }}
-          path: ${{ steps.test-run.outputs.debug-files-output-dir }}
-
-      - name: "Save Docker images to cache"
-        if: ${{ !cancelled() && (failure() || !steps.docker-cache.cache.hit) }}
-        run: ./tools/azure-pipelines/cache_docker_images.sh -f ${{ github.workspace }}/${{ env.DOCKER_IMAGES_CACHE_FOLDER }} save
-
-  e2e-prereq-check:
-    name: "Check: Code modified"
-    needs: compile
-    runs-on: ubuntu-latest
-    outputs:
-      skip-e2e: ${{ steps.docs-only-pr-check.skip-e2e }}
-    steps:
-      - name: "Flink Checkout"
-        uses: actions/checkout@v3
-        with:
-          persist-credentials: false
-
-      # Skip e2e test execution if this is a documentation only pull request (master / release builds will still be checked regularly)
-      - name: "Check if it's a docs-only PR (i.e. e2e tests can be skipped)"
-        id: docs-only-pr-check
-        run: |
-          source ./tools/azure-pipelines/build_properties.sh
-          if is_docs_only_pullrequest; then
-            echo "This is a documentation-only change. Skipping e2e execution."
-            echo "skip-e2e=true" >> $GITHUB_OUTPUT
-          else
-            echo "This is a regular CI build. Continuing ..."
-          fi
-        shell: bash
-
-  e2e:
-    name: "E2E (group ${{ matrix.group }})"
-    needs: [compile, e2e-prereq-check]
-    runs-on: ubuntu-latest
-    if: ${{ needs.e2e-prereq-check.outputs.skip-e2e != 'true' }}
-    # timeout-minutes: 310
-    env:
-      E2E_CACHE_FOLDER: ${{ github.workspace }}/.e2e-cache
-      E2E_TARBALL_CACHE: ${{ github.workspace }}/.e2e-tarbal-cache
-    strategy:
-      fail-fast: false
-      matrix:
-        group: [1, 2]
-
-    steps:
-      - name: "Flink Checkout"
-        uses: actions/checkout@v3
-        with:
-          persist-credentials: false
-
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "temurin"
-          java-version: ${{ inputs.jdk-version }}
-
-      - name: "Setup Maven package cache"
-        uses: actions/cache@v3
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: ${{ runner.os }}-maven-
-
-      - name: "Setup Maven package cache"
-        uses: actions/cache@v3
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: ${{ runner.os }}-maven-
-
-      - name: "Install missing packages"
-        run: sudo apt-get install -y net-tools docker-compose
-
-      - name: "Download build artifacts from compile job"
-        uses: actions/download-artifact@v3
-        with:
-          name: build-artifacts-${{ needs.compile.outputs.stringified-workflow-name }}-${{ github.run_number }}
-          path: ${{ runner.temp }}
-
-      - name: "Unpack build artifact"
-        run: ./tools/azure-pipelines/unpack_build_artifact.sh -f ${{ runner.temp }}/${{ env.FLINK_ARTIFACT_FILENAME }} -t ${{ github.workspace }}
-
-      # the cache task does not create directories on a cache miss, and can later fail when trying to tar the directory if the test haven't created it
-      # this may for example happen if a given directory is only used by a subset of tests, which are run in a different 'group'
-      - name: "Create cache directories"
-        run: |
-          mkdir -p ${{ env.E2E_CACHE_FOLDER }}
-          mkdir -p ${{ env.E2E_TARBALL_CACHE }}
-          mkdir -p ${{ github.workspace}}/${{ env.DOCKER_IMAGES_CACHE_FOLDER }}
-
-      - name: "Load E2E files from Cache"
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.E2E_CACHE_FOLDER }}
-          key: e2e-cache-${{ matrix.group }}-${{ hashFiles('**/flink-end-to-end-tests/**/*.java', '!**/avro/**') }}
-
-      - name: "Load E2E artifacts from Cache"
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.E2E_TARBALL_CACHE }}
-          key: e2e-artifact-cache-${{ matrix.group }}-${{ hashFiles('**/flink-end-to-end-tests/**/*.sh') }}
-          restore-keys: e2e-artifact-cache-${{ matrix.group }}
-
-      - name: "Try loading Docker images from Cache"
-        id: docker-cache
-        uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/${{ env.DOCKER_IMAGES_CACHE_FOLDER }}
-          key: e2e-${{ matrix.group }}-docker-${{ runner.os }}-${{ hashFiles('**/cache_docker_images.sh', '**/flink-test-utils-parent/**/DockerImageVersions.java') }}
-
-      - name: "Load Docker images if not present in Cache, yet"
-        if: ${{ !cancelled() && !steps.docker-cache.cache.hit }}
-        run: ./tools/azure-pipelines/cache_docker_images.sh -f ${{ github.workspace}}/${{ env.DOCKER_IMAGES_CACHE_FOLDER }} load
-
-      - name: "Build Flink"
-        run: ${{ inputs.environment }} ./tools/ci/validate.sh "install -DskipTests -Dfast $PROFILE -Pskip-webui-build"
-
-      - name: "Run E2E Tests"
-        id: test-run
-        env:
-          IT_CASE_S3_BUCKET: ${{ secrets.s3_bucket }}
-          IT_CASE_S3_ACCESS_KEY: ${{ secrets.s3_access_key }}
-          IT_CASE_S3_SECRET_KEY: ${{ secrets.s3_secret_key }}
-          IT_CASE_GLUE_SCHEMA_ACCESS_KEY: ${{ secrets.glue_schema_access_key }}
-          IT_CASE_GLUE_SCHEMA_SECRET_KEY: ${{ secrets.glue_schema_secret_key }}
-        # timeout-minutes: 310
-        run: |
-          ${{ inputs.environment }} FLINK_DIR=`pwd`/build-target ./tools/azure-pipelines/uploading_watchdog.sh \
-            -a ${{ github.job }} \
-            -d ${{ github.workspace }} \
-            -t 310 \
-            flink-end-to-end-tests/run-nightly-tests.sh ${{ matrix.group }}
-
-      - name: "Upload Logs"
-        uses: actions/upload-artifact@v3
-        if: ${{ failure() && steps.test-run.outputs.debug-files-output-dir }} != ''
-        with:
-          name: logs-e2e-${{ needs.compile.outputs.stringified-workflow-name }}-${{ github.run_number }}-${{ matrix.group }}-${{ steps.test-run.outputs.debug-files-name }}
-          path: ${{ steps.test-run.outputs.debug-files-output-dir }}
-
-      - name: "Save Docker images to Cache"
-        if: ${{ !cancelled() && (failure() || !steps.docker-cache.cache.hit) }}
-        run: ./tools/azure-pipelines/cache_docker_images.sh -f ${{ github.workspace }}/${{ env.DOCKER_IMAGES_CACHE_FOLDER }} save
diff --git a/pom.xml b/pom.xml
@@ -62,6 +62,7 @@ under the License.
 		</repository>
 	</repositories>
 
+	<!-- random comment to trigger new cache key -->
 	<modules>
 		<module>flink-annotations</module>
 		<module>flink-architecture-tests</module>

diff --git a/tools/ci/maven-utils.sh b/tools/ci/maven-utils.sh
@@ -68,7 +68,7 @@ export MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR="$MAVEN_ARGS "
 # see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html
 MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-Dmaven.wagon.http.pool=false "
 # logging 
-MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn "
+MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS "
 # suppress snapshot updates
 MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="--no-snapshot-updates "
 # enable non-interactive batch mode