From b3756d8312d3c84014960e9288bf8e3913133271 Mon Sep 17 00:00:00 2001 From: Guillaume Smet Date: Mon, 19 Aug 2024 17:09:05 +0200 Subject: [PATCH] Try a new approach for caching Maven local repository We used to cache a repo once a week but only after building the Java 17 build. The new approach is the following: - We use dependency:go-offline to populate the cache, hoping the quality will be better - We drop the cache entirely every month - We have one cache entry per branch and we keep updating it during the month by creating new caches - We use restore-keys to find the best cache possible for a given run - We share this cache with all the workflows --- .github/workflows/ci-actions-incremental.yml | 75 +++++++++++++------ .github/workflows/deploy-snapshots.yml | 23 +++--- .github/workflows/doc-build.yml | 22 ++++-- .../workflows/native-it-selected-graalvm.yml | 18 +++-- .github/workflows/owasp-check.yml | 17 +++++ .github/workflows/podman-build.yml | 20 +++-- .github/workflows/sonarcloud.yml | 25 ++++--- 7 files changed, 141 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci-actions-incremental.yml b/.github/workflows/ci-actions-incremental.yml index dc91ffe82d1b2..276b4531a83a9 100644 --- a/.github/workflows/ci-actions-incremental.yml +++ b/.github/workflows/ci-actions-incremental.yml @@ -105,7 +105,8 @@ jobs: outputs: gib_args: ${{ steps.get-gib-args.outputs.gib_args }} gib_impacted: ${{ steps.get-gib-impacted.outputs.impacted_modules }} - m2-cache-key: ${{ steps.cache-key.outputs.m2-cache-key }} + m2-monthly-branch-cache-key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }} + m2-monthly-cache-key: ${{ steps.cache-key.outputs.m2-monthly-cache-key }} quarkus-metadata-cache-key: ${{ steps.cache-key.outputs.quarkus-metadata-cache-key }} quarkus-metadata-cache-key-default: ${{ steps.cache-key.outputs.quarkus-metadata-cache-key-default }} steps: @@ -129,8 +130,12 @@ jobs: - name: Generate cache key id: cache-key run: | + CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}" + CURRENT_MONTH=$(/bin/date -u "+%Y-%m") + ROOT_CACHE_KEY="m2-cache" + echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT + echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT CURRENT_WEEK=$(/bin/date -u "+%Y-%U") - echo "m2-cache-key=m2-cache-${CURRENT_WEEK}" >> $GITHUB_OUTPUT echo "quarkus-metadata-cache-key=quarkus-metadata-cache-${CURRENT_WEEK}-${{ github.ref_name }}" >> $GITHUB_OUTPUT echo "quarkus-metadata-cache-key-default=quarkus-metadata-cache-${CURRENT_WEEK}-${{ github.event.repository.default_branch }}" >> $GITHUB_OUTPUT - name: Cache Maven Repository @@ -138,8 +143,16 @@ jobs: uses: actions/cache@v4 with: path: ~/.m2/repository - # refresh cache every week to avoid unlimited growth - key: ${{ steps.cache-key.outputs.m2-cache-key }} + # This will never work but it should fallback to the restore key below and get the latest cache for this branch. + # The cache is dropped monthly to prevent unlimited growth. + # The cache is per branch. + key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}- + ${{ steps.cache-key.outputs.m2-monthly-cache-key }}- + - name: Populate the cache + run: | + ./mvnw -T2C $COMMON_MAVEN_ARGS dependency:go-offline - name: Verify native-tests.json run: ./.github/verify-tests-json.sh native-tests.json integration-tests/ - name: Verify virtual-threads-tests.json @@ -362,8 +375,10 @@ jobs: uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every week to avoid unlimited growth - key: ${{ needs.build-jdk17.outputs.m2-cache-key }} + key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}- + ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}- - name: Download .m2/repository/io/quarkus uses: actions/download-artifact@v4 with: @@ -467,8 +482,10 @@ jobs: uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every week to avoid unlimited growth - key: ${{ needs.build-jdk17.outputs.m2-cache-key }} + key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}- + ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}- - name: Download .m2/repository/io/quarkus uses: actions/download-artifact@v4 with: @@ -567,8 +584,10 @@ jobs: uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every week to avoid unlimited growth - key: ${{ needs.build-jdk17.outputs.m2-cache-key }} + key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}- + ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}- - name: Download .m2/repository/io/quarkus uses: actions/download-artifact@v4 with: @@ -657,8 +676,10 @@ jobs: uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every week to avoid unlimited growth - key: ${{ needs.build-jdk17.outputs.m2-cache-key }} + key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}- + ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}- - name: Download .m2/repository/io/quarkus uses: actions/download-artifact@v4 with: @@ -751,8 +772,10 @@ jobs: uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every week to avoid unlimited growth - key: ${{ needs.build-jdk17.outputs.m2-cache-key }} + key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}- + ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}- - name: Download .m2/repository/io/quarkus uses: actions/download-artifact@v4 with: @@ -832,8 +855,10 @@ jobs: uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every week to avoid unlimited growth - key: ${{ needs.build-jdk17.outputs.m2-cache-key }} + key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}- + ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}- - name: Download .m2/repository/io/quarkus uses: actions/download-artifact@v4 with: @@ -914,8 +939,10 @@ jobs: uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every week to avoid unlimited growth - key: ${{ needs.build-jdk17.outputs.m2-cache-key }} + key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}- + ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}- - name: Download .m2/repository/io/quarkus uses: actions/download-artifact@v4 with: @@ -996,8 +1023,10 @@ jobs: uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every week to avoid unlimited growth - key: ${{ needs.build-jdk17.outputs.m2-cache-key }} + key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}- + ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}- - name: Download .m2/repository/io/quarkus uses: actions/download-artifact@v4 with: @@ -1107,8 +1136,10 @@ jobs: uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every week to avoid unlimited growth - key: ${{ needs.build-jdk17.outputs.m2-cache-key }} + key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}- + ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}- - name: Download .m2/repository/io/quarkus uses: actions/download-artifact@v4 with: diff --git a/.github/workflows/deploy-snapshots.yml b/.github/workflows/deploy-snapshots.yml index 796059f6aca48..f17ed78f505d9 100644 --- a/.github/workflows/deploy-snapshots.yml +++ b/.github/workflows/deploy-snapshots.yml @@ -25,18 +25,23 @@ jobs: with: distribution: temurin java-version: 17 - - name: Get Date - id: get-date + - name: Generate cache key + id: cache-key run: | - echo "date=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT - shell: bash - - name: Cache Maven Repository - id: cache-maven - uses: actions/cache@v4 + CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}" + CURRENT_MONTH=$(/bin/date -u "+%Y-%m") + ROOT_CACHE_KEY="m2-cache" + echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT + echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT + - name: Restore Maven Repository + uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every month to avoid unlimited growth - key: deploy-snapshots-${{ steps.get-date.outputs.date }} + # This will always fail but we will default to the best matching restore keys + key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}- + ${{ steps.cache-key.outputs.m2-monthly-cache-key }}- - name: Build and Deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_API_TOKEN }} diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 363c5ccf4adea..b5383fb3fbbfd 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -58,17 +58,23 @@ jobs: with: distribution: temurin java-version: 17 - - name: Get Date - id: get-date + - name: Generate cache key + id: cache-key run: | - echo "date=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT - - name: Cache Maven Repository - id: cache-maven - uses: actions/cache@v4 + CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}" + CURRENT_MONTH=$(/bin/date -u "+%Y-%m") + ROOT_CACHE_KEY="m2-cache" + echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT + echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT + - name: Restore Maven Repository + uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every month to avoid unlimited growth - key: q2maven-doc-${{ steps.get-date.outputs.date }} + # This will always fail but we will default to the best matching restore keys + key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}- + ${{ steps.cache-key.outputs.m2-monthly-cache-key }}- - name: Build run: | ./mvnw -DquicklyDocs -B --settings .github/mvn-settings.xml diff --git a/.github/workflows/native-it-selected-graalvm.yml b/.github/workflows/native-it-selected-graalvm.yml index fa732d6acaa5b..89fc7b6d53daf 100644 --- a/.github/workflows/native-it-selected-graalvm.yml +++ b/.github/workflows/native-it-selected-graalvm.yml @@ -77,17 +77,23 @@ jobs: - name: Generate cache key id: cache-key run: | + CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}" + CURRENT_MONTH=$(/bin/date -u "+%Y-%m") + ROOT_CACHE_KEY="m2-cache" + echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT + echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT CURRENT_WEEK=$(/bin/date -u "+%Y-%U") - echo "m2-cache-key=m2-cache-${CURRENT_WEEK}" >> $GITHUB_OUTPUT echo "quarkus-metadata-cache-key=quarkus-metadata-cache-${CURRENT_WEEK}-${{ github.ref_name }}" >> $GITHUB_OUTPUT echo "quarkus-metadata-cache-key-default=quarkus-metadata-cache-${CURRENT_WEEK}-${{ github.event.repository.default_branch }}" >> $GITHUB_OUTPUT - - name: Cache Maven Repository - id: cache-maven - uses: actions/cache@v4 + - name: Restore Maven Repository + uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every week to avoid unlimited growth - key: ${{ steps.cache-key.outputs.m2-cache-key }} + # This will always fail but we will default to the best matching restore keys + key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}- + ${{ steps.cache-key.outputs.m2-monthly-cache-key }}- - name: Verify native-tests.json run: ./.github/verify-tests-json.sh native-tests.json integration-tests/ - name: Verify virtual-threads-tests.json diff --git a/.github/workflows/owasp-check.yml b/.github/workflows/owasp-check.yml index 87af61bffc1d4..eb3f640ce02fc 100644 --- a/.github/workflows/owasp-check.yml +++ b/.github/workflows/owasp-check.yml @@ -26,6 +26,23 @@ jobs: distribution: temurin java-version: 17 + - name: Generate cache key + id: cache-key + run: | + CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}" + CURRENT_MONTH=$(/bin/date -u "+%Y-%m") + ROOT_CACHE_KEY="m2-cache" + echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT + echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT + - name: Restore Maven Repository + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + # This will always fail but we will default to the best matching restore keys + key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}- + ${{ steps.cache-key.outputs.m2-monthly-cache-key }}- - name: Build Java run: ./mvnw -B --settings .github/mvn-settings.xml -Dquickly-ci install diff --git a/.github/workflows/podman-build.yml b/.github/workflows/podman-build.yml index 3ff2bfc27ab54..59fdbe2e21dd2 100644 --- a/.github/workflows/podman-build.yml +++ b/.github/workflows/podman-build.yml @@ -81,13 +81,23 @@ jobs: - name: Set up testcontainers for podman run: | echo ryuk.container.privileged=true > ~/.testcontainers.properties - - name: Cache Maven Repository - id: cache-maven - uses: actions/cache@v4 + - name: Generate cache key + id: cache-key + run: | + CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}" + CURRENT_MONTH=$(/bin/date -u "+%Y-%m") + ROOT_CACHE_KEY="m2-cache" + echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT + echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT + - name: Restore Maven Repository + uses: actions/cache/restore@v4 with: path: ~/.m2/repository - # refresh cache every month to avoid unlimited growth - key: q2maven-${{ steps.get-date.outputs.date }} + # This will always fail but we will default to the best matching restore keys + key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}- + ${{ steps.cache-key.outputs.m2-monthly-cache-key }}- - name: Initial build run: | ./mvnw -T1C $COMMON_MAVEN_ARGS -DskipTests -DskipITs -DskipDocs -Dinvoker.skip -Dskip.gradle.tests -Djbang.skip -Dtruststore.skip -Dno-format -Dtcks -Prelocations clean install diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 617ca03f97061..8d420f4546344 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -16,21 +16,28 @@ jobs: with: distribution: temurin java-version: 17 - - name: Get Date - id: get-date + - name: Generate cache key + id: cache-key run: | - echo "date=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT - shell: bash + CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}" + CURRENT_MONTH=$(/bin/date -u "+%Y-%m") + ROOT_CACHE_KEY="m2-cache" + echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT + echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT + - name: Restore Maven Repository + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + # This will always fail but we will default to the best matching restore keys + key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }} + restore-keys: | + ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}- + ${{ steps.cache-key.outputs.m2-monthly-cache-key }}- - name: Cache SonarCloud packages uses: actions/cache@v4 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar-${{ steps.get-date.outputs.date }} - - name: Cache Maven packages - uses: actions/cache@v4 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ steps.get-date.outputs.date }} - name: Build and analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}