Merge pull request #35421 from gsmet/ci-cache-handling

Adjust how the m2 cache is handled on CI
quarkusio · Aug 20, 2023 · ef9284f · ef9284f
2 parents f708308 + c378870
commit ef9284f
Showing 1 changed file with 97 additions and 48 deletions.
diff --git a/.github/workflows/ci-actions-incremental.yml b/.github/workflows/ci-actions-incremental.yml
@@ -100,6 +100,7 @@ jobs:
     outputs:
       gib_args: ${{ steps.get-gib-args.outputs.gib_args }}
       gib_impacted: ${{ steps.get-gib-impacted.outputs.impacted_modules }}
+      m2-cache-key: ${{ steps.m2-cache-key.outputs.key }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -114,18 +115,18 @@ jobs:
         with:
           distribution: temurin
           java-version: 11
-      - name: Get Date
-        id: get-date
+      - name: Generate .m2 cache key
+        id: m2-cache-key
         run: |
-          echo "date=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT
+          echo "key=m2-cache-$(/bin/date -u "+%Y-%U")" >> $GITHUB_OUTPUT
         shell: bash
       - name: Cache Maven Repository
         id: cache-maven
         uses: actions/cache@v3
         with:
           path: ~/.m2/repository
-          # refresh cache every month to avoid unlimited growth
-          key: q2maven-${{ steps.get-date.outputs.date }}
+          # refresh cache every week to avoid unlimited growth
+          key: ${{ steps.m2-cache-key.outputs.key }}
       - name: Build
         run: |
           ./mvnw -T1C $COMMON_MAVEN_ARGS -DskipTests -DskipITs -DskipDocs -Dinvoker.skip -Dno-format -Dtcks -Prelocations clean install
@@ -172,18 +173,18 @@ jobs:
           echo 'impacted_modules<<EOF' >> $GITHUB_OUTPUT
           echo "${GIB_IMPACTED}" >> $GITHUB_OUTPUT
           echo 'EOF' >> $GITHUB_OUTPUT
-      - name: Tar Maven Repo
+      - name: Tar .m2/repository/io/quarkus
         shell: bash
-        run: tar -I 'pigz -9' -cf maven-repo.tgz -C ~ .m2/repository
-      - name: Persist Maven Repo
+        run: tar -I 'pigz -9' -cf m2-io-quarkus.tgz -C ~ .m2/repository/io/quarkus
+      - name: Upload .m2/repository/io/quarkus
         uses: actions/upload-artifact@v3
         with:
-          name: maven-repo
-          path: maven-repo.tgz
+          name: m2-io-quarkus
+          path: m2-io-quarkus.tgz
           retention-days: 7
-      - name: Delete Local Artifacts From Cache
+      - name: Delete snapshots artifacts from cache
         shell: bash
-        run: rm -r ~/.m2/repository/io/quarkus
+        run: find ~/.m2 -name \*-SNAPSHOT -type d -exec rm -rf {} +
       - name: Upload build reports (if build failed)
         uses: actions/upload-artifact@v3
         if: ${{ failure() || cancelled() }}
@@ -306,14 +307,20 @@ jobs:
           java-version: ${{ matrix.java.java-version }}
           architecture: ${{ matrix.java.architecture || 'x64' }}
 
-      - name: Download Maven Repo
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v3
+        with:
+          path: ~/.m2/repository
+          # refresh cache every week to avoid unlimited growth
+          key: ${{ needs.build-jdk11.outputs.m2-cache-key }}
+      - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v3
         with:
-          name: maven-repo
+          name: m2-io-quarkus
           path: .
-      - name: Extract Maven Repo
+      - name: Extract .m2/repository/io/quarkus
         shell: bash
-        run: tar -xzf maven-repo.tgz -C ~
+        run: tar -xzf m2-io-quarkus.tgz -C ~
       - name: Build
         shell: bash
         # Despite the pre-calculated run_jvm flag, GIB has to be re-run here to figure out the exact submodules to build.
@@ -363,7 +370,7 @@ jobs:
   maven-tests:
     name: Maven Tests - JDK ${{matrix.java.name}}
     runs-on: ${{ matrix.java.os-name }}
-    needs: calculate-test-jobs
+    needs: [build-jdk11, calculate-test-jobs]
     env:
       MAVEN_OPTS: -Xmx2g -XX:MaxMetaspaceSize=1g
     # Skip main in forks
@@ -388,14 +395,20 @@ jobs:
         if: "startsWith(matrix.java.os-name, 'windows')"
         run: git config --global core.longpaths true
       - uses: actions/checkout@v3
-      - name: Download Maven Repo
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v3
+        with:
+          path: ~/.m2/repository
+          # refresh cache every week to avoid unlimited growth
+          key: ${{ needs.build-jdk11.outputs.m2-cache-key }}
+      - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v3
         with:
-          name: maven-repo
+          name: m2-io-quarkus
           path: .
-      - name: Extract Maven Repo
+      - name: Extract .m2/repository/io/quarkus
         shell: bash
-        run: tar -xzf maven-repo.tgz -C ~
+        run: tar -xzf m2-io-quarkus.tgz -C ~
       - name: Set up JDK ${{ matrix.java.java-version }}
         uses: actions/setup-java@v3
         with:
@@ -437,7 +450,7 @@ jobs:
   gradle-tests:
     name: Gradle Tests - JDK ${{matrix.java.name}}
     runs-on: ${{matrix.java.os-name}}
-    needs: calculate-test-jobs
+    needs: [build-jdk11, calculate-test-jobs]
     env:
       # leave more space for the actual gradle execution (which is just wrapped by maven)
       MAVEN_OPTS: -Xmx1g
@@ -463,14 +476,20 @@ jobs:
         if: "startsWith(matrix.java.os-name, 'windows')"
         run: git config --global core.longpaths true
       - uses: actions/checkout@v3
-      - name: Download Maven Repo
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v3
+        with:
+          path: ~/.m2/repository
+          # refresh cache every week to avoid unlimited growth
+          key: ${{ needs.build-jdk11.outputs.m2-cache-key }}
+      - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v3
         with:
-          name: maven-repo
+          name: m2-io-quarkus
           path: .
-      - name: Extract Maven Repo
+      - name: Extract .m2/repository/io/quarkus
         shell: bash
-        run: tar -xzf maven-repo.tgz -C ~
+        run: tar -xzf m2-io-quarkus.tgz -C ~
       - name: Set up JDK ${{ matrix.java.java-version }}
         uses: actions/setup-java@v3
         with:
@@ -501,7 +520,7 @@ jobs:
   devtools-tests:
     name: Devtools Tests - JDK ${{matrix.java.name}}
     runs-on: ${{matrix.java.os-name}}
-    needs: calculate-test-jobs
+    needs: [build-jdk11, calculate-test-jobs]
     # Skip main in forks
     if: "needs.calculate-test-jobs.outputs.run_devtools == 'true' && (github.repository == 'quarkusio/quarkus' || !endsWith(github.ref, '/main'))"
     timeout-minutes: 60
@@ -529,14 +548,20 @@ jobs:
         if: "startsWith(matrix.java.os-name, 'windows')"
         run: git config --global core.longpaths true
       - uses: actions/checkout@v3
-      - name: Download Maven Repo
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v3
+        with:
+          path: ~/.m2/repository
+          # refresh cache every week to avoid unlimited growth
+          key: ${{ needs.build-jdk11.outputs.m2-cache-key }}
+      - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v3
         with:
-          name: maven-repo
+          name: m2-io-quarkus
           path: .
-      - name: Extract Maven Repo
+      - name: Extract .m2/repository/io/quarkus
         shell: bash
-        run: tar -xzf maven-repo.tgz -C ~
+        run: tar -xzf m2-io-quarkus.tgz -C ~
       - name: Set up JDK ${{ matrix.java.java-version }}
         uses: actions/setup-java@v3
         with:
@@ -585,14 +610,20 @@ jobs:
           }
     steps:
       - uses: actions/checkout@v3
-      - name: Download Maven Repo
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v3
+        with:
+          path: ~/.m2/repository
+          # refresh cache every week to avoid unlimited growth
+          key: ${{ needs.build-jdk11.outputs.m2-cache-key }}
+      - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v3
         with:
-          name: maven-repo
+          name: m2-io-quarkus
           path: .
-      - name: Extract Maven Repo
+      - name: Extract .m2/repository/io/quarkus
         shell: bash
-        run: tar -xzf maven-repo.tgz -C ~
+        run: tar -xzf m2-io-quarkus.tgz -C ~
       - name: Set up JDK ${{ matrix.java.java-version }}
         uses: actions/setup-java@v3
         with:
@@ -626,14 +657,20 @@ jobs:
       matrix: ${{ fromJson(needs.calculate-test-jobs.outputs.virtual_threads_matrix) }}
     steps:
       - uses: actions/checkout@v3
-      - name: Download Maven Repo
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v3
+        with:
+          path: ~/.m2/repository
+          # refresh cache every week to avoid unlimited growth
+          key: ${{ needs.build-jdk11.outputs.m2-cache-key }}
+      - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v3
         with:
-          name: maven-repo
+          name: m2-io-quarkus
           path: .
-      - name: Extract Maven Repo
+      - name: Extract .m2/repository/io/quarkus
         shell: bash
-        run: tar -xzf maven-repo.tgz -C ~
+        run: tar -xzf m2-io-quarkus.tgz -C ~
       - name: Set up JDK 20
         uses: actions/setup-java@v3
         with:
@@ -682,14 +719,20 @@ jobs:
         with:
           distribution: temurin
           java-version: 11
-      - name: Download Maven Repo
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v3
+        with:
+          path: ~/.m2/repository
+          # refresh cache every week to avoid unlimited growth
+          key: ${{ needs.build-jdk11.outputs.m2-cache-key }}
+      - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v3
         with:
-          name: maven-repo
+          name: m2-io-quarkus
           path: .
-      - name: Extract Maven Repo
+      - name: Extract .m2/repository/io/quarkus
         shell: bash
-        run: tar -xzf maven-repo.tgz -C ~
+        run: tar -xzf m2-io-quarkus.tgz -C ~
       - name: Verify
         # Important: keep -pl ... in sync with "Calculate run flags"!
         # Despite the pre-calculated run_tcks flag, GIB has to be re-run here to figure out the exact tcks submodules to build.
@@ -722,7 +765,7 @@ jobs:
 
   native-tests:
     name: Native Tests - ${{matrix.category}}
-    needs: calculate-test-jobs
+    needs: [build-jdk11, calculate-test-jobs]
     runs-on: ${{matrix.os-name}}
     env:
       # leave more space for the actual native compilation and execution
@@ -766,14 +809,20 @@ jobs:
         shell: bash
         run: |
           cat <<< $(jq '.HttpHeaders += {"User-Agent": "Quarkus-CI-Docker-Client"}' ~/.docker/config.json) > ~/.docker/config.json
-      - name: Download Maven Repo
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v3
+        with:
+          path: ~/.m2/repository
+          # refresh cache every week to avoid unlimited growth
+          key: ${{ needs.build-jdk11.outputs.m2-cache-key }}
+      - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v3
         with:
-          name: maven-repo
+          name: m2-io-quarkus
           path: .
-      - name: Extract Maven Repo
+      - name: Extract .m2/repository/io/quarkus
         shell: bash
-        run: tar -xzf maven-repo.tgz -C ~
+        run: tar -xzf m2-io-quarkus.tgz -C ~
       - name: Build
         shell: bash
         env: