diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d7a849467..fdd92ca31 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,6 +21,18 @@ jobs: - platform: hifive_unmatched bits: 32 + # Output cache keys that were used so we can consolidate them later. Note + # that this is a matrix job, and job outputs for these are not well supported + # at all in Github Actions (https://github.com/orgs/community/discussions/26639). + # Essentially, the last job to set these output variables will win, which is + # not always great. In our case, though, this is actually fine since we don't + # necessarily need "precise" matching here -- any job's output should be good + # enough to serve as a future key into the cache. + outputs: + buildroot-dl-key: ${{ steps.restore-buildroot-dl.outputs.cache-primary-key }} + ccache-key: ${{ steps.restore-ccache.outputs.cache-primary-key }} + ymdh: ${{ steps.cache-keys.outputs.YMDH }} + steps: ########### @@ -54,26 +66,33 @@ jobs: # much always be using the max Github Action cache limit (10GB), but this is okay # since we really only care about keeping the latest cache anyways. - name: Restore buildroot packages - uses: actions/cache@v3 + id: restore-buildroot-dl + uses: actions/cache/restore@v3 with: - path: buildroot/dl - key: buildroot-dl-${{ matrix.platform }}${{ matrix.bits }}-${{ steps.cache-keys.outputs.YMDH }} + path: dl.tar + key: buildroot-dl-${{ steps.cache-keys.outputs.YMDH }} restore-keys: | - buildroot-dl-${{ matrix.platform }}${{ matrix.bits }}-${{ steps.cache-keys.outputs.YMD }} - buildroot-dl-${{ matrix.platform }}${{ matrix.bits }}-${{ steps.cache-keys.outputs.YM }} - buildroot-dl-${{ matrix.platform }}${{ matrix.bits }}-${{ steps.cache-keys.outputs.Y }} - buildroot-dl-${{ matrix.platform }}${{ matrix.bits }}- + buildroot-dl-${{ steps.cache-keys.outputs.YMD }} + buildroot-dl-${{ steps.cache-keys.outputs.YM }} + buildroot-dl-${{ steps.cache-keys.outputs.Y }} + buildroot-dl- - name: Restore ccache - uses: actions/cache@v3 + id: restore-ccache + uses: actions/cache/restore@v3 with: - path: ~/.buildroot-ccache - key: ccache-${{ matrix.platform }}${{ matrix.bits }}-${{ steps.cache-keys.outputs.YMDH }} + path: ccache.tar.xz + key: ccache-${{ steps.cache-keys.outputs.YMDH }} restore-keys: | - ccache-${{ matrix.platform }}${{ matrix.bits }}-${{ steps.cache-keys.outputs.YMD }} - ccache-${{ matrix.platform }}${{ matrix.bits }}-${{ steps.cache-keys.outputs.YM }} - ccache-${{ matrix.platform }}${{ matrix.bits }}-${{ steps.cache-keys.outputs.Y }} - ccache-${{ matrix.platform }}${{ matrix.bits }}- + ccache-${{ steps.cache-keys.outputs.YMD }} + ccache-${{ steps.cache-keys.outputs.YM }} + ccache-${{ steps.cache-keys.outputs.Y }} + ccache- + + - name: Decompress caches + run: | + if [[ -f dl.tar ]] ; then tar -xf dl.tar -C buildroot ; fi + if [[ -f ccache.tar.xz ]]; then tar -xf ccache.tar.xz -C ~ ; fi ############## ## Keystone ## @@ -108,12 +127,95 @@ jobs: tar -cf - $COMPRESSDIRS | xz -9 -T0 > build.tar.xz + - name: Compress cache directories + run: | + tar -C buildroot --exclude='**/git' -cf dl.tar dl/ + tar -C ~ -cf - .buildroot-ccache | xz -9 -T0 > ccache.tar.xz + - name: Upload build directory uses: actions/upload-artifact@v4 with: name: keystone-${{ matrix.platform }}${{ matrix.bits }}-builddir path: build.tar.xz retention-days: 1 + compression-level: 0 + + - name: Upload buildroot download directory + uses: actions/upload-artifact@v4 + with: + name: keystone-${{ matrix.platform }}${{ matrix.bits }}-buildroot-dl + path: dl.tar + retention-days: 1 + compression-level: 0 + + - name: Upload ccache directory + uses: actions/upload-artifact@v4 + with: + name: keystone-${{ matrix.platform }}${{ matrix.bits }}-ccache + path: ccache.tar.xz + retention-days: 1 + compression-level: 0 + +############### +## Utilities ## +############### + + # Combine cache directories to save space + combine-caches: + runs-on: ubuntu-latest + needs: build + steps: + - name: Install dependencies + run: | + sudo apt-get -y update && sudo apt-get -y install ccache + + # First, fetch the caches themselves + - name: Prepare output directories + run: | + mkdir -p buildroot/dl/ ~/.buildroot-ccache/ + + - name: Fetch buildroot caches + uses: actions/download-artifact@v4 + with: + pattern: keystone-*-buildroot-dl + + - name: Fetch ccache caches + uses: actions/download-artifact@v4 + with: + pattern: keystone-*-ccache + + # Then, combine the caches + - name: Merge caches + run: | + for d in keystone-*-buildroot-dl; do + tar --skip-old-files -xf "$d/dl.tar" -C buildroot + done + + RESULTDIR=""$HOME/.buildroot-ccache" + for d in keystone-*-ccache; do + TMPDIR=$(mktemp -d) + tar -xf "$d/ccache.tar.xz" -C "$TMPDIR" + ( cd "$TMPDIR/.buildroot-ccache" ; cp -a --parents ? "$RESULTDIR" ) + rm -rf "$TMPDIR" + done + ccache -d "$RESULTDIR" -c + + - name: Recompress caches + run: | + tar -C buildroot --exclude='**/git' -cf dl.tar dl/ + tar -C ~ -cf - .buildroot-ccache | xz -9 -T0 > ccache.tar.xz + + - name: Save buildroot download cache + uses: actions/cache/save@v3 + with: + path: dl.tar + key: ${{ needs.build.outputs.buildroot-dl-key }} + + - name: Save ccache + uses: actions/cache/save@v3 + with: + path: ccache.tar.xz + key: ${{ needs.build.outputs.ccache-key }} ########### ## Tests ## diff --git a/mkutils/plat/generic/run.mk b/mkutils/plat/generic/run.mk index 6b02e6f8f..45db1bb5e 100644 --- a/mkutils/plat/generic/run.mk +++ b/mkutils/plat/generic/run.mk @@ -34,6 +34,7 @@ call: $(call log,info,Calling command in QEMU) ssh -i $(BUILDROOT_BUILDDIR)/target/root/.ssh/id-rsa \ -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 \ -p $(QEMU_PORT) root@localhost $(KEYSTONE_COMMAND) 2>&1 | \ grep -v "Warning: Permanently added" | tee -a $(CALL_LOGFILE)