From 11a5134c27f896ac5ec0b8a5b98eff1bc6355c1e Mon Sep 17 00:00:00 2001 From: Sanjay Vasandani Date: Mon, 15 May 2023 15:37:46 -0700 Subject: [PATCH] Update WFA actions to v2. This uses the new action to load the Bazel cache for workflows which use Bazel. --- .github/workflows/build-test.yml | 79 +++++++++++++++++++------ .github/workflows/create-cmm-images.yml | 24 ++++++-- .github/workflows/lint.yml | 20 +------ .github/workflows/run-k8s-tests.yml | 22 +++++++ 4 files changed, 106 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 9ce2660a7bb..d2f1ab89e80 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -27,13 +27,44 @@ jobs: name: Build and test runs-on: ubuntu-20.04 env: - CONTAINER_REGISTRY: "localhost:5001" - IMAGE_REPO_PREFIX: halo + HOST_PLATFORM: //build/platforms:ubuntu_20_04 CLUSTER_LOGS_PATH: cluster-logs steps: - name: Check out revision uses: actions/checkout@v3 + - name: Get Bazel cache params + id: get-cache-params + uses: world-federation-of-advertisers/actions/bazel-get-cache-params@v2 + with: + cache-version: 1 + bazel-options: |- + --compilation_mode=opt + + - name: Restore Bazel cache + uses: actions/cache/restore@v3 + with: + path: ${{ steps.get-cache-params.outputs.cache-path }} + key: ${{ steps.get-cache-params.outputs.cache-key }} + restore-keys: |- + ${{ steps.get-cache-params.outputs.restore-key }} + + - name: Build + run: > + bazelisk build //... + --compilation_mode=opt + --host_platform="$HOST_PLATFORM" + + - name: Run tests + id: run-tests + run: > + bazelisk test //... + --compilation_mode=opt + --host_platform="$HOST_PLATFORM" + + # Shut down Bazel server to reclaim memory. + - run: bazelisk shutdown + - name: Create cluster id: create-cluster run: | @@ -42,24 +73,24 @@ jobs: chmod +x kind-with-registry.sh ./kind-with-registry.sh - - name: Bazel build and test - uses: world-federation-of-advertisers/actions/bazel-build-test@v1 - with: - cache-version: 1 - workspace-path: . - target-patterns: | - //... - //src/test/kotlin/org/wfanet/measurement/integration/k8s:EmptyClusterCorrectnessTest - build-options: | - --compilation_mode=opt - --host_platform=//build/platforms:ubuntu_20_04 - --define=container_registry=${{ env.CONTAINER_REGISTRY }} - --define=image_repo_prefix=${{ env.IMAGE_REPO_PREFIX }} - --define=image_tag=latest + - name: Run correctness test + id: run-correctness-test + env: + CONTAINER_REGISTRY: "localhost:5001" + IMAGE_REPO_PREFIX: halo + run: > + bazelisk test --test_output=streamed + //src/test/kotlin/org/wfanet/measurement/integration/k8s:EmptyClusterCorrectnessTest + --compilation_mode=opt + --host_platform="$HOST_PLATFORM" + --define=container_registry="$CONTAINER_REGISTRY" + --define=image_repo_prefix="$IMAGE_REPO_PREFIX" + --define=image_tag=latest - name: Export cluster logs id: export-cluster-logs - if: failure() && steps.create-cluster.outcome == 'success' + if: failure() && steps.run-correctness-test.outcome == 'failure' + continue-on-error: true run: | mkdir -p "$CLUSTER_LOGS_PATH" kind export logs "$CLUSTER_LOGS_PATH" @@ -67,6 +98,20 @@ jobs: - name: Upload cluster logs artifact if: failure() && steps.export-cluster-logs.outcome == 'success' uses: actions/upload-artifact@v3 + continue-on-error: true with: name: cluster-logs path: ${{ env.CLUSTER_LOGS_PATH }} + + - name: Save Bazel cache + if: github.event_name == 'push' + uses: actions/cache/save@v3 + continue-on-error: true + with: + path: ${{ steps.get-cache-params.outputs.cache-path }} + key: ${{ steps.get-cache-params.outputs.cache-key }} + + - name: Upload Bazel testlogs + continue-on-error: true + uses: world-federation-of-advertisers/actions/bazel-upload-testlogs@v2 + if: failure() && (steps.run-tests.outcome == 'failure' || steps.run-correctness-test.outcome == 'failure') diff --git a/.github/workflows/create-cmm-images.yml b/.github/workflows/create-cmm-images.yml index 59c461bedf3..1d94c794ca0 100644 --- a/.github/workflows/create-cmm-images.yml +++ b/.github/workflows/create-cmm-images.yml @@ -22,14 +22,14 @@ on: description: "Tag of container images" value: ${{ jobs.push-images.outputs.image-tag }} -env: - CONTAINER_REGISTRY: ghcr.io - jobs: push-images: runs-on: ubuntu-20.04 outputs: image-tag: ${{ steps.output-image-tag.outputs.image-tag }} + env: + CONTAINER_REGISTRY: ghcr.io + HOST_PLATFORM: //build/platforms:ubuntu_20_04 steps: - uses: actions/checkout@v3 @@ -40,6 +40,22 @@ jobs: id: output-image-tag run: echo "image-tag=$IMAGE_TAG" >> $GITHUB_OUTPUT + - name: Get Bazel cache params + id: get-cache-params + uses: world-federation-of-advertisers/actions/bazel-get-cache-params@v2 + with: + cache-version: 1 + bazel-options: |- + --compilation_mode=opt + + - name: Restore Bazel cache + uses: actions/cache/restore@v3 + with: + path: ${{ steps.get-cache-params.outputs.cache-path }} + key: ${{ steps.get-cache-params.outputs.cache-key }} + restore-keys: |- + ${{ steps.get-cache-params.outputs.restore-key }} + - name: Authenticate to GitHub Container Registry uses: docker/login-action@v2 with: @@ -51,7 +67,7 @@ jobs: run: > bazelisk run //src/main/docker:push_all_gke_images --compilation_mode=opt - --platforms=//build/platforms:ubuntu_20_04 + --host_platform="$HOST_PLATFORM" --define "container_registry=$CONTAINER_REGISTRY" --define "image_repo_prefix=$GITHUB_REPOSITORY_OWNER" --define "image_tag=$IMAGE_TAG" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d119187040c..108b9b54629 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,28 +9,12 @@ jobs: lint: name: Lint changed files runs-on: ubuntu-20.04 - defaults: - run: - shell: bash steps: - - name: Check out revision - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - java-version: '11' - java-package: jdk - architecture: x64 - distribution: 'zulu' - - - name: Set up linters - uses: world-federation-of-advertisers/actions/setup-linters@v1 - - - name: Lint - uses: world-federation-of-advertisers/actions/lint@v1 + - uses: world-federation-of-advertisers/actions/lint@v2 with: license-type: apache copyright-holder: The Cross-Media Measurement Authors diff --git a/.github/workflows/run-k8s-tests.yml b/.github/workflows/run-k8s-tests.yml index c4381054828..30818f8e318 100644 --- a/.github/workflows/run-k8s-tests.yml +++ b/.github/workflows/run-k8s-tests.yml @@ -38,6 +38,22 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Get Bazel cache params + id: get-cache-params + uses: world-federation-of-advertisers/actions/bazel-get-cache-params@v2 + with: + cache-version: 1 + bazel-options: |- + --compilation_mode=opt + + - name: Restore Bazel cache + uses: actions/cache/restore@v3 + with: + path: ${{ steps.get-cache-params.outputs.cache-path }} + key: ${{ steps.get-cache-params.outputs.cache-key }} + restore-keys: |- + ${{ steps.get-cache-params.outputs.restore-key }} + # Authenticate to Google Cloud. This will export some environment # variables, including GCLOUD_PROJECT. - name: Authenticate to Google Cloud @@ -47,6 +63,7 @@ jobs: service_account: ${{ vars.SIMULATOR_SERVICE_ACCOUNT }} - name: Run tests + id: run-tests env: STORAGE_BUCKET: ${{ vars.STORAGE_BUCKET }} KINGDOM_PUBLIC_API_TARGET: ${{ vars.KINGDOM_PUBLIC_API_TARGET }} @@ -61,3 +78,8 @@ jobs: --define "simulator_storage_bucket=$STORAGE_BUCKET" --define "mc_name=$MC_NAME" --define "mc_api_key=$MC_API_KEY" + + - name: Upload Bazel testlogs + continue-on-error: true + uses: world-federation-of-advertisers/actions/bazel-upload-testlogs@v2 + if: failure() && steps.run-tests.outcome == 'failure'