diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 9ce2660a7bb..860a3eb458c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -27,13 +27,45 @@ jobs: name: Build and test runs-on: ubuntu-20.04 env: - CONTAINER_REGISTRY: "localhost:5001" - IMAGE_REPO_PREFIX: halo CLUSTER_LOGS_PATH: cluster-logs steps: - name: Check out revision uses: actions/checkout@v3 + - name: Write ~/.bazelrc + run: | + cat << EOF > ~/.bazelrc + build -c opt + build --host_platform //build/platforms:ubuntu_20_04 + build --define container_registry=localhost:5001 + build --define image_repo_prefix=halo + build --define image_tag=latest + EOF + + - name: Get Bazel cache params + id: get-cache-params + uses: world-federation-of-advertisers/actions/bazel-get-cache-params@v2 + with: + cache-version: 1 + + - name: Restore Bazel cache + uses: actions/cache/restore@v3 + with: + path: ${{ steps.get-cache-params.outputs.cache-path }} + key: ${{ steps.get-cache-params.outputs.cache-key }} + restore-keys: |- + ${{ steps.get-cache-params.outputs.restore-key }} + + - name: Build + run: bazelisk build //... + + # Shut down Bazel server to reclaim memory. + - run: bazelisk shutdown + + - name: Run tests + id: run-tests + run: bazelisk test //... + - name: Create cluster id: create-cluster run: | @@ -42,24 +74,17 @@ jobs: chmod +x kind-with-registry.sh ./kind-with-registry.sh - - name: Bazel build and test - uses: world-federation-of-advertisers/actions/bazel-build-test@v1 - with: - cache-version: 1 - workspace-path: . - target-patterns: | - //... - //src/test/kotlin/org/wfanet/measurement/integration/k8s:EmptyClusterCorrectnessTest - build-options: | - --compilation_mode=opt - --host_platform=//build/platforms:ubuntu_20_04 - --define=container_registry=${{ env.CONTAINER_REGISTRY }} - --define=image_repo_prefix=${{ env.IMAGE_REPO_PREFIX }} - --define=image_tag=latest + - name: Run correctness test + id: run-correctness-test + run: > + bazelisk test + //src/test/kotlin/org/wfanet/measurement/integration/k8s:EmptyClusterCorrectnessTest + --test_output=streamed - name: Export cluster logs id: export-cluster-logs - if: failure() && steps.create-cluster.outcome == 'success' + if: failure() && steps.run-correctness-test.outcome == 'failure' + continue-on-error: true run: | mkdir -p "$CLUSTER_LOGS_PATH" kind export logs "$CLUSTER_LOGS_PATH" @@ -67,6 +92,26 @@ jobs: - name: Upload cluster logs artifact if: failure() && steps.export-cluster-logs.outcome == 'success' uses: actions/upload-artifact@v3 + continue-on-error: true with: name: cluster-logs path: ${{ env.CLUSTER_LOGS_PATH }} + + # Delete large files that are cheap to re-create. + - name: Trim Bazel cache + run: | + bazel_bin="$(bazelisk info -c opt bazel-bin)" + find "${bazel_bin}" -type f -iname '*.tar' -delete + + - name: Save Bazel cache + if: github.event_name == 'push' + uses: actions/cache/save@v3 + continue-on-error: true + with: + path: ${{ steps.get-cache-params.outputs.cache-path }} + key: ${{ steps.get-cache-params.outputs.cache-key }} + + - name: Upload Bazel testlogs + continue-on-error: true + uses: world-federation-of-advertisers/actions/bazel-upload-testlogs@v2 + if: failure() && (steps.run-tests.outcome == 'failure' || steps.run-correctness-test.outcome == 'failure') diff --git a/.github/workflows/create-cmm-images.yml b/.github/workflows/create-cmm-images.yml index 59c461bedf3..6a27ade43d3 100644 --- a/.github/workflows/create-cmm-images.yml +++ b/.github/workflows/create-cmm-images.yml @@ -22,14 +22,13 @@ on: description: "Tag of container images" value: ${{ jobs.push-images.outputs.image-tag }} -env: - CONTAINER_REGISTRY: ghcr.io - jobs: push-images: runs-on: ubuntu-20.04 outputs: image-tag: ${{ steps.output-image-tag.outputs.image-tag }} + env: + CONTAINER_REGISTRY: ghcr.io steps: - uses: actions/checkout@v3 @@ -40,6 +39,30 @@ jobs: id: output-image-tag run: echo "image-tag=$IMAGE_TAG" >> $GITHUB_OUTPUT + - name: Write ~/.bazelrc + run: | + cat << EOF > ~/.bazelrc + build -c opt + build --host_platform //build/platforms:ubuntu_20_04 + build --define container_registry=$CONTAINER_REGISTRY + build --define image_repo_prefix=$GITHUB_REPOSITORY_OWNER + build --define image_tag=$IMAGE_TAG + EOF + + - name: Get Bazel cache params + id: get-cache-params + uses: world-federation-of-advertisers/actions/bazel-get-cache-params@v2 + with: + cache-version: 1 + + - name: Restore Bazel cache + uses: actions/cache/restore@v3 + with: + path: ${{ steps.get-cache-params.outputs.cache-path }} + key: ${{ steps.get-cache-params.outputs.cache-key }} + restore-keys: |- + ${{ steps.get-cache-params.outputs.restore-key }} + - name: Authenticate to GitHub Container Registry uses: docker/login-action@v2 with: @@ -48,10 +71,4 @@ jobs: password: ${{ github.token }} - name: Push images - run: > - bazelisk run //src/main/docker:push_all_gke_images - --compilation_mode=opt - --platforms=//build/platforms:ubuntu_20_04 - --define "container_registry=$CONTAINER_REGISTRY" - --define "image_repo_prefix=$GITHUB_REPOSITORY_OWNER" - --define "image_tag=$IMAGE_TAG" + run: bazelisk run //src/main/docker:push_all_gke_images diff --git a/.github/workflows/run-k8s-tests.yml b/.github/workflows/run-k8s-tests.yml index c4381054828..5f3a6c6f68a 100644 --- a/.github/workflows/run-k8s-tests.yml +++ b/.github/workflows/run-k8s-tests.yml @@ -46,18 +46,46 @@ jobs: workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ vars.SIMULATOR_SERVICE_ACCOUNT }} - - name: Run tests + - name: Write ~/.bazelrc env: STORAGE_BUCKET: ${{ vars.STORAGE_BUCKET }} KINGDOM_PUBLIC_API_TARGET: ${{ vars.KINGDOM_PUBLIC_API_TARGET }} MC_NAME: ${{ vars.MC_NAME }} MC_API_KEY: ${{ secrets.MC_API_KEY }} + run: | + cat << EOF > ~/.bazelrc + build -c opt + build --host_platform //build/platforms:ubuntu_20_04 + build --define google_cloud_project=$GCLOUD_PROJECT + build --define kingdom_public_api_target=$KINGDOM_PUBLIC_API_TARGET + build --define simulator_storage_bucket=$STORAGE_BUCKET + build --define mc_name=$MC_NAME + build --define mc_api_key=$MC_API_KEY + build --test_env=GOOGLE_APPLICATION_CREDENTIALS + EOF + + - name: Get Bazel cache params + id: get-cache-params + uses: world-federation-of-advertisers/actions/bazel-get-cache-params@v2 + with: + cache-version: 1 + + - name: Restore Bazel cache + uses: actions/cache/restore@v3 + with: + path: ${{ steps.get-cache-params.outputs.cache-path }} + key: ${{ steps.get-cache-params.outputs.cache-key }} + restore-keys: |- + ${{ steps.get-cache-params.outputs.restore-key }} + + - name: Run tests + id: run-tests run: > - bazelisk test -c opt --test_output=streamed - --test_env=GOOGLE_APPLICATION_CREDENTIALS + bazelisk test //src/test/kotlin/org/wfanet/measurement/integration/k8s:GcsCorrectnessTest - --define "google_cloud_project=$GCLOUD_PROJECT" - --define "kingdom_public_api_target=$KINGDOM_PUBLIC_API_TARGET" - --define "simulator_storage_bucket=$STORAGE_BUCKET" - --define "mc_name=$MC_NAME" - --define "mc_api_key=$MC_API_KEY" + --test_output=streamed + + - name: Upload Bazel testlogs + continue-on-error: true + uses: world-federation-of-advertisers/actions/bazel-upload-testlogs@v2 + if: failure() && steps.run-tests.outcome == 'failure'