e2e test runtime reproducibility #46
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: e2e test runtime reproducibility | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 19 * * 0" # 7pm UTC on Sundays | |
| jobs: | |
| os-matrix: | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04, ubuntu-20.04] | |
| # As we do not use the Cachix cache containing the artifacts built by developers in this workflow, | |
| # building the node-installer-image ensures that the whole transitive closure of the packages we build (i.e. everything | |
| # except what's in cache.nixos.org). Therefore, building this ensures that the kernel, image, IGVM, runtime and the image itself | |
| # is reproducible across individual builds (as the --rebuild flag is used, causing Nix to rebuild the node-installer-image derivation) | |
| # and across independent builds on Ubuntu 20.04 and 22.04 (which also test the reproducibility of the transitive closure of our packages, as no shared | |
| # cache is present between the two machines) | |
| # | |
| # If adjusting the build-target, remember to also adjust the matrix for the collect-checksums job | |
| build-target: ["microsoft.contrast-node-installer-image", "kata.contrast-node-installer-image"] | |
| fail-fast: false | |
| # Usually we would define the matrix outputs here, but as GitHub Actions don't seem to allow per-combination outputs, | |
| # we'll write the outputs without defining them here. See https://github.com/orgs/community/discussions/17245#discussioncomment-3814009. | |
| runs-on: ${{ matrix.os }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - uses: ./.github/actions/setup_nix | |
| with: | |
| githubToken: ${{ secrets.GITHUB_TOKEN }} | |
| cachixToken: "" # Don't use the cachix cache | |
| - uses: nicknovitski/nix-develop@9be7cfb4b10451d3390a75dc18ad0465bed4932a # v1.2.1 | |
| - name: Build | |
| id: build | |
| run: | | |
| nix build .#${{ matrix.build-target }} --option substituters https://cache.nixos.org --builders "" | |
| reference_checksum="$(jq -r '.manifests[0].digest' result/index.json)" | |
| echo "$reference_checksum" > ${{ matrix.build-target }}-${{ matrix.os }}-reference_checksum.txt | |
| nix build .#${{ matrix.build-target }} --rebuild --option substituters https://cache.nixos.org --builders "" -o rebuild | |
| rebuild_checksum="$(jq -r '.manifests[0].digest' rebuild/index.json)" | |
| echo "$rebuild_checksum" > ${{ matrix.build-target }}-${{ matrix.os }}-rebuild_checksum.txt | |
| - name: Upload Build Artifacts | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: ${{ matrix.build-target }}-${{ matrix.os }} | |
| path: result | |
| - name: Upload Build Artifacts (Rebuild) | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: ${{ matrix.build-target }}-${{ matrix.os }}-rebuild | |
| path: rebuild | |
| - name: Upload checksums | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: ${{ matrix.build-target }}-${{ matrix.os }}-checksums | |
| path: ${{ matrix.build-target }}-${{ matrix.os }}-*_checksum.txt | |
| - name: Notify teams channel of failure | |
| if: ${{ failure() && github.ref == 'main' && github.run_attempt == 1 }} | |
| uses: ./.github/actions/post_to_teams | |
| with: | |
| webhook: ${{ secrets.TEAMS_CI_WEBHOOK }} | |
| title: "Runtime reproducibility test failed" | |
| message: "Runtime reproducibility test failed on ${{ matrix.build-target}}-${{matrix.os }}" | |
| additionalFields: '[{"title": "Build target", "value": "${{ matrix.build-target }}"}, {"title": "OS", "value": "${{ matrix.os }}"}]' | |
| collect-checksums: | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| contents: read | |
| needs: os-matrix | |
| strategy: | |
| matrix: | |
| build-target: ["microsoft.contrast-node-installer-image", "kata.contrast-node-installer-image"] | |
| steps: | |
| - name: Download all checksum artifacts | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| pattern: "${{matrix.build-target}}-*-checksums" | |
| path: "./checksums" | |
| merge-multiple: true | |
| - name: Collect checksums | |
| id: collect | |
| shell: python | |
| run: | | |
| import json, os, pathlib | |
| seen = {} | |
| for file in os.listdir("./checksums"): | |
| checksum = pathlib.Path("./checksums", file).read_text() | |
| if not checksum in seen: | |
| seen[checksum] = [] | |
| seen[checksum].append(file) | |
| assert len(seen) > 0 | |
| if len(seen) > 1: | |
| print("At least one checksum mismatched:") | |
| print(json.dumps(seen, indent=2)) | |
| exit(1) | |
| print("All checksums were equal") | |
| - name: Notify teams channel of failure | |
| if: ${{ failure() && github.ref == 'main' && github.run_attempt == 1 }} | |
| uses: ./.github/actions/post_to_teams | |
| with: | |
| webhook: ${{ secrets.TEAMS_CI_WEBHOOK }} | |
| title: "Runtime reproducibility test failed" | |
| message: "failed to collect checksums" | |
| additionalFields: '[{"title": "Build target", "value": "${{matrix.build-target}}"}]' |