flatcar-3033: backport build dev packages #676
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Run build" | |
| on: | |
| pull_request: | |
| # Run when the PR is ready and each time a review is re-requested | |
| # (i.e. after feedback has been addressed). | |
| types: [review_requested, ready_for_review] | |
| workflow_dispatch: | |
| inputs: | |
| image_formats: | |
| description: | | |
| Space-separated vendor formats to build. | |
| required: true | |
| default: qemu_uefi | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} | |
| cancel-in-progress: true | |
| permissions: | |
| pull-requests: write | |
| jobs: | |
| packages: | |
| # Do not run when still in draft mode but a review was requested anyway | |
| if: github.event.pull_request.draft == false | |
| name: "Build Flatcar packages" | |
| runs-on: | |
| - self-hosted | |
| - debian | |
| - build | |
| - x64 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: ["amd64", "arm64"] | |
| defaults: | |
| run: | |
| working-directory: scripts | |
| steps: | |
| - name: Prepare machine | |
| shell: bash | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| sudo rm /bin/sh | |
| sudo ln -s /bin/bash /bin/sh | |
| sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-user-static git | |
| sudo mkdir -p /etc/apt/keyrings | |
| curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg | |
| echo \ | |
| "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ | |
| $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null | |
| sudo apt-get update | |
| sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin | |
| - uses: actions/checkout@v3 | |
| with: | |
| path: scripts | |
| fetch-depth: 0 | |
| # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR | |
| # instead of its head commit. That commit is not connected to any branch. | |
| # This causes breakage downstream e.g. when the devcontainer test wants to check out | |
| # the ref in the scripts repo that corresponds to this build. | |
| - name: If this is a PR build, use head commit instead of the merge commit | |
| if: ${{ github.event.pull_request.head.sha }} | |
| shell: bash | |
| run: | | |
| exec 2>&1 | |
| set -x | |
| set -euo pipefail | |
| git checkout ${{ github.event.pull_request.head.sha }} | |
| git submodule update | |
| - name: Set environment | |
| shell: bash | |
| run: | | |
| arch="${{ matrix.arch }}" | |
| echo "arch=${arch}" >> $GITHUB_ENV | |
| IMAGE_FORMATS="qemu_uefi" | |
| [ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}" | |
| echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV | |
| # Artifact root for images and torcx tarball as seen from within the container | |
| echo "CI_CONTAINER_ARTIFACT_ROOT=/home/sdk/trunk/src/scripts/artifacts" >> $GITHUB_ENV | |
| echo "CI_CONTAINER_TORCX_ROOT=/home/sdk/trunk/src/scripts/artifacts/torcx" >> $GITHUB_ENV | |
| mkdir -p artifacts/torcx | |
| # Placeholder URL for run-kola-tests.yaml, "Extract artifacts" step which will replace | |
| # this with its IP address. | |
| echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV | |
| - name: Build packages | |
| shell: bash | |
| run: | | |
| exec 2>&1 | |
| set -x | |
| set -euo pipefail | |
| source ci-automation/ci_automation_common.sh | |
| source sdk_container/.repo/manifests/version.txt | |
| version="alpha-$FLATCAR_VERSION_ID" | |
| check_version_string "$version" | |
| sdk_version="${FLATCAR_SDK_VERSION}" | |
| sdk_name="flatcar-sdk-${arch}" | |
| docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")" | |
| docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}" | |
| sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")" | |
| container_name="flatcar-ci-build" | |
| echo "container_name=${container_name}" >> "$GITHUB_ENV" | |
| # Create version file | |
| ( | |
| source sdk_lib/sdk_container_common.sh | |
| create_versionfile "$sdk_version" "$version" | |
| ) | |
| # Run the packages build. This will create the ci build container | |
| # which will be re-used by subsequent build steps. | |
| ./run_sdk_container -n "${container_name}" -v "${version}" \ | |
| -C "${sdk_image}" \ | |
| ./build_packages --board="${arch}-usr" \ | |
| --torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \ | |
| --torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}" | |
| # Create binpkgs tarball for archiving as artifact later | |
| ./run_sdk_container -n "${container_name}" \ | |
| tar -C "/build/${arch}-usr/var/lib/portage/pkgs/" \ | |
| -cvf binpkgs.tar . | |
| - name: Extract build logs | |
| if: always() | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| set -x | |
| # Copy logs | |
| ./run_sdk_container -n "${container_name}" \ | |
| tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \ | |
| /build/${arch}-usr/var/tmp/portage | |
| - name: Upload build logs | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| retention-days: 7 | |
| name: ${{ matrix.arch }}-build-logs | |
| path: | | |
| scripts/ebuild_logs.tar.xz | |
| - name: Build image | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| set -x | |
| echo 'channel="developer"' >> $GITHUB_ENV | |
| channel="developer" | |
| source ci-automation/ci_automation_common.sh | |
| official_arg="--noofficial" | |
| ./run_sdk_container -n "${container_name}" \ | |
| ./set_official --board="${arch}-usr" "${official_arg}" | |
| ./run_sdk_container -n "${container_name}" \ | |
| ./build_image --board="${arch}-usr" --group="${channel}" \ | |
| --output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \ | |
| --torcx_root="${CI_CONTAINER_TORCX_ROOT}" prodtar container | |
| - name: Build VM image(s) | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| set -x | |
| source ci-automation/ci_automation_common.sh | |
| images_out="images" | |
| has_packet=0 | |
| has_pxe=0 | |
| formats="${IMAGE_FORMATS}" | |
| for format in "${formats}";do | |
| [[ "${format}" = 'packet' ]] || [[ "${format}" = 'equinix_metal' ]] && has_packet=1 | |
| [[ "${format}" = 'pxe' ]] && has_pxe=1 | |
| done | |
| [[ ${has_packet} -eq 1 ]] && [[ ${has_pxe} -eq 0 ]] && set -- 'pxe' "${@}" | |
| if echo "$formats" | tr ' ' '\n' | grep -q '^vmware'; then | |
| formats=$(echo "$formats" | tr ' ' '\n' | sed '/vmware.*/d') | |
| formats+=" vmware vmware_insecure vmware_ova vmware_raw" | |
| fi | |
| if echo "$formats" | tr ' ' '\n' | grep -q -P '^(ami|aws)'; then | |
| formats=$(echo "$formats" | tr ' ' '\n' | sed '/ami.*/d' | sed '/aws/d') | |
| formats+=" ami ami_vmdk" | |
| fi | |
| # Keep compatibility with SDK scripts where "equinix_metal" remains unknown. | |
| formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g') | |
| for format in ${formats}; do | |
| echo " ################### VENDOR '${format}' ################### " | |
| ./run_sdk_container -n "${container_name}" \ | |
| ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \ | |
| --from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \ | |
| --image_compression_formats=bz2 | |
| done | |
| # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really) | |
| # so we move things around. | |
| mkdir -p artifacts/images | |
| ( | |
| cd artifacts/${arch}-usr/latest/ | |
| mv * ../../images/ | |
| ) | |
| # create a tarball for torcx package + JSON file because upload-artifacts cannot handle filenames containing colons | |
| # (such as "docker:20.10.torcx.tgz") | |
| mv artifacts/torcx/${arch}-usr/latest/torcx_manifest.json artifacts/torcx/pkgs/ | |
| tar -C artifacts/torcx/pkgs/ -cvf torcx.tar . | |
| - name: Upload binpkgs | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| retention-days: 7 | |
| name: ${{ matrix.arch }}-binpkgs | |
| path: | | |
| scripts/binpkgs.tar | |
| - name: Upload update image (used with kola tests later) | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| retention-days: 7 | |
| name: ${{ matrix.arch }}-test-update | |
| path: | | |
| scripts/artifacts/images/flatcar_test_update.gz | |
| - name: Upload generic image | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| retention-days: 7 | |
| name: ${{ matrix.arch }}-generic-image | |
| path: | | |
| scripts/artifacts/images/flatcar_production_image.bin.bz2 | |
| scripts/artifacts/images/flatcar_production_image.grub | |
| scripts/artifacts/images/flatcar_production_image.shim | |
| scripts/artifacts/images/flatcar_production_image.vmlinuz | |
| scripts/artifacts/images/flatcar_production_image*.txt | |
| scripts/artifacts/images/flatcar_production_image*.json | |
| scripts/artifacts/images/flatcar_production_image_pcr_policy.zip | |
| scripts/artifacts/images/flatcar_production_*_efi_*.fd | |
| - name: Upload developer container | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| retention-days: 7 | |
| name: ${{ matrix.arch }}-devcontainer | |
| path: | | |
| scripts/artifacts/images/flatcar_developer_container* | |
| - name: Upload torcx tarball | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| retention-days: 7 | |
| name: ${{ matrix.arch }}-torcx | |
| path: | | |
| scripts/torcx.tar | |
| # Clean up what we uploaded already so the "vendor images" wildcard | |
| # works when uploading artifacts in the next step. | |
| - name: Remove update, generic and devcontainer images | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| set -x | |
| rm -f artifacts/images/flatcar_test_update.gz \ | |
| artifacts/images/flatcar_production_image* \ | |
| artifacts/images/flatcar_developer_container* \ | |
| artifacts/images/flatcar_production_update* | |
| - name: Upload vendor images | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| retention-days: 7 | |
| name: ${{ matrix.arch }}-vm-images | |
| path: | | |
| scripts/artifacts/images/*.img.bz2 | |
| scripts/artifacts/images/*.bin.bz2 | |
| scripts/artifacts/images/flatcar_production_*_efi_*.fd | |
| scripts/artifacts/images/*.txt | |
| scripts/artifacts/images/flatcar_production_*.sh | |
| test: | |
| needs: packages | |
| name: "Run kola tests" | |
| uses: ./.github/workflows/run-kola-tests.yaml |