Skip to content

Image - cjdcordeiro - refs/pull/235/merge #1212

Image - cjdcordeiro - refs/pull/235/merge

Image - cjdcordeiro - refs/pull/235/merge #1212

Workflow file for this run

name: Image
run-name: 'Image - ${{ inputs.oci-image-name || github.triggering_actor }} - ${{ github.ref }}'
on:
push:
paths:
- "oci/*/image.y*ml"
- "!oci/mock*"
pull_request:
paths:
- "oci/*/image.y*ml"
- "!oci/mock*"
workflow_dispatch:
inputs:
oci-image-name:
description: "OCI image to build and test"
required: true
b64-image-trigger:
description: "(Base64 encoded) Pass the image.yaml as an argument"
required: false
type: string
upload:
description: "Upload image to GHCR"
required: true
type: boolean
default: false
external_ref_id: #(1)
description: 'Optional ID for unique run detection'
required: false
type: string
default: "default-id"
workflow_call:
inputs:
oci-image-name:
description: "OCI image to build and test"
type: string
required: true
upload:
description: "Upload image to GHCR"
required: true
type: boolean
default: false
env:
VULNERABILITY_REPORT_SUFFIX: ".vulnerability-report.json"
jobs:
prepare-build:
runs-on: ubuntu-22.04
name: Prepare build
outputs:
build-matrix: ${{ steps.prepare-matrix.outputs.build-matrix }}
release-to: ${{ steps.prepare-matrix.outputs.release-to }}
oci-img-path: ${{ steps.validate-image.outputs.img-path }}
oci-img-name: ${{ steps.validate-image.outputs.img-name }}
steps:
- name: ${{ inputs.external_ref_id }} #(2)
if: ${{ github.event_name == 'workflow_dispatch' }}
run: echo 'Started by ${{ inputs.external_ref_id }}' >> "$GITHUB_STEP_SUMMARY"
- uses: actions/checkout@v4
- name: Infer number of image triggers
uses: tj-actions/changed-files@v35
id: changed-files
if: github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call'
with:
separator: ","
dir_names: "true"
files: |
oci/*/image.y*ml
- name: Validate image from dispatch
id: validate-image
run: |
set -ex
# check if this is coming from a workflow dispatch/call
# as checking github.event_name isn't reliable here
if [ "${{ inputs.oci-image-name }}" != "" ]
then
img_path="oci/${{ inputs.oci-image-name }}"
else
img_path="${{ steps.changed-files.outputs.all_changed_files }}"
occurrences="${img_path//[^,]}"
if [ ${#occurrences} -ne 0 ]
then
echo "ERR: can only build 1 image at a time, but trying to trigger ${img_path}"
exit 1
fi
fi
test -d "${img_path}"
echo "img-name=$(basename ${img_path})" >> "$GITHUB_OUTPUT"
echo "img-path=${img_path}" >> "$GITHUB_OUTPUT"
- name: Use custom image trigger
if: ${{ inputs.b64-image-trigger != '' }}
run: echo ${{ inputs.b64-image-trigger }} | base64 -d > ${{ steps.validate-image.outputs.img-path }}/image.yaml
- uses: actions/setup-python@v5
with:
python-version: "3.x"
- run: pip install -r src/image/requirements.txt
- name: Validate and prepare builds matrix
id: prepare-matrix
env:
DATA_DIR: "revision-data"
run: |
mkdir ${{ env.DATA_DIR }}
./src/image/prepare_single_image_build_matrix.py \
--oci-path ${{ steps.validate-image.outputs.img-path }} \
--revision-data-dir ${{ env.DATA_DIR }} \
run-build:
needs: [prepare-build]
strategy:
fail-fast: true
matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
uses: ./.github/workflows/Build-Rock.yaml
with:
oci-archive-name: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
oci-factory-path: ${{ matrix.path }}
rock-name: ${{ matrix.name }}
rock-repo: ${{ matrix.source }}
rock-repo-commit: ${{ matrix.commit }}
rockfile-directory: ${{ matrix.directory }}
secrets: inherit
test:
needs: [prepare-build, run-build]
name: Test
strategy:
fail-fast: true
matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
uses: ./.github/workflows/Tests.yaml
with:
oci-image-name: "${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}"
oci-image-path: "oci/${{ matrix.name }}"
test-from: "cache"
cache-key: ${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
secrets: inherit
prepare-upload:
runs-on: ubuntu-22.04
needs: [prepare-build, run-build, test]
name: Prepare upload
if: ${{ inputs.upload || (github.ref_name == 'main' && github.event_name == 'push') }}
env:
OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
DATA_DIR: "revision-data"
outputs:
build-matrix: ${{ steps.prepare-matrix.outputs.build-matrix }}
revision-data-cache-key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
steps:
- uses: actions/checkout@v4
- name: Use custom image trigger
if: ${{ inputs.b64-image-trigger != '' }}
run: echo ${{ inputs.b64-image-trigger }} | base64 -d > ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml
- uses: actions/setup-python@v5
with:
python-version: "3.x"
- run: |
./src/uploads/requirements.sh
pip install -r src/image/requirements.txt -r src/uploads/requirements.txt
- name: Upload the lockfile for the image
id: swift-lock
run: |
./src/uploads/swift_lockfile_lock.sh \
${{ needs.prepare-build.outputs.oci-img-name }}
# Here starts the critical section, have to be executed in sequence outside of matrix.
- name: Get next revision number
id: get-next-revision
run: ./src/image/define_image_revision.sh
- name: Prepare builds matrix for upload
id: prepare-matrix
run: |
set -ex
mkdir ${{ env.DATA_DIR }}
./src/image/prepare_single_image_build_matrix.py \
--oci-path ${{ needs.prepare-build.outputs.oci-img-path }} \
--revision-data-dir ${{ env.DATA_DIR }} \
--next-revision ${{ steps.get-next-revision.outputs.revision }} \
--infer-image-track
echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}-$(date +%s)" >> "$GITHUB_OUTPUT"
- name: Preempt Swift slot
run: |
./src/uploads/preempt_swift_slots.sh ${{ env.DATA_DIR }}
# Here leaves the critical section.
# The lock will be removed even if the steps above fail,
# or the workflow is cancelled.
- name: Remove the lockfile for the image
# Failing to lock the swift container can mean there are multiple
# workflows trying to upload the same image at the same time.
# Therefore we should not remove the lockfile if the swift lock failed.
if: ${{ always() && steps.swift-lock.outcome != 'failure' }}
run: |
./src/uploads/swift_lockfile_unlock.sh \
${{ needs.prepare-build.outputs.oci-img-name }}
# The revision files have to be sanitised before merging,
# since the `track` field should not be present.
- name: Sanitise revision files
run: |
set -ex
for revision_file in `ls ${{ env.DATA_DIR }}`
do
jq 'del(.track, .base)' ${{ env.DATA_DIR }}/$revision_file > ${{ env.DATA_DIR }}/$revision_file.tmp
mv ${{ env.DATA_DIR }}/$revision_file.tmp ${{ env.DATA_DIR }}/$revision_file
done
- uses: actions/cache/save@v4
with:
path: ${{ steps.prepare-matrix.outputs.revision-data-dir }}
key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
upload:
runs-on: ubuntu-22.04
needs: [prepare-build, prepare-upload]
name: Upload
strategy:
fail-fast: true
matrix: ${{ fromJSON(needs.prepare-upload.outputs.build-matrix) }}
env:
OCI_ARCHIVE_NAME: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
outputs:
artefacts-hashes: ${{ steps.artefacts-hashes.outputs.hashes }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Setup environment
env:
ROCKS_DEV_LP_SSH_PRIVATE: ${{ secrets.ROCKS_DEV_LP_SSH_PRIVATE }}
ROCKS_DEV_LP_USERNAME: ${{ secrets.ROCKS_DEV_LP_USERNAME }}
CPC_BUILD_TOOLS_REPO: git.launchpad.net/~cloudware/cloudware/+git/cpc_build_tools
# CPC_BUILD_TOOLS_REPO_REF: 9b716ed8a8ba728d036b54b1bb17a8f49dbda434
SKOPEO_BRANCH: "v1.9.1"
SKOPEO_URL: "https://github.com/containers/skopeo"
run: |
./src/image/requirements.sh
./src/uploads/requirements.sh
pip install -r src/uploads/requirements.txt -r src/image/requirements.txt
- uses: actions/cache/restore@v4
with:
path: ${{ env.OCI_ARCHIVE_NAME }}
key: ${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
fail-on-cache-miss: true
- name: Name output artefact
id: rename-oci-archive
run: |
# Rename the OCI archive tarball
canonical_tag="${{ matrix.track }}_${{ matrix.revision }}"
name="${{ matrix.name }}_${canonical_tag}"
mv ${{ env.OCI_ARCHIVE_NAME }} $name
echo "name=${name}" >> "$GITHUB_OUTPUT"
echo "canonical-tag=${canonical_tag}" >> "$GITHUB_OUTPUT"
- uses: actions/cache/save@v4
with:
path: ${{ steps.rename-oci-archive.outputs.name }}
key: ${{ github.run_id }}-${{ steps.rename-oci-archive.outputs.name }}
- name: Install Syft
uses: anchore/sbom-action/download-syft@v0
with:
syft-version: "v0.75.0"
- name: Infer architectures
id: get-arches
run: |
set -ex
arches=$(skopeo inspect --raw \
oci-archive:${{ steps.rename-oci-archive.outputs.name }} \
| jq -r 'if has("manifests") then .manifests[].platform.architecture else "${{ runner.arch }}" end' \
| jq -nRcr '[inputs] | join(",")')
echo "Detected architectures: ${arches}"
echo "arches='${arches}'" >> "$GITHUB_OUTPUT"
- name: Generate SBOMs
id: generate-sboms
run: |
set -ex
echo "Generating SBOMs for ${{ steps.get-arches.outputs.arches }}"
IFS=',' read -ra arch_list <<< ${{ steps.get-arches.outputs.arches }}
for arch in "${arch_list[@]}"; do
if [[ "${arch}" == "unknown" ]]; then
continue
fi
echo "Generate SBOM for ${arch}"
skopeo --override-arch $arch copy \
oci-archive:${{ steps.rename-oci-archive.outputs.name }} \
oci-archive:${{ steps.rename-oci-archive.outputs.name }}.${arch}
syft oci-archive:${{ steps.rename-oci-archive.outputs.name }}.${arch} \
-o spdx-json \
--file ${{ steps.rename-oci-archive.outputs.name }}.${arch}.sbom.spdx.json
done
all_sboms_zip="${{ steps.rename-oci-archive.outputs.name }}.sbom.spdx.zip"
zip "${all_sboms_zip}" ${{ steps.rename-oci-archive.outputs.name }}.*.sbom.spdx.json
echo "sboms=${all_sboms_zip}" >> "$GITHUB_OUTPUT"
- name: Fetch vulnerability artifacts for hashing
uses: actions/cache/restore@v4
with:
path: ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
key: ${{ github.run_id }}-${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
fail-on-cache-miss: true
# https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/generic/README.md
- name: Calculate artefacts hashes
id: artefacts-hashes
env:
VULN_REPORT: ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
SBOMS: ${{ steps.generate-sboms.outputs.sboms }}
OCI_IMAGE_ARCHIVE: ${{ steps.rename-oci-archive.outputs.name }}
run: |
set -ex
echo "hashes=$(sha256sum ${{ env.VULN_REPORT }} ${{ env.OCI_IMAGE_ARCHIVE }} ${{ env.SBOMS }} | base64 -w0)"
- name: Login to GHCR
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Upload SBOM
uses: actions/upload-artifact@v4
with:
name: ${{ steps.generate-sboms.outputs.sboms }}
path: ${{ steps.generate-sboms.outputs.sboms }}
if-no-files-found: error
- name: Upload image
uses: actions/upload-artifact@v4
with:
name: ${{ steps.rename-oci-archive.outputs.name }}
path: ${{ steps.rename-oci-archive.outputs.name }}
if-no-files-found: error
- name: Upload to GHCR
id: upload-image
env:
GHCR_REPO: ghcr.io/${{ github.repository_owner }}/oci-factory
GHCR_USERNAME: ${{ github.actor }}
GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
run: |
oci_images="${PWD}/images-oci"
rm -fr $oci_images
mkdir -p $oci_images
tar -xf ${{ steps.rename-oci-archive.outputs.name }} -C $oci_images
source="oci:${oci_images}"
digest="$(skopeo inspect $source | jq -r .Digest)"
echo "digest=$digest" >> $GITHUB_OUTPUT
./src/image/tag_and_publish.sh "${source}" \
${{ matrix.name }} \
${{ steps.rename-oci-archive.outputs.canonical-tag }}
- name: Upload build metadata to Swift
env:
OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
IMAGE_NAME: ${{ matrix.name }}
SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
run: |
jq --arg base "${{ matrix.base }}" \
--arg digest "${{ steps.upload-image.outputs.digest }}" \
'. + {base: $base, digest: $digest}' <<< '${{ toJSON(matrix) }}' > build_metadata.json
./src/uploads/upload_to_swift.sh \
${{ matrix.name }} \
${{ matrix.track }} \
${{ matrix.revision }} \
build_metadata.json \
${{ steps.generate-sboms.outputs.sboms }} \
${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
- name: Create Git tag
uses: rickstaa/action-create-tag@v1
with:
tag: "${{ steps.rename-oci-archive.outputs.name }}"
message: "upload(${{ matrix.name }}): Build and upload new image revision ${{ matrix.revision }}"
force_push_tag: true
github_token: ${{ secrets.ROCKSBOT_TOKEN }}
# We need this job because we want to make the releases all in one go,
# so that we know which revisions to release, and so that we can update
# and commit the _releases.json file in a single commit, outside a matrix job
prepare-releases:
name: Prepare releases
needs: [prepare-build, prepare-upload, upload]
runs-on: ubuntu-22.04
if: ${{ needs.prepare-build.outputs.release-to != '' }}
concurrency:
group: ${{ needs.prepare-build.outputs.oci-img-path }}
cancel-in-progress: true
env:
REVISION_DATA_DIR: revision-data
steps:
- uses: actions/setup-python@v5
with:
python-version: "3.x"
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Use custom image trigger
if: ${{ inputs.b64-image-trigger != '' }}
run: echo ${{ inputs.b64-image-trigger }} | base64 -d > ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml
- uses: actions/cache/restore@v4
with:
path: ${{ env.REVISION_DATA_DIR }}
key: ${{ needs.prepare-upload.outputs.revision-data-cache-key }}
fail-on-cache-miss: true
- run: pip install -r src/image/requirements.txt
- name: Merge release requests
id: merge-release-requests
env:
PYTHONUNBUFFERED: 1
run: |
set -ex
for revision_file in `ls ${{ env.REVISION_DATA_DIR }}`
do
ret=0
release=$(jq -er .release < ${{ env.REVISION_DATA_DIR }}/$revision_file) || ret=1
if [ $ret -eq 1 ]
then
echo "Revision $revision_file not marked for release"
continue
fi
echo "Merge revision $revision_file with requested releases"
./src/image/merge_release_info.py \
--image-trigger "${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml" \
--revision-data-file "${{ env.REVISION_DATA_DIR }}/${revision_file}"
done
- uses: actions/cache/save@v4
with:
path: ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml
key: ${{ github.run_id }}-image-trigger
release:
name: Release
needs: [prepare-build, prepare-releases]
uses: ./.github/workflows/Release.yaml
with:
oci-image-name: "${{ needs.prepare-build.outputs.oci-img-name }}"
image-trigger-cache-key: "${{ github.run_id }}-image-trigger"
secrets: inherit
generate-provenance:
name: Generate SLSA provenance
needs: [upload]
runs-on: ubuntu-22.04
steps:
- uses: actions/download-artifact@v4
id: download
with:
path: artifacts
- name: Generate provenance
uses: philips-labs/[email protected]
with:
command: generate
subcommand: files
arguments: --artifact-path ${{ steps.download.outputs.download-path }} --output-path provenance.json
# - uses: sigstore/[email protected]
# with:
# cosign-release: 'v2.0.0'
# - name: Sign provenance file
# env:
# PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
# COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
# PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }}
# run: |
# echo "${PRIVATE_KEY}" > cosign.key
# echo "${PUBLIC_KEY}" > cosign.pub
# cosign sign-blob --key cosign.key --output-signature provenance.json.sig provenance.json
# - name: Upload verification keys
# uses: actions/upload-artifact@v4
# with:
# name: verify
# path: |
# cosign.pub
# provenance.json.sig
# if-no-files-found: error
# - name: Generate checksums
# working-directory: ${{ steps.download.outputs.download-path }}
# run: sha256sum * > SHA256SUMS || true
# - name: Upload SHA256SUMS file
# uses: actions/upload-artifact@v4
# with:
# name: SHA256SUMS
# path: SHA256SUMS
# if-no-files-found: error
- name: Upload provenance file
uses: actions/upload-artifact@v4
with:
name: provenance
path: provenance.json
if-no-files-found: error
# generate-provenance:
# name: Generate SLSA provenance
# needs: [upload]
# permissions:
# actions: read # Needed for detection of GitHub Actions environment.
# id-token: write # Needed for provenance signing and ID
# contents: write # Needed for release uploads
# uses: slsa-framework/slsa-github-generator/.github/workflows/[email protected]
# with:
# base64-subjects: "${{ needs.upload.outputs.artefacts-hashes }}"
notify:
runs-on: ubuntu-22.04
name: Notify
needs: [prepare-build, run-build, upload, prepare-releases, generate-provenance]
if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name != 'pull_request' }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.x'
- name: Summarize workflow failure message
id: get-summary
run: |
echo '${{ toJson(needs) }}' > jobs.json
./src/notifications/summarize_workflow_results.py --jobs-file jobs.json
- name: Get contacts for ${{ needs.prepare-build.outputs.oci-img-name }}
id: get-contacts
working-directory: ${{ needs.prepare-build.outputs.oci-img-path }}
run: |
mm_channels=$(yq -r '.notify | ."mattermost-channels" | join(",")' < contacts.y*ml)
echo "mattermost-channels=${mm_channels}" >> "$GITHUB_OUTPUT"
- name: Notify via Mattermost
env:
MM_BOT_TOKEN: ${{ secrets.MM_BOT_TOKEN }}
FINAL_STATUS: failure
MM_SERVER: ${{ secrets.MM_SERVER }}
URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SUMMARY: ${{ steps.get-summary.outputs.summary }}
FOOTER: "Triggered by ${{ github.triggering_actor }}. Ref: ${{ github.ref }}. Attempts: ${{ github.run_attempt }}"
TITLE: '${{ needs.prepare-build.outputs.oci-img-name }}: failed to build->upload->release'
run: |
for channel in $(echo ${{ steps.get-contacts.outputs.mattermost-channels }} | tr ',' ' ')
do
MM_CHANNEL_ID="${channel}" ./src/notifications/send_to_mattermost.sh
done