Skip to content

image: mono refs/heads/edge, buildroot - #4002

image: mono refs/heads/edge, buildroot -

image: mono refs/heads/edge, buildroot - #4002

Workflow file for this run

name: 'Build OT2 image on github workflows'
run-name: 'image: mono ${{ inputs.monorepo-ref }}, buildroot ${{ inputs.buildroot-ref }}'
on:
workflow_dispatch:
inputs:
monorepo-ref:
description: |
Ref of https://github.com/opentrons/opentrons to build. This MUST be a full ref, e.g. refs/heads/edge, or '-' to indicate not-specified. If not specified, will be determined from the oe-core ref if specified, and then default to edge.
required: true
default: '-'
buildroot-ref:
description: |
Ref of https://github.com/opentrons/buildroot to build. This is different from the ref specified in the github api/webUI when starting this workflow - that ref is what contains this workflow, this ref specifies what gets built. It MUST be a full ref, e.g. refs/heads/main, or '-' to indicate not-specified. If not specified, will be decided based on the monorepo ref; if that isn't specified, will be main.
required: true
default: '-'
infra-stage:
description: |
What infra stage to run on. This should almost always be prod; dev is useful when you explicitly want to test that stage.
required: true
type: choice
options:
- 'stage-prod'
- 'stage-dev'
default: 'stage-prod'
env:
CI: true
jobs:
decide-refs:
name: 'Decide refs to build'
runs-on: 'ubuntu-latest'
outputs:
buildroot: ${{ steps.build-refs.outputs.buildroot }}
monorepo: ${{ steps.build-refs.outputs.monorepo }}
variant: ${{ steps.build-refs.outputs.variant }}
build-type: ${{ steps.build-refs.outputs.build-type }}
steps:
- name: Check out sources for action
uses: 'actions/checkout@v3'
with:
path: ./buildroot-for-workflow
- name: Get the refs
id: build-refs
uses: './buildroot-for-workflow/.github/actions/build-refs'
with:
token: ${{ github.token }}
monorepo: ${{ inputs.monorepo-ref }}
buildroot: ${{ inputs.buildroot-ref }}
run-build:
needs: decide-refs
strategy:
matrix:
build_env: [ '${{ inputs.infra-stage }}', ]
name: 'Building ${{needs.decide-refs.outputs.variant}} images on ${{ matrix.build_env }}'
timeout-minutes: 480
runs-on: ['self-hosted', '${{matrix.build_env}}', '${{needs.decide-refs.outputs.variant}}']
concurrency:
group: ${{needs.decide-refs.outputs.monorepo}} ${{needs.decide-refs.outputs.buildroot}} ${{needs.decide-refs.outputs.variant}} ${{needs.decide-refs.outputs.build-type}} ${{inputs.infra-stage}}
cancel-in-progress: false
steps:
- name: Set up vm requirements
run: |
echo 'fs.inotify.max_user_watches=655360' | tee -a /etc/sysctl.conf
echo 'fs.inotify.max_user_instances=1280' | tee -a /etc/sysctl.conf
echo 'fs.file-max=100000' | tee -a /etc/sysctl.conf
sysctl -p
- name: Fetch initial sources for action
uses: 'actions/checkout@v3'
with:
fetch-depth: 0
path: ./buildroot-for-workflow
- name: Fetch buildroot source
uses: 'actions/checkout@v3'
with:
fetch-depth: 0
ref: ${{needs.decide-refs.outputs.buildroot}}
path: ./buildroot
- name: Fetch monorepo source
uses: 'actions/checkout@v3'
with:
fetch-depth: 0
ref: ${{ needs.decide-refs.outputs.monorepo }}
repository: Opentrons/opentrons
path: ./opentrons
- name: Configure AWS Credentials
uses: './buildroot-for-workflow/.github/actions/aws-credentials'
id: aws
with:
access_key_id: ${{ secrets.ROBOT_STACK_AWS_ACCESS_KEY_ID }}
secret_access_key: ${{ secrets.ROBOT_STACK_AWS_SECRET_ACCESS_KEY }}
region: us-east-2
stage: ${{ matrix.build_env }}
- name: Pull or create cache locations
id: cache-setup
run: |
cachedir=${LOCAL_CACHE:-./cache}
for cachetype in downloads output ; do
absdir="${cachedir}/${cachetype}"
mkdir -p ${absdir}
echo "${cachetype}=${absdir}" >> $GITHUB_OUTPUT
here=$(pwd)
reldir=$(realpath --relative-to=${here} ${cachedir}/${cachetype})
echo "Created ${cachetype} cache at ${absdir} (${reldir})"
echo "${cachetype}-rel=${reldir}" >> $GITHUB_OUTPUT
done
- name: Set up docker environment file
id: docker-env
run: |
cat <<EOF >./docker-envfile
OT_BUILD_TYPE=${{needs.decide-refs.outputs.build-type}}
FORCE_UNSAFE_CONFIGURE=1
BR2_DL_DIR=/downloads
EOF
echo "envfile=$(pwd)/docker-envfile" >> $GITHUB_OUTPUT
- name: Build or download docker container
id: get-image
run: |
cd buildroot
imgname=$(./opentrons-build-container.sh pull || ./opentrons-build-container.sh build)
echo "image-name=$imgname" >> $GITHUB_OUTPUT
- name: Set up signing
if: needs.decide-refs.outputs.build-type == 'release'
run: |
echo "${{secrets.ROBOT_SIGNING_KEY}}" > ./buildroot/.signing-key
- name: Set up docker args
id: docker-args
run: |
if [ ${{needs.decide-refs.outputs.variant}} = "release" ] ; then
_project=robot-stack
else
_project=ot3
fi
echo "Stack name ${_project} from variant ${{needs.decide-refs.outputs.variant}}"
mount_args="type=bind,consistency=delegated"
bind_br="--mount source=$(pwd)/buildroot,destination=/buildroot,${mount_args}"
bind_ot="--mount source=$(pwd)/opentrons,destination=/opentrons,${mount_args}"
bind_dl="--mount source=${{steps.cache-setup.outputs.downloads}},destination=/downloads,${mount_args}"
bind_op="--mount source=${{steps.cache-setup.outputs.output}},destination=/output,${mount_args}"
binds="${bind_br} ${bind_ot} ${bind_dl} ${bind_op}"
env="--env-file ${{steps.docker-env.outputs.envfile}}"
echo "run-binds=${binds}" >> $GITHUB_OUTPUT
echo "env=${env}" >> $GITHUB_OUTPUT
args="${binds} ${env} ${{steps.get-image.outputs.image-name}} O=/output OPENTRONS_PROJECT=${_project}"
echo "args=${args}" >> $GITHUB_OUTPUT
- name: Configure
run: |
cd buildroot
docker run ${{steps.docker-args.outputs.args}} ot2_defconfig
- name: Download package sources
run: |
cd buildroot
docker run ${{steps.docker-args.outputs.args}} source
- name: Run build
run: |
cd buildroot
docker run ${{steps.docker-args.outputs.args}} all
- name: Upload build log result
if: always()
uses: actions/upload-artifact@v3
with:
name: buildlog.txt
path: ./buildroot/buildlog.txt
- name: Prep artifacts for upload
id: artifact-copy
run: |
artifact_dir="${{steps.cache-setup.outputs.output}}/images"
prepped_artifact_dir="${{steps.cache-setup.outputs.output}}/unversioned_artifacts"
mkdir -p ${prepped_artifact_dir}
echo "prepped_artifact_dir=${prepped_artifact_dir}" >> $GITHUB_OUTPUT
versioned_artifact_dir_rel="./versioned_artifacts"
versioned_artifact_dir=$(realpath ${versioned_artifact_dir_rel})
mkdir -p ${versioned_artifact_dir}
echo "versioned_artifact_dir=${versioned_artifact_dir}" >> $GITHUB_OUTPUT
echo "versioned_artifact_dir_rel=${versioned_artifact_dir_rel}" >> $GITHUB_OUTPUT
_fulltag="${{needs.decide-refs.outputs.monorepo}}"
echo "monorepo_shorttag=${_fulltag:10}" >> $GITHUB_OUTPUT
_vers_tag=${{needs.decide-refs.outputs.variant == 'release' && '${_fulltag:11}' || '${_fulltag:14}'}}
cp ${artifact_dir}/ot2-system.zip ${prepped_artifact_dir}/ot2-system.zip
echo "ot2_system=${prepped_artifact_dir}/ot2-system.zip" >> $GITHUB_OUTPUT
cp ${artifact_dir}/ot2-fullimage.zip ${prepped_artifact_dir}/ot2-fullimage.zip
echo "ot2_fullimage=${prepped_artifact_dir}/ot2-fullimage.zip" >> $GITHUB_OUTPUT
cp ${artifact_dir}/VERSION.json ${prepped_artifact_dir}/VERSION.json
echo "version_json=${prepped_artifact_dir}/VERSION.json" >> $GITHUB_OUTPUT
cp ${artifact_dir}/release-notes.md ${prepped_artifact_dir}/release-notes.md
echo "release_notes=${prepped_artifact_dir}/release-notes.md" >> $GITHUB_OUTPUT
versioned_system_zip="ot2-system-${_vers_tag}.zip"
cp ${artifact_dir}/ot2-system.zip "${versioned_artifact_dir}/${versioned_system_zip}"
echo "ot2_system_versioned=${versioned_artifact_dir}/${versioned_system_zip}" >> $GITHUB_OUTPUT
versioned_fullimage_zip="ot2-fullimage-${_vers_tag}.zip"
cp ${artifact_dir}/ot2-fullimage.zip ${versioned_artifact_dir}/${versioned_fullimage_zip}
echo "ot2_fullimage_versioned=${versioned_artifact_dir}/${versioned_fullimage_zip}" >> $GITHUB_OUTPUT
versioned_version_json="VERSION-ot2-${_vers_tag}.json"
cp ${artifact_dir}/VERSION.json ${versioned_artifact_dir}/${versioned_version_json}
echo "version_json_versioned=${versioned_artifact_dir}/${versioned_version_json}" >> $GITHUB_OUTPUT
echo "Prepared unversioned artifacts: $(find ${prepped_artifact_dir}/)"
echo "Prepared versioned artifacts: $(find ${versioned_artifact_dir}/)"
- name: Upload results to S3
shell: bash
id: 'upload-results'
run: |
prefix=ot2-br
arnslug=${S3_ARTIFACT_ARN/arn:aws:s3:::/}
root_uri=${arnslug}/${prefix}/${{github.run_id}}
pushd ${{steps.artifact-copy.outputs.prepped_artifact_dir}}
aws --profile=${{ steps.aws.outputs.profile_name }} s3 cp --recursive --acl=public-read . s3://${root_uri}/
root_url=https://${root_uri}
echo "console_url=https://s3.console.aws.amazon.com/s3/buckets/${arnslug}?prefix=${{github.run_id}}" >> $GITHUB_OUTPUT
echo "version_file_url=$root_url/VERSION.json" >> $GITHUB_OUTPUT
echo "release_notes_file_url=$root_url/release-notes.md" >> $GITHUB_OUTPUT
echo "system_url=$root_url/ot2-system.zip" >> $GITHUB_OUTPUT
echo "fullimage_url=$root_url/ot2-fullimage.zip" >> $GITHUB_OUTPUT
echo "arnslug=$arnslug" >> $GITHUB_OUTPUT
echo "root_uri=$root_uri" >> $GITHUB_OUTPUT
echo "root_url=$root_url" >> $GITHUB_OUTPUT
echo "root_prefix=ot2-br" >> $GITHUB_OUTPUT
echo "machine_root=${arnslug}/${prefix}" >> $GITHUB_OUTPUT
popd
- name: Handle release manifest
if: ${{ needs.decide-refs.outputs.build-type == 'release' }}
shell: bash
run: |
pushd buildroot
aws --profile=${{ steps.aws.outputs.profile_name }} s3 cp --acl=public-read s3://${{steps.upload-results.outputs.machine_root}}/releases.json releases.json
python3 update_releases_file.py --releases-file releases.json --version-file ${{steps.artifact-copy.outputs.version_json}} --base-url ${{ steps.upload-results.outputs.root_url }}
aws --profile=${{ steps.aws.outputs.profile_name }} s3 cp --acl=public-read releases.json s3://${{steps.upload-results.outputs.machine_root}}/releases.json
popd
- name: Upload system zip to monorepo release
if: needs.decide-refs.outputs.build-type == 'release'
uses: 'ncipollo/[email protected]'
with:
allowUpdates: true
omitBody: true
omitName: true
omitPrereleaseDuringUpdate: true
omitDraftDuringUpdate: true
repo: opentrons
tag: ${{ steps.artifact-copy.outputs.monorepo_shorttag }}
artifacts: "${{steps.artifact-copy.outputs.versioned_artifact_dir_rel}}/ot2-*.zip"
artifactContentType: application/zip
token: ${{secrets.MONOREPO_RELEASE_ARTIFACT_UPLOAD_TOKEN}}
- name: Upload version json to monorepo release
if: needs.decide-refs.outputs.build-type == 'release'
uses: 'ncipollo/[email protected]'
with:
allowUpdates: true
omitBody: true
omitName: true
omitPrereleaseDuringUpdate: true
omitDraftDuringUpdate: true
repo: opentrons
tag: ${{ steps.artifact-copy.outputs.monorepo_shorttag }}
artifacts: "${{steps.artifact-copy.outputs.versioned_artifact_dir_rel}}/VERSION-*.json"
artifactContentType: application/json
token: ${{secrets.MONOREPO_RELEASE_ARTIFACT_UPLOAD_TOKEN}}
- name: Post results as internal-release
if: matrix.build_env == 'stage-prod' && needs.decide-refs.outputs.variant == 'internal-release'
uses: slackapi/[email protected]
with:
payload: "{\"s3-url\":\"${{ steps.upload-results.outputs.console_url }}/\",\"type\":\"branch\", \"reflike\":\"${{ needs.decide-refs.outputs.buildroot }}\",\"monorepo-reflike\":\"${{ needs.decide-refs.outputs.monorepo }}\",\"full-image\":\"${{ steps.upload-results.outputs.fullimage_url }}\", \"system-update\":\"${{ steps.upload-results.outputs.system_url }}\", \"version-file\":\"${{ steps.upload-results.outputs.version_file_url }}\", \"release-notes\":\"${{ steps.upload-results.outputs.release_notes_file_url }}\"}"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_INTERNAL_RELEASE }}
- name: Post results as release
if: matrix.build_env == 'stage-prod' && needs.decide-refs.outputs.variant == 'release'
uses: slackapi/[email protected]
with:
payload: "{\"s3-url\":\"${{ steps.upload-results.outputs.console_url }}/\",\"type\":\"branch\", \"reflike\":\"${{ needs.decide-refs.outputs.buildroot }}\",\"monorepo-reflike\":\"${{ needs.decide-refs.outputs.monorepo }}\",\"full-image\":\"${{ steps.upload-results.outputs.fullimage_url }}\", \"system-update\":\"${{ steps.upload-results.outputs.system_url }}\", \"version-file\":\"${{ steps.upload-results.outputs.version_file_url }}\", \"release-notes\":\"${{ steps.upload-results.outputs.release_notes_file_url }}\"}"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_RELEASE }}
- name: remove build data
if: always()
run: |
rm -rf ./*
output_check=${{steps.cache-setup.outputs.output}}
rm -rf ${output_check:-/does/not/exist}