Consolidate image & executable build into exe-in-image #1
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Build, test and deploy | |
on: | |
pull_request: | |
push: | |
tags: | |
- '**' | |
jobs: | |
build-executable-x64: | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout Code | |
uses: actions/checkout@v3 | |
with: | |
fetch-depth: 0 | |
submodules: true | |
- name: Get and verify tag value | |
run: | | |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV | |
./scripts/cmp_tags.sh | |
# TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/[email protected]" | |
- name: Build gProfiler executable | |
# Using DOCKER_BUILDKIT=1 although it has another cache mechanism which is not supported by satackey/[email protected] | |
# We tried to cache using buildx cache (cache-from, cache-to flags) and got strange behavior when caching, | |
# decided not to use buildkit here to get nice caches. | |
# if we enable caching again, we can disable DOCKER_BUILDKIT. | |
run: | | |
mkdir -p output | |
./scripts/build_x86_64_executable.sh | |
mv build/x86_64/gprofiler output/gprofiler_x86_64 | |
cp output/gprofiler_x86_64 output/gprofiler # for backwards compatibility | |
- name: Upload the executables as job artifacts | |
uses: actions/upload-artifact@v2 | |
with: | |
name: gprofiler | |
path: output/ | |
retention-days: 1 | |
test-executable-x64: | |
if: ${{ !startsWith(github.ref, 'refs/tags/') }} | |
runs-on: ubuntu-20.04 # the tests which use ruby/node/python etc and run at non-root privs, fail to read the files when running | |
# on ubuntu-22.04/ubuntu-latest: | |
# stderr: ruby: Permission denied -- /home/runner/work/gprofiler/gprofiler/tests/containers/ruby/fibonacci.rb (LoadError) | |
needs: build-executable-x64 | |
strategy: | |
fail-fast: false | |
matrix: | |
containers: | |
- alpine | |
- ubuntu:14.04 | |
- ubuntu:16.04 | |
- ubuntu:18.04 | |
- ubuntu:20.04 | |
- ubuntu:20.10 | |
- ubuntu:22.04 | |
- centos:6 | |
- centos:7 | |
- centos:8 | |
- debian:8 | |
- debian:9 | |
- debian:10 | |
- debian:11 | |
steps: | |
- name: Set up Python 3.8 | |
uses: actions/setup-python@v2 | |
with: | |
python-version: "3.8" | |
- name: Install Java | |
uses: actions/setup-java@v1 | |
with: | |
java-version: '8.0.275' | |
java-package: jdk | |
architecture: x64 | |
- name: Install Node.JS | |
uses: actions/setup-node@v2 | |
with: | |
# same version as used in tests/containers/nodejs/Dockerfile | |
node-version: 10.x | |
- name: Checkout Code | |
uses: actions/checkout@v3 | |
with: | |
submodules: true | |
- name: Download the executable from previous job | |
uses: actions/download-artifact@v2 | |
with: | |
name: gprofiler | |
path: dist/ | |
- name: Run gProfiler tests | |
run: | | |
mv dist/gprofiler_x86_64 dist/gprofiler | |
chmod +x dist/gprofiler | |
NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable | |
build-executable-aarch64: | |
if: startsWith(github.ref, 'refs/tags/') | |
runs-on: ["self-hosted", "ARM64"] | |
steps: | |
- name: Checkout Code | |
uses: actions/checkout@v3 | |
with: | |
fetch-depth: 0 | |
submodules: true | |
- name: Set up requirements | |
run: scripts/setup_runner_requirements.sh | |
- name: Get and verify tag value | |
run: | | |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV | |
./scripts/cmp_tags.sh | |
- name: Build gProfiler executable | |
run: | | |
mkdir -p output | |
./scripts/build_aarch64_executable.sh | |
mv build/aarch64/gprofiler output/gprofiler_aarch64 | |
- name: Upload the executables as job artifacts | |
uses: actions/upload-artifact@v2 | |
with: | |
name: gprofiler | |
path: output/ | |
retention-days: 1 | |
deploy-executable: | |
if: startsWith(github.ref, 'refs/tags/') | |
runs-on: ubuntu-20.04 | |
needs: | |
- build-executable-x64 | |
- build-executable-aarch64 | |
steps: | |
- name: Download executables from the previous job | |
uses: actions/download-artifact@v2 | |
with: | |
name: gprofiler | |
path: output/ | |
- name: Get tag value | |
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV | |
- name: Release | |
uses: softprops/action-gh-release@v1 | |
with: | |
files: | | |
output/gprofiler_x86_64 | |
output/gprofiler_aarch64 | |
output/gprofiler | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
build-container-x64: | |
runs-on: ubuntu-latest | |
needs: | |
- build-executable-x64 | |
steps: | |
- name: Checkout Code | |
uses: actions/checkout@v3 | |
with: | |
submodules: true | |
fetch-depth: 0 | |
- name: Download executables from the previous job | |
uses: actions/download-artifact@v2 | |
with: | |
name: gprofiler | |
path: output/ | |
# the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs | |
# more nicely. the tests will then be able to use the built image. | |
- name: Build gProfiler image | |
run: ./scripts/build_x86_64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler | |
- name: Export gProfiler image | |
run: mkdir -p output && docker image save gprofiler > output/gprofiler.img | |
- name: Upload the image artifact | |
uses: actions/upload-artifact@v2 | |
with: | |
name: gprofiler.img | |
path: output/ | |
retention-days: 1 | |
test-container-x64: | |
if: ${{ !startsWith(github.ref, 'refs/tags/') }} | |
needs: build-container-x64 | |
runs-on: ubuntu-20.04 # for 3.10.0 python tag | |
strategy: | |
fail-fast: false # helps detecting flakiness / errors specific to one Python version | |
matrix: | |
python-version: | |
- "3.8" | |
- "3.9" | |
- "3.10.0" # see https://github.com/Granulate/gprofiler/issues/502, we need 3.10.0, others fail PyPerf tests. | |
steps: | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v2 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: Install Java | |
uses: actions/setup-java@v1 | |
with: | |
java-version: '8.0.275' | |
java-package: jdk | |
architecture: x64 | |
- name: Install Node.JS | |
uses: actions/setup-node@v2 | |
with: | |
# same version as used in tests/containers/nodejs/Dockerfile | |
node-version: 10.x | |
- name: Checkout Code | |
uses: actions/checkout@v3 | |
with: | |
submodules: true | |
- name: Download the executable from previous job | |
uses: actions/download-artifact@v2 | |
with: | |
name: gprofiler | |
path: output/ | |
- name: Download the image from previous job | |
uses: actions/download-artifact@v2 | |
with: | |
name: gprofiler.img | |
path: output/ | |
- name: Add +x to gprofiler | |
run: chmod +x ./output/gprofiler | |
- name: Extract resources from gProfiler executable | |
run: sudo ./output/gprofiler extract-resources --resources-dest=./gprofiler/resources | |
# used in the tests | |
- name: Import gProfiler image | |
run: docker image load < output/gprofiler.img | |
# TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/[email protected]" | |
- name: Run gProfiler tests | |
run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py | |
build-container-aarch64: | |
if: startsWith(github.ref, 'refs/tags/') | |
needs: build-executable-aarch64 | |
runs-on: ["self-hosted", "ARM64"] | |
steps: | |
- name: Checkout Code | |
uses: actions/checkout@v3 | |
with: | |
fetch-depth: 0 | |
submodules: true | |
- name: Set up runner requirements | |
run: scripts/setup_runner_requirements.sh | |
- name: Get and verify tag value | |
run: ./scripts/verify_tag.sh | |
- name: Login to DockerHub | |
uses: docker/login-action@v1 | |
with: | |
username: ${{ secrets.DOCKERHUB_USERNAME }} | |
password: ${{ secrets.DOCKERHUB_TOKEN }} | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v1 | |
- name: Download executables from the previous job | |
uses: actions/download-artifact@v2 | |
with: | |
name: gprofiler | |
path: output/ | |
# TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/[email protected]" | |
- name: Build and push | |
run: | | |
set -x | |
export DOCKER_BUILDKIT=1 | |
BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" | |
LATEST_IMAGE="${{ env.GH_REPO }}:latest" | |
AARCH64_IMAGE="$BASE_IMAGE-aarch64" | |
# build & push Aarch64 | |
./scripts/build_aarch64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler-aarch64 -t "$AARCH64_IMAGE" --push | |
deploy-container: | |
if: startsWith(github.ref, 'refs/tags/') | |
runs-on: ubuntu-latest | |
needs: | |
- build-container-x64 | |
- build-container-aarch64 | |
steps: | |
# needed to set up env.GH_REPO etc | |
- name: Checkout Code | |
uses: actions/checkout@v3 | |
with: | |
fetch-depth: 0 | |
submodules: true | |
- name: Download the image from previous job | |
uses: actions/download-artifact@v2 | |
with: | |
name: gprofiler.img | |
path: output/ | |
- name: Get and verify tag value | |
run: ./scripts/verify_tag.sh | |
- name: Login to DockerHub | |
uses: docker/login-action@v1 | |
with: | |
username: ${{ secrets.DOCKERHUB_USERNAME }} | |
password: ${{ secrets.DOCKERHUB_TOKEN }} | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v1 | |
- name: Push manifest | |
run: | | |
set -x | |
export DOCKER_BUILDKIT=1 | |
BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" | |
LATEST_IMAGE="${{ env.GH_REPO }}:latest" | |
AARCH64_IMAGE="$BASE_IMAGE-aarch64" | |
X86_64_IMAGE="$BASE_IMAGE-x86_64" | |
docker pull $AARCH64_IMAGE | |
docker image load < output/gprofiler.img | |
docker tag gprofiler $X86_64_IMAGE | |
docker push $X86_64_IMAGE | |
# create manifests for the tag + for 'latest' | |
docker manifest create "$BASE_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" | |
docker manifest push "$BASE_IMAGE" | |
docker manifest create "$LATEST_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" | |
docker manifest push "$LATEST_IMAGE" |