Building and testing #345
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| run-name: Building and testing | |
| on: | |
| workflow_call: | |
| inputs: | |
| release: | |
| required: false | |
| type: boolean | |
| default: false | |
| release-to-pypi: | |
| required: false | |
| type: boolean | |
| default: false | |
| outputs: | |
| release-commit-objects: | |
| description: release-commit-objects (for importing release-commit) | |
| value: ${{ jobs.version.outputs.release-commit-objects }} | |
| release-commit-digest: | |
| description: release-commit's digest | |
| value: ${{ jobs.version.outputs.release-commit-digest }} | |
| effective-version: | |
| description: effective version used during build | |
| value: ${{ jobs.version.outputs.effective_version }} | |
| next-version: | |
| description: next version (for bump-commits) | |
| value: ${{ jobs.version.outputs.next_version }} | |
| push: | |
| jobs: | |
| params: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| ocm_repository: ${{ steps.params.outputs.ocm_repository }} | |
| oci_repository: ${{ steps.params.outputs.oci_repository }} | |
| oci_platforms: ${{ steps.params.outputs.oci_platforms }} | |
| steps: | |
| - name: params | |
| id: params | |
| run: | | |
| repo_base=europe-docker.pkg.dev/gardener-project | |
| snapshots_repo="${repo_base}/snapshots" | |
| releases_repo="${repo_base}/releases" | |
| if ${{ inputs.release || false }}; then | |
| ocm_repository=${releases_repo} | |
| oci_repository=${releases_repo} | |
| else | |
| ocm_repository=${snapshots_repo} | |
| oci_repository=${snapshots_repo} | |
| fi | |
| echo "ocm_repository=${ocm_repository}" >> "${GITHUB_OUTPUT}" | |
| echo "oci_repository=${oci_repository}" >> "${GITHUB_OUTPUT}" | |
| echo "oci_platforms=linux/amd64,linux/arm64" >> "${GITHUB_OUTPUT}" | |
| version: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| effective_version: ${{ steps.version.outputs.effective_version }} | |
| next_version: ${{ steps.version.outputs.next_version }} | |
| repo_version: ${{ steps.version.outputs.repo_version }} | |
| setuptools_version: ${{ steps.version.outputs.setuptools_version }} | |
| release-commit-objects: ${{ steps.capture-commit.outputs.commit-objects }} | |
| release-commit-digest: ${{ steps.capture-commit.outputs.commit-digest }} | |
| steps: | |
| - name: install git | |
| run: | | |
| if which git &>/dev/null; then exit 0; fi | |
| apt-get install -y git | |
| - uses: actions/checkout@v4 | |
| - name: calculate-effective-version | |
| id: version | |
| run: | | |
| src_version=$(.ci/read-version) | |
| commit=${{ github.sha }} | |
| echo "commit-digest: ${commit}" | |
| major="$(echo ${src_version} | cut -d. -f1)" | |
| minor="$(echo ${src_version} | cut -d. -f2)" | |
| patch="$(echo ${src_version} | cut -d. -f3 | cut -d- -f1)" | |
| if ${{ inputs.release || false }}; then | |
| effective_version=${major}.${minor}.${patch} | |
| setuptools_version=${effective_version} | |
| # hardcode to bumping "minor" for now | |
| next_minor=$(( $minor + 1 )) | |
| next_version="${major}.${next_minor}.${patch}-dev" | |
| echo "next_version=${next_version}" >> "${GITHUB_OUTPUT}" | |
| echo "next_version=${next_version}" | |
| else | |
| effective_version=${major}.${minor}.${patch}-${commit} | |
| setuptools_version=${src_version} | |
| fi | |
| echo "effective-version: ${effective_version}" | |
| echo "effective_version=${effective_version}" >> "${GITHUB_OUTPUT}" | |
| echo "repo_version=${src_version}" >> "${GITHUB_OUTPUT}" | |
| echo "setuptools_version=${setuptools_version}" >> "${GITHUB_OUTPUT}" | |
| - name: create commit with effective version | |
| if: ${{ inputs.release }} | |
| run: | | |
| git config --global --add safe.directory $PWD | |
| git config user.name 'Gardener-CICD Bot' | |
| git config user.email '[email protected]' | |
| echo ${{ steps.version.outputs.effective_version }} | .ci/write-version | |
| touch /tmp/timestamp-ref | |
| git add . | |
| git commit -m "Release ${{ steps.version.outputs.effective_version }}" | |
| - name: capture commit with effective version | |
| if: ${{ inputs.release }} | |
| uses: ./.github/actions/capture-commit | |
| id: capture-commit | |
| with: | |
| timestamp-reference: /tmp/timestamp-ref | |
| package: | |
| runs-on: ubuntu-latest | |
| environment: build | |
| outputs: | |
| ocm_resources: ${{ steps.package.outputs.ocm_resources }} | |
| needs: | |
| - version | |
| container: | |
| image: python:alpine | |
| steps: | |
| - name: Install git, setuptools, node (for upload-artifact) | |
| run: | | |
| apk add \ | |
| bash \ | |
| file \ | |
| git \ | |
| nodejs \ | |
| xz | |
| pip3 install --root-user-action ignore \ | |
| setuptools \ | |
| pyyaml | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 # required by upload-artifact / `act` | |
| # see: https://github.com/nektos/act/issues/973 | |
| - name: create distribution package | |
| id: package | |
| run: | | |
| set -eu | |
| version=${{ needs.version.outputs.setuptools_version }} | |
| echo "version: ${version}" | |
| echo "${version}" | .ci/write-version | |
| pkg_dir=dist | |
| mkdir -p ${pkg_dir} | |
| pkg_dir="$(readlink -f dist)" | |
| echo "pkgdir: ${pkg_dir}" | |
| for path in \ | |
| setup.py \ | |
| setup.gha.py \ | |
| setup.oci.py \ | |
| setup.ocm.py \ | |
| setup.whd.py \ | |
| ; do | |
| echo "building distribution package from ${path}" | |
| python3 ${path} \ | |
| bdist_wheel \ | |
| --dist-dir ${pkg_dir} | |
| set -x | |
| rm -rf build | |
| set +x | |
| done | |
| # special-case: cli-package (need to chdir in order to not confuse setuptools) | |
| ( | |
| cd cli | |
| python3 setup.py \ | |
| bdist_wheel \ | |
| --dist-dir ${pkg_dir} | |
| ) | |
| echo "Built packages" | |
| ls "${pkg_dir}" | |
| blobs_dir="${pkg_dir}/blobs.d" | |
| mkdir ${blobs_dir} | |
| resources_file=resources.yaml | |
| for package in \ | |
| gardener-cicd-whd \ | |
| gardener-cicd-cli \ | |
| gardener-cicd-libs \ | |
| gardener-gha-libs \ | |
| gardener-oci \ | |
| gardener-ocm \ | |
| ; do | |
| path="$(echo $package | tr - _)-*" | |
| prefix=dist | |
| access_type='localBlob' | |
| outf="${pkg_dir}/${package}.tar.gz" | |
| tar cJf ${outf} -C ${pkg_dir} $(cd ${pkg_dir}; ls ${path}) | |
| mimetype=$(file -i ${outf} | cut -d: -f2 | cut -d' ' -f2-) | |
| leng=$(stat -c"%s" ${outf}) | |
| digest="$(sha256sum ${outf} | cut -d' ' -f1)" | |
| echo "\ | |
| - name: ${package} | |
| version: ${version} | |
| type: ${mimetype} | |
| relation: local | |
| access: | |
| type: localBlob | |
| localReference: sha256:${digest} | |
| size: ${leng} | |
| mediaType: ${mimetype}" \ | |
| >> ${resources_file} | |
| mv ${outf} ${blobs_dir}/${digest} | |
| done | |
| cp "${resources_file}" dist/ocm_resources.yaml | |
| echo "ocm_resources=dist/ocm_resources.yaml" >> "${GITHUB_OUTPUT}" | |
| find "${pkg_dir}" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: dist/ | |
| component_descriptor: | |
| name: Generate + Publish OCM-Component-Descriptor | |
| runs-on: ubuntu-latest | |
| container: | |
| image: python:alpine | |
| permissions: | |
| contents: write | |
| id-token: write | |
| needs: | |
| - version | |
| - params | |
| - package | |
| - images | |
| - lint | |
| - unittests | |
| - documentation | |
| outputs: | |
| ocm_repository: ${{ steps.params.outputs.ocm_repository }} | |
| steps: | |
| - name: Install Packages | |
| run: | | |
| apk add --no-cache \ | |
| bash \ | |
| git | |
| git config --global --add safe.directory $PWD | |
| - uses: actions/checkout@v4 | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: Retrieve Linting Logs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: linting-logs # targetpath: bandit.tar.gz | |
| path: /tmp/linting-logs | |
| - name: GAR-Auth | |
| id: auth | |
| uses: ./.github/actions/gar-auth | |
| - name: component-descriptor | |
| run: | | |
| set -eu | |
| pip3 install --no-cache \ | |
| aiohttp \ | |
| dacite \ | |
| deprecated \ | |
| pyaml \ | |
| python-dateutil \ | |
| requests \ | |
| www-authenticate \ | |
| &>/dev/null | |
| python -c "import oci" | |
| version=${{ needs.version.outputs.effective_version }} | |
| ocm_repo=${{ needs.params.outputs.ocm_repository }} | |
| echo "generating component-descriptor" | |
| python3 -m ocm create \ | |
| --name github.com/${{ github.repository }} \ | |
| --version ${version} \ | |
| --ocm-repo ${ocm_repo} \ | |
| --provider sap-se \ | |
| --label '{ | |
| "name": "cloud.gardener.cnudie/responsibles", | |
| "value": [ | |
| { | |
| "type": "githubTeam", | |
| "teamname": "gardener/ci-maintainers", | |
| "github_hostname": "github.com" | |
| } | |
| ] | |
| }' \ | |
| > component-descriptor.yaml | |
| if ${{ inputs.release || false }}; then | |
| commit_digest=${{ needs.version.outputs.release-commit-digest }} | |
| else | |
| commit_digest=${{ github.sha }} | |
| fi | |
| echo "adding main source" | |
| echo "\ | |
| name: main-source | |
| version: ${version} | |
| type: git | |
| labels: | |
| - name: cloud.gardener/cicd/source | |
| value: | |
| repository-classification: main | |
| access: | |
| type: github | |
| repoUrl: github.com/${{ github.repository }} | |
| version: ${version} | |
| commit: ${commit_digest} | |
| ref: ${{ github.ref }} | |
| " \ | |
| | python3 -m ocm append source \ | |
| --file component-descriptor.yaml | |
| echo "adding resources" | |
| ocm_resources=${{ needs.package.outputs.ocm_resources }} | |
| echo "ocm-resources-file: /tmp/${ocm_resources}" | |
| cat "/tmp/${ocm_resources}" | \ | |
| python3 -m ocm append resource \ | |
| --file component-descriptor.yaml | |
| echo "adding linting-evidence resource" | |
| linting_evidence=/tmp/linting-logs/bandit.tar.gz | |
| linting_digest=$(sha256sum ${linting_evidence} | cut -d' ' -f1) | |
| cp ${linting_evidence} /tmp/dist/blobs.d/${linting_digest} | |
| cat << EOF > linting_evidence.ocm-resource | |
| name: sast-linting-evidence | |
| version: ${version} | |
| type: application/gzip | |
| relation: local | |
| access: | |
| type: localBlob | |
| localReference: sha256:${linting_digest} | |
| size: $(stat -c"%s" ${linting_evidence}) | |
| labels: | |
| - name: gardener.cloud/purposes | |
| value: | |
| - lint | |
| - sast | |
| - pybandit | |
| - name: gardener.cloud/comment | |
| value: | | |
| we use bandit (linter) for SAST scans | |
| see: https://bandit.readthedocs.io/en/latest/ | |
| EOF | |
| cat linting_evidence.ocm-resource | \ | |
| python3 -m ocm append resource \ | |
| --file component-descriptor.yaml | |
| echo "adding image-resource" | |
| echo "${{ needs.images.outputs.ocm_resources }} " | \ | |
| python3 -m ocm append resource \ | |
| --file component-descriptor.yaml | |
| echo "component-descriptor to be uploaded:" | |
| cat component-descriptor.yaml | |
| echo "uploading component-descriptor" | |
| python -m ocm upload \ | |
| --file component-descriptor.yaml \ | |
| --blobs-dir /tmp/dist/blobs.d | |
| lint: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - package | |
| container: | |
| image: python:3.12-alpine | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: lint | |
| run: | | |
| # debug | |
| which python | |
| which python3 | |
| python3 --version | |
| echo "install dependencies for python-packages" | |
| if ! apk add --no-cache $(cat gardener-cicd-libs.apk-packages) >/tmp/apk.log; then | |
| echo "error while trying to install apk-packages:" | |
| cat /tmp/apk.log | |
| exit 1 | |
| fi | |
| echo "installing linters" | |
| export CFLAGS='-Wno-int-conversion' | |
| if ! pip3 install --upgrade --break-system-packages \ | |
| --find-links /tmp/dist \ | |
| gardener-cicd-libs \ | |
| gardener-cicd-cli \ | |
| gardener-oci \ | |
| gardener-ocm \ | |
| bandit \ | |
| flake8 \ | |
| setuptools \ | |
| pylama \ | |
| pylint \ | |
| > /tmp/pip3-install.log; then | |
| echo "error while trying to install packages:" | |
| cat /tmp/pip3-install.log | |
| fi | |
| echo "running linters" | |
| bandit_logfile=bandit.log | |
| bandit_evidence=bandit.tar.gz | |
| bandit_extra_args="-f txt -o ${bandit_logfile}" \ | |
| .ci/lint | |
| # pass bandit.log + used cfg (pyproject.toml) as evidence | |
| tar czf $bandit_evidence $bandit_logfile pyproject.toml | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: linting-logs | |
| path: bandit.tar.gz | |
| unittests: | |
| needs: | |
| - package | |
| permissions: | |
| contents: read | |
| runs-on: ubuntu-latest | |
| container: | |
| image: python:alpine | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: run-tests | |
| run: | | |
| set -eu | |
| echo "install dependencies for python-packages" | |
| apk add --no-cache $(cat gardener-cicd-libs.apk-packages) | |
| echo "install packages" | |
| export CFLAGS='-Wno-int-conversion' | |
| if ! pip3 install --break-system-packages \ | |
| --find-links /tmp/dist \ | |
| gardener-cicd-libs \ | |
| gardener-cicd-cli \ | |
| gardener-oci \ | |
| gardener-ocm \ | |
| pytest \ | |
| setuptools \ | |
| > /tmp/pip3-install.log; then | |
| echo "error while trying to install packages:" | |
| cat /tmp/pip3-install.log | |
| fi | |
| pip3 list | |
| echo "running tests" | |
| mkdir /tmp/fake-cfg.d | |
| touch /tmp/fake-cfg.d/config_types.yaml | |
| export CC_CONFIG_DIR=/tmp/fake-cfg.d | |
| .ci/test | |
| images: | |
| name: Build OCI Images | |
| needs: | |
| - version | |
| - package | |
| - params | |
| outputs: | |
| oci_image_ref: ${{ steps.image-build.outputs.image_reference }} | |
| ocm_resources: ${{ steps.image-build.outputs.ocm_resource }} | |
| runs-on: ubuntu-latest | |
| environment: build | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: prepare build-filesystem | |
| id: prepare | |
| run: | | |
| cp -r /tmp/dist . | |
| ls -lta | |
| setuptools_version=${{ needs.version.outputs.setuptools_version }} | |
| # workaround: set repository-version to setuptools-version so installation of | |
| # packages will succeed | |
| echo "${setuptools_version}" | .ci/write-version | |
| - name: Build OCI Image (using ocm-oci-build-action) | |
| uses: ./.github/actions/ocm-oci-build | |
| id: image-build | |
| with: | |
| name: job-image | |
| repository: ${{ needs.params.outputs.oci_repository }}/cicd/job-image | |
| version: ${{ needs.version.outputs.effective_version }} | |
| oci_platforms: ${{ needs.params.outputs.oci_platforms }} | |
| context: . # pass modified path rather than clean checkout | |
| ocm_labels: | | |
| - name: cloud.gardener.cnudie/dso/scanning-hints/package-versions | |
| value: | |
| - name: containerd | |
| version: v1.6.15 | |
| - name: gardener.cloud/cve-categorisation | |
| value: | |
| authentication_enforced: true | |
| availability_requirement: low | |
| confidentiality_requirement: high | |
| integrity_requirement: high | |
| network_exposure: protected | |
| user_interaction: gardener-operator | |
| documentation: | |
| name: Generate Documentation | |
| needs: | |
| - images | |
| runs-on: ubuntu-latest | |
| environment: build | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Docker-Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Generate Documentation | |
| run: | | |
| image_ref=${{ needs.images.outputs.oci_image_ref }} | |
| mkdir documentation-out.d | |
| docker run -v$PWD:/src \ | |
| -e GH_PAGES_PATH=/src/documentation-out.d \ | |
| ${image_ref} \ | |
| /src/.ci/generate_documentation | |
| ls documentation-out.d | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: documentation | |
| path: documentation-out.d |