diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..5ec5a72 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,53 @@ +# Reusable GitHub Actions workflows + + +## Test components + +``` {yaml} +name: Test components + +on: + pull_request: + push: + branches: + - main + - master + +jobs: + test: + uses: viash-io/viash-actions/.github/workflows/test.yaml@v6 +``` + +## Build components + +``` {yaml} +name: Build components + +on: + push: + branches: + - main + - master + workflow_dispatch: + inputs: + version: + description: | + The version of the project to build. Example: `1.0.3`. + + If not provided, a development build with a version name + based on the branch name will be built. Otherwise, a release + build with the provided version will be built. + required: false + retag_image_tag: + description: | + A previously known tag that was used to build the Docker + images. When provided, these images will be retagged and + reused for the current build. This is useful when creating + bug fix releases where the Docker images are known to be + working. + required: false + +jobs: + test: + uses: viash-io/viash-actions/.github/workflows/build.yaml@v6 +``` diff --git a/.github/workflows/README.qmd b/.github/workflows/README.qmd new file mode 100644 index 0000000..b7058e1 --- /dev/null +++ b/.github/workflows/README.qmd @@ -0,0 +1,56 @@ +--- +title: Reusable GitHub Actions workflows +format: gfm +--- + +## Test components + +```{yaml} +name: Test components + +on: + pull_request: + push: + branches: + - main + - master + +jobs: + test: + uses: viash-io/viash-actions/.github/workflows/test.yaml@v6 +``` + + +## Build components + +```{yaml} +name: Build components + +on: + push: + branches: + - main + - master + workflow_dispatch: + inputs: + version: + description: | + The version of the project to build. Example: `1.0.3`. + + If not provided, a development build with a version name + based on the branch name will be built. Otherwise, a release + build with the provided version will be built. + required: false + retag_image_tag: + description: | + A previously known tag that was used to build the Docker + images. When provided, these images will be retagged and + reused for the current build. This is useful when creating + bug fix releases where the Docker images are known to be + working. + required: false + +jobs: + test: + uses: viash-io/viash-actions/.github/workflows/build.yaml@v6 +``` diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..75c9090 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,147 @@ +name: Build + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs.version }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + version: + type: string + description: | + The version of the project to build. Example: `1.0.3`. + + If not provided, a development build with a version name + based on the branch name will be built. Otherwise, a release + build with the provided version will be built. + required: false + retag_image_tag: + type: string + description: | + A previously known tag that was used to build the Docker + images. When provided, these images will be retagged and + reused for the current build. This is useful when creating + bug fix releases where the Docker images are known to be + working. + required: false + +jobs: + # phase 1 + target: + name: Build target branch + runs-on: ubuntu-latest + permissions: + contents: write + + outputs: + target_branch: ${{ steps.build-target.outputs.target_branch }} + version: ${{ steps.build-target.outputs.version }} + docker_matrix: ${{ steps.build-target.outputs.docker_matrix }} + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: 'recursive' + fetch-depth: 0 + + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 + + - name: Determine variables + id: variables + run: | + VERSION="${{ github.event.inputs.version }}" + SOURCE_BRANCH=$(echo "$GITHUB_REF" | sed 's/refs\/heads\///') + + if [[ -z $VERSION ]]; then + TARGET_BRANCH="build/$SOURCE_BRANCH" + VERSION=$(echo "$TARGET_BRANCH" | sed 's/[^a-zA-Z0-9_]/_/g') + else + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+.*$ ]]; then + echo "Version '$VERSION' does not match PEP440" + exit 1 + fi + TARGET_BRANCH="release/${VERSION%.*}.x" + fi + + echo "Set version of Viash package to '$VERSION'" + echo "version=$VERSION" >> $GITHUB_OUTPUT + + echo "Set target branch to '$TARGET_BRANCH'" + echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT + + + - uses: viash-io/viash-actions/project/build-target@v6 + id: build-target + with: + target_branch: ${{ steps.variables.outputs.target_branch }} + version: ${{ steps.variables.outputs.version }} + + - name: Deploy to target branch + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_branch: ${{ steps.variables.outputs.target_branch }} + publish_dir: . + + # phase 2 + docker: + name: Build + needs: target + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + fail-fast: false + matrix: + component: ${{ fromJson(needs.target.outputs.docker_matrix) }} + + steps: + # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' + - uses: data-intuitive/reclaim-the-bytes@v2 + + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + ref: ${{ needs.target.outputs.target_branch }} + + - uses: viash-io/viash-actions/setup@v6 + + - name: Login to container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build image + if: ${{ inputs.retag_image_tag == '' }} + run: | + ${{matrix.component.executable}} ---engine docker ---setup build ---verbose + + - name: Retag image + if: ${{ inputs.retag_image_tag != '' }} + run: | + source_image_tag="${{ inputs.retag_image_tag }}" + + # get image id from executable + # format: VIASH_DOCKER_IMAGE_ID='annotate/popv:latest' + dest_image_id=$(grep -oP "VIASH_DOCKER_IMAGE_ID='[^']+'" ${{matrix.component.executable}} | cut -d"'" -f2) + + # for viash 0.9 and later, use the following line instead + # dest_image_id=$(${{matrix.component.executable}} ---engine docker ---docker_image_id) + + # strip tag from dest_image_id and replace with source_image_tag using bash variable substitution + source_image_id="${dest_image_id%:*}:${source_image_tag}" + + echo "Retagging image '$source_image_id' to '$dest_image_id'" + docker tag "$source_image_id" "$dest_image_id" + + - name: Push image + run: | + ${{matrix.component.executable}} ---engine docker ---setup push ---verbose diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..d0f2b60 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,111 @@ +name: Test + +on: + workflow_call: + inputs: + timeout: + type: number + description: | + The maximum time in minutes that a test is allowed to run. + Default: 30. + required: false + default: 30 + num_cpus: + type: number + description: | + The number of CPUs to use for testing. + Default: 2. + required: false + default: 2 + memory: + type: string + description: | + The amount of memory to use for testing. + Default: 14gb. + required: false + default: "14gb" + +jobs: + + # phase 1 + list: + name: List components to test + runs-on: ubuntu-latest + + outputs: + matrix: ${{ steps.ns_list_filter_nextflow.outputs.output_matrix }} + cache_key: ${{ steps.cache.outputs.cache_key }} + dest_paths: ${{ steps.cache.outputs.dest_paths }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: 'recursive' + + - uses: viash-io/viash-actions/setup@v6 + + - uses: viash-io/viash-actions/project/sync-and-cache@main + id: cache + + - id: ns_list + uses: viash-io/viash-actions/ns-list@v6 + with: + format: json + runner: executable + + - id: ns_list_filter_changed + uses: viash-io/viash-actions/project/detect-changed-components@v6 + with: + input_file: "${{ steps.ns_list.outputs.output_file }}" + + - name: Filter out Nextflow scripts (testing currently not supported) + id: ns_list_filter_nextflow + run: | + OUTPUT_MATRIX=$(echo '${{ steps.ns_list_filter_changed.outputs.output_matrix }}' | jq -c 'map(select(.main_script_type != "nextflow_script"))') + echo "output_matrix=$OUTPUT_MATRIX" >> $GITHUB_OUTPUT + + # phase 2 + test: + name: Test + needs: list + if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }} + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + component: ${{ fromJson(needs.list.outputs.matrix) }} + + steps: + # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' + - name: Clear space + uses: data-intuitive/reclaim-the-bytes@v2 + + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: 'recursive' + fetch-depth: 0 + + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 + + # use cache + - name: Cache resources data + if: ${{ needs.list.outputs.cache_key != '' }} + uses: actions/cache/restore@v4 + timeout-minutes: 10 + with: + path: ${{ needs.list.outputs.dest_paths }} + key: ${{ needs.list.outputs.cache_key }} + + - name: Run test + timeout-minutes: ${{ github.event.inputs.timeout }} + env: + RUNNER_TEMP: "${{ runner.temp }}/viash_temp" + run: | + viash test \ + "${{ matrix.component.config }}" \ + --cpus ${{ github.event.inputs.num_cpus }} \ + --memory ${{ github.event.inputs.memory }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b0d990..8c8bea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,16 @@ -# viash-actions v6.3.2 +# viash-actions v6.4.0 + +## New functionality + +`.github/workflows/build.yaml`: Added a callable workflow for building Viash components (PR #38). + +`.github/workflows/test.yaml`: Added a callable workflow for testing Viash components (PR #38). ## Bug fixes * `project/detect-changed-components`: Fix realpath not being able to resolve changed files (PR #39). + # viash-actions v6.3.1 ## Bug fixes