Add callable workflows (#38)

* Skip caching steps in action if no test resources are found * return empty cachekey and dest_paths if no test resources are detected * update changelog * update md * add callable workflows for building and testing Viash components * update changelog * add names * fix branch * already some minor improvements * Update CHANGELOG.md * support retagging an image * make resources configurable * clean up actions * add more info * add readme
viash-io · Jul 17, 2024 · a47da7a · a47da7a
1 parent 3fcb79a
commit a47da7a
Show file tree

Hide file tree

Showing 5 changed files with 375 additions and 1 deletion.
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
@@ -0,0 +1,53 @@
+# Reusable GitHub Actions workflows
+
+
+## Test components
+
+``` {yaml}
+name: Test components
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - master
+
+jobs:
+  test:
+    uses: viash-io/viash-actions/.github/workflows/test.yaml@v6
+```
+
+## Build components
+
+``` {yaml}
+name: Build components
+
+on:
+  push:
+    branches:
+      - main
+      - master
+  workflow_dispatch:
+    inputs:
+      version:
+        description: |
+          The version of the project to build. Example: `1.0.3`.
+
+          If not provided, a development build with a version name
+          based on the branch name will be built. Otherwise, a release
+          build with the provided version will be built.
+        required: false
+      retag_image_tag:
+        description: |
+          A previously known tag that was used to build the Docker
+          images. When provided, these images will be retagged and
+          reused for the current build. This is useful when creating
+          bug fix releases where the Docker images are known to be
+          working.
+        required: false
+
+jobs:
+  test:
+    uses: viash-io/viash-actions/.github/workflows/build.yaml@v6
+```
diff --git a/.github/workflows/README.qmd b/.github/workflows/README.qmd
@@ -0,0 +1,56 @@
+---
+title: Reusable GitHub Actions workflows
+format: gfm
+---
+
+## Test components
+
+```{yaml}
+name: Test components
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - master
+
+jobs:
+  test:
+    uses: viash-io/viash-actions/.github/workflows/test.yaml@v6
+```
+
+
+## Build components
+
+```{yaml}
+name: Build components
+
+on:
+  push:
+    branches:
+      - main
+      - master
+  workflow_dispatch:
+    inputs:
+      version:
+        description: |
+          The version of the project to build. Example: `1.0.3`.
+
+          If not provided, a development build with a version name
+          based on the branch name will be built. Otherwise, a release
+          build with the provided version will be built.
+        required: false
+      retag_image_tag:
+        description: |
+          A previously known tag that was used to build the Docker
+          images. When provided, these images will be retagged and
+          reused for the current build. This is useful when creating
+          bug fix releases where the Docker images are known to be
+          working.
+        required: false
+
+jobs:
+  test:
+    uses: viash-io/viash-actions/.github/workflows/build.yaml@v6
+```
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -0,0 +1,147 @@
+name: Build
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs.version }}
+  cancel-in-progress: true
+
+on:
+  workflow_call: 
+    inputs:
+      version:
+        type: string
+        description: |
+          The version of the project to build. Example: `1.0.3`.
+          
+          If not provided, a development build with a version name
+          based on the branch name will be built. Otherwise, a release
+          build with the provided version will be built.
+        required: false
+      retag_image_tag:
+        type: string
+        description: |
+          A previously known tag that was used to build the Docker
+          images. When provided, these images will be retagged and
+          reused for the current build. This is useful when creating
+          bug fix releases where the Docker images are known to be
+          working.
+        required: false
+
+jobs:
+  # phase 1
+  target:
+    name: Build target branch
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    outputs:
+      target_branch: ${{ steps.build-target.outputs.target_branch }}
+      version: ${{ steps.build-target.outputs.version }}
+      docker_matrix: ${{ steps.build-target.outputs.docker_matrix }}
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          fetch-depth: 0
+
+      - name: Install Viash
+        uses: viash-io/viash-actions/setup@v6
+
+      - name: Determine variables
+        id: variables
+        run: |
+          VERSION="${{ github.event.inputs.version }}"
+          SOURCE_BRANCH=$(echo "$GITHUB_REF" | sed 's/refs\/heads\///')
+
+          if [[ -z $VERSION ]]; then
+            TARGET_BRANCH="build/$SOURCE_BRANCH"
+            VERSION=$(echo "$TARGET_BRANCH" | sed 's/[^a-zA-Z0-9_]/_/g')
+          else
+            if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+.*$ ]]; then
+              echo "Version '$VERSION' does not match PEP440"
+              exit 1
+            fi
+            TARGET_BRANCH="release/${VERSION%.*}.x"
+          fi
+
+          echo "Set version of Viash package to '$VERSION'"
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+          echo "Set target branch to '$TARGET_BRANCH'"
+          echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT
+          
+
+      - uses: viash-io/viash-actions/project/build-target@v6
+        id: build-target
+        with:
+          target_branch: ${{ steps.variables.outputs.target_branch }}
+          version: ${{ steps.variables.outputs.version }}
+
+      - name: Deploy to target branch
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_branch: ${{ steps.variables.outputs.target_branch }}
+          publish_dir: .
+
+  # phase 2
+  docker:
+    name: Build
+    needs: target
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        component: ${{ fromJson(needs.target.outputs.docker_matrix) }}
+
+    steps:
+      # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
+      - uses: data-intuitive/reclaim-the-bytes@v2
+
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          ref: ${{ needs.target.outputs.target_branch }}
+
+      - uses: viash-io/viash-actions/setup@v6
+
+      - name: Login to container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build image
+        if: ${{ inputs.retag_image_tag == '' }}
+        run: |
+          ${{matrix.component.executable}} ---engine docker ---setup build ---verbose
+        
+      - name: Retag image
+        if: ${{ inputs.retag_image_tag != '' }}
+        run: |
+          source_image_tag="${{ inputs.retag_image_tag }}"
+
+          # get image id from executable
+          # format: VIASH_DOCKER_IMAGE_ID='annotate/popv:latest'
+          dest_image_id=$(grep -oP "VIASH_DOCKER_IMAGE_ID='[^']+'" ${{matrix.component.executable}} | cut -d"'" -f2)
+
+          # for viash 0.9 and later, use the following line instead
+          # dest_image_id=$(${{matrix.component.executable}} ---engine docker ---docker_image_id)
+
+          # strip tag from dest_image_id and replace with source_image_tag using bash variable substitution
+          source_image_id="${dest_image_id%:*}:${source_image_tag}"
+
+          echo "Retagging image '$source_image_id' to '$dest_image_id'"
+          docker tag "$source_image_id" "$dest_image_id"
+  
+      - name: Push image
+        run: |
+          ${{matrix.component.executable}} ---engine docker ---setup push ---verbose
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -0,0 +1,111 @@
+name: Test
+
+on:
+  workflow_call:
+    inputs:
+      timeout:
+        type: number
+        description: |
+          The maximum time in minutes that a test is allowed to run.
+          Default: 30.
+        required: false
+        default: 30
+      num_cpus:
+        type: number
+        description: |
+          The number of CPUs to use for testing.
+          Default: 2.
+        required: false
+        default: 2
+      memory:
+        type: string
+        description: |
+          The amount of memory to use for testing.
+          Default: 14gb.
+        required: false
+        default: "14gb"
+
+jobs:
+
+  # phase 1
+  list:
+    name: List components to test
+    runs-on: ubuntu-latest
+
+    outputs:
+      matrix: ${{ steps.ns_list_filter_nextflow.outputs.output_matrix }}
+      cache_key: ${{ steps.cache.outputs.cache_key }}
+      dest_paths: ${{ steps.cache.outputs.dest_paths }}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        submodules: 'recursive'
+
+    - uses: viash-io/viash-actions/setup@v6
+
+    - uses: viash-io/viash-actions/project/sync-and-cache@main
+      id: cache
+
+    - id: ns_list
+      uses: viash-io/viash-actions/ns-list@v6
+      with:
+        format: json
+        runner: executable
+
+    - id: ns_list_filter_changed
+      uses: viash-io/viash-actions/project/detect-changed-components@v6
+      with:
+        input_file: "${{ steps.ns_list.outputs.output_file }}"
+
+    - name: Filter out Nextflow scripts (testing currently not supported)
+      id: ns_list_filter_nextflow
+      run: |
+        OUTPUT_MATRIX=$(echo '${{ steps.ns_list_filter_changed.outputs.output_matrix }}' | jq -c 'map(select(.main_script_type != "nextflow_script"))')
+        echo "output_matrix=$OUTPUT_MATRIX" >> $GITHUB_OUTPUT
+
+  # phase 2
+  test:
+    name: Test
+    needs: list
+    if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        component: ${{ fromJson(needs.list.outputs.matrix) }}
+
+    steps:
+    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
+    - name: Clear space
+      uses: data-intuitive/reclaim-the-bytes@v2
+
+    - name: Check out repository
+      uses: actions/checkout@v4
+      with:
+        submodules: 'recursive'
+        fetch-depth: 0
+
+    - name: Install Viash
+      uses: viash-io/viash-actions/setup@v6
+
+    # use cache
+    - name: Cache resources data
+      if: ${{ needs.list.outputs.cache_key != '' }}
+      uses: actions/cache/restore@v4
+      timeout-minutes: 10
+      with:
+        path: ${{ needs.list.outputs.dest_paths }}
+        key: ${{ needs.list.outputs.cache_key }}
+
+    - name: Run test
+      timeout-minutes: ${{ github.event.inputs.timeout }}
+      env:
+        RUNNER_TEMP: "${{ runner.temp }}/viash_temp"
+      run: |
+        viash test \
+          "${{ matrix.component.config }}" \
+          --cpus ${{ github.event.inputs.num_cpus }} \
+          --memory ${{ github.event.inputs.memory }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,9 +1,16 @@
-# viash-actions v6.3.2
+# viash-actions v6.4.0
+
+## New functionality
+
+`.github/workflows/build.yaml`: Added a callable workflow for building Viash components (PR #38).
+
+`.github/workflows/test.yaml`: Added a callable workflow for testing Viash components (PR #38).
 
 ## Bug fixes
 
 * `project/detect-changed-components`: Fix realpath not being able to resolve changed files (PR #39).
 
+
 # viash-actions v6.3.1
 
 ## Bug fixes