Merge pull request #1 from openproblems-bio/bugfix/no-ref/fix-test

Update docker images

.github/workflows/build.yaml

@@ -1,122 +1,21 @@
-name: build
+name: Build
 
 on:
   push:
     branches: [ 'main' ]
   workflow_dispatch:
     inputs:
-      target_branch:
-        description: 'Branch to deploy to. If not specified, `build-${BRANCH_NAME}` will be used.'
-        required: false
       version:
-        description: 'Version name to use for the build. If not specified, `build-${BRANCH_NAME}` will be used.'
+        description: |
+          The version of the project to build. Example: `1.0.3`.
+          
+          If not provided, a development build with a version name
+          based on the branch name will be built. Otherwise, a release
+          build with the provided version will be built.
         required: false
 
 jobs:
-  # phase 1
-  list:
-    runs-on: ubuntu-latest
-
-    outputs:
-      target_branch: ${{ steps.defaults.outputs.target_branch }}
-      version: ${{ steps.defaults.outputs.version }}
-      component_matrix: ${{ steps.set_matrix.outputs.matrix }}
-
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: 'recursive'
-
-    - uses: viash-io/viash-actions/setup@v5
-
-    - name: Determine version tag from branch name
-      id: defaults
-      run: |
-        BRANCH_NAME=$(echo $GITHUB_REF | sed 's/refs\/heads\///')
-
-        VERSION=${{ github.event.inputs.version }}
-        if [ -z "$VERSION" ]; then
-          VERSION="build-$BRANCH_NAME"
-        fi
-        echo "version=$VERSION" >> $GITHUB_OUTPUT
-
-        TARGET_BRANCH=${{ github.event.inputs.target_branch }}
-        if [ -z "$TARGET_BRANCH" ]; then
-          TARGET_BRANCH="build-$BRANCH_NAME"
-        fi
-        echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT
-
-    - name: Remove target folder from .gitignore
-      run: |
-        # allow publishing the target folder
-        sed -i '/^\/target.*/d' .gitignore
-
-    - uses: viash-io/viash-actions/ns-build@v5
-      with:
-        config_mod: .version := '${{ steps.defaults.outputs.version }}'
-        parallel: true
-
-    - name: Deploy to target branch
-      uses: peaceiris/actions-gh-pages@v4
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        publish_dir: .
-        publish_branch: ${{ steps.defaults.outputs.target_branch }}
-
-    - id: ns_list
-      uses: viash-io/viash-actions/ns-list@v5
-      with:
-        platform: docker
-        src: src
-        format: json
-
-    - id: set_matrix
-      run: |
-        echo "matrix=$(jq -c '[ .[] | 
-          { 
-            "name": (.namespace + "/" + .name),
-            "dir": .info.config | capture("^(?<dir>.*\/)").dir
-          }
-        ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT
-
-  # phase 2
   build:
-    needs: list
-
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        component: ${{ fromJson(needs.list.outputs.component_matrix) }}
-
-    steps:
-    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
-    - uses: data-intuitive/reclaim-the-bytes@v2
-
-    - uses: actions/checkout@v4
-
-    - uses: viash-io/viash-actions/setup@v5
-
-    - name: Build container
-      uses: viash-io/viash-actions/ns-build@v5
-      with:
-        config_mod: .version := '${{ needs.list.outputs.version }}'
-        platform: docker
-        src: ${{ matrix.component.dir }}
-        setup: build
-
-    - name: Login to container registry
-      uses: docker/login-action@v3
-      with:
-        registry: ghcr.io
-        username: ${{ secrets.GTHB_USER }}
-        password: ${{ secrets.GTHB_PAT }}
-
-    - name: Push container
-      uses: viash-io/viash-actions/ns-build@v5
-      with:
-        config_mod: .version := '${{ needs.list.outputs.version }}'
-        platform: docker
-        src: ${{ matrix.component.dir }}
-        setup: push
+    uses: openproblems-bio/actions/.github/workflows/build.yml@main
+    with:
+      version: ${{ github.event.inputs.version }}

.github/workflows/test.yaml

@@ -1,113 +1,9 @@
-name: test
+name: Test
 
 on:
-  pull_request:
   push:
-    branches: [ '**' ]
+  pull_request:
 
 jobs:
-  run_ci_check_job:
-    runs-on: ubuntu-latest
-    outputs:
-      run_ci: ${{ steps.github_cli.outputs.check }}
-    steps:
-      - name: 'Check if branch has an existing pull request and the trigger was a push'
-        id: github_cli
-        run: |
-          pull_request=$(gh pr list -R ${{ github.repository }} -H ${{ github.ref_name }} --json url --state open --limit 1 | jq '.[0].url')
-          # If the branch has a PR and this run was triggered by a push event, do not run
-          if [[ "$pull_request" != "null" && "$GITHUB_REF_NAME" != "main" && "${{ github.event_name == 'push' }}" == "true" && "${{ !contains(github.event.head_commit.message, 'ci force') }}" == "true" ]]; then
-            echo "check=false" >> $GITHUB_OUTPUT
-          else
-            echo "check=true" >> $GITHUB_OUTPUT
-          fi
-        env:
-          GITHUB_TOKEN: ${{ secrets.GTHB_PAT }}
-
-  # phase 1
-  list:
-    needs: run_ci_check_job
-    env:
-      s3_bucket: s3://openproblems-data/resources_test
-    runs-on: ubuntu-latest
-    if: ${{ needs.run_ci_check_job.outputs.run_ci == 'true' }}
-
-    outputs:
-      matrix: ${{ steps.set_matrix.outputs.matrix }}
-      cache_key: ${{ steps.cache.outputs.cache_key }}
-
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        submodules: 'recursive'
-
-    - uses: christian-ci/action-yaml-github-output@v2
-      with:
-        file_path: _viash.yaml
-
-    - uses: viash-io/viash-actions/setup@v5
-
-    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v5
-      id: cache
-      with:
-        s3_bucket: $s3_bucket/$NAME
-        dest_path: resources
-        cache_key_prefix: resources__
-
-    - id: ns_list
-      uses: viash-io/viash-actions/ns-list@v5
-      with:
-        engine: docker
-        format: json
-
-    - id: ns_list_filtered
-      uses: viash-io/viash-actions/project/detect-changed-components@v5
-      with:
-        input_file: "${{ steps.ns_list.outputs.output_file }}"
-
-    - id: set_matrix
-      run: |
-        echo "matrix=$(jq -c '[ .[] | 
-          { 
-            "name": (.namespace + "/" + .name),
-            "config": .info.config
-          }
-        ]' ${{ steps.ns_list_filtered.outputs.output_file }} )" >> $GITHUB_OUTPUT
-
-  # phase 2
-  viash_test:
-    needs: list
-    if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }}
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        component: ${{ fromJson(needs.list.outputs.matrix) }}
-
-    steps:
-    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
-    - uses: data-intuitive/reclaim-the-bytes@v2
-
-    - uses: actions/checkout@v4
-      with:
-        submodules: 'recursive'
-
-    - uses: viash-io/viash-actions/setup@v5
-
-    # use cache
-    - name: Cache resources data
-      uses: actions/cache@v4
-      timeout-minutes: 10
-      with:
-        path: resources
-        key: ${{ needs.list.outputs.cache_key }}
-
-    - name: Run test
-      timeout-minutes: 30
-      run: |
-        VIASH_TEMP=$RUNNER_TEMP/viash viash test \
-          "${{ matrix.component.config }}" \
-          --cpus 2 \
-          --memory "16gb"
+  build:
+    uses: openproblems-bio/actions/.github/workflows/test.yml@main

_viash.yaml

@@ -10,6 +10,16 @@ links:
   repository: https://github.com/openproblems-bio/task_denoising
   docker_registry: ghcr.io/openproblems-bio
 
+info:
+  test_resources:
+    - type: s3
+      path: s3://openproblems-data/resources_test/denoising/
+      dest: resources_test/denoising
+    - type: s3
+      path: s3://openproblems-data/resources_test/common/
+      dest: resources_test/common
+
+
 version: dev
 
 config_mods: |

scripts/run_benchmark.sh

@@ -6,7 +6,7 @@ publish_dir="s3://openproblems-data/resources/denoising/results/${RUN_ID}"
 # make sure only log_cp10k is used
 cat > /tmp/params.yaml << HERE
 input_states: s3://openproblems-data/resources/denoising/datasets/**/log_cp10k/state.yaml
-rename_keys: 'input_train:output_train,input_test:output_test'
+rename_keys: 'input_train:output_train;input_test:output_test'
 output_state: "state.yaml"
 publish_dir: "$publish_dir"
 HERE

scripts/run_benchmark_test.sh

@@ -2,7 +2,7 @@
 
 cat > /tmp/params.yaml << 'HERE'
 input_states: s3://openproblems-data/resources_test/denoising/**/state.yaml
-rename_keys: 'input_train:output_train,input_test:output_test'
+rename_keys: 'input_train:output_train;input_test:output_test'
 output_state: "state.yaml"
 publish_dir: s3://openproblems-nextflow/temp/denoising/
 HERE

src/api/comp_process_dataset.yaml

@@ -1,4 +1,4 @@
-namespace: ""
+namespace: "process_dataset"
 info:
   type: process_dataset
   type_info:

src/control_methods/no_denoising/config.vsh.yaml

@@ -16,7 +16,7 @@ resources:
 
 engines:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: ghcr.io/openproblems-bio/base_images/python:1.1.0
 
 runners:
   - type: executable

src/control_methods/perfect_denoising/config.vsh.yaml

@@ -17,7 +17,7 @@ resources:
 
 engines:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: ghcr.io/openproblems-bio/base_images/python:1.1.0
 
 runners:
   - type: executable

src/methods/alra/config.vsh.yaml

@@ -33,7 +33,7 @@ resources:
     path: script.R
 engines:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: ghcr.io/openproblems-bio/base_images/r:1.1.0
     setup:
       - type: r
         cran: [ Matrix, rsvd ]

src/methods/knn_smoothing/config.vsh.yaml

@@ -30,7 +30,7 @@ resources:
 
 engines:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: ghcr.io/openproblems-bio/base_images/python:1.1.0
     setup:
       - type: python
         packages:

src/methods/magic/config.vsh.yaml

@@ -53,7 +53,7 @@ resources:
     path: script.py
 engines:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: ghcr.io/openproblems-bio/base_images/python:1.1.0
     setup:
       - type: python
         pip: [scprep, magic-impute, scipy, scikit-learn<1.2]

src/methods/saver/config.vsh.yaml

@@ -23,7 +23,7 @@ resources:
     path: script.R
 engines:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: ghcr.io/openproblems-bio/base_images/r:1.1.0
     setup:
       - type: r
         github: mohuangx/SAVER

src/metrics/mse/config.vsh.yaml

@@ -18,10 +18,10 @@ resources:
     path: script.py
 engines:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: ghcr.io/openproblems-bio/base_images/python:1.1.0
     setup:
       - type: python
-        packages:
+        pypi:
           - scikit-learn
           - scprep
 runners:

src/metrics/mse/script.py

@@ -6,7 +6,7 @@
 ## VIASH START
 par = {
     'input_test': 'resources_test/denoising/pancreas/test.h5ad',
-    'input_denoised': 'resources_test/denoising/pancreas/magic.h5ad',
+    'input_prediction': 'resources_test/denoising/pancreas/denoised.h5ad',
     'output': 'output_mse.h5ad'
 }
 meta = {
@@ -15,7 +15,7 @@
 ## VIASH END
 
 print("Load data", flush=True)
-input_denoised = ad.read_h5ad(par['input_denoised'], backed="r")
+input_denoised = ad.read_h5ad(par['input_prediction'], backed="r")
 input_test = ad.read_h5ad(par['input_test'], backed="r")
 
 test_data = ad.AnnData(X=input_test.layers["counts"], dtype="float")
@@ -26,7 +26,7 @@
 # scaling and transformation
 target_sum = 10000
 
-sc.pp.normalize_total(test_data, target_sum)
+sc.pp.normalize_total(test_data, target_sum=target_sum)
 sc.pp.log1p(test_data)
 
 sc.pp.normalize_total(denoised_data, target_sum)