Add basic template files

.github/build.yaml

@@ -0,0 +1,122 @@
+name: build
+
+on:
+  push:
+    branches: [ 'main' ]
+  workflow_dispatch:
+    inputs:
+      target_branch:
+        description: 'Branch to deploy to. If not specified, `build-${BRANCH_NAME}` will be used.'
+        required: false
+      version:
+        description: 'Version name to use for the build. If not specified, `build-${BRANCH_NAME}` will be used.'
+        required: false
+
+jobs:
+  # phase 1
+  list:
+    runs-on: ubuntu-latest
+
+    outputs:
+      target_branch: ${{ steps.defaults.outputs.target_branch }}
+      version: ${{ steps.defaults.outputs.version }}
+      component_matrix: ${{ steps.set_matrix.outputs.matrix }}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: 'recursive'
+
+    - uses: viash-io/viash-actions/setup@v5
+
+    - name: Determine version tag from branch name
+      id: defaults
+      run: |
+        BRANCH_NAME=$(echo $GITHUB_REF | sed 's/refs\/heads\///')
+
+        VERSION=${{ github.event.inputs.version }}
+        if [ -z "$VERSION" ]; then
+          VERSION="build-$BRANCH_NAME"
+        fi
+        echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+        TARGET_BRANCH=${{ github.event.inputs.target_branch }}
+        if [ -z "$TARGET_BRANCH" ]; then
+          TARGET_BRANCH="build-$BRANCH_NAME"
+        fi
+        echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT
+
+    - name: Remove target folder from .gitignore
+      run: |
+        # allow publishing the target folder
+        sed -i '/^\/target.*/d' .gitignore
+
+    - uses: viash-io/viash-actions/ns-build@v5
+      with:
+        config_mod: .functionality.version := '${{ steps.defaults.outputs.version }}'
+        parallel: true
+
+    - name: Deploy to target branch
+      uses: peaceiris/actions-gh-pages@v4
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: .
+        publish_branch: ${{ steps.defaults.outputs.target_branch }}
+
+    - id: ns_list
+      uses: viash-io/viash-actions/ns-list@v5
+      with:
+        platform: docker
+        src: src
+        format: json
+
+    - id: set_matrix
+      run: |
+        echo "matrix=$(jq -c '[ .[] | 
+          { 
+            "name": (.functionality.namespace + "/" + .functionality.name),
+            "dir": .info.config | capture("^(?<dir>.*\/)").dir
+          }
+        ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT
+
+  # phase 2
+  build:
+    needs: list
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        component: ${{ fromJson(needs.list.outputs.component_matrix) }}
+
+    steps:
+    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
+    - uses: data-intuitive/reclaim-the-bytes@v2
+
+    - uses: actions/checkout@v4
+
+    - uses: viash-io/viash-actions/setup@v5
+
+    - name: Build container
+      uses: viash-io/viash-actions/ns-build@v5
+      with:
+        config_mod: .functionality.version := '${{ needs.list.outputs.version }}'
+        platform: docker
+        src: ${{ matrix.component.dir }}
+        setup: build
+
+    - name: Login to container registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ secrets.GTHB_USER }}
+        password: ${{ secrets.GTHB_PAT }}
+
+    - name: Push container
+      uses: viash-io/viash-actions/ns-build@v5
+      with:
+        config_mod: .functionality.version := '${{ needs.list.outputs.version }}'
+        platform: docker
+        src: ${{ matrix.component.dir }}
+        setup: push

.github/test.yaml

@@ -0,0 +1,109 @@
+name: test
+
+on:
+  pull_request:
+  push:
+    branches: [ '**' ]
+
+jobs:
+  run_ci_check_job:
+    runs-on: ubuntu-latest
+    outputs:
+      run_ci: ${{ steps.github_cli.outputs.check }}
+    steps:
+      - name: 'Check if branch has an existing pull request and the trigger was a push'
+        id: github_cli
+        run: |
+          pull_request=$(gh pr list -R ${{ github.repository }} -H ${{ github.ref_name }} --json url --state open --limit 1 | jq '.[0].url')
+          # If the branch has a PR and this run was triggered by a push event, do not run
+          if [[ "$pull_request" != "null" && "$GITHUB_REF_NAME" != "main" && "${{ github.event_name == 'push' }}" == "true" && "${{ !contains(github.event.head_commit.message, 'ci force') }}" == "true" ]]; then
+            echo "check=false" >> $GITHUB_OUTPUT
+          else
+            echo "check=true" >> $GITHUB_OUTPUT
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GTHB_PAT }}
+
+  # phase 1
+  list:
+    needs: run_ci_check_job
+    env:
+      s3_bucket: s3://openproblems-data/resources_test/cell_cell_communication
+    runs-on: ubuntu-latest
+    if: ${{ needs.run_ci_check_job.outputs.run_ci == 'true' }}
+
+    outputs:
+      matrix: ${{ steps.set_matrix.outputs.matrix }}
+      cache_key: ${{ steps.cache.outputs.cache_key }}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        submodules: 'recursive'
+
+    - uses: viash-io/viash-actions/setup@v5
+
+    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v5
+      id: cache
+      with:
+        s3_bucket: $s3_bucket
+        dest_path: resources
+        cache_key_prefix: resources__
+
+    - id: ns_list
+      uses: viash-io/viash-actions/ns-list@v5
+      with:
+        platform: docker
+        format: json
+
+    - id: ns_list_filtered
+      uses: viash-io/viash-actions/project/detect-changed-components@v5
+      with:
+        input_file: "${{ steps.ns_list.outputs.output_file }}"
+
+    - id: set_matrix
+      run: |
+        echo "matrix=$(jq -c '[ .[] | 
+          { 
+            "name": (.functionality.namespace + "/" + .functionality.name),
+            "config": .info.config
+          }
+        ]' ${{ steps.ns_list_filtered.outputs.output_file }} )" >> $GITHUB_OUTPUT
+
+  # phase 2
+  viash_test:
+    needs: list
+    if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        component: ${{ fromJson(needs.list.outputs.matrix) }}
+
+    steps:
+    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
+    - uses: data-intuitive/reclaim-the-bytes@v2
+
+    - uses: actions/checkout@v4
+      with:
+        submodules: 'recursive'
+
+    - uses: viash-io/viash-actions/setup@v5
+
+    # use cache
+    - name: Cache resources data
+      uses: actions/cache@v4
+      timeout-minutes: 10
+      with:
+        path: resources
+        key: ${{ needs.list.outputs.cache_key }}
+
+    - name: Run test
+      timeout-minutes: 30
+      run: |
+        VIASH_TEMP=$RUNNER_TEMP/viash viash test \
+          "${{ matrix.component.config }}" \
+          --cpus 2 \
+          --memory "16gb"

.gitignore

@@ -0,0 +1,9 @@
+resources
+work
+.nextflow*
+target
+.vscode
+.DS_Store
+output
+trace-*
+.ipynb_checkpoints

INSTRUCTIONS.md

@@ -0,0 +1,9 @@
+# task-template
+This repo is a template to create a new task that has the correct files and structure needed to start a new task.
+
+
+<!-- Add to readme 
+* update _viash.yaml
+* update src/api/task_info.yaml
+* update scripts/download_resources
+-->

README.md

@@ -1,2 +1,30 @@
-# task-template
-This repo is a template to create a new task that has the correct files and structure needed to start a new task.
+# Task Template
+This repo is a template to create a new task for the OpenProblems v2. This repo contains several example files and components that can be used when updated with the task info.
+
+Check out the [instructions](INSTRUCTIONS.md) for more information on how to update the example files and components. These instructions also contain information on how to build out the task and basic commands.
+For more information on the OpenProblems v2, check out the [OpenProblems.bio](openproblems.bio) website.
+
+**Warning:** This README will be overwritten when performing the `create_task_readme` script.
+
+## Create a repository from this template
+
+**Important**  
+Before creating a new repository, make sure yoou are part of the openProblems task team. This will be done when you create an issue for the task and you got the go ahead to create the task
+
+The instructions below will guide you through creating a new repository from this template ([creating-a-repository-from-a-template](https://docs.github.com/en/repositories/creating-and-managing-repositories/creating-a-repository-from-a-template#creating-a-repository-from-a-template)).
+
+
+* Click the "Use this template" button on the top right of the repository.
+* Use the Owner dropdown menu to select the `openproblems-bio` account.
+* Type a name for your repository (task_...), and a description.
+* Set the repository visibility to public.
+* Click "Create repository from template".
+
+
+<!-- Add to readme 
+* update _viash.yaml
+* update src/api/task_info.yaml
+* update scripts/download_resources
+-->
+
+

_viash.yaml

@@ -0,0 +1,15 @@
+viash_version: 0.9.0
+
+source: src
+target: target
+
+config_mods: |
+  .functionality.version := 'dev'
+  .functionality.arguments[.multiple == true].multiple_sep := ';'
+  .platforms[.type == 'docker'].target_registry := 'ghcr.io'
+  .platforms[.type == 'docker'].target_organization := 'openproblems-bio/task-template'
+  .platforms[.type == 'docker'].target_image_source := 'https://github.com/openproblems-bio/task-template'
+  .platforms[.type == "nextflow"].directives.tag := "$id"
+  .platforms[.type == "nextflow"].auto.simplifyOutput := false
+  .platforms[.type == "nextflow"].config.labels := { lowmem : "memory = 20.Gb", midmem : "memory = 50.Gb", highmem : "memory = 100.Gb", lowcpu : "cpus = 5", midcpu : "cpus = 15", highcpu : "cpus = 30", lowtime : "time = 1.h", midtime : "time = 4.h", hightime : "time = 8.h", veryhightime : "time = 24.h" }
+  .platforms[.type == "nextflow"].config.script := "process.errorStrategy = 'ignore'"

main.nf

@@ -0,0 +1,3 @@
+workflow {
+    print("This is a dummy placeholder for pipeline execution. Please use the corresponding nf files for running pipelines.")
+}

nextflow.config

@@ -0,0 +1 @@
+process.container = 'nextflow/bash:latest'

scripts/add_a_control_method.sh

@@ -0,0 +1,8 @@
+component_name="my_control_method"
+component_lang="python" # change this to "r" if need be
+
+common/create_component/create_component \
+  --language "$component_lang" \
+  --name "$component_name" \
+  --api_file src/api/comp_control_method.yaml \
+  --output "src/control_methods/$component_name"

scripts/add_a_method.sh

@@ -0,0 +1,6 @@
+component_name="my_method"
+component_lang="python" # change this to "r" if need be
+
+common/create_component/create_component \
+  --language "$component_lang" \
+  --name "$component_name"

scripts/add_a_metric.sh

@@ -0,0 +1,8 @@
+component_name="my_metric"
+component_lang="python" # change this to "r" if need be
+
+common/create_component/create_component \
+  --language "$component_lang" \
+  --name "$component_name" \
+  --api_file src/api/comp_metric.yaml \
+  --output "src/metrics/$component_name"

scripts/create_readme.sh

@@ -0,0 +1,4 @@
+common/create_task_readme/create_task_readme \
+    --task_dir src \
+    --github_url https://github.com/openproblems-bio/task-template \
+    --output README.md

scripts/download_resources.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+echo ">> Downloading resources"
+
+viash run common/src/sync_resources/config.vsh.yaml -- \
+  --input "s3://openproblems-bio/public/neurips-2023-competition/workflow-resources/" \
+  --output "resources" \
+  --delete

src/api/comp_control_method.yaml

@@ -0,0 +1,29 @@
+namespace: control_methods
+info:
+  type: control_method
+  type_info:
+    label: Control Method
+    summary: A control method.
+    description: |
+      A control method to predict perturbation effects.
+arguments:
+  - name: --train_h5ad
+    __merge__: file_train_h5ad.yaml
+    required: false
+    direction: input
+  - name: --test_h5ad
+    __merge__: file_test_h5ad.yaml
+    required: true
+    direction: input
+  - name: --output
+    __merge__: file_prediction.yaml
+    required: true
+    direction: output
+test_resources:
+  - type: python_script
+    path: /common/src/component_tests/run_and_check_output.py
+  - type: python_script
+    path: /common/component_tests/check_method_config.py
+  - path: /common/library.bib
+  - path: #TODO: fill in e.g. /resources/denoising/pancreas
+    dest: #TODO: fill in e.g. resources/denoising/pancreas