Merge pull request #8 from beardymcjohnface/dev

DEV: Can now double-handle --profile

.github/workflows/codecov.yml

@@ -0,0 +1,69 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Codecov
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - '.github/**'
+      - 'build/**'
+      - 'tests/**'
+      - '{{cookiecutter.project_slug}}/**'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - '.github/**'
+      - 'build/**'
+      - 'tests/**'
+      - '{{cookiecutter.project_slug}}/**'
+
+permissions:
+  contents: read
+
+jobs:
+  tests:
+    name: "Codecov using python ${{ matrix.python-version }} on ${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.10"]
+
+    steps:
+      - uses: "actions/checkout@v3"
+        with:
+          fetch-depth: 0
+
+      # Setup env
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: snaketool
+          environment-file: build/environment.yaml
+          python-version: ${{ matrix.python-version }}
+          auto-activate-base: false
+
+      - name: "Setup Snaketool on ${{ matrix.os }} for Python ${{ matrix.python-version }}"
+        run: |
+          cookiecutter --no-input ./
+          cd my_snaketool/
+          python -m pip install --upgrade pip
+          pip install .
+
+      - name: "Generate coverage report on ${{ matrix.os }} for Python ${{ matrix.python-version }}"
+        run: |
+          pip install pytest pytest-cov
+          pytest --cov=./ --cov-report=xml --cov-append
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: coverage.xml
+          fail_ci_if_error: true

.github/workflows/python-app.yml

@@ -24,7 +24,7 @@ permissions:
 
 jobs:
   tests:
-    name: "Python ${{ matrix.python-version }}"
+    name: "Python ${{ matrix.python-version }} on ${{ matrix.os }}"
     runs-on: ${{ matrix.os }}
 
     defaults:
@@ -60,10 +60,3 @@ jobs:
         run: |
           pip install pytest pytest-cov
           pytest --cov=./ --cov-report=xml --cov-append
-          
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          file: coverage.xml
-          fail_ci_if_error: true

tests/test_snaketool.py

@@ -41,10 +41,11 @@ def test_snaketool_cli(tmp_dir):
     exec_command("my_snaketool -h")
     exec_command("my_snaketool run -h")
     exec_command("my_snaketool config -h")
+    exec_command("my_snaketool --version")
 
 
 def test_snaketool_commands(tmp_dir):
     """test Snaketool"""
-    exec_command("my_snaketool run --input yeet")
+    exec_command("my_snaketool run --input yeet --no-use-conda ")
     exec_command("my_snaketool config")
-    exec_command("my_snaketool citation")
+    exec_command("my_snaketool citation")

{{cookiecutter.project_slug}}/setup.py

@@ -52,7 +52,7 @@ def get_data_files():
     data_files=get_data_files(),
     py_modules=["{{cookiecutter.project_slug}}"],
     install_requires=[
-        "snaketool-utils>=0.0.2",
+        "snaketool-utils>=0.0.4",
         "snakemake{{cookiecutter.snakemake_version}}",
         "pyyaml{{cookiecutter.pyyaml_version}}",
         "Click{{cookiecutter.click_version}}",

{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/__main__.py

@@ -59,6 +59,12 @@ def common_options(func):
         click.option(
             "--threads", help="Number of threads to use", default=1, show_default=True
         ),
+        click.option(
+            "--profile",
+            default=None,
+            help="Snakemake profile to use",
+            show_default=False,
+        ),
         click.option(
             "--use-conda/--no-use-conda",
             default=True,
@@ -76,7 +82,6 @@ def common_options(func):
             "--snake-default",
             multiple=True,
             default=[
-                "--rerun-incomplete",
                 "--printshellcmds",
                 "--nolock",
                 "--show-failed-logs",
@@ -90,6 +95,11 @@ def common_options(func):
             callback=default_to_output,
             hidden=True,
         ),
+        click.option(
+            "--system-config",
+            default=snake_base(os.path.join("config", "config.yaml")),
+            hidden=True,
+        ),
         click.argument("snake_args", nargs=-1),
     ]
     for option in reversed(options):
@@ -137,31 +147,29 @@ def cli():
 )
 @click.option("--input", "_input", help="Input file/directory", type=str, required=True)
 @common_options
-def run(_input, output, log, **kwargs):
+def run(**kwargs):
     """Run {{cookiecutter.project_name}}"""
     # Config to add or update in configfile
     merge_config = {
-        "input": _input,
-        "output": output,
-        "log": log
+        "{{cookiecutter.project_slug}}": {
+            "args": kwargs
+        }
     }
 
     # run!
     run_snakemake(
         # Full path to Snakefile
         snakefile_path=snake_base(os.path.join("workflow", "Snakefile")),
-        system_config=snake_base(os.path.join("config", "config.yaml")),
         merge_config=merge_config,
-        log=log,
         **kwargs
     )
 
 
 @click.command()
 @common_options
-def config(configfile, **kwargs):
+def config(configfile, system_config, **kwargs):
     """Copy the system default config file"""
-    copy_config(configfile, system_config=snake_base(os.path.join("config", "config.yaml")))
+    copy_config(configfile, system_config=system_config)
 
 
 @click.command()

{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/config/config.yaml

@@ -1,4 +1,6 @@
-# Snakemake config
-input:
-output: '{{cookiecutter.project_slug}}.out/'
-log: '{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}.log'
+# Namespaced config file example
+{{cookiecutter.project_slug}}:
+    args: # Command line args will be (over)written here at runtime
+        input:
+        output:
+    # Add customisable config here

{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/config/system_config.yaml

@@ -0,0 +1,2 @@
+{{cookiecutter.project_slug}}:
+    # Config you don't want users to change

{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/workflow/Snakefile

@@ -1,49 +1,24 @@
-import glob
 
+# Update default config with runtime config
+configfile: os.path.join(workflow.basedir, "../", "config", "config.yaml")
+configfile: os.path.join(workflow.basedir, "../", "config", "system_config.yaml")
+config.update(config["{{cookiecutter.project_slug}}"]) # convenience if using namespaced config
 
-configfile: os.path.join(workflow.basedir, '../', 'config', 'config.yaml')
 
-
-# Concatenate Snakemake's own log file with the master log file
-def copy_log_file():
-    files = glob.glob(os.path.join(".snakemake", "log", "*.snakemake.log"))
-    if not files:
-        return None
-    current_log = max(files, key=os.path.getmtime)
-    shell("cat " + current_log + " >> " + config['log'])
-
-onsuccess:
-    copy_log_file()
-
-onerror:
-    copy_log_file()
-
-
-# Target file
-outTouch = os.path.join(config['output'], config['input'])
-
-
-# Mark target rules
-target_rules = []
-def targetRule(fn):
-    assert fn.__name__.startswith('__')
-    target_rules.append(fn.__name__[2:])
-    return fn
+# Rules files
+include: os.path.join(workflow.basedir, "rules", "preflight.smk")
+include: os.path.join(workflow.basedir, "rules", "example.smk")
 
 
+# Target rules
 @targetRule
 rule all:
     input:
-        outTouch
+        targets
 
 
 @targetRule
 rule print_targets:
     run:
         print("\nTop level rules are: \n", file=sys.stderr)
         print("* " + "\n* ".join(target_rules) + "\n\n", file=sys.stderr)
-
-
-rule yeet:
-    output:
-        touch(outTouch)

{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/workflow/envs/example.yaml

@@ -0,0 +1,5 @@
+name: python
+channels:
+    - defaults
+dependencies:
+    - python>=3.9

{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/workflow/rules/example.smk

@@ -0,0 +1,11 @@
+rule example:
+    output:
+        os.path.join(dirs["results"], "example.done")
+    conda:
+        os.path.join(dirs["envs"], "example.yaml")
+    benchmark:
+        os.path.join(dirs["bench"], "example.txt")
+    log:
+        os.path.join(dirs["logs"], "example.err")
+    script:
+        os.path.join(dirs["scripts"], "example.py")

{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/workflow/rules/preflight.smk

@@ -0,0 +1,43 @@
+
+# Directories
+dirs = {
+    "logs": os.path.join(config["args"]["output"], "logs"),
+    "bench": os.path.join(config["args"]["output"], "bench"),
+    "results": os.path.join(config["args"]["output"], "results"),
+    "envs": os.path.join(workflow.basedir, "envs"),
+    "scripts": os.path.join(workflow.basedir, "scripts")
+}
+
+
+# Targets
+targets = [
+    os.path.join(dirs["results"], "example.done")
+]
+
+
+# Misc
+target_rules = []
+
+
+def targetRule(fn):
+    """Mark rules as target rules for rule print_targets"""
+    assert fn.__name__.startswith("__")
+    target_rules.append(fn.__name__[2:])
+    return fn
+
+
+def copy_log_file():
+    """Concatenate Snakemake log to output log file"""
+    import glob
+
+    files = glob.glob(os.path.join(".snakemake", "log", "*.snakemake.log"))
+    if files:
+        current_log = max(files, key=os.path.getmtime)
+        shell("cat " + current_log + " >> " + config["args"]["log"])
+
+
+onsuccess:
+    copy_log_file()
+
+onerror:
+    copy_log_file()

{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/workflow/scripts/example.py

@@ -0,0 +1,7 @@
+
+def main(**kwargs):
+    open(kwargs["output"], "w").close()
+
+
+if __name__ == "__main__":
+    main(output=snakemake.output[0],)