feat: integration of worker "seqvars ingest" (#1189)

.github/workflows/main.yml

@@ -28,13 +28,79 @@ jobs:
           --health-retries 10
         ports:
           - 5432:5432
+
+      # We launch a minio instance for testing.  Note that we use the bitnami
+      # image because of the issue lined out in this SO discussion:
+      #
+      # - https://stackoverflow.com/questions/64031598
+      minio:
+        image: bitnami/minio:latest
+        env:
+          MINIO_ROOT_USER: minioadmin
+          MINIO_ROOT_PASSWORD: minio-root-password
+        options: >-
+          --name=minio
+          --health-cmd "curl http://localhost:9000/minio/health/live"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+        ports:
+          - 9000:9000
+
     env:
       CELERY_BROKER_URL: redis://0.0.0.0:6379/0
       DATABASE_URL: 'postgres://varfish_web:[email protected]/varfish_web'
       POSTGRES_HOST: 0.0.0.0
       POSTGRES_PORT: 5432
+      VARFISH_CASE_IMPORT_INTERNAL_STORAGE: |
+        {
+          "bucket": "varfish-server-test",
+          "host": "minio",
+          "port": 9000,
+          "access_key": "varfish-server-test",
+          "secret_key": "varfish-server-test"
+        }
 
     steps:
+      - name: Perform minio client setup
+        run: |
+          set -x
+          # create host alias for minio
+          echo "127.0.0.1 minio" | sudo tee -a /etc/hosts
+          # install minio client and configure default alias
+          wget -O /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc
+          chmod +x /usr/local/bin/mc
+          mc alias set minio/ http://minio:9000 minioadmin minio-root-password
+          # setup bucket and access key for tests
+          mc mb minio/varfish-server-test
+          mc admin user add minio varfish-server-test varfish-server-test
+          # write policy file for bucket access, add it to server, and associate with
+          # access key created above
+          cat >/tmp/policy.json <<"EOF"
+          {
+            "Version": "2012-10-17",
+            "Statement": [
+              {
+                "Action": [
+                      "s3:DeleteObject",
+                      "s3:GetBucketLocation",
+                      "s3:GetObject",
+                      "s3:ListBucket",
+                      "s3:PutObject"
+                  ],
+                "Effect": "Allow",
+                "Resource": [
+                  "arn:aws:s3:::varfish-server-test/*",
+                  "arn:aws:s3:::varfish-server-test"
+                ],
+                "Sid": "BucketAccessForUser"
+              }
+            ]
+          }
+          EOF
+          mc admin policy create minio varfish-server-test-policy /tmp/policy.json
+          mc admin policy attach minio varfish-server-test-policy --user varfish-server-test
+
       - name: Install system dependencies
         run: |
           sudo apt-get update
@@ -50,7 +116,9 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          # We need to fix the patch version here otherwise, snapshot tests
+          # with randomness will / may fail.
+          python-version: "3.10.13"
 
       - name: Install pip and Pipenv
         run: |

cases/migrations/0002_auto_20231025_1003.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.22 on 2023-10-25 10:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("cases", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="individual",
+            name="affected",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="individual",
+            name="father",
+            field=models.CharField(blank=True, max_length=128, null=True),
+        ),
+        migrations.AddField(
+            model_name="individual",
+            name="mother",
+            field=models.CharField(blank=True, max_length=128, null=True),
+        ),
+    ]

cases/models.py

@@ -1,3 +1,4 @@
+import typing
 import uuid as uuid_object
 
 from django.db import models
@@ -88,6 +89,12 @@ class Individual(models.Model):
 
     #: The name of the individual.
     name = models.CharField(max_length=128)
+    #: The name of the father, if any.
+    father = models.CharField(max_length=128, null=True, blank=True)
+    #: The name of the mother, if any.
+    mother = models.CharField(max_length=128, null=True, blank=True)
+    #: Whether the individual is assumed to be affected.
+    affected = models.BooleanField(default=False)
     #: The "sex assigned at birth" of the individual.
     sex = models.CharField(max_length=128, choices=SEX_CHOICES)
     #: The karyotypic sex.
@@ -132,3 +139,35 @@ class Disease(TermCore):
 
 class PhenotypicFeature(TermCore):
     """Phenotypic feature associated with an ``Individual``."""
+
+
+def write_pedigree_as_plink(pedigree: Pedigree, outputf: typing.TextIO, family_name="FAM"):
+    """Write a pedigree as a PLINK file.
+
+    :param pedigree: The pedigree to write.
+    :param outputf: The output file.
+    """
+    sex_map = {
+        Individual.SEX_UNKNOWN: "0",
+        Individual.SEX_MALE: "1",
+        Individual.SEX_FEMALE: "2",
+        Individual.SEX_OTHER: "0",
+    }
+    affected_map = {
+        False: "1",
+        True: "2",
+        None: "0",
+    }
+    for individual in pedigree.individual_set.order_by("name"):
+        row = [
+            family_name,
+            individual.name,
+            individual.father or "0",
+            individual.mother or "0",
+            sex_map.get(individual.sex, "0"),
+            affected_map.get(individual.affected, "0"),
+        ]
+        print(
+            "\t".join(row),
+            file=outputf,
+        )

cases/tests/factories.py

@@ -136,6 +136,9 @@ class IndividualFactory(factory.django.DjangoModelFactory):
 
     pedigree = factory.SubFactory(PedigreeFactory)
     name = factory.Sequence(lambda n: f"individual-{n}")
+    father = None
+    mother = None
+    affected = True
     sex = Individual.SEX_MALE
     karyotypic_sex = Individual.KARYOTYPE_XY
     assay = Individual.ASSAY_WES

cases/tests/snapshots/snap_test_models.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+# snapshottest: v1 - https://goo.gl/zC4yUc
+from __future__ import unicode_literals
+
+from snapshottest import Snapshot
+
+snapshots = Snapshot()
+
+snapshots[
+    "TestWritePedigreeAsPlink::testRun PLINK ped file"
+] = """FAM\tindividual-0\t0\t0\t1\t2
+"""

cases/tests/test_models.py

@@ -0,0 +1,22 @@
+import tempfile
+
+from snapshottest.unittest import TestCase as TestCaseSnapshot
+from test_plus import TestCase
+
+from cases.models import write_pedigree_as_plink
+from cases.tests.factories import IndividualFactory
+
+
+class TestWritePedigreeAsPlink(TestCaseSnapshot, TestCase):
+    def setUp(self):
+        super().setUp()
+        self.individual = IndividualFactory()
+        self.pedigree = self.individual.pedigree
+
+    def testRun(self):
+        with tempfile.TemporaryFile(mode="w+t") as tmpf:
+            write_pedigree_as_plink(self.pedigree, tmpf)
+            tmpf.flush()
+            tmpf.seek(0)
+            fcontents = tmpf.read()
+            self.assertMatchSnapshot(fcontents, "PLINK ped file")