Change to multisample data

.gitattributes

@@ -0,0 +1 @@
+*.rds.gz filter=lfs diff=lfs merge=lfs -text

README.md

@@ -29,6 +29,7 @@ To shut down the development containers, you can use:
 
 ### Prerequisites
 
+#### Remote - Containers
 Development is done inside a development container that is automatically built,
 run, and managed by Visual Studio Code. You do not need R, R Studio, or a Python
 virtual environment to be installed locally.
@@ -39,6 +40,24 @@ extension installed. Make sure you restart VS Code after installing to make sure
 loads successfully. You should see a green icon in the leftmost part of the status bar,
 which indicates that the remote container plugin has been installed.
 
+#### Git LFS
+File(s) under `data/test` are downloaded by [inframock](https://github.com/biomage-ltd/inframock), uploaded to mock S3 and used by the workers. As some of these files are over Github's file size limit (100 MB), they are stored using [Git LFS](https://git-lfs.github.com/). Follow the installation instructions on their website to setup Git LFS locally.
+
+Once you have installed Git LFS, you can open the worker root directory in a terminal and run 
+
+```
+git lfs install
+```
+
+If Git LFS is installed successfully, it should print
+
+```
+Updated git hooks.
+Git LFS initialized.
+```
+
+You can see the list of files tracked by Git LFS in `.gitattributes`.
+
 ### Setup
 
 To open the R workspace, you can type `code r/r.workspace` while in the terminal inside

docker-compose.linux-dev.yaml

@@ -0,0 +1,45 @@
+version: "3.2"
+services:
+  r:
+    container_name: "biomage-worker-r"
+    build:
+      context: r/
+      target: dev
+      args:
+        - GITHUB_PAT=${GITHUB_PAT}
+    volumes:
+      - ./r:/r:cached
+      - ./data:/data:cached
+    environment:
+      - EXPERIMENT_ID=e52b39624588791a7889e39c617f669e
+    expose:
+      - "4000"
+    ports:
+      - "4000:4000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+  pipeline-server:
+    container_name: "biomage-pipeline-server"
+    build:
+      context: pipeline-server/
+      target: dev
+    volumes:
+      - ./pipeline-server:/pipeline-server:cached
+    expose:
+      - "6969"
+    ports:
+      - "6969:6969"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+  python:
+    container_name: "biomage-worker-python"
+    build:
+      context: python/
+      target: dev
+    volumes:
+      - ./python:/python:cached
+      - ./data:/data:cached
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    environment:
+      - EXPERIMENT_ID=e52b39624588791a7889e39c617f669e

docker-compose.yml

@@ -7,14 +7,15 @@ services:
       target: dev
       args:
         - GITHUB_PAT=${GITHUB_PAT}
-        - EXPERIMENT_ID=5928a56c7cbff9de78974ab50765ed20
     volumes:
       - ./r:/r:cached
       - ./data:/data:cached
     expose:
       - "4000"
     ports:
       - "4000:4000"
+    environment:
+      - EXPERIMENT_ID=e52b39624588791a7889e39c617f669e
   python:
     container_name: "biomage-worker-python"
     build:
@@ -23,3 +24,5 @@ services:
     volumes:
       - ./python:/python:cached
       - ./data:/data:cached
+    environment:
+      - EXPERIMENT_ID=e52b39624588791a7889e39c617f669e

python/src/config/__init__.py

@@ -14,7 +14,7 @@ def get_config():
 
     aws_account_id = os.getenv("AWS_ACCOUNT_ID", default="242905224710")
     aws_region = os.getenv("AWS_DEFAULT_REGION", default="eu-west-1")
-    experiment_id = os.getenv("EXPERIMENT_ID", default="5928a56c7cbff9de78974ab50765ed20")
+    experiment_id = os.getenv("EXPERIMENT_ID", default="e52b39624588791a7889e39c617f669e")
 
     # set up cluster env based on gitlab env if one was not specified
     # this is only run if `kube_env` is specified, i.e. when the system

python/src/tests/tasks/test_cluster_cells.py

@@ -19,7 +19,7 @@ def open_test_adata(self):
     @pytest.fixture(autouse=True)
     def load_correct_definition(self):
         self.correct_request = {
-            "experimentId": "5928a56c7cbff9de78974ab50765ed20",
+            "experimentId": "e52b39624588791a7889e39c617f669e",
             "timeout": "2099-12-31 00:00:00",
             "body": {
                 "name": "ClusterCells",
@@ -51,7 +51,7 @@ def test_louvain_clustering_works(self):
 
     def test_leiden_clustering_works(self):
         alternative_request = {
-            "experimentId": "5928a56c7cbff9de78974ab50765ed20",
+            "experimentId": "e52b39624588791a7889e39c617f669e",
             "timeout": "2099-12-31 00:00:00",
             "body": {
                 "name": "ClusterCells",

python/src/tests/tasks/test_differential_expression.py

@@ -128,7 +128,7 @@ def get_request(
         self, cellSet="cluster1", compareWith="rest", basis="all", maxNum=None
     ):
         request = {
-            "experimentId": "5928a56c7cbff9de78974ab50765ed20",
+            "experimentId": "e52b39624588791a7889e39c617f669e",
             "timeout": "2099-12-31 00:00:00",
             "body": {
                 "name": "DifferentialExpression",
@@ -258,17 +258,6 @@ def test_appropriate_genes_returned_when_a_limit_is_specified(
 
         assert len(res) <= request["body"]["maxNum"]
 
-    @responses.activate
-    def test_all_genes_returned_when_no_limit_is_specified(self, mock_dynamo_get):
-        m, dynamodb = mock_dynamo_get
-        m.return_value = dynamodb
-
-        res = DifferentialExpression(self.get_request(), self._adata).compute()
-        res = res[0].result
-        res = json.loads(res)["rows"]
-
-        assert len(res) <= len(self._adata.raw.var.index)
-
     @responses.activate
     def test_cells_in_sets_intersection_are_filtered_out(self, mock_dynamo_get):
         m, dynamodb = mock_dynamo_get

python/src/tests/tasks/test_gene_expression.py

@@ -20,7 +20,7 @@ def open_test_adata(self):
     @pytest.fixture(autouse=True)
     def load_correct_definition(self):
         self.correct_request = {
-            "experimentId": "5928a56c7cbff9de78974ab50765ed20",
+            "experimentId": "e52b39624588791a7889e39c617f669e",
             "timeout": "2099-12-31 00:00:00",
             "body": {
                 "name": "GeneExpression",

r/Dockerfile

@@ -50,8 +50,5 @@ FROM builder AS dev
 # when source files change
 RUN pip install -U jedi radian PyYAML watchdog[watchmedo]
 
-ARG EXPERIMENT_ID
-ENV EXPERIMENT_ID $EXPERIMENT_ID
-
 WORKDIR /r/src
 CMD watchmedo auto-restart --directory=. --pattern=* --recursive -- Rscript work.r