use iac config for scaledown cron

chart-infra/templates/scale-to-zero.yaml

@@ -1,4 +1,4 @@
-{{- if (eq .Values.kubernetes.env "staging") }}
+{{- if or (eq .Values.kubernetes.env "staging") (eq .Values.myAccount.accountId "160782110667") }}
 apiVersion: batch/v1
 kind: CronJob
 metadata:

python/Dockerfile

@@ -1,13 +1,12 @@
 # pull official base image
-FROM python:3.8-buster AS builder
+FROM python:3.8-slim-buster AS builder
 
 # set working directory
 WORKDIR /src
 
 # install app dependencies
 COPY requirements.txt ./
-RUN pip3 install -r requirements.txt
-
+RUN pip3 install --no-cache-dir -r requirements.txt
 
 # ---------------------------------------------------
 # PRODUCTION BUILD

python/requirements.txt

@@ -16,7 +16,6 @@ PyYAML==5.4
 requests==2.24.0
 responses==0.12.0
 sinfo==0.3.1
-umap-learn==0.4.6
 watchdog==0.10.3
 backoff==1.10.0
 socket_io_emitter==0.1.5.1

python/src/worker/__main__.py

@@ -23,6 +23,8 @@
 def main():
     if config.IGNORE_TIMEOUT:
         info("Worker configured to ignore timeout, will run forever...")
+    else:
+        info(f"Worker timeout is {config.TIMEOUT} (s)...")
 
     while not config.EXPERIMENT_ID:
         info("Experiment not yet assigned, waiting...")

python/src/worker/config/__init__.py

@@ -2,20 +2,16 @@
 import re
 import types
 from functools import cached_property
+from .domain_specific import get_domain_specific
 
 import boto3
 import redis
 from aws_xray_sdk import core, global_sdk_config
 
+
 kube_env = os.getenv("K8S_ENV")
 cluster_env = os.getenv("CLUSTER_ENV")
-
-timeout = int(
-    os.getenv(
-        "WORK_TIMEOUT",
-        default=str(60 * 10),
-    )
-)
+timeout = get_domain_specific().get(kube_env, {}).get('timeout', 10 * 60)
 
 ignore_timeout = os.getenv("IGNORE_TIMEOUT") == "true"
 aws_account_id = os.getenv("AWS_ACCOUNT_ID")

python/src/worker/config/domain_specific.py

@@ -0,0 +1,23 @@
+import os
+
+ACCOUNT_ID = {
+  'BIOMAGE': '242905224710',
+  'HMS': '160782110667',
+}
+
+domain_specific = {
+    'HMS': {'production': {'timeout': 40 * 60}},
+    'BIOMAGE': {},
+    'BIOMAGE_PRIVATE': {}
+}
+
+
+def get_domain_specific():
+    aws_account_id = os.getenv('AWS_ACCOUNT_ID')
+
+    if aws_account_id == ACCOUNT_ID['HMS']:
+        return domain_specific['HMS']
+    elif aws_account_id == ACCOUNT_ID['BIOMAGE']:
+        return domain_specific['BIOMAGE']
+    else:
+        return domain_specific['BIOMAGE_PRIVATE']

r/Dockerfile

@@ -34,14 +34,6 @@ RUN Rscript restore_renv.R
 # deactivate so that above .libPaths used
 RUN R -q -e "renv::deactivate()"
 
-# install miniconda and python umap-learn for RunUMAP
-# clean conda
-ENV RETICULATE_MINICONDA_PATH=/src/r-miniconda
-RUN R -q -e "reticulate::install_miniconda()" && \
-    R -q -e "reticulate::conda_install(packages = 'umap-learn=0.5.3', python_version='3.10')" && \
-    CONDA_PATH=$(R -q -s -e "cat(reticulate::conda_binary())") && \
-    $CONDA_PATH clean --force-pkgs-dirs -y
-
 # use renv::snapshot() while R dependency updates are quick to build
 COPY ./renv.lock .
 RUN Rscript restore_renv.R
@@ -63,7 +55,6 @@ RUN Rscript check_package_licenses.R
 # ---------------------------------------------------
 FROM rocker/r-ver:4.2.2 AS common
 WORKDIR /src/worker
-ENV RETICULATE_MINICONDA_PATH=/src/r-miniconda
 
 # get source code and R packages
 COPY --from=builder /src /src
@@ -73,14 +64,16 @@ ENV RENV_LIB=/src/lib
 RUN echo ".libPaths(c('$RENV_LIB', .libPaths()))" >> $(R RHOME)/etc/Rprofile.site
 
 # install runtime system deps
+# python3-dev prevents 'reticulate can only bind to copies of Python built with --enable-shared'
 # cleanup setup files
-RUN echo "python3-pip" >> sysdeps_run.txt && \
+RUN echo "python3-pip python3-venv python3-dev" >> sysdeps_run.txt && \
     cat sysdeps_run.txt | xargs ./install_debian_packages.sh && \
     rm -rf *
 
-# link system libstdc++ to conda installed version
-RUN rm /usr/lib/x86_64-linux-gnu/libstdc++.so.6 && \
-    ln -s /src/r-miniconda/envs/r-reticulate/lib/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so.6
+# install umap-learn for RunUMAP
+ENV WORKON_HOME=/src/.virtualenvs
+RUN R -q -e "reticulate::virtualenv_create('r-reticulate')" && \
+    R -q -e "reticulate::virtualenv_install('r-reticulate', 'umap-learn==0.5.3', pip_options='--no-cache-dir')" 
 
 # ---------------------------------------------------
 # PRODUCTION BUILD