♻️ Migrate to docker stack config

.env-devel

@@ -25,6 +25,7 @@ AUTOSCALING_EC2_ACCESS=null
 AUTOSCALING_EC2_INSTANCES=null
 AUTOSCALING_NODES_MONITORING=null
 AUTOSCALING_POLL_INTERVAL=10
+AUTOSCALING_LOGLEVEL=DEBUG
 
 
 BF_API_KEY=none
@@ -43,13 +44,14 @@ CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null
 CLUSTERS_KEEPER_TASK_INTERVAL=30
 CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null
 CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX=""
-
+CLUSTERS_KEEPER_LOGLEVEL=INFO
 
 DASK_SCHEDULER_HOST=dask-scheduler
 DASK_SCHEDULER_PORT=8786
 DASK_TLS_CA_FILE=/home/scu/.dask/dask-crt.pem
 DASK_TLS_KEY=/home/scu/.dask/dask-key.pem
 DASK_TLS_CERT=/home/scu/.dask/dask-crt.pem
+CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"${DASK_TLS_CERT}","tls_client_cert":"${DASK_TLS_CERT}","tls_client_key":"${DASK_TLS_KEY}"}'
 
 DIRECTOR_REGISTRY_CACHING_TTL=900
 DIRECTOR_REGISTRY_CACHING=True
@@ -143,12 +145,6 @@ S3_ENDPOINT=172.17.0.1:9001
 S3_SECRET_KEY=12345678
 S3_SECURE=0
 
-S3_EXPORT_ACCESS_KEY=87654321
-S3_EXPORT_BUCKET_NAME=simcore-export
-S3_EXPORT_ENDPOINT=172.17.0.1:9001
-S3_EXPORT_SECRET_KEY=87654321
-S3_EXPORT_SECURE=0
-
 SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1
 SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key
 

.pre-commit-config.yaml

@@ -52,3 +52,10 @@ repos:
       language: script
       types: [file, python]
       entry: scripts/precommit/pytest-testit.bash
+  - repo: local
+    hooks:
+    - id: validate-docker-compose
+      name: validate-docker-compose
+      language: script
+      types: [file]
+      entry: scripts/precommit/validate-docker-compose.bash

scripts/docker/docker-compose-config.bash

@@ -61,55 +61,30 @@ if docker compose version --short | grep --quiet "^2\." ; then
   if [[ -z "$version" ]]; then
     version="3.9"  # Default to 3.9 if version is not found in file
   fi
-
+# shellcheck disable=SC2002
   docker_command="\
-docker \
---log-level=ERROR \
-compose \
---env-file ${env_file}"
-
-  if [ "$project_directory" ]; then
-    docker_command+=" --project-directory ${project_directory}"
-  fi
+set -o allexport && \
+. ${env_file} && set +o allexport && \
+docker stack config"
 
   for compose_file_path in "$@"
   do
-    docker_command+=" --file=${compose_file_path}"
+    docker_command+=" --compose-file ${compose_file_path}"
   done
-  docker_command+="\
- config \
+  # WE CANNOT DO THIS:
+  # docker_command+=" --skip-interpolation"
+  # because docker stack compose will *validate* that e.g. `replicas: ${SIMCORE_SERVICES_POSTGRES_REPLICAS}` is a valid number, which it is not if it is read as a literal string.
+
+  docker_command+=" \
 | sed '/published:/s/\"//g' \
 | sed '/size:/s/\"//g' \
-| sed '1 { /name:.*/d ; }' \
-| sed '1 i version: \"${version}\"' \
-| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'"
+| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/' \
+| sed 's/\\\$/\\\$\\\$/g'" # mitigates https://stackoverflow.com/questions/40619582/how-can-i-escape-a-dollar-sign-in-a-docker-compose-file
 
   # Execute the command
   show_info "Executing Docker command: ${docker_command}"
-  eval ${docker_command}
+  eval "${docker_command}"
 else
-  show_warning "docker compose V2 is not available, trying V1 instead... please update your docker engine."
-  if docker-compose version --short | grep --quiet "^1\." ; then
-    show_info "Running compose V1"
-    docker_command="\
-docker-compose \
---log-level=ERROR \
---env-file ${env_file}"
-    for compose_file_path in "$@"
-    do
-      docker_command+=" --file=${compose_file_path} "
-    done
-    if [ "$project_directory" ]; then
-      docker_command+=" --project-directory ${project_directory}"
-    fi
-    docker_command+=" \
-config \
-| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'"
-    # Execute the command
-    show_info "Executing Docker command: ${docker_command}"
-    eval ${docker_command}
-  else
-    show_error "docker-compose V1 is not available. It is impossible to run this script!"
-    exit 1
-  fi
+  show_warning "docker compose V2 is not available... please update your docker engine."
+  exit 1
 fi

scripts/precommit/validate-docker-compose.bash

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Get a list of all staged files
+staged_files=$(git diff --cached --name-only --diff-filter=ACM)
+
+# Loop through each file
+for file in $staged_files
+do
+  # Check if the file name contains "docker-compose" and is a .yml or .yaml file
+  if [[ $file == *docker-compose*.yml || $file == *docker-compose*.yaml ]]; then
+    echo "Checking $file" 1>&2
+    # Check the file for lines with more than one dollar sign
+    if grep -n -P '\$\{[^}]*\$\{[^}]*\}[^}]*\}' "$file"; then
+      echo "Error: $file contains a line with more than one dollar sign."
+      exit 1
+    elif grep -n -P '\$[a-zA-Z_][a-zA-Z0-9_]*' "$file"; then
+      echo "Error: $file contains a line with an environment variable not wrapped in curly braces."
+      exit 1
+    fi
+  fi
+done
+
+
+# If no errors were found, allow the commit
+exit 0

services/catalog/docker-compose-extra.yml

@@ -60,7 +60,7 @@ services:
       - S3_SECRET_KEY=${S3_SECRET_KEY}
       - S3_BUCKET_NAME=${S3_BUCKET_NAME}
       - STORAGE_ENDPOINT=${STORAGE_ENDPOINT}
-      - EXTRA_HOSTS_SUFFIX=${EXTRA_HOSTS_SUFFIX:-undefined}
+      - EXTRA_HOSTS_SUFFIX=undefined
       - SIMCORE_SERVICES_NETWORK_NAME=interactive_services_subnet
       - MONITORING_ENABLED=${MONITORING_ENABLED:-True}
       - TRACING_ENABLED=${TRACING_ENABLED:-True}

services/docker-compose.yml

@@ -23,7 +23,7 @@ services:
       - DIRECTOR_V2_HOST=${DIRECTOR_V2_HOST}
       - DIRECTOR_V2_PORT=${DIRECTOR_V2_PORT}
 
-      - API_SERVER_LOGLEVEL=${API_SERVER_LOGLEVEL:-${LOG_LEVEL:-WARNING}}
+      - API_SERVER_LOGLEVEL=${API_SERVER_LOGLEVEL:-WARNING}
 
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_HOST=${POSTGRES_HOST}
@@ -69,7 +69,7 @@ services:
     networks:
       - autoscaling_subnet
     environment:
-      - AUTOSCALING_LOGLEVEL=${AUTOSCALING_LOGLEVEL:-${LOG_LEVEL:-WARNING}}
+      - AUTOSCALING_LOGLEVEL=${AUTOSCALING_LOGLEVEL}
       - AUTOSCALING_POLL_INTERVAL=${AUTOSCALING_POLL_INTERVAL}
       - AUTOSCALING_DRAIN_NODES_WITH_LABELS=${AUTOSCALING_DRAIN_NODES_WITH_LABELS}
 
@@ -92,7 +92,7 @@ services:
       - EC2_INSTANCES_KEY_NAME=${EC2_INSTANCES_KEY_NAME}
       - EC2_INSTANCES_CUSTOM_TAGS=${EC2_INSTANCES_CUSTOM_TAGS}
 
-      - AUTOSCALING_NODES_MONITORING=${AUTOSCALING_NODES_MONITORING} # dyn autoscaling
+      - AUTOSCALING_NODES_MONITORING=${AUTOSCALING_NODES_MONITORING} # dyn autoscaling envvar
       - NODES_MONITORING_NODE_LABELS=${NODES_MONITORING_NODE_LABELS}
       - NODES_MONITORING_SERVICE_LABELS=${NODES_MONITORING_SERVICE_LABELS}
       - NODES_MONITORING_NEW_NODES_LABELS=${NODES_MONITORING_NEW_NODES_LABELS}
@@ -154,12 +154,12 @@ services:
       - default
     environment:
       - CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=${CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG}
-      - CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH=${CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH:-{"type":"tls","tls_ca_file":"${DASK_TLS_CERT}","tls_client_cert":"${DASK_TLS_CERT}","tls_client_key":"${DASK_TLS_KEY}"}}
+      - CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH=${CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH}
       - CLUSTERS_KEEPER_DASK_NTHREADS=${CLUSTERS_KEEPER_DASK_NTHREADS}
       - CLUSTERS_KEEPER_DASK_WORKER_SATURATION=${CLUSTERS_KEEPER_DASK_WORKER_SATURATION}
       - CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=${CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION}
       - CLUSTERS_KEEPER_TASK_INTERVAL=${CLUSTERS_KEEPER_TASK_INTERVAL}
-      - CLUSTERS_KEEPER_LOGLEVEL=${CLUSTERS_KEEPER_LOGLEVEL:-${LOG_LEVEL:-WARNING}}
+      - CLUSTERS_KEEPER_LOGLEVEL=${CLUSTERS_KEEPER_LOGLEVEL}
       - CLUSTERS_KEEPER_EC2_ACCESS=${CLUSTERS_KEEPER_EC2_ACCESS}
       - CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID=${CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID}
       - CLUSTERS_KEEPER_EC2_ENDPOINT=${CLUSTERS_KEEPER_EC2_ENDPOINT}
@@ -201,16 +201,16 @@ services:
     init: true
     hostname: "{{.Node.Hostname}}-{{.Task.Slot}}"
     environment:
-      - DEFAULT_MAX_MEMORY=${DEFAULT_MAX_MEMORY:-0}
-      - DEFAULT_MAX_NANO_CPUS=${DEFAULT_MAX_NANO_CPUS:-0}
+      - DEFAULT_MAX_MEMORY=${DIRECTOR_DEFAULT_MAX_MEMORY:-0}
+      - DEFAULT_MAX_NANO_CPUS=${DIRECTOR_DEFAULT_MAX_NANO_CPUS:-0}
       - DIRECTOR_REGISTRY_CACHING_TTL=${DIRECTOR_REGISTRY_CACHING_TTL}
       - DIRECTOR_REGISTRY_CACHING=${DIRECTOR_REGISTRY_CACHING}
       - DIRECTOR_SELF_SIGNED_SSL_FILENAME=${DIRECTOR_SELF_SIGNED_SSL_FILENAME}
       - DIRECTOR_SELF_SIGNED_SSL_SECRET_ID=${DIRECTOR_SELF_SIGNED_SSL_SECRET_ID}
       - DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME=${DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME}
       - DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=${DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS}
       - DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS=${DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS}
-      - EXTRA_HOSTS_SUFFIX=${EXTRA_HOSTS_SUFFIX:-undefined}
+      - EXTRA_HOSTS_SUFFIX=undefined
       - LOGLEVEL=${LOG_LEVEL:-WARNING}
       - MONITORING_ENABLED=${MONITORING_ENABLED:-True}
       - PUBLISHED_HOST_NAME=${MACHINE_FQDN}
@@ -256,7 +256,7 @@ services:
 
       - COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE=${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE}
       - COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL=${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL}
-      - COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH=${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH:-{"type":"tls","tls_ca_file":"${DASK_TLS_CERT}","tls_client_cert":"${DASK_TLS_CERT}","tls_client_key":"${DASK_TLS_KEY}"}}
+      - COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH=${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH}
       - COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE=${COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE}
       - COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE=${COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE}
 
@@ -420,11 +420,6 @@ services:
       RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC: ${RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC}
       RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL: ${RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL}
       RESOURCE_USAGE_TRACKER_S3: ${RESOURCE_USAGE_TRACKER_S3}
-      S3_ACCESS_KEY: ${S3_EXPORT_ACCESS_KEY}
-      S3_BUCKET_NAME: ${S3_EXPORT_BUCKET_NAME}
-      S3_ENDPOINT: ${S3_EXPORT_ENDPOINT}
-      S3_SECRET_KEY: ${S3_EXPORT_SECRET_KEY}
-      S3_SECURE: ${S3_EXPORT_SECURE}
 
   dynamic-schdlr:
     image: ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:${DOCKER_IMAGE_TAG:-latest}
@@ -481,7 +476,7 @@ services:
         - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.healthcheck.interval=500s
         - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.healthcheck.timeout=1ms
         # see [#2718](https://github.com/ITISFoundation/osparc-simcore/issues/2718)
-        # catchall for dy-sidecar powered-services (this happens if a backend disappears and a frontend tries to reconnect, the right return value is a 503)
+        # catchall for dynamic-sidecar powered-services (this happens if a backend disappears and a frontend tries to reconnect, the right return value is a 503)
         - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.service=${SWARM_STACK_NAME}_modern_services_catchall
         # the priority is a bit higher than webserver, the webserver is the fallback to everything and has prio 2
         - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.priority=3

services/web/server/src/simcore_service_webserver/application_settings.py

@@ -81,7 +81,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     # Release information: Passed by the osparc-ops-autodeployer
     SIMCORE_VCS_RELEASE_TAG: str | None = Field(
         default=None,
-        description="Name of the tag that makrs this release or None if undefined",
+        description="Name of the tag that marks this release, or None if undefined",
         example="ResistanceIsFutile10",
     )
 

tests/e2e/Makefile

@@ -14,8 +14,8 @@ define _up_simcore
 # set some parameters to allow for e2e to run
 echo LOGIN_REGISTRATION_INVITATION_REQUIRED=0 >> $(SIMCORE_DOT_ENV)
 echo LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0 >> $(SIMCORE_DOT_ENV)
-echo DEFAULT_MAX_NANO_CPUS=1000000000 >> $(SIMCORE_DOT_ENV)
-echo DEFAULT_MAX_MEMORY=134217728 >> $(SIMCORE_DOT_ENV)
+echo DIRECTOR_DEFAULT_MAX_NANO_CPUS=1000000000 >> $(SIMCORE_DOT_ENV)
+echo DIRECTOR_DEFAULT_MAX_MEMORY=134217728 >> $(SIMCORE_DOT_ENV)
 echo SIDECAR_FORCE_CPU_NODE=1 >> $(SIMCORE_DOT_ENV)
 $(MAKE_C) $(REPO_BASE_DIR) up-prod ops_ci=1
 endef