From 49df4009f3b77ed8b4150c9941efab3873a266bf Mon Sep 17 00:00:00 2001
From: kaiser <kaiser@itis.swiss>
Date: Wed, 20 Mar 2024 12:34:04 +0100
Subject: [PATCH 01/15] Remove double defaults that are unsupported by docker
 stack compose

---
 services/docker-compose.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/services/docker-compose.yml b/services/docker-compose.yml
index 2c1a30f9573..439d3223cfa 100644
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -23,7 +23,8 @@ services:
       - DIRECTOR_V2_HOST=${DIRECTOR_V2_HOST}
       - DIRECTOR_V2_PORT=${DIRECTOR_V2_PORT}
 
-      - API_SERVER_LOGLEVEL=${API_SERVER_LOGLEVEL:-${LOG_LEVEL:-WARNING}}
+      - API_SERVER_LOGLEVEL=${API_SERVER_LOGLEVEL:WARNING}
+
 
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_HOST=${POSTGRES_HOST}

From e2e1c27d77540191218f8525d4d514b5ea38cc30 Mon Sep 17 00:00:00 2001
From: kaiser <kaiser@itis.swiss>
Date: Wed, 20 Mar 2024 12:35:48 +0100
Subject: [PATCH 02/15] fix

---
 services/docker-compose.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/services/docker-compose.yml b/services/docker-compose.yml
index 439d3223cfa..f216f584137 100644
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -23,8 +23,7 @@ services:
       - DIRECTOR_V2_HOST=${DIRECTOR_V2_HOST}
       - DIRECTOR_V2_PORT=${DIRECTOR_V2_PORT}
 
-      - API_SERVER_LOGLEVEL=${API_SERVER_LOGLEVEL:WARNING}
-
+      - API_SERVER_LOGLEVEL=${API_SERVER_LOGLEVEL:-WARNING}
 
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_HOST=${POSTGRES_HOST}

From 423b04c73ce1e5804cd9dbac20c0a74b1a295347 Mon Sep 17 00:00:00 2001
From: kaiser <kaiser@itis.swiss>
Date: Fri, 22 Mar 2024 16:35:04 +0100
Subject: [PATCH 03/15] Phase out docker compose v1, update
 docker-compose-configs.bash

---
 scripts/docker/docker-compose-config.bash | 60 ++++++-----------------
 1 file changed, 15 insertions(+), 45 deletions(-)

diff --git a/scripts/docker/docker-compose-config.bash b/scripts/docker/docker-compose-config.bash
index a9327b18745..3dddfc64de8 100755
--- a/scripts/docker/docker-compose-config.bash
+++ b/scripts/docker/docker-compose-config.bash
@@ -17,16 +17,12 @@ show_error() {
 
 
 env_file=".env"
-project_directory=""
 # Parse command line arguments
-while getopts ":e:p:" opt; do
+while getopts ":e:" opt; do
   case $opt in
     e)
       env_file="$OPTARG"
       ;;
-    p)
-      project_directory="$OPTARG"
-      ;;
     \?)
       show_error "Invalid option: -$OPTARG"
       exit 1
@@ -61,55 +57,29 @@ if docker compose version --short | grep --quiet "^2\." ; then
   if [[ -z "$version" ]]; then
     version="3.9"  # Default to 3.9 if version is not found in file
   fi
-
+# shellcheck disable=SC2002
   docker_command="\
-docker \
---log-level=ERROR \
-compose \
---env-file ${env_file}"
-
-  if [ "$project_directory" ]; then
-    docker_command+=" --project-directory ${project_directory}"
-  fi
+set -o allexport && \
+. ${env_file} && set +o allexport && \
+docker stack config"
 
   for compose_file_path in "$@"
   do
-    docker_command+=" --file=${compose_file_path}"
+    docker_command+=" --compose-file ${compose_file_path}"
   done
-  docker_command+="\
- config \
+
+  #docker_command+=" --skip-interpolation"
+
+  docker_command+=" \
 | sed '/published:/s/\"//g' \
 | sed '/size:/s/\"//g' \
-| sed '1 { /name:.*/d ; }' \
-| sed '1 i version: \"${version}\"' \
-| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'"
+| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/' \
+| sed 's/\\\$/\\\$\\\$/g'" # mitigates https://stackoverflow.com/questions/40619582/how-can-i-escape-a-dollar-sign-in-a-docker-compose-file
 
   # Execute the command
   show_info "Executing Docker command: ${docker_command}"
-  eval ${docker_command}
+  eval "${docker_command}"
 else
-  show_warning "docker compose V2 is not available, trying V1 instead... please update your docker engine."
-  if docker-compose version --short | grep --quiet "^1\." ; then
-    show_info "Running compose V1"
-    docker_command="\
-docker-compose \
---log-level=ERROR \
---env-file ${env_file}"
-    for compose_file_path in "$@"
-    do
-      docker_command+=" --file=${compose_file_path} "
-    done
-    if [ "$project_directory" ]; then
-      docker_command+=" --project-directory ${project_directory}"
-    fi
-    docker_command+=" \
-config \
-| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'"
-    # Execute the command
-    show_info "Executing Docker command: ${docker_command}"
-    eval ${docker_command}
-  else
-    show_error "docker-compose V1 is not available. It is impossible to run this script!"
-    exit 1
-  fi
+  show_warning "docker compose V2 is not available... please update your docker engine."
+  exit 1
 fi

From bb8e6e2a79b2f83539ceda0d0ecc8634b757f978 Mon Sep 17 00:00:00 2001
From: kaiser <kaiser@itis.swiss>
Date: Fri, 22 Mar 2024 16:38:39 +0100
Subject: [PATCH 04/15] fix

---
 scripts/docker/docker-compose-config.bash | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/docker/docker-compose-config.bash b/scripts/docker/docker-compose-config.bash
index 3dddfc64de8..a078153c04b 100755
--- a/scripts/docker/docker-compose-config.bash
+++ b/scripts/docker/docker-compose-config.bash
@@ -17,12 +17,16 @@ show_error() {
 
 
 env_file=".env"
+project_directory=""
 # Parse command line arguments
-while getopts ":e:" opt; do
+while getopts ":e:p:" opt; do
   case $opt in
     e)
       env_file="$OPTARG"
       ;;
+    p)
+      project_directory="$OPTARG"
+      ;;
     \?)
       show_error "Invalid option: -$OPTARG"
       exit 1

From 2e46b82b5bcaee775124694f2ed429298962d6bc Mon Sep 17 00:00:00 2001
From: kaiser <kaiser@itis.swiss>
Date: Tue, 26 Mar 2024 16:18:32 +0100
Subject: [PATCH 05/15] Handle
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH

---
 .env-devel                                | 1 +
 scripts/docker/docker-compose-config.bash | 6 +-----
 services/docker-compose.yml               | 4 ++--
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/.env-devel b/.env-devel
index a1b4f229acd..1b7669b05a5 100644
--- a/.env-devel
+++ b/.env-devel
@@ -50,6 +50,7 @@ DASK_SCHEDULER_PORT=8786
 DASK_TLS_CA_FILE=/home/scu/.dask/dask-crt.pem
 DASK_TLS_KEY=/home/scu/.dask/dask-key.pem
 DASK_TLS_CERT=/home/scu/.dask/dask-crt.pem
+CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"${DASK_TLS_CERT}","tls_client_cert":"${DASK_TLS_CERT}","tls_client_key":"${DASK_TLS_KEY}"}'
 
 DIRECTOR_REGISTRY_CACHING_TTL=900
 DIRECTOR_REGISTRY_CACHING=True
diff --git a/scripts/docker/docker-compose-config.bash b/scripts/docker/docker-compose-config.bash
index a078153c04b..3dddfc64de8 100755
--- a/scripts/docker/docker-compose-config.bash
+++ b/scripts/docker/docker-compose-config.bash
@@ -17,16 +17,12 @@ show_error() {
 
 
 env_file=".env"
-project_directory=""
 # Parse command line arguments
-while getopts ":e:p:" opt; do
+while getopts ":e:" opt; do
   case $opt in
     e)
       env_file="$OPTARG"
       ;;
-    p)
-      project_directory="$OPTARG"
-      ;;
     \?)
       show_error "Invalid option: -$OPTARG"
       exit 1
diff --git a/services/docker-compose.yml b/services/docker-compose.yml
index 2c518858538..f0aae72d00a 100644
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -154,7 +154,7 @@ services:
       - default
     environment:
       - CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=${CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG}
-      - CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH=${CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH:-{"type":"tls","tls_ca_file":"${DASK_TLS_CERT}","tls_client_cert":"${DASK_TLS_CERT}","tls_client_key":"${DASK_TLS_KEY}"}}
+      - CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH=${CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH}
       - CLUSTERS_KEEPER_DASK_NTHREADS=${CLUSTERS_KEEPER_DASK_NTHREADS}
       - CLUSTERS_KEEPER_DASK_WORKER_SATURATION=${CLUSTERS_KEEPER_DASK_WORKER_SATURATION}
       - CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=${CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION}
@@ -256,7 +256,7 @@ services:
 
       - COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE=${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE}
       - COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL=${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL}
-      - COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH=${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH:-{"type":"tls","tls_ca_file":"${DASK_TLS_CERT}","tls_client_cert":"${DASK_TLS_CERT}","tls_client_key":"${DASK_TLS_KEY}"}}
+      - COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH=${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH}
       - COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE=${COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE}
       - COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE=${COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE}
 

From 219c45b65e42059a9a1a657d05c7dd2af2842410 Mon Sep 17 00:00:00 2001
From: kaiser <kaiser@itis.swiss>
Date: Tue, 26 Mar 2024 17:34:54 +0100
Subject: [PATCH 06/15] WIP: pre-commit hook

---
 .env-devel                                    |  3 ++-
 .pre-commit-config.yaml                       |  7 ++++++
 .../precommit/validate-docker-compose.bash    | 25 +++++++++++++++++++
 services/docker-compose.yml                   |  6 ++---
 4 files changed, 37 insertions(+), 4 deletions(-)
 create mode 100755 scripts/precommit/validate-docker-compose.bash

diff --git a/.env-devel b/.env-devel
index 1b7669b05a5..efe60c4e828 100644
--- a/.env-devel
+++ b/.env-devel
@@ -25,6 +25,7 @@ AUTOSCALING_EC2_ACCESS=null
 AUTOSCALING_EC2_INSTANCES=null
 AUTOSCALING_NODES_MONITORING=null
 AUTOSCALING_POLL_INTERVAL=10
+AUTOSCALING_LOGLEVEL=DEBUG
 
 
 BF_API_KEY=none
@@ -43,7 +44,7 @@ CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null
 CLUSTERS_KEEPER_TASK_INTERVAL=30
 CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null
 CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX=""
-
+CLUSTERS_KEEPER_LOGLEVEL=INFO
 
 DASK_SCHEDULER_HOST=dask-scheduler
 DASK_SCHEDULER_PORT=8786
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index bf135727ee2..a8bb2f6113d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -52,3 +52,10 @@ repos:
       language: script
       types: [file, python]
       entry: scripts/precommit/pytest-testit.bash
+  - repo: local
+    hooks:
+    - id: validate-docker-compose
+      name: validate-docker-compose
+      language: script
+      types: [file]
+      entry: scripts/precommit/validate-docker-compose.bash
diff --git a/scripts/precommit/validate-docker-compose.bash b/scripts/precommit/validate-docker-compose.bash
new file mode 100755
index 00000000000..102842d1a31
--- /dev/null
+++ b/scripts/precommit/validate-docker-compose.bash
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Get a list of all staged files
+staged_files=$(git diff --cached --name-only --diff-filter=ACM)
+
+# Loop through each file
+for file in $staged_files
+do
+  # Check if the file name contains "docker-compose" and is a .yml or .yaml file
+  if [[ $file == *docker-compose*.yml || $file == *docker-compose*.yaml ]]; then
+    echo "Checking $file" 1>&2
+    # Check the file for lines with more than one dollar sign
+    if grep -n -P '\$\{[^}]*\$\{[^}]*\}[^}]*\}' "$file"; then
+      echo "Error: $file contains a line with more than one dollar sign."
+      exit 1
+    elif
+      grep -n -P '\$\{[^}]*:-[^}]*\}' "$file"; then
+      echo "Error: $file contains at least one line with an environment variable with a default value."
+      exit 1
+    fi
+  fi
+done
+
+# If no errors were found, allow the commit
+exit 0
diff --git a/services/docker-compose.yml b/services/docker-compose.yml
index f0aae72d00a..46748e6dead 100644
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -69,7 +69,7 @@ services:
     networks:
       - autoscaling_subnet
     environment:
-      - AUTOSCALING_LOGLEVEL=${AUTOSCALING_LOGLEVEL:-${LOG_LEVEL:-WARNING}}
+      - AUTOSCALING_LOGLEVEL=${AUTOSCALING_LOGLEVEL}
       - AUTOSCALING_POLL_INTERVAL=${AUTOSCALING_POLL_INTERVAL}
       - AUTOSCALING_DRAIN_NODES_WITH_LABELS=${AUTOSCALING_DRAIN_NODES_WITH_LABELS}
 
@@ -92,7 +92,7 @@ services:
       - EC2_INSTANCES_KEY_NAME=${EC2_INSTANCES_KEY_NAME}
       - EC2_INSTANCES_CUSTOM_TAGS=${EC2_INSTANCES_CUSTOM_TAGS}
 
-      - AUTOSCALING_NODES_MONITORING=${AUTOSCALING_NODES_MONITORING} # dyn autoscaling
+      - AUTOSCALING_NODES_MONITORING=${AUTOSCALING_NODES_MONITORING} # dyn autoscaling envvar
       - NODES_MONITORING_NODE_LABELS=${NODES_MONITORING_NODE_LABELS}
       - NODES_MONITORING_SERVICE_LABELS=${NODES_MONITORING_SERVICE_LABELS}
       - NODES_MONITORING_NEW_NODES_LABELS=${NODES_MONITORING_NEW_NODES_LABELS}
@@ -159,7 +159,7 @@ services:
       - CLUSTERS_KEEPER_DASK_WORKER_SATURATION=${CLUSTERS_KEEPER_DASK_WORKER_SATURATION}
       - CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=${CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION}
       - CLUSTERS_KEEPER_TASK_INTERVAL=${CLUSTERS_KEEPER_TASK_INTERVAL}
-      - CLUSTERS_KEEPER_LOGLEVEL=${CLUSTERS_KEEPER_LOGLEVEL:-${LOG_LEVEL:-WARNING}}
+      - CLUSTERS_KEEPER_LOGLEVEL=${CLUSTERS_KEEPER_LOGLEVEL}
       - CLUSTERS_KEEPER_EC2_ACCESS=${CLUSTERS_KEEPER_EC2_ACCESS}
       - CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID=${CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID}
       - CLUSTERS_KEEPER_EC2_ENDPOINT=${CLUSTERS_KEEPER_EC2_ENDPOINT}

From 0be3d51bb64de28938167ea4380e9960847485d2 Mon Sep 17 00:00:00 2001
From: kaiser <kaiser@itis.swiss>
Date: Wed, 27 Mar 2024 08:51:06 +0100
Subject: [PATCH 07/15] fix custom pre-commit hook

---
 scripts/precommit/validate-docker-compose.bash | 6 +++---
 services/docker-compose.yml                    | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/precommit/validate-docker-compose.bash b/scripts/precommit/validate-docker-compose.bash
index 102842d1a31..887837e4811 100755
--- a/scripts/precommit/validate-docker-compose.bash
+++ b/scripts/precommit/validate-docker-compose.bash
@@ -13,13 +13,13 @@ do
     if grep -n -P '\$\{[^}]*\$\{[^}]*\}[^}]*\}' "$file"; then
       echo "Error: $file contains a line with more than one dollar sign."
       exit 1
-    elif
-      grep -n -P '\$\{[^}]*:-[^}]*\}' "$file"; then
-      echo "Error: $file contains at least one line with an environment variable with a default value."
+    elif grep -n -P '\$[a-zA-Z_][a-zA-Z0-9_]*' "$file"; then
+      echo "Error: $file contains a line with an environment variable not wrapped in curly braces."
       exit 1
     fi
   fi
 done
 
+
 # If no errors were found, allow the commit
 exit 0
diff --git a/services/docker-compose.yml b/services/docker-compose.yml
index 46748e6dead..5793ef758ca 100644
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -481,7 +481,7 @@ services:
         - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.healthcheck.interval=500s
         - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.healthcheck.timeout=1ms
         # see [#2718](https://github.com/ITISFoundation/osparc-simcore/issues/2718)
-        # catchall for dy-sidecar powered-services (this happens if a backend disappears and a frontend tries to reconnect, the right return value is a 503)
+        # catchall for dynamic-sidecar powered-services (this happens if a backend disappears and a frontend tries to reconnect, the right return value is a 503)
         - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.service=${SWARM_STACK_NAME}_modern_services_catchall
         # the priority is a bit higher than webserver, the webserver is the fallback to everything and has prio 2
         - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.priority=3

From 3d7e66f50e091d80e0a8533a92b91b917978cb0d Mon Sep 17 00:00:00 2001
From: kaiser <kaiser@itis.swiss>
Date: Wed, 27 Mar 2024 11:38:44 +0100
Subject: [PATCH 08/15] Fix env vars

---
 .env-devel                                            |  6 ------
 services/catalog/docker-compose-extra.yml             |  2 +-
 services/docker-compose.yml                           | 11 +++--------
 .../simcore_service_webserver/application_settings.py |  2 +-
 tests/e2e/Makefile                                    |  4 ++--
 5 files changed, 7 insertions(+), 18 deletions(-)

diff --git a/.env-devel b/.env-devel
index efe60c4e828..b708cfbbbe5 100644
--- a/.env-devel
+++ b/.env-devel
@@ -145,12 +145,6 @@ S3_ENDPOINT=172.17.0.1:9001
 S3_SECRET_KEY=12345678
 S3_SECURE=0
 
-S3_EXPORT_ACCESS_KEY=87654321
-S3_EXPORT_BUCKET_NAME=simcore-export
-S3_EXPORT_ENDPOINT=172.17.0.1:9001
-S3_EXPORT_SECRET_KEY=87654321
-S3_EXPORT_SECURE=0
-
 SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1
 SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key
 
diff --git a/services/catalog/docker-compose-extra.yml b/services/catalog/docker-compose-extra.yml
index 68fc041013a..a3a5912b910 100644
--- a/services/catalog/docker-compose-extra.yml
+++ b/services/catalog/docker-compose-extra.yml
@@ -60,7 +60,7 @@ services:
       - S3_SECRET_KEY=${S3_SECRET_KEY}
       - S3_BUCKET_NAME=${S3_BUCKET_NAME}
       - STORAGE_ENDPOINT=${STORAGE_ENDPOINT}
-      - EXTRA_HOSTS_SUFFIX=${EXTRA_HOSTS_SUFFIX:-undefined}
+      - EXTRA_HOSTS_SUFFIX=undefined
       - SIMCORE_SERVICES_NETWORK_NAME=interactive_services_subnet
       - MONITORING_ENABLED=${MONITORING_ENABLED:-True}
       - TRACING_ENABLED=${TRACING_ENABLED:-True}
diff --git a/services/docker-compose.yml b/services/docker-compose.yml
index 5793ef758ca..75dc5150154 100644
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -201,8 +201,8 @@ services:
     init: true
     hostname: "{{.Node.Hostname}}-{{.Task.Slot}}"
     environment:
-      - DEFAULT_MAX_MEMORY=${DEFAULT_MAX_MEMORY:-0}
-      - DEFAULT_MAX_NANO_CPUS=${DEFAULT_MAX_NANO_CPUS:-0}
+      - DEFAULT_MAX_MEMORY=${DIRECTOR_DEFAULT_MAX_MEMORY:-0}
+      - DEFAULT_MAX_NANO_CPUS=${DIRECTOR_DEFAULT_MAX_NANO_CPUS:-0}
       - DIRECTOR_REGISTRY_CACHING_TTL=${DIRECTOR_REGISTRY_CACHING_TTL}
       - DIRECTOR_REGISTRY_CACHING=${DIRECTOR_REGISTRY_CACHING}
       - DIRECTOR_SELF_SIGNED_SSL_FILENAME=${DIRECTOR_SELF_SIGNED_SSL_FILENAME}
@@ -210,7 +210,7 @@ services:
       - DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME=${DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME}
       - DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=${DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS}
       - DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS=${DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS}
-      - EXTRA_HOSTS_SUFFIX=${EXTRA_HOSTS_SUFFIX:-undefined}
+      - EXTRA_HOSTS_SUFFIX=undefined
       - LOGLEVEL=${LOG_LEVEL:-WARNING}
       - MONITORING_ENABLED=${MONITORING_ENABLED:-True}
       - PUBLISHED_HOST_NAME=${MACHINE_FQDN}
@@ -420,11 +420,6 @@ services:
       RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC: ${RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC}
       RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL: ${RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL}
       RESOURCE_USAGE_TRACKER_S3: ${RESOURCE_USAGE_TRACKER_S3}
-      S3_ACCESS_KEY: ${S3_EXPORT_ACCESS_KEY}
-      S3_BUCKET_NAME: ${S3_EXPORT_BUCKET_NAME}
-      S3_ENDPOINT: ${S3_EXPORT_ENDPOINT}
-      S3_SECRET_KEY: ${S3_EXPORT_SECRET_KEY}
-      S3_SECURE: ${S3_EXPORT_SECURE}
 
   dynamic-schdlr:
     image: ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:${DOCKER_IMAGE_TAG:-latest}
diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py
index 0132f73b212..4098ba8bfde 100644
--- a/services/web/server/src/simcore_service_webserver/application_settings.py
+++ b/services/web/server/src/simcore_service_webserver/application_settings.py
@@ -81,7 +81,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     # Release information: Passed by the osparc-ops-autodeployer
     SIMCORE_VCS_RELEASE_TAG: str | None = Field(
         default=None,
-        description="Name of the tag that makrs this release or None if undefined",
+        description="Name of the tag that marks this release, or None if undefined",
         example="ResistanceIsFutile10",
     )
 
diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile
index 9f2d343253b..30d457eca3b 100644
--- a/tests/e2e/Makefile
+++ b/tests/e2e/Makefile
@@ -14,8 +14,8 @@ define _up_simcore
 # set some parameters to allow for e2e to run
 echo LOGIN_REGISTRATION_INVITATION_REQUIRED=0 >> $(SIMCORE_DOT_ENV)
 echo LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0 >> $(SIMCORE_DOT_ENV)
-echo DEFAULT_MAX_NANO_CPUS=1000000000 >> $(SIMCORE_DOT_ENV)
-echo DEFAULT_MAX_MEMORY=134217728 >> $(SIMCORE_DOT_ENV)
+echo DIRECTOR_DEFAULT_MAX_NANO_CPUS=1000000000 >> $(SIMCORE_DOT_ENV)
+echo DIRECTOR_DEFAULT_MAX_MEMORY=134217728 >> $(SIMCORE_DOT_ENV)
 echo SIDECAR_FORCE_CPU_NODE=1 >> $(SIMCORE_DOT_ENV)
 $(MAKE_C) $(REPO_BASE_DIR) up-prod ops_ci=1
 endef

From 0897c6b9416eb1875bd9300825efdd4ef032cf21 Mon Sep 17 00:00:00 2001
From: kaiser <kaiser@itis.swiss>
Date: Wed, 27 Mar 2024 11:43:28 +0100
Subject: [PATCH 09/15] fix docker-compose-config.bash

---
 scripts/docker/docker-compose-config.bash | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/scripts/docker/docker-compose-config.bash b/scripts/docker/docker-compose-config.bash
index 3dddfc64de8..bd3c966034f 100755
--- a/scripts/docker/docker-compose-config.bash
+++ b/scripts/docker/docker-compose-config.bash
@@ -17,12 +17,16 @@ show_error() {
 
 
 env_file=".env"
+project_directory=""
 # Parse command line arguments
-while getopts ":e:" opt; do
+while getopts ":e:p:" opt; do
   case $opt in
     e)
       env_file="$OPTARG"
       ;;
+    p)
+      project_directory="$OPTARG"
+      ;;
     \?)
       show_error "Invalid option: -$OPTARG"
       exit 1
@@ -67,8 +71,9 @@ docker stack config"
   do
     docker_command+=" --compose-file ${compose_file_path}"
   done
-
-  #docker_command+=" --skip-interpolation"
+  # WE CANNOT DO THIS:
+  # docker_command+=" --skip-interpolation"
+  # because docker stack compose will *validate* that e.g. `replicas: ${SIMCORE_SERVICES_POSTGRES_REPLICAS}` is a valid number, which it is not if it is read as a literal string.
 
   docker_command+=" \
 | sed '/published:/s/\"//g' \

From 7cb5df4dcaebf4c97e707a1fb2fa246e7cd1b18b Mon Sep 17 00:00:00 2001
From: Sylvain <35365065+sanderegg@users.noreply.github.com>
Date: Tue, 2 Apr 2024 10:32:22 +0200
Subject: [PATCH 10/15] Some proposed changes and should fix the
 integration/system tests (#15)

* revert changes

* have docker stack config for clarity

* replaced docker compose config by docker stack config

* replaced docker compose config by docker stack config

* fix issue

* add a note

* delete file

* fix note
---
 Makefile                                      | 12 +--
 .../src/pytest_simcore/docker_compose.py      | 11 ++-
 .../pytest_simcore/helpers/utils_docker.py    | 27 +-----
 scripts/docker/docker-compose-config.bash     | 90 -------------------
 scripts/docker/docker-stack-config.bash       | 76 ++++++++++++++++
 services/autoscaling/tests/manual/Makefile    |  4 +-
 services/osparc-gateway-server/Makefile       |  2 +-
 .../test_used_docker_compose.py               |  2 +-
 8 files changed, 97 insertions(+), 127 deletions(-)
 delete mode 100755 scripts/docker/docker-compose-config.bash
 create mode 100755 scripts/docker/docker-stack-config.bash

diff --git a/Makefile b/Makefile
index 5b63c8cb50b..d36ed2d27ff 100644
--- a/Makefile
+++ b/Makefile
@@ -238,7 +238,7 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 	@export DOCKER_REGISTRY=local && \
 	export DOCKER_IMAGE_TAG=development && \
 	export DEV_PC_CPU_COUNT=${CPU_COUNT} && \
-	scripts/docker/docker-compose-config.bash -e .env \
+	scripts/docker/docker-stack-config.bash -e .env \
 		services/docker-compose.yml \
 		services/docker-compose.local.yml \
 		services/docker-compose.devel.yml \
@@ -248,7 +248,7 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 	# Creating config for stack with 'local/{service}:production' to $@
 	@export DOCKER_REGISTRY=local && \
 	export DOCKER_IMAGE_TAG=production && \
-	scripts/docker/docker-compose-config.bash -e .env \
+	scripts/docker/docker-stack-config.bash -e .env \
 		services/docker-compose.yml \
 		services/docker-compose.local.yml \
 		> $@
@@ -258,7 +258,7 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 	# Creating config for stack with 'local/{service}:production' (except of static-webserver -> static-webserver:development) to $@
 	@export DOCKER_REGISTRY=local && \
 	export DOCKER_IMAGE_TAG=production && \
-	scripts/docker/docker-compose-config.bash -e $< \
+	scripts/docker/docker-stack-config.bash -e $< \
 		services/docker-compose.yml \
 		services/docker-compose.local.yml \
 		services/docker-compose.devel-frontend.yml \
@@ -266,7 +266,7 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 
 .stack-simcore-version.yml: .env $(docker-compose-configs)
 	# Creating config for stack with '$(DOCKER_REGISTRY)/{service}:${DOCKER_IMAGE_TAG}' to $@
-	@scripts/docker/docker-compose-config.bash -e .env \
+	@scripts/docker/docker-stack-config.bash -e .env \
 		services/docker-compose.yml \
 		services/docker-compose.local.yml \
 		> $@
@@ -276,13 +276,13 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 	# Creating config for ops stack to $@
 ifdef ops_ci
 	@$(shell \
-		scripts/docker/docker-compose-config.bash -e .env \
+		scripts/docker/docker-stack-config.bash -e .env \
 		services/docker-compose-ops-ci.yml \
 		> $@ \
 	)
 else
 	@$(shell \
-		scripts/docker/docker-compose-config.bash -e .env \
+		scripts/docker/docker-stack-config.bash -e .env \
 		services/docker-compose-ops.yml \
 		> $@ \
 	)
diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py
index c011620e62e..57ea7d0a0f5 100644
--- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py
+++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py
@@ -101,14 +101,19 @@ def env_file_for_testing(
     #   https://docs.docker.com/compose/environment-variables/#the-env-file
 
     env_test_path = temp_folder / ".env.test"
-
     with env_test_path.open("wt") as fh:
         print(
             f"# Auto-generated from env_file_for_testing in {__file__}",
             file=fh,
         )
-        for key in sorted(testing_environ_vars.keys()):
-            print(f"{key}={testing_environ_vars[key]}", file=fh)
+        for key, value in sorted(testing_environ_vars.items()):
+            # NOTE: python-dotenv parses JSON encoded strings correctly, but
+            # writing them back shows an issue. if the original ENV is something like MY_ENV='{"correct": "encodedjson"}'
+            # it goes to MY_ENV={"incorrect": "encodedjson"}!
+            if value.startswith(("{", "[")) and value.endswith(("}", "]")):
+                print(f"{key}='{value}'", file=fh)
+            else:
+                print(f"{key}={value}", file=fh)
 
     #
     # WARNING: since compose files have references to ../.env we MUST create .env
diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py
index eda49ff4dc4..dcccfa55141 100644
--- a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py
+++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py
@@ -140,13 +140,8 @@ def run_docker_compose_config(
             ".yaml",
         ], "Expected yaml/yml file as destination path"
 
-    # SEE https://docs.docker.com/compose/reference/
-    bash_options = [
-        "-p",
-        str(project_dir),  # Specify an alternate working directory
-    ]
     # https://docs.docker.com/compose/environment-variables/#using-the---env-file--option
-    bash_options += [
+    bash_options = [
         "-e",
         str(env_file_path),  # Custom environment variables
     ]
@@ -158,15 +153,13 @@ def run_docker_compose_config(
         bash_options += [os.path.relpath(docker_compose_path, project_dir)]
 
     # SEE https://docs.docker.com/compose/reference/config/
-    docker_compose_path = scripts_dir / "docker" / "docker-compose-config.bash"
+    docker_compose_path = scripts_dir / "docker" / "docker-stack-config.bash"
     assert docker_compose_path.exists()
-
     args = [f"{docker_compose_path}", *bash_options]
     print(" ".join(args))
 
     process = subprocess.run(
         args,
-        shell=False,
         cwd=project_dir,
         capture_output=True,
         check=True,
@@ -176,20 +169,6 @@ def run_docker_compose_config(
     compose_file_str = process.stdout.decode("utf-8")
     compose_file: dict[str, Any] = yaml.safe_load(compose_file_str)
 
-    def _remove_top_level_name_attribute_generated_by_compose_v2(
-        compose: dict[str, Any]
-    ) -> dict[str, Any]:
-        """docker compose V2 CLI config adds a top level name attribute
-        https://docs.docker.com/compose/compose-file/#name-top-level-element
-        but it is incompatible with docker stack deploy...
-        """
-        compose.pop("name", None)
-        return compose
-
-    compose_file = _remove_top_level_name_attribute_generated_by_compose_v2(
-        compose_file
-    )
-
     if destination_path:
         #
         # NOTE: This step could be avoided and reading instead from stdout
@@ -282,7 +261,7 @@ def save_docker_infos(destination_dir: Path):
                             json.dumps(container.attrs, indent=2)
                         )
 
-            except Exception as err:  # pylint: disable=broad-except
+            except Exception as err:  # pylint: disable=broad-except  # noqa: PERF203
                 if container.status != ContainerStatus.created:
                     print(
                         f"Error while dumping {container.name=}, {container.status=}.\n\t{err=}"
diff --git a/scripts/docker/docker-compose-config.bash b/scripts/docker/docker-compose-config.bash
deleted file mode 100755
index bd3c966034f..00000000000
--- a/scripts/docker/docker-compose-config.bash
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/bin/bash
-# generated using chatgpt
-show_info() {
-    local message="$1"
-    echo -e "\e[37mInfo:\e[0m $message" >&2
-}
-
-show_warning() {
-  local message="$1"
-  echo -e "\e[31mWarning:\e[0m $message" >&2
-}
-
-show_error() {
-  local message="$1"
-  echo -e "\e[31mError:\e[0m $message" >&2
-}
-
-
-env_file=".env"
-project_directory=""
-# Parse command line arguments
-while getopts ":e:p:" opt; do
-  case $opt in
-    e)
-      env_file="$OPTARG"
-      ;;
-    p)
-      project_directory="$OPTARG"
-      ;;
-    \?)
-      show_error "Invalid option: -$OPTARG"
-      exit 1
-      ;;
-    :)
-      show_error "Option -$OPTARG requires an argument."
-      exit 1
-      ;;
-  esac
-done
-shift $((OPTIND-1))
-
-if [[ "$#" -eq 0 ]]; then
-    show_error "No compose files specified!"
-    exit 1
-fi
-
-# REFERENCE: https://github.com/docker/compose/issues/9306
-# composeV2 defines specifications for docker compose to run
-# they are not 100% compatible with what docker stack deploy command expects
-# some parts have to be modified
-
-
-
-
-# check if docker-compose V2 is available
-if docker compose version --short | grep --quiet "^2\." ; then
-  show_info "Running compose V2"
-  # V2 does not write the version anymore, so we take it from the first compose file
-  first_compose_file="${1}"
-  version=$(grep --max-count=1 "^version:" "${first_compose_file}" | cut --delimiter=' ' --fields=2 | tr --delete \"\')
-  if [[ -z "$version" ]]; then
-    version="3.9"  # Default to 3.9 if version is not found in file
-  fi
-# shellcheck disable=SC2002
-  docker_command="\
-set -o allexport && \
-. ${env_file} && set +o allexport && \
-docker stack config"
-
-  for compose_file_path in "$@"
-  do
-    docker_command+=" --compose-file ${compose_file_path}"
-  done
-  # WE CANNOT DO THIS:
-  # docker_command+=" --skip-interpolation"
-  # because docker stack compose will *validate* that e.g. `replicas: ${SIMCORE_SERVICES_POSTGRES_REPLICAS}` is a valid number, which it is not if it is read as a literal string.
-
-  docker_command+=" \
-| sed '/published:/s/\"//g' \
-| sed '/size:/s/\"//g' \
-| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/' \
-| sed 's/\\\$/\\\$\\\$/g'" # mitigates https://stackoverflow.com/questions/40619582/how-can-i-escape-a-dollar-sign-in-a-docker-compose-file
-
-  # Execute the command
-  show_info "Executing Docker command: ${docker_command}"
-  eval "${docker_command}"
-else
-  show_warning "docker compose V2 is not available... please update your docker engine."
-  exit 1
-fi
diff --git a/scripts/docker/docker-stack-config.bash b/scripts/docker/docker-stack-config.bash
new file mode 100755
index 00000000000..62fb15fc40e
--- /dev/null
+++ b/scripts/docker/docker-stack-config.bash
@@ -0,0 +1,76 @@
+#!/bin/bash
+# generated using chatgpt
+show_info() {
+  local message="$1"
+  echo -e "\e[37mInfo:\e[0m $message" >&2
+}
+
+show_warning() {
+  local message="$1"
+  echo -e "\e[31mWarning:\e[0m $message" >&2
+}
+
+show_error() {
+  local message="$1"
+  echo -e "\e[31mError:\e[0m $message" >&2
+}
+
+env_file=".env"
+# Parse command line arguments
+while getopts ":e:" opt; do
+  case $opt in
+  e)
+    env_file="$OPTARG"
+    ;;
+  \?)
+    show_error "Invalid option: -$OPTARG"
+    exit 1
+    ;;
+  :)
+    show_error "Option -$OPTARG requires an argument."
+    exit 1
+    ;;
+  esac
+done
+shift $((OPTIND - 1))
+
+if [[ "$#" -eq 0 ]]; then
+  show_error "No compose files specified!"
+  exit 1
+fi
+
+# Check if Docker version is greater than or equal to 25.0.3
+version_check=$(docker --version | grep --extended-regexp --only-matching '[0-9]+\.[0-9]+\.[0-9]+')
+IFS='.' read -r -a version_parts <<<"$version_check"
+
+if [[ "${version_parts[0]}" -gt 25 ]] ||
+  { [[ "${version_parts[0]}" -eq 25 ]] && [[ "${version_parts[1]}" -gt 0 ]]; } ||
+  { [[ "${version_parts[0]}" -eq 25 ]] && [[ "${version_parts[1]}" -eq 0 ]] && [[ "${version_parts[2]}" -ge 3 ]]; }; then
+  show_info "Running Docker version $version_check"
+else
+  show_error "Docker version 25.0.3 or higher is required."
+  exit 1
+fi
+
+# shellcheck disable=SC2002
+docker_command="\
+set -o allexport && \
+. ${env_file} && set +o allexport && \
+docker stack config"
+
+for compose_file_path in "$@"; do
+  docker_command+=" --compose-file ${compose_file_path}"
+done
+# WE CANNOT DO THIS:
+# docker_command+=" --skip-interpolation"
+# because docker stack compose will *validate* that e.g. `replicas: ${SIMCORE_SERVICES_POSTGRES_REPLICAS}` is a valid number, which it is not if it is read as a literal string.
+
+docker_command+=" \
+| sed '/published:/s/\"//g' \
+| sed '/size:/s/\"//g' \
+| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/' \
+| sed 's/\\\$/\\\$\\\$/g'" # mitigates https://stackoverflow.com/questions/40619582/how-can-i-escape-a-dollar-sign-in-a-docker-compose-file
+
+# Execute the command
+show_info "Executing Docker command: ${docker_command}"
+eval "${docker_command}"
diff --git a/services/autoscaling/tests/manual/Makefile b/services/autoscaling/tests/manual/Makefile
index c34b2bf7b53..f6d8b97e4bb 100644
--- a/services/autoscaling/tests/manual/Makefile
+++ b/services/autoscaling/tests/manual/Makefile
@@ -6,12 +6,12 @@ include ../../../../scripts/common-service.Makefile
 
 
 .stack-devel.yml: .env
-	../../../../scripts/docker/docker-compose-config.bash -e .env \
+	../../../../scripts/docker/docker-stack-config.bash -e .env \
 		docker-compose.yml \
 		> $@
 
 .stack-computational-devel.yml: .env
-	../../../../scripts/docker/docker-compose-config.bash -e .env \
+	../../../../scripts/docker/docker-stack-config.bash -e .env \
 		docker-compose.yml \
 		docker-compose-computational.yml \
 		> $@
diff --git a/services/osparc-gateway-server/Makefile b/services/osparc-gateway-server/Makefile
index fdc4f28e417..73a8327b8c8 100644
--- a/services/osparc-gateway-server/Makefile
+++ b/services/osparc-gateway-server/Makefile
@@ -60,7 +60,7 @@ PHONY: .init-swarm
 .PHONY: config
 export OSPARC_GATEWAY_CONFIG_FILE_HOST = .osparc-dask-gateway-config.py
 export SWARM_STACK_NAME ?= dask-gateway
-docker-compose-config-cmd=../../scripts/docker/docker-compose-config.bash
+docker-compose-config-cmd=../../scripts/docker/docker-stack-config.bash
 docker-compose-configs = $(wildcard services/docker-compose*.yml)
 
 $(OSPARC_GATEWAY_CONFIG_FILE_HOST): $(CURDIR)/config/default_config.py  ## creates config file from defaults in /config/default_config.py
diff --git a/tests/environment-setup/test_used_docker_compose.py b/tests/environment-setup/test_used_docker_compose.py
index d213cce1ed6..bacf178492c 100644
--- a/tests/environment-setup/test_used_docker_compose.py
+++ b/tests/environment-setup/test_used_docker_compose.py
@@ -96,7 +96,7 @@ def _skip_osparc_gateway_server(p) -> bool:
 @pytest.fixture
 def docker_compose_config_bash(osparc_simcore_scripts_dir: Path) -> Path:
     docker_compose_config_script = (
-        osparc_simcore_scripts_dir / "docker" / "docker-compose-config.bash"
+        osparc_simcore_scripts_dir / "docker" / "docker-stack-config.bash"
     )
     assert docker_compose_config_script.exists()
     return docker_compose_config_script

From a522c0a93c57e5e73d1dd3f03fa45bcdd5ad08f4 Mon Sep 17 00:00:00 2001
From: Sylvain <35365065+sanderegg@users.noreply.github.com>
Date: Tue, 2 Apr 2024 14:59:24 +0200
Subject: [PATCH 11/15] San/proposals and fixes2 (#16)

* remove sed stuff

* remove sed

* go for info

* missing variable

* dask stack config does not support nested ENV definitions
---
 .env-devel                              | 5 +++--
 scripts/docker/docker-stack-config.bash | 6 ------
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/.env-devel b/.env-devel
index b708cfbbbe5..8c5739d3edd 100644
--- a/.env-devel
+++ b/.env-devel
@@ -25,7 +25,7 @@ AUTOSCALING_EC2_ACCESS=null
 AUTOSCALING_EC2_INSTANCES=null
 AUTOSCALING_NODES_MONITORING=null
 AUTOSCALING_POLL_INTERVAL=10
-AUTOSCALING_LOGLEVEL=DEBUG
+AUTOSCALING_LOGLEVEL=INFO
 
 
 BF_API_KEY=none
@@ -51,13 +51,14 @@ DASK_SCHEDULER_PORT=8786
 DASK_TLS_CA_FILE=/home/scu/.dask/dask-crt.pem
 DASK_TLS_KEY=/home/scu/.dask/dask-key.pem
 DASK_TLS_CERT=/home/scu/.dask/dask-crt.pem
-CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"${DASK_TLS_CERT}","tls_client_cert":"${DASK_TLS_CERT}","tls_client_key":"${DASK_TLS_KEY}"}'
+CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
 
 DIRECTOR_REGISTRY_CACHING_TTL=900
 DIRECTOR_REGISTRY_CACHING=True
 DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 
 COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL=tls://dask-scheduler:8786
+COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
 COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE=S3
 COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE=PRESIGNED
 COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE=PRESIGNED
diff --git a/scripts/docker/docker-stack-config.bash b/scripts/docker/docker-stack-config.bash
index 62fb15fc40e..6b95bc73647 100755
--- a/scripts/docker/docker-stack-config.bash
+++ b/scripts/docker/docker-stack-config.bash
@@ -65,12 +65,6 @@ done
 # docker_command+=" --skip-interpolation"
 # because docker stack compose will *validate* that e.g. `replicas: ${SIMCORE_SERVICES_POSTGRES_REPLICAS}` is a valid number, which it is not if it is read as a literal string.
 
-docker_command+=" \
-| sed '/published:/s/\"//g' \
-| sed '/size:/s/\"//g' \
-| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/' \
-| sed 's/\\\$/\\\$\\\$/g'" # mitigates https://stackoverflow.com/questions/40619582/how-can-i-escape-a-dollar-sign-in-a-docker-compose-file
-
 # Execute the command
 show_info "Executing Docker command: ${docker_command}"
 eval "${docker_command}"

From 7d796a129235c670f70e249c67623a1ac5f3c907 Mon Sep 17 00:00:00 2001
From: Sylvain <35365065+sanderegg@users.noreply.github.com>
Date: Tue, 2 Apr 2024 15:27:30 +0200
Subject: [PATCH 12/15] check for 24.0.9 since the CI has that

---
 scripts/docker/docker-stack-config.bash | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/docker/docker-stack-config.bash b/scripts/docker/docker-stack-config.bash
index 6b95bc73647..b41f6361d4b 100755
--- a/scripts/docker/docker-stack-config.bash
+++ b/scripts/docker/docker-stack-config.bash
@@ -39,13 +39,13 @@ if [[ "$#" -eq 0 ]]; then
   exit 1
 fi
 
-# Check if Docker version is greater than or equal to 25.0.3
+# Check if Docker version is greater than or equal to 24.0.9
 version_check=$(docker --version | grep --extended-regexp --only-matching '[0-9]+\.[0-9]+\.[0-9]+')
 IFS='.' read -r -a version_parts <<<"$version_check"
 
-if [[ "${version_parts[0]}" -gt 25 ]] ||
-  { [[ "${version_parts[0]}" -eq 25 ]] && [[ "${version_parts[1]}" -gt 0 ]]; } ||
-  { [[ "${version_parts[0]}" -eq 25 ]] && [[ "${version_parts[1]}" -eq 0 ]] && [[ "${version_parts[2]}" -ge 3 ]]; }; then
+if [[ "${version_parts[0]}" -gt 24 ]] ||
+  { [[ "${version_parts[0]}" -eq 24 ]] && [[ "${version_parts[1]}" -gt 0 ]]; } ||
+  { [[ "${version_parts[0]}" -eq 24 ]] && [[ "${version_parts[1]}" -eq 0 ]] && [[ "${version_parts[2]}" -ge 9 ]]; }; then
   show_info "Running Docker version $version_check"
 else
   show_error "Docker version 25.0.3 or higher is required."

From 58d73da39ce49cc31ec1328aafd1d4f8811efcef Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Tue, 2 Apr 2024 15:55:17 +0200
Subject: [PATCH 13/15] cleanup

---
 services/clusters-keeper/Makefile             | 34 -----------
 services/clusters-keeper/docker-compose.yml   | 59 -------------------
 .../test_used_docker_compose.py               |  8 ++-
 3 files changed, 5 insertions(+), 96 deletions(-)
 delete mode 100644 services/clusters-keeper/Makefile
 delete mode 100644 services/clusters-keeper/docker-compose.yml

diff --git a/services/clusters-keeper/Makefile b/services/clusters-keeper/Makefile
deleted file mode 100644
index 0a6d6966df7..00000000000
--- a/services/clusters-keeper/Makefile
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-# DEVELOPMENT recipes for Auto scaling service
-#
-include ../../scripts/common.Makefile
-include ../../scripts/common-service.Makefile
-
-
-
-.PHONY: up-devel down
-up-devel: .env ## starts local test application (running bare metal against AWS)
-	# setting up dependencies
-	@docker compose up
-
-down: .env ## stops local test app dependencies (running bare metal against AWS)
-	-@docker compose down
-
-
-PHONY: .init-swarm
-.init-swarm:
-	# Ensures swarm is initialized (careful we use a default pool of 172.20.0.0/14. Ensure you do not use private IPs in that range!)
-	$(if $(SWARM_HOSTS),,docker swarm init --advertise-addr=$(get_my_ip) --default-addr-pool 172.20.0.0/14)
-
-
-.PHONY: test-dask-scheduler-deploy
-deploy-dask-stack: .init-swarm ## deploy the dask stack for local testing
-	# using local/dask-sidecar:production images
-	@DOCKER_IMAGE_TAG=production \
-	DOCKER_REGISTRY=local \
-	LOG_LEVEL=INFO \
-	docker stack deploy --with-registry-auth --compose-file=src/simcore_service_clusters_keeper/data/docker-compose.yml osparc_dask_stack
-
-down-dask-stack: ## removes the dask stack
-	# stopping dask stack
-	-@docker stack rm osparc_dask_stack
diff --git a/services/clusters-keeper/docker-compose.yml b/services/clusters-keeper/docker-compose.yml
deleted file mode 100644
index 1d8d5126194..00000000000
--- a/services/clusters-keeper/docker-compose.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-version: "3.8"
-services:
-  rabbit:
-    image: itisfoundation/rabbitmq:3.11.2-management
-    init: true
-    ports:
-      - "5672:5672"
-      - "15672:15672"
-      - "15692"
-    environment:
-      - RABBITMQ_DEFAULT_USER=${RABBIT_USER}
-      - RABBITMQ_DEFAULT_PASS=${RABBIT_PASSWORD}
-    healthcheck:
-      # see https://www.rabbitmq.com/monitoring.html#individual-checks for info about health-checks available in rabbitmq
-      test: rabbitmq-diagnostics -q status
-      interval: 5s
-      timeout: 30s
-      retries: 5
-      start_period: 5s
-
-  redis:
-    image: "redis:6.2.6@sha256:4bed291aa5efb9f0d77b76ff7d4ab71eee410962965d052552db1fb80576431d"
-    init: true
-    ports:
-      - "6379:6379"
-    healthcheck:
-      test: [ "CMD", "redis-cli", "ping" ]
-      interval: 5s
-      timeout: 30s
-      retries: 50
-
-  redis-commander:
-    image: rediscommander/redis-commander:latest
-    init: true
-    ports:
-      - "18081:8081"
-    environment:
-      - >-
-        REDIS_HOSTS=
-        resources:${REDIS_HOST}:${REDIS_PORT}:0,
-        locks:${REDIS_HOST}:${REDIS_PORT}:1,
-        validation_codes:${REDIS_HOST}:${REDIS_PORT}:2,
-        scheduled_maintenance:${REDIS_HOST}:${REDIS_PORT}:3,
-        user_notifications:${REDIS_HOST}:${REDIS_PORT}:4,
-        announcements:${REDIS_HOST}:${REDIS_PORT}:5,
-        distributed_identifiers:${REDIS_HOST}:${REDIS_PORT}:6
-      # If you add/remove a db, do not forget to update the --databases entry in the docker-compose.yml
-
-  clusters-keeper:
-    image: local/clusters-keeper:development
-    init: true
-    ports:
-      - "8010:8000"
-      - "3015:3000"
-    env_file:
-      - .env
-    volumes:
-      - ./:/devel/services/clusters-keeper
-      - ../../packages:/devel/packages
diff --git a/tests/environment-setup/test_used_docker_compose.py b/tests/environment-setup/test_used_docker_compose.py
index bacf178492c..08066b771d0 100644
--- a/tests/environment-setup/test_used_docker_compose.py
+++ b/tests/environment-setup/test_used_docker_compose.py
@@ -75,12 +75,12 @@ def ensure_env_file(env_devel_file: Path) -> Iterable[Path]:
 repo_dir = current_dir.parent.parent
 
 
-def _skip_osparc_gateway_server(p) -> bool:
-    return "osparc-gateway-server" not in f"{p}"
+def _skip_not_useful_docker_composes(p) -> bool:
+    return "osparc-gateway-server" not in f"{p}" and "manual" not in f"{p}"
 
 
 compose_paths = filter(
-    _skip_osparc_gateway_server,
+    _skip_not_useful_docker_composes,
     chain(
         *[
             repo_dir.rglob(glob)
@@ -117,6 +117,8 @@ def test_validate_compose_file(
         str(compose_path.relative_to(repo_dir)), "-> version=", compose.get("version")
     )
 
+    # NOTE: with docker stack config, the .env file MUST be alongside the docker-compose file
+
     subprocess.run(
         " ".join(
             [

From 66fc6f3e067731240022b7a15f96580f7f9e10da Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Tue, 2 Apr 2024 16:13:06 +0200
Subject: [PATCH 14/15] too much deletion

---
 services/clusters-keeper/Makefile | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 services/clusters-keeper/Makefile

diff --git a/services/clusters-keeper/Makefile b/services/clusters-keeper/Makefile
new file mode 100644
index 00000000000..ea96e6102ed
--- /dev/null
+++ b/services/clusters-keeper/Makefile
@@ -0,0 +1,5 @@
+#
+# DEVELOPMENT recipes for Clusters-keeper service
+#
+include ../../scripts/common.Makefile
+include ../../scripts/common-service.Makefile

From bda54a0771ccf337d9f2e5993fc2654990c9f42e Mon Sep 17 00:00:00 2001
From: kaiser <kaiser@itis.swiss>
Date: Wed, 3 Apr 2024 08:40:55 +0200
Subject: [PATCH 15/15] Fix: docker stack config warning - secrets dont support
 mode, uid, gid

---
 services/docker-compose.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/docker-compose.yml b/services/docker-compose.yml
index 7eb4115ad1d..8623318df5f 100644
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -2,10 +2,10 @@ version: "3.8"
 x-dask-tls-secrets: &dask_tls_secrets
   - source: dask_tls_key
     target: ${DASK_TLS_KEY}
-    mode: 444
+    # mode: 444 # not supported by docker stack compose as of 26.0.0
   - source: dask_tls_cert
     target: ${DASK_TLS_CERT}
-    mode: 444
+    # mode: 444 # not supported by docker stack compose as of 26.0.0
 
 services:
   api-server: