Skip to content

Commit

Permalink
♻️ Migrate to docker stack config (#5523)
Browse files Browse the repository at this point in the history
Co-authored-by: kaiser <[email protected]>
Co-authored-by: Sylvain <[email protected]>
  • Loading branch information
3 people authored Apr 3, 2024
1 parent 3afe0b9 commit 7f25edf
Show file tree
Hide file tree
Showing 17 changed files with 149 additions and 272 deletions.
11 changes: 4 additions & 7 deletions .env-devel
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ AUTOSCALING_EC2_ACCESS=null
AUTOSCALING_EC2_INSTANCES=null
AUTOSCALING_NODES_MONITORING=null
AUTOSCALING_POLL_INTERVAL=10
AUTOSCALING_LOGLEVEL=INFO


BF_API_KEY=none
Expand All @@ -43,19 +44,21 @@ CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null
CLUSTERS_KEEPER_TASK_INTERVAL=30
CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null
CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX=""

CLUSTERS_KEEPER_LOGLEVEL=INFO

DASK_SCHEDULER_HOST=dask-scheduler
DASK_SCHEDULER_PORT=8786
DASK_TLS_CA_FILE=/home/scu/.dask/dask-crt.pem
DASK_TLS_KEY=/home/scu/.dask/dask-key.pem
DASK_TLS_CERT=/home/scu/.dask/dask-crt.pem
CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'

DIRECTOR_REGISTRY_CACHING_TTL=900
DIRECTOR_REGISTRY_CACHING=True
DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'

COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL=tls://dask-scheduler:8786
COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE=S3
COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE=PRESIGNED
COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE=PRESIGNED
Expand Down Expand Up @@ -142,12 +145,6 @@ S3_ENDPOINT=172.17.0.1:9001
S3_SECRET_KEY=12345678
S3_SECURE=0

S3_EXPORT_ACCESS_KEY=87654321
S3_EXPORT_BUCKET_NAME=simcore-export
S3_EXPORT_ENDPOINT=172.17.0.1:9001
S3_EXPORT_SECRET_KEY=87654321
S3_EXPORT_SECURE=0

SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1
SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key

Expand Down
7 changes: 7 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,10 @@ repos:
language: script
types: [file, python]
entry: scripts/precommit/pytest-testit.bash
- repo: local
hooks:
- id: validate-docker-compose
name: validate-docker-compose
language: script
types: [file]
entry: scripts/precommit/validate-docker-compose.bash
12 changes: 6 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
@export DOCKER_REGISTRY=local && \
export DOCKER_IMAGE_TAG=development && \
export DEV_PC_CPU_COUNT=${CPU_COUNT} && \
scripts/docker/docker-compose-config.bash -e .env \
scripts/docker/docker-stack-config.bash -e .env \
services/docker-compose.yml \
services/docker-compose.local.yml \
services/docker-compose.devel.yml \
Expand All @@ -248,7 +248,7 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
# Creating config for stack with 'local/{service}:production' to $@
@export DOCKER_REGISTRY=local && \
export DOCKER_IMAGE_TAG=production && \
scripts/docker/docker-compose-config.bash -e .env \
scripts/docker/docker-stack-config.bash -e .env \
services/docker-compose.yml \
services/docker-compose.local.yml \
> $@
Expand All @@ -258,15 +258,15 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
# Creating config for stack with 'local/{service}:production' (except of static-webserver -> static-webserver:development) to $@
@export DOCKER_REGISTRY=local && \
export DOCKER_IMAGE_TAG=production && \
scripts/docker/docker-compose-config.bash -e $< \
scripts/docker/docker-stack-config.bash -e $< \
services/docker-compose.yml \
services/docker-compose.local.yml \
services/docker-compose.devel-frontend.yml \
> $@

.stack-simcore-version.yml: .env $(docker-compose-configs)
# Creating config for stack with '$(DOCKER_REGISTRY)/{service}:${DOCKER_IMAGE_TAG}' to $@
@scripts/docker/docker-compose-config.bash -e .env \
@scripts/docker/docker-stack-config.bash -e .env \
services/docker-compose.yml \
services/docker-compose.local.yml \
> $@
Expand All @@ -276,13 +276,13 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
# Creating config for ops stack to $@
ifdef ops_ci
@$(shell \
scripts/docker/docker-compose-config.bash -e .env \
scripts/docker/docker-stack-config.bash -e .env \
services/docker-compose-ops-ci.yml \
> $@ \
)
else
@$(shell \
scripts/docker/docker-compose-config.bash -e .env \
scripts/docker/docker-stack-config.bash -e .env \
services/docker-compose-ops.yml \
> $@ \
)
Expand Down
11 changes: 8 additions & 3 deletions packages/pytest-simcore/src/pytest_simcore/docker_compose.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,14 +101,19 @@ def env_file_for_testing(
# https://docs.docker.com/compose/environment-variables/#the-env-file

env_test_path = temp_folder / ".env.test"

with env_test_path.open("wt") as fh:
print(
f"# Auto-generated from env_file_for_testing in {__file__}",
file=fh,
)
for key in sorted(testing_environ_vars.keys()):
print(f"{key}={testing_environ_vars[key]}", file=fh)
for key, value in sorted(testing_environ_vars.items()):
# NOTE: python-dotenv parses JSON encoded strings correctly, but
# writing them back shows an issue. if the original ENV is something like MY_ENV='{"correct": "encodedjson"}'
# it goes to MY_ENV={"incorrect": "encodedjson"}!
if value.startswith(("{", "[")) and value.endswith(("}", "]")):
print(f"{key}='{value}'", file=fh)
else:
print(f"{key}={value}", file=fh)

#
# WARNING: since compose files have references to ../.env we MUST create .env
Expand Down
27 changes: 3 additions & 24 deletions packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,13 +140,8 @@ def run_docker_compose_config(
".yaml",
], "Expected yaml/yml file as destination path"

# SEE https://docs.docker.com/compose/reference/
bash_options = [
"-p",
str(project_dir), # Specify an alternate working directory
]
# https://docs.docker.com/compose/environment-variables/#using-the---env-file--option
bash_options += [
bash_options = [
"-e",
str(env_file_path), # Custom environment variables
]
Expand All @@ -158,15 +153,13 @@ def run_docker_compose_config(
bash_options += [os.path.relpath(docker_compose_path, project_dir)]

# SEE https://docs.docker.com/compose/reference/config/
docker_compose_path = scripts_dir / "docker" / "docker-compose-config.bash"
docker_compose_path = scripts_dir / "docker" / "docker-stack-config.bash"
assert docker_compose_path.exists()

args = [f"{docker_compose_path}", *bash_options]
print(" ".join(args))

process = subprocess.run(
args,
shell=False,
cwd=project_dir,
capture_output=True,
check=True,
Expand All @@ -176,20 +169,6 @@ def run_docker_compose_config(
compose_file_str = process.stdout.decode("utf-8")
compose_file: dict[str, Any] = yaml.safe_load(compose_file_str)

def _remove_top_level_name_attribute_generated_by_compose_v2(
compose: dict[str, Any]
) -> dict[str, Any]:
"""docker compose V2 CLI config adds a top level name attribute
https://docs.docker.com/compose/compose-file/#name-top-level-element
but it is incompatible with docker stack deploy...
"""
compose.pop("name", None)
return compose

compose_file = _remove_top_level_name_attribute_generated_by_compose_v2(
compose_file
)

if destination_path:
#
# NOTE: This step could be avoided and reading instead from stdout
Expand Down Expand Up @@ -282,7 +261,7 @@ def save_docker_infos(destination_dir: Path):
json.dumps(container.attrs, indent=2)
)

except Exception as err: # pylint: disable=broad-except
except Exception as err: # pylint: disable=broad-except # noqa: PERF203
if container.status != ContainerStatus.created:
print(
f"Error while dumping {container.name=}, {container.status=}.\n\t{err=}"
Expand Down
115 changes: 0 additions & 115 deletions scripts/docker/docker-compose-config.bash

This file was deleted.

70 changes: 70 additions & 0 deletions scripts/docker/docker-stack-config.bash
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/bin/bash
# generated using chatgpt
show_info() {
local message="$1"
echo -e "\e[37mInfo:\e[0m $message" >&2
}

show_warning() {
local message="$1"
echo -e "\e[31mWarning:\e[0m $message" >&2
}

show_error() {
local message="$1"
echo -e "\e[31mError:\e[0m $message" >&2
}

env_file=".env"
# Parse command line arguments
while getopts ":e:" opt; do
case $opt in
e)
env_file="$OPTARG"
;;
\?)
show_error "Invalid option: -$OPTARG"
exit 1
;;
:)
show_error "Option -$OPTARG requires an argument."
exit 1
;;
esac
done
shift $((OPTIND - 1))

if [[ "$#" -eq 0 ]]; then
show_error "No compose files specified!"
exit 1
fi

# Check if Docker version is greater than or equal to 24.0.9
version_check=$(docker --version | grep --extended-regexp --only-matching '[0-9]+\.[0-9]+\.[0-9]+')
IFS='.' read -r -a version_parts <<<"$version_check"

if [[ "${version_parts[0]}" -gt 24 ]] ||
{ [[ "${version_parts[0]}" -eq 24 ]] && [[ "${version_parts[1]}" -gt 0 ]]; } ||
{ [[ "${version_parts[0]}" -eq 24 ]] && [[ "${version_parts[1]}" -eq 0 ]] && [[ "${version_parts[2]}" -ge 9 ]]; }; then
show_info "Running Docker version $version_check"
else
show_error "Docker version 25.0.3 or higher is required."
exit 1
fi

# shellcheck disable=SC2002
docker_command="\
set -o allexport && \
. ${env_file} && set +o allexport && \
docker stack config"

for compose_file_path in "$@"; do
docker_command+=" --compose-file ${compose_file_path}"
done
# WE CANNOT DO THIS:
# docker_command+=" --skip-interpolation"
# because docker stack compose will *validate* that e.g. `replicas: ${SIMCORE_SERVICES_POSTGRES_REPLICAS}` is a valid number, which it is not if it is read as a literal string.

# Execute the command
show_info "Executing Docker command: ${docker_command}"
eval "${docker_command}"
Loading

0 comments on commit 7f25edf

Please sign in to comment.