♻️ add osparc-gateway-server to repo (⚠️ DEVOPS) (#3960)

.github/workflows/ci-testing-deploy.yml

@@ -65,6 +65,7 @@ jobs:
       dynamic-sidecar: ${{ steps.filter.outputs.dynamic-sidecar }}
       invitations: ${{ steps.filter.outputs.invitations }}
       migration: ${{ steps.filter.outputs.migration }}
+      osparc-gateway-server: ${{ steps.filter.outputs.osparc-gateway-server }}
       static-webserver: ${{ steps.filter.outputs.static-webserver }}
       storage: ${{ steps.filter.outputs.storage }}
       webserver: ${{ steps.filter.outputs.webserver }}
@@ -154,6 +155,10 @@ jobs:
               - 'packages/**'
               - 'services/migration/**'
               - 'services/docker-compose*'
+            osparc-gateway-server:
+              - 'packages/**'
+              - 'services/osparc-gateway-server/**'
+              - 'services/docker-compose*'
             static-webserver:
               - 'services/static-webserver/**'
               - 'services/docker-compose*'
@@ -837,6 +842,60 @@ jobs:
         with:
           flags: unittests #optional
 
+  unit-test-osparc-gateway-server:
+    needs: changes
+    if: ${{ needs.changes.outputs.osparc-gateway-server == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 18 # if this timeout gets too small, then split the tests
+    name: "[unit] osparc-gateway-server"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.9"]
+        os: [ubuntu-20.04]
+        docker_buildx: [v0.8.2]
+        docker_compose: [1.29.1]
+        include:
+          - docker_compose: 1.29.1
+            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: ${{ matrix.docker_buildx }}
+          driver: docker-container
+      - name: setup docker-compose
+        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
+      - name: setup python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          cache: "pip"
+          cache-dependency-path: "services/osparc-gateway-server/requirements/ci.txt"
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - name: install
+        run: |
+          make devenv
+          source .venv/bin/activate && \
+          pushd services/osparc-gateway-server && \
+          make install-ci
+      - name: typecheck
+        run: |
+          source .venv/bin/activate && \
+          pushd services/osparc-gateway-server && \
+          make mypy
+      - name: test
+        run: |
+          source .venv/bin/activate && \
+          pushd services/osparc-gateway-server && \
+          make test-ci-unit
+      - uses: codecov/[email protected]
+        with:
+          flags: unittests #optional
+
   unit-test-dynamic-sidecar:
     needs: changes
     if: ${{ needs.changes.outputs.dynamic-sidecar == 'true' || github.event_name == 'push' }}
@@ -1282,6 +1341,7 @@ jobs:
         unit-test-dynamic-sidecar,
         unit-test-frontend,
         unit-test-models-library,
+        unit-test-osparc-gateway-server,
         unit-test-postgres-database,
         unit-test-python-linting,
         unit-test-service-integration,
@@ -1546,6 +1606,81 @@ jobs:
         with:
           flags: integrationtests #optional
 
+  integration-test-osparc-gateway-server:
+    needs: [changes, build-test-images]
+    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.osparc-gateway-server == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 30 # if this timeout gets too small, then split the tests
+    name: "[int] osparc-gateway-server"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.9"]
+        os: [ubuntu-20.04]
+        docker_buildx: [v0.8.2]
+        docker_compose: [1.29.1]
+        include:
+          - docker_compose: 1.29.1
+            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: ${{ matrix.docker_buildx }}
+          driver: docker-container
+
+      - name: setup docker-compose
+        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
+      - name: setup python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          cache: "pip"
+          cache-dependency-path: "services/osparc-gateway-server/requirements/ci.txt"
+      - name: expose github runtime for buildx
+        uses: crazy-max/ghaction-github-runtime@v2
+      - name: download docker images
+        uses: actions/download-artifact@v3
+        with:
+          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+          path: /${{ runner.temp }}/build
+      - name: load docker images
+        run: make load-images local-src=/${{ runner.temp }}/build
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - name: install
+        run: |
+          make devenv && \
+          source .venv/bin/activate && \
+          pushd services/osparc-gateway-server && \
+          make install-ci
+      - name: integration-test
+        run: |
+          source .venv/bin/activate && \
+          pushd services/osparc-gateway-server && \
+          make test-ci-integration
+      - name: system-test
+        run: |
+          source .venv/bin/activate && \
+          pushd services/osparc-gateway-server && \
+          make test-system
+      - name: upload failed tests logs
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.job }}_docker_logs
+          path: ./services/director-v2/test_failures
+      - name: cleanup
+        if: always()
+        run: |
+          pushd services/osparc-gateway-server && \
+          make down
+      - uses: codecov/[email protected]
+        with:
+          flags: integrationtests #optional
+
   integration-test-simcore-sdk:
     needs: [changes, build-test-images]
     if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }}
@@ -1613,6 +1748,7 @@ jobs:
       [
         integration-test-director-v2-01,
         integration-test-director-v2-02,
+        integration-test-osparc-gateway-server,
         integration-test-simcore-sdk,
         integration-test-webserver-01,
         integration-test-webserver-02,

Makefile

@@ -42,6 +42,7 @@ SERVICES_NAMES_TO_BUILD := \
   dynamic-sidecar \
 	invitations \
   migration \
+	osparc-gateway-server \
   service-integration \
   static-webserver \
   storage \
@@ -162,7 +163,7 @@ docker buildx bake \
 		)\
 	)\
 	$(if $(push),--push,) \
-	$(if $(push),--file docker-bake.hcml,) --file docker-compose-build.yml $(if $(target),$(target),) \
+	$(if $(push),--file docker-bake.hcl,) --file docker-compose-build.yml $(if $(target),$(target),) \
 	$(if $(findstring -nc,$@),--no-cache,\
 		$(foreach service, $(SERVICES_NAMES_TO_BUILD),\
 			--set $(service).cache-to=type=gha$(comma)mode=max$(comma)scope=$(service) \
@@ -229,52 +230,45 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 
 .stack-simcore-development.yml: .env $(docker-compose-configs)
 	# Creating config for stack with 'local/{service}:development' to $@
-	@export DOCKER_REGISTRY=local \
-	export DOCKER_IMAGE_TAG=development; \
-	export DEV_PC_CPU_COUNT=${CPU_COUNT}; \
-	docker-compose \
-		--env-file .env \
-		--file services/docker-compose.yml \
-		--file services/docker-compose.local.yml \
-		--file services/docker-compose.devel.yml \
-		--log-level=ERROR \
-		config | sed --regexp-extended "s/cpus: ([0-9\\.]+)/cpus: '\\1'/" > $@
+	@export DOCKER_REGISTRY=local && \
+	export DOCKER_IMAGE_TAG=development && \
+	export DEV_PC_CPU_COUNT=${CPU_COUNT} && \
+	scripts/docker/docker-compose-config.bash -e .env \
+		services/docker-compose.yml \
+		services/docker-compose.local.yml \
+		services/docker-compose.devel.yml \
+		> $@
 
 .stack-simcore-production.yml: .env $(docker-compose-configs)
 	# Creating config for stack with 'local/{service}:production' to $@
-	@export DOCKER_REGISTRY=local;       \
-	export DOCKER_IMAGE_TAG=production; \
-	docker-compose \
-		--env-file .env \
-		--file services/docker-compose.yml \
-		--file services/docker-compose.local.yml \
-		--log-level=ERROR \
-		config | sed --regexp-extended "s/cpus: ([0-9\\.]+)/cpus: '\\1'/" > $@
+	@export DOCKER_REGISTRY=local && \
+	export DOCKER_IMAGE_TAG=production && \
+	scripts/docker/docker-compose-config.bash -e .env \
+		services/docker-compose.yml \
+		services/docker-compose.local.yml \
+		> $@
 
 .stack-simcore-version.yml: .env $(docker-compose-configs)
 	# Creating config for stack with '$(DOCKER_REGISTRY)/{service}:${DOCKER_IMAGE_TAG}' to $@
-	@docker-compose \
-		--env-file .env \
-		--file services/docker-compose.yml \
-		--file services/docker-compose.local.yml \
-		--log-level=ERROR \
-		config | sed --regexp-extended "s/cpus: ([0-9\\.]+)/cpus: '\\1'/" > $@
+	@scripts/docker/docker-compose-config.bash -e .env \
+		services/docker-compose.yml \
+		services/docker-compose.local.yml \
+		> $@
+
 
 .stack-ops.yml: .env $(docker-compose-configs)
 	# Compiling config file for filestash
-	$(eval TMP_PATH_TO_FILESTASH_CONFIG=$(shell set -o allexport; \
-	source $(CURDIR)/.env; \
-	set +o allexport; \
+	$(eval TMP_PATH_TO_FILESTASH_CONFIG=$(shell set -o allexport && \
+	source $(CURDIR)/.env && \
+	set +o allexport && \
 	python3 scripts/filestash/create_config.py))
 	# Creating config for ops stack to $@
 	# -> filestash config at $(TMP_PATH_TO_FILESTASH_CONFIG)
 	@$(shell \
 		export TMP_PATH_TO_FILESTASH_CONFIG="${TMP_PATH_TO_FILESTASH_CONFIG}" && \
-		docker-compose \
-			--env-file .env \
-			--file services/docker-compose-ops.yml \
-			--log-level=DEBUG \
-			config | sed --regexp-extended "s/cpus: ([0-9\\.]+)/cpus: '\\1'/" > $@ \
+		scripts/docker/docker-compose-config.bash -e .env \
+		services/docker-compose-ops.yml \
+		> $@ \
 	)
 
 
@@ -557,28 +551,35 @@ rm-registry: ## remove the registry and changes to host/file
 		echo removing entry in /etc/hosts...;\
 		sudo sed -i "/127.0.0.1 $(LOCAL_REGISTRY_HOSTNAME)/d" /etc/hosts,\
 		echo /etc/hosts is already cleaned)
-	@$(if $(shell grep "{\"insecure-registries\": \[\"$(LOCAL_REGISTRY_HOSTNAME):5000\"\]}" /etc/docker/daemon.json),\
+	@$(if $(shell jq -e '.["insecure-registries"]? | index("http://$(LOCAL_REGISTRY_HOSTNAME):5000")? // empty' /etc/docker/daemon.json),\
 		echo removing entry in /etc/docker/daemon.json...;\
-		sudo sed -i '/{"insecure-registries": \["$(LOCAL_REGISTRY_HOSTNAME):5000"\]}/d' /etc/docker/daemon.json;,\
-		echo /etc/docker/daemon.json is already cleaned)
+		jq 'if .["insecure-registries"] then .["insecure-registries"] |= map(select(. != "http://$(LOCAL_REGISTRY_HOSTNAME):5000")) else . end' /etc/docker/daemon.json > /tmp/daemon.json && \
+		sudo mv /tmp/daemon.json /etc/docker/daemon.json &&\
+		echo restarting engine... &&\
+		sudo service docker restart &&\
+		echo done,\
+		echo /etc/docker/daemon.json already cleaned)
 	# removing container and volume
-	-docker rm --force $(LOCAL_REGISTRY_HOSTNAME)
-	-docker volume rm $(LOCAL_REGISTRY_VOLUME)
+	-@docker rm --force $(LOCAL_REGISTRY_HOSTNAME)
+	-@docker volume rm $(LOCAL_REGISTRY_VOLUME)
 
 local-registry: .env ## creates a local docker registry and configure simcore to use it (NOTE: needs admin rights)
 	@$(if $(shell grep "127.0.0.1 $(LOCAL_REGISTRY_HOSTNAME)" /etc/hosts),,\
 					echo configuring host file to redirect $(LOCAL_REGISTRY_HOSTNAME) to 127.0.0.1; \
 					sudo echo 127.0.0.1 $(LOCAL_REGISTRY_HOSTNAME) | sudo tee -a /etc/hosts;\
 					echo done)
-	@$(if $(shell grep "{\"insecure-registries\": \[\"registry:5000\"\]}" /etc/docker/daemon.json),,\
+	@$(if $(shell jq -e '.["insecure-registries"]? | index("http://$(LOCAL_REGISTRY_HOSTNAME):5000")? // empty' /etc/docker/daemon.json),,\
 					echo configuring docker engine to use insecure local registry...; \
-					sudo echo {\"insecure-registries\": [\"$(LOCAL_REGISTRY_HOSTNAME):5000\"]} | sudo tee -a /etc/docker/daemon.json; \
-					echo restarting engine...; \
-					sudo service docker restart;\
+					jq 'if .["insecure-registries"] | index("http://$(LOCAL_REGISTRY_HOSTNAME):5000") then . else .["insecure-registries"] += ["http://$(LOCAL_REGISTRY_HOSTNAME):5000"] end' /etc/docker/daemon.json > /tmp/daemon.json &&\
+					sudo mv /tmp/daemon.json /etc/docker/daemon.json &&\
+					echo restarting engine... &&\
+					sudo service docker restart &&\
 					echo done)
+
 	@$(if $(shell docker ps --format="{{.Names}}" | grep registry),,\
-					echo starting registry on $(LOCAL_REGISTRY_HOSTNAME):5000...; \
-					docker run --detach \
+					echo starting registry on http://$(LOCAL_REGISTRY_HOSTNAME):5000...; \
+					docker run \
+							--detach \
 							--init \
 							--env REGISTRY_STORAGE_DELETE_ENABLED=true \
 							--publish 5000:5000 \

packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py

@@ -242,7 +242,6 @@ def save_docker_infos(destination_path: Path):
                 destination_path.mkdir(parents=True, exist_ok=True)
 
         for container in all_containers:
-
             try:
                 container_name = safe_artifact_name(container.name)
 

scripts/common.Makefile

@@ -47,6 +47,9 @@ VENV_DIR      := $(abspath $(REPO_BASE_DIR)/.venv)
 # environment variables files
 DOT_ENV_FILE = $(abspath $(REPO_BASE_DIR)/.env)
 
+# utils
+get_my_ip := $(shell hostname --all-ip-addresses | cut --delimiter=" " --fields=1)
+
 #
 # SHORTCUTS
 #