diff --git a/packages/postgres-database/docker/Dockerfile b/packages/postgres-database/docker/Dockerfile index d06eef136e9..db1634bfdf5 100644 --- a/packages/postgres-database/docker/Dockerfile +++ b/packages/postgres-database/docker/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 FROM python:3.6-slim as base LABEL maintainer=sanderegg @@ -16,8 +17,8 @@ FROM base as build RUN apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - git \ + build-essential \ + git \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -25,7 +26,8 @@ RUN apt-get update \ # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" -RUN pip --no-cache-dir install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools diff --git a/packages/postgres-database/docker/Makefile b/packages/postgres-database/docker/Makefile index 8ab8a17c010..d6864030bae 100644 --- a/packages/postgres-database/docker/Makefile +++ b/packages/postgres-database/docker/Makefile @@ -23,7 +23,8 @@ SIMCORE_NETWORK ?= $(SWARM_STACK_NAME)_default .PHONY: build build: ## build postgres migration image # use GIT_REPOSITORY='$(GIT_REPOSITORY)' GIT_BRANCH='$(GIT_BRANCH)' - @docker build --file Dockerfile \ + @docker buildx build --file Dockerfile \ + --load \ --tag $(MIGRATER_TAG_NAME) \ --build-arg GIT_REPOSITORY=$(GIT_REPOSITORY) \ --build-arg GIT_BRANCH=$(GIT_BRANCH) \ diff --git a/packages/postgres-database/scripts/erd/Dockerfile b/packages/postgres-database/scripts/erd/Dockerfile index dc4642356f2..c69273908d6 100644 --- a/packages/postgres-database/scripts/erd/Dockerfile +++ b/packages/postgres-database/scripts/erd/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG PYTHON_VERSION="3.10.10" FROM python:${PYTHON_VERSION}-slim-buster as base @@ -14,14 +15,16 @@ RUN apt-get update \ && apt-get clean -RUN pip --no-cache-dir install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools # devenv -RUN pip install --no-cache-dir \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ pyparsing \ pydot \ sqlalchemy_schemadisplay diff --git a/packages/postgres-database/scripts/erd/Makefile b/packages/postgres-database/scripts/erd/Makefile index 745eb5db1be..043b09662ab 100644 --- a/packages/postgres-database/scripts/erd/Makefile +++ b/packages/postgres-database/scripts/erd/Makefile @@ -20,7 +20,8 @@ IMAGE_NAME:=local/postgres-database-scripts-erd:${PYTHON_VERSION} # SEE https://medium.com/faun/set-current-host-user-for-docker-container-4e521cef9ffc .PHONY: build build build-nc: ## builds tooling image ${IMAGE_NAME} - docker build $(if $(findstring -nc,$@),--no-cache,) \ + docker buildx build $(if $(findstring -nc,$@),--no-cache,) \ + --load \ --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ --tag ${IMAGE_NAME} . diff --git a/packages/service-integration/Dockerfile b/packages/service-integration/Dockerfile index a4d55e5ac0c..00dc5156850 100644 --- a/packages/service-integration/Dockerfile +++ b/packages/service-integration/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG PYTHON_VERSION="3.10.10" FROM python:${PYTHON_VERSION}-slim-buster as base @@ -51,7 +52,8 @@ RUN apt-get update \ # packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" -RUN pip install --no-cache-dir --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools @@ -63,10 +65,11 @@ COPY --chown=scu:scu packages/service-integration packages/service-integration # WARNING: keep synced with `make install-prod` (did not use it directly because if would require copying scripts/common.Makefile and other parts of the repo) -RUN cd packages/service-integration \ - && pip install --no-cache-dir -r requirements/_base.txt \ - && pip install --no-cache-dir ../models-library/ \ - && pip install --no-cache-dir . +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + cd packages/service-integration \ + && pip install -r requirements/_base.txt \ + && pip install ../models-library/ \ + && pip install . # -------------------------- Build stage ------------------- diff --git a/requirements/tools/Dockerfile b/requirements/tools/Dockerfile index 17ccbc3695a..5e5576bbeb0 100644 --- a/requirements/tools/Dockerfile +++ b/requirements/tools/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # NOTE: This is a first step towards a devcontainer # to perform operations like pip-compile or auto-formatting # that preserves identical environment across developer machines @@ -21,14 +22,16 @@ RUN apt-get update \ # SEE bug with pip==22.1 https://github.com/jazzband/pip-tools/issues/1617 -RUN pip --no-cache-dir install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools # devenv -RUN pip install \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ pip-tools \ pipreqs \ pipdeptree && \ diff --git a/requirements/tools/Makefile b/requirements/tools/Makefile index bf9f194f1a7..dd5c82e191f 100644 --- a/requirements/tools/Makefile +++ b/requirements/tools/Makefile @@ -84,7 +84,8 @@ IMAGE_NAME:=local/python-devkit:${PYTHON_VERSION} # SEE https://medium.com/faun/set-current-host-user-for-docker-container-4e521cef9ffc .PHONY: build build build-nc: ## builds tooling image ${IMAGE_NAME} - docker build $(if $(findstring -nc,$@),--no-cache,) \ + docker buildx build $(if $(findstring -nc,$@),--no-cache,) \ + --load \ --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ --tag ${IMAGE_NAME} . diff --git a/scripts/act.bash b/scripts/act.bash index 8a9bdadeb21..73d4d77e0d3 100755 --- a/scripts/act.bash +++ b/scripts/act.bash @@ -14,7 +14,7 @@ DOCKER_IMAGE_NAME=dind-act-runner ACT_RUNNER=ubuntu-20.04=catthehacker/ubuntu:act-20.04 ACT_VERSION_TAG=v0.2.20 # from https://github.com/nektos/act/releases -docker build -t $DOCKER_IMAGE_NAME - <> /home/scu/.config/dask/distributed.yaml echo "$INFO" "Starting as dask scheduler:${scheduler_version}..." if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ]; then diff --git a/services/datcore-adapter/Dockerfile b/services/datcore-adapter/Dockerfile index 6f5663f67f6..80e494110af 100644 --- a/services/datcore-adapter/Dockerfile +++ b/services/datcore-adapter/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG PYTHON_VERSION="3.10.10" FROM python:${PYTHON_VERSION}-slim-buster as base @@ -11,12 +12,15 @@ FROM python:${PYTHON_VERSION}-slim-buster as base LABEL maintainer=sanderegg -RUN set -eux && \ +RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux && \ apt-get update && \ - apt-get install -y --no-install-recommends gosu && \ - rm -rf /var/lib/apt/lists/* && \ + apt-get install -y --no-install-recommends \ + gosu \ + && apt-get clean -y \ # verify that the binary works - gosu nobody true + && gosu nobody true # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) ENV SC_USER_ID=8004 \ @@ -56,18 +60,20 @@ FROM base as build ENV SC_BUILD_TARGET=build -RUN apt-get update \ +RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux \ + && apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + build-essential # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" -RUN pip --no-cache-dir install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools @@ -77,7 +83,9 @@ WORKDIR /build # install base 3rd party dependencies # NOTE: copies to /build to avoid overwriting later which would invalidate this layer COPY --chown=scu:scu services/datcore-adapter/requirements/_base.txt . -RUN pip --no-cache-dir install -r _base.txt +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ + --requirement _base.txt # --------------------------Prod-depends-only stage ------------------- @@ -95,8 +103,10 @@ COPY --chown=scu:scu services/datcore-adapter /build/services/datcore-adapter WORKDIR /build/services/datcore-adapter -RUN pip3 --no-cache-dir install -r requirements/prod.txt \ - && pip3 --no-cache-dir list -v +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ + --requirement requirements/prod.txt \ + && pip list -v # --------------------------Production stage ------------------- @@ -115,6 +125,8 @@ ENV PYTHONOPTIMIZE=TRUE WORKDIR /home/scu +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu # Starting from clean base image, copies pre-installed virtualenv from prod-only-deps COPY --chown=scu:scu --from=prod-only-deps ${VIRTUAL_ENV} ${VIRTUAL_ENV} @@ -123,10 +135,10 @@ COPY --chown=scu:scu services/datcore-adapter/docker services/datcore-adapter/do RUN chmod +x services/datcore-adapter/docker/*.sh -HEALTHCHECK --interval=30s \ - --timeout=20s \ - --start-period=30s \ - --retries=3 \ +HEALTHCHECK --interval=10s \ + --timeout=5s \ + --start-period=5s \ + --retries=5 \ CMD ["python3", "services/datcore-adapter/docker/healthcheck.py", "http://localhost:8000/v0/live"] ENTRYPOINT [ "/bin/sh", "services/datcore-adapter/docker/entrypoint.sh" ] diff --git a/services/director-v2/Dockerfile b/services/director-v2/Dockerfile index 313b6e1f5e8..fa3292d574c 100644 --- a/services/director-v2/Dockerfile +++ b/services/director-v2/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG PYTHON_VERSION="3.10.10" FROM python:${PYTHON_VERSION}-slim-buster as base @@ -11,12 +12,15 @@ FROM python:${PYTHON_VERSION}-slim-buster as base LABEL maintainer=pcrespov -RUN set -eux && \ +RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux && \ apt-get update && \ - apt-get install -y --no-install-recommends gosu && \ - rm -rf /var/lib/apt/lists/* && \ + apt-get install -y --no-install-recommends \ + gosu \ + && apt-get clean -y \ # verify that the binary works - gosu nobody true + && gosu nobody true # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) ENV SC_USER_ID=8004 \ @@ -56,18 +60,20 @@ FROM base as build ENV SC_BUILD_TARGET=build -RUN apt-get update \ +RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux \ + && apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + build-essential # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" -RUN pip --no-cache-dir install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools @@ -77,7 +83,9 @@ WORKDIR /build # install base 3rd party dependencies # NOTE: copies to /build to avoid overwriting later which would invalidate this layer COPY --chown=scu:scu services/director-v2/requirements/_base.txt . -RUN pip --no-cache-dir install -r _base.txt +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ + --requirement _base.txt # --------------------------Prod-depends-only stage ------------------- @@ -95,8 +103,10 @@ COPY --chown=scu:scu services/director-v2 /build/services/director-v2 WORKDIR /build/services/director-v2 -RUN pip3 --no-cache-dir install -r requirements/prod.txt \ - && pip3 --no-cache-dir list -v +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ + --requirement requirements/prod.txt \ + && pip list -v # --------------------------Production stage ------------------- @@ -115,6 +125,8 @@ ENV PYTHONOPTIMIZE=TRUE WORKDIR /home/scu +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu # Starting from clean base image, copies pre-installed virtualenv from prod-only-deps COPY --chown=scu:scu --from=prod-only-deps ${VIRTUAL_ENV} ${VIRTUAL_ENV} diff --git a/services/director/Dockerfile b/services/director/Dockerfile index 1df4e033ac2..5aff2a24348 100644 --- a/services/director/Dockerfile +++ b/services/director/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG PYTHON_VERSION="3.6.10" FROM python:${PYTHON_VERSION}-slim-buster as base # @@ -10,46 +11,49 @@ FROM python:${PYTHON_VERSION}-slim-buster as base LABEL maintainer=sanderegg -RUN set -eux && \ +RUN --mount=type=cache,id=basecache36,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt36,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux && \ apt-get update && \ - apt-get install -y gosu && \ - rm -rf /var/lib/apt/lists/* && \ + apt-get install -y --no-install-recommends \ + gosu \ + && apt-get clean -y \ # verify that the binary works - gosu nobody true + && gosu nobody true # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) ENV SC_USER_ID=8004 \ - SC_USER_NAME=scu \ - SC_BUILD_TARGET=base \ - SC_BOOT_MODE=default + SC_USER_NAME=scu \ + SC_BUILD_TARGET=base \ + SC_BOOT_MODE=default RUN adduser \ - --uid ${SC_USER_ID} \ - --disabled-password \ - --gecos "" \ - --shell /bin/sh \ - --home /home/${SC_USER_NAME} \ - ${SC_USER_NAME} + --uid ${SC_USER_ID} \ + --disabled-password \ + --gecos "" \ + --shell /bin/sh \ + --home /home/${SC_USER_NAME} \ + ${SC_USER_NAME} # Sets utf-8 encoding for Python et al ENV LANG=C.UTF-8 # Turns off writing .pyc files; superfluous on an ephemeral container. ENV PYTHONDONTWRITEBYTECODE=1 \ - VIRTUAL_ENV=/home/scu/.venv + VIRTUAL_ENV=/home/scu/.venv # Ensures that the python and pip executables used # in the image will be those from our virtualenv. ENV PATH="${VIRTUAL_ENV}/bin:$PATH" # environment variables ENV REGISTRY_AUTH='' \ - REGISTRY_USER='' \ - REGISTRY_PW='' \ - REGISTRY_URL='' \ - REGISTRY_VERSION='v2' \ - PUBLISHED_HOST_NAME='' \ - SIMCORE_SERVICES_NETWORK_NAME='' \ - EXTRA_HOSTS_SUFFIX='undefined' + REGISTRY_USER='' \ + REGISTRY_PW='' \ + REGISTRY_URL='' \ + REGISTRY_VERSION='v2' \ + PUBLISHED_HOST_NAME='' \ + SIMCORE_SERVICES_NETWORK_NAME='' \ + EXTRA_HOSTS_SUFFIX='undefined' EXPOSE 8080 @@ -64,34 +68,37 @@ FROM base as build ENV SC_BUILD_TARGET=build -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - git \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* +RUN --mount=type=cache,id=basecache36,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt36,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + git # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" -RUN pip --no-cache-dir install --upgrade \ - pip~=21.3 \ - wheel \ - setuptools +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ + pip~=21.3 \ + wheel \ + setuptools # copy director and dependencies COPY --chown=scu:scu packages /build/packages COPY --chown=scu:scu services/director /build/services/director # install base 3rd party dependencies (NOTE: this speeds up devel mode) -RUN pip --no-cache-dir install -r /build/services/director/requirements/_base.txt +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install -r /build/services/director/requirements/_base.txt # FIXME: # necessary to prevent duplicated files. # Will be removed when director is refactored using cookiecutter as this will not be necessary anymore COPY --chown=scu:scu api/specs/common/schemas/node-meta-v0.0.1.json \ - /build/services/director/src/simcore_service_director/api/v0/oas-parts/schemas/node-meta-v0.0.1.json + /build/services/director/src/simcore_service_director/api/v0/oas-parts/schemas/node-meta-v0.0.1.json # --------------------------Prod-depends-only stage ------------------- # This stage is for production only dependencies that get partially wiped out afterwards (final docker image concerns) @@ -103,7 +110,8 @@ FROM build as prod-only-deps WORKDIR /build/services/director ENV SC_BUILD_TARGET=prod-only-deps -RUN pip --no-cache-dir install -r requirements/prod.txt +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install -r requirements/prod.txt # --------------------------Production stage ------------------- # Final cleanup up to reduce image size and startup setup @@ -115,11 +123,13 @@ RUN pip --no-cache-dir install -r requirements/prod.txt FROM base as production ENV SC_BUILD_TARGET=production \ - SC_BOOT_MODE=production + SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE WORKDIR /home/scu +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu # bring installed package without build tools COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} @@ -127,11 +137,12 @@ COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY --chown=scu:scu services/director/docker services/director/docker RUN chmod +x services/director/docker/*.sh + HEALTHCHECK --interval=30s \ - --timeout=120s \ - --start-period=30s \ - --retries=3 \ - CMD ["python3", "/home/scu/services/director/docker/healthcheck.py", "http://localhost:8080/v0/"] + --timeout=120s \ + --start-period=30s \ + --retries=3 \ + CMD ["python3", "/home/scu/services/director/docker/healthcheck.py", "http://localhost:8080/v0/"] ENTRYPOINT [ "services/director/docker/entrypoint.sh" ] CMD ["services/director/docker/boot.sh"] diff --git a/services/docker-compose.yml b/services/docker-compose.yml index f1ac2d27408..9c90a49b15c 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -187,8 +187,17 @@ services: networks: - default environment: - - RESOURCE_USAGE_TRACKER_LOGLEVEL=${LOG_LEVEL:-INFO} - LOG_FORMAT_LOCAL_DEV_ENABLED=${LOG_FORMAT_LOCAL_DEV_ENABLED} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT} + - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_PORT=${POSTGRES_PORT} + - POSTGRES_USER=${POSTGRES_USER} + - PROMETHEUS_URL=${PROMETHEUS_URL} + - PROMETHEUS_USERNAME=${PROMETHEUS_USERNAME} + - PROMETHEUS_PASSWORD=${PROMETHEUS_PASSWORD} + - RESOURCE_USAGE_TRACKER_LOGLEVEL=${LOG_LEVEL:-INFO} static-webserver: image: ${DOCKER_REGISTRY:-itisfoundation}/static-webserver:${DOCKER_IMAGE_TAG:-latest} diff --git a/services/dynamic-sidecar/Dockerfile b/services/dynamic-sidecar/Dockerfile index 3238f77e415..644531b51f4 100644 --- a/services/dynamic-sidecar/Dockerfile +++ b/services/dynamic-sidecar/Dockerfile @@ -78,17 +78,15 @@ RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=lock set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + build-essential # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" \ && mkdir -p "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}" -RUN --mount=type=cache,mode=0777,target=/root/.cache/pip \ - pip --no-cache-dir install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools @@ -98,7 +96,8 @@ WORKDIR /build # install base 3rd party dependencies # NOTE: copies to /build to avoid overwriting later which would invalidate this layer COPY --chown=scu:scu services/dynamic-sidecar/requirements/_base.txt . -RUN pip --no-cache-dir install -r _base.txt +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install -r _base.txt # copy utility devops scripts COPY --chown=scu:scu services/dynamic-sidecar/scripts/Makefile /home/scu @@ -119,10 +118,10 @@ COPY --chown=scu:scu services/dynamic-sidecar /build/services/dynamic-sidecar WORKDIR /build/services/dynamic-sidecar -RUN --mount=type=cache,mode=0777,target=/root/.cache/pip \ - pip --no-cache-dir install \ - --requirement requirements/prod.txt &&\ - pip --no-cache-dir list -v +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ + --requirement requirements/prod.txt \ + && pip list -v # --------------------------Production stage ------------------- # Final cleanup up to reduce image size and startup setup @@ -140,6 +139,8 @@ ENV PYTHONOPTIMIZE=TRUE WORKDIR /home/scu +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu # Starting from clean base image, copies pre-installed virtualenv from prod-only-deps COPY --chown=scu:scu --from=prod-only-deps ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY --chown=scu:scu --from=prod-only-deps ${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR} ${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR} diff --git a/services/invitations/Dockerfile b/services/invitations/Dockerfile index 76da5508d58..6c778948312 100644 --- a/services/invitations/Dockerfile +++ b/services/invitations/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG PYTHON_VERSION="3.10.10" FROM python:${PYTHON_VERSION}-slim-buster as base @@ -11,12 +12,15 @@ FROM python:${PYTHON_VERSION}-slim-buster as base LABEL maintainer=pcrespov -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends gosu; \ - rm -rf /var/lib/apt/lists/*; \ +RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + gosu \ + && apt-get clean -y \ # verify that the binary works - gosu nobody true + && gosu nobody true # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) ENV SC_USER_ID=8004 \ @@ -55,18 +59,20 @@ FROM base as build ENV SC_BUILD_TARGET=build -RUN apt-get update \ +RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux \ + && apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + build-essential # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" -RUN pip --no-cache-dir install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools @@ -76,7 +82,10 @@ WORKDIR /build # install base 3rd party dependencies # NOTE: copies to /build to avoid overwriting later which would invalidate this layer COPY --chown=scu:scu services/invitations/requirements/_base.txt . -RUN pip --no-cache-dir install -r _base.txt +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ + --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -94,8 +103,10 @@ COPY --chown=scu:scu services/invitations /build/services/invitations WORKDIR /build/services/invitations -RUN pip3 --no-cache-dir install -r requirements/prod.txt \ - && pip3 --no-cache-dir list -v +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ + --requirement requirements/prod.txt \ + && pip list -v # --------------------------Production stage ------------------- @@ -114,6 +125,8 @@ ENV PYTHONOPTIMIZE=TRUE WORKDIR /home/scu +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu # Starting from clean base image, copies pre-installed virtualenv from prod-only-deps COPY --chown=scu:scu --from=prod-only-deps ${VIRTUAL_ENV} ${VIRTUAL_ENV} diff --git a/services/migration/Dockerfile b/services/migration/Dockerfile index 6fa7c745620..ef33d75a6c3 100644 --- a/services/migration/Dockerfile +++ b/services/migration/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG PYTHON_VERSION="3.10.10" FROM python:${PYTHON_VERSION}-slim-buster as base @@ -31,18 +32,20 @@ ENV PATH="${VIRTUAL_ENV}/bin:$PATH" # -------------------------------------------- FROM base as build -RUN apt-get update \ +RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux \ + && apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ - git \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + git # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" -RUN pip --no-cache-dir install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools @@ -51,7 +54,7 @@ WORKDIR /build # install only base 3rd party dependencies COPY --chown=scu:scu packages/postgres-database/ . -RUN pip --no-cache-dir --quiet \ +RUN pip --quiet \ install \ -r requirements/prod.txt \ && pip freeze @@ -76,8 +79,10 @@ WORKDIR /home/scu USER ${SC_USER_NAME} +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu # bring installed package without build tools -COPY --from=build ${VIRTUAL_ENV} ${VIRTUAL_ENV} +COPY --chown=scu:scu --from=build ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY --chown=scu:scu services/migration/docker services/migration/docker HEALTHCHECK \ diff --git a/services/osparc-gateway-server/Dockerfile b/services/osparc-gateway-server/Dockerfile index b6fd96d1b21..c058c75b385 100644 --- a/services/osparc-gateway-server/Dockerfile +++ b/services/osparc-gateway-server/Dockerfile @@ -16,8 +16,6 @@ RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=loc gosu \ libffi-dev \ libffi7 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -67,15 +65,13 @@ RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=loc && apt-get install -y --no-install-recommends \ build-essential \ git \ - golang-go \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + golang-go # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0777,target=/root/.cache/pip \ - pip install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools @@ -84,7 +80,7 @@ WORKDIR /build # install base 3rd party dependencies (NOTE: this speeds up devel mode) COPY --chown=scu:scu services/osparc-gateway-server/requirements/_base.txt . -RUN --mount=type=cache,mode=0777,target=/root/.cache/pip \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ pip install \ --requirement _base.txt @@ -111,7 +107,7 @@ COPY --chown=scu:scu services/osparc-gateway-server/ /build/services/osparc-gate WORKDIR /build/services/osparc-gateway-server -RUN --mount=type=cache,mode=0777,target=/root/.cache/pip \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ pip install \ --requirement requirements/prod.txt @@ -130,7 +126,8 @@ ENV SC_BUILD_TARGET=production \ ENV PYTHONOPTIMIZE=TRUE WORKDIR /home/scu - +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu # bring installed package without build tools COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} # copy docker entrypoint and boot scripts diff --git a/services/resource-usage-tracker/Dockerfile b/services/resource-usage-tracker/Dockerfile index 297fae1a5a9..f22e2babf0a 100644 --- a/services/resource-usage-tracker/Dockerfile +++ b/services/resource-usage-tracker/Dockerfile @@ -14,13 +14,11 @@ LABEL maintainer=sanderegg RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ - set -eux; \ - apt-get update; \ + set -eux && \ + apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ - && apt-get autoclean -y\ - && apt-get autoremove -y\ - && rm -rf /var/lib/apt/lists/* \ + && apt-get clean -y \ # verify that the binary works && gosu nobody true @@ -67,17 +65,15 @@ RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=lock set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + build-essential # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0777,target=/root/.cache/pip \ - pip --no-cache-dir install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools @@ -87,8 +83,8 @@ WORKDIR /build # install base 3rd party dependencies # NOTE: copies to /build to avoid overwriting later which would invalidate this layer COPY --chown=scu:scu services/resource-usage-tracker/requirements/_base.txt . -RUN --mount=type=cache,mode=0777,target=/root/.cache/pip \ - pip --no-cache-dir install \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ --requirement _base.txt @@ -107,10 +103,10 @@ COPY --chown=scu:scu services/resource-usage-tracker /build/services/resource-us WORKDIR /build/services/resource-usage-tracker -RUN --mount=type=cache,mode=0777,target=/root/.cache/pip \ - pip --no-cache-dir install \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ --requirement requirements/prod.txt \ - && pip --no-cache-dir list -v + && pip list -v # --------------------------Production stage ------------------- @@ -129,6 +125,8 @@ ENV PYTHONOPTIMIZE=TRUE WORKDIR /home/scu +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu # Starting from clean base image, copies pre-installed virtualenv from prod-only-deps COPY --chown=scu:scu --from=prod-only-deps ${VIRTUAL_ENV} ${VIRTUAL_ENV} @@ -136,7 +134,6 @@ COPY --chown=scu:scu --from=prod-only-deps ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY --chown=scu:scu services/resource-usage-tracker/docker services/resource-usage-tracker/docker RUN chmod +x services/resource-usage-tracker/docker/*.sh - HEALTHCHECK --interval=10s \ --timeout=5s \ --start-period=5s \ diff --git a/services/static-webserver/client/Makefile b/services/static-webserver/client/Makefile index fc276f02dd6..4950e9a7c34 100644 --- a/services/static-webserver/client/Makefile +++ b/services/static-webserver/client/Makefile @@ -45,6 +45,7 @@ follow-dev-logs: ## follow the logs of the qx compiler compile: ## qx compiles host' 'source' -> image's 'build-output' # qx compile 'source' within $(docker_image) image [itisfoundation/qooxdoo-kit:${QOOXDOO_KIT_TAG}] @docker buildx build --file $(docker_file) --tag $(docker_image) \ + --load \ --build-arg tag=${QOOXDOO_KIT_TAG} \ --build-arg VCS_REF=${VCS_REF} \ --build-arg VCS_REF_CLIENT=${VCS_REF_CLIENT} \ @@ -54,7 +55,7 @@ compile: ## qx compiles host' 'source' -> image's 'build-output' touch: ## minimal image build with /project/output-build inside # touch /project/output-build such that multi-stage 'services/web/Dockerfile' can build development target (fixes #1097) - @docker buildx build --file $(docker_file) --tag $(docker_image) --build-arg tag=${QOOXDOO_KIT_TAG} --target=touch . + @docker buildx build --load --file $(docker_file) --tag $(docker_image) --build-arg tag=${QOOXDOO_KIT_TAG} --target=touch . upgrade: ## upgrade to official version of the tool # upgrading to ${QOOXDOO_KIT_TAG} diff --git a/services/storage/Dockerfile b/services/storage/Dockerfile index fbef7dec104..3303c91aa44 100644 --- a/services/storage/Dockerfile +++ b/services/storage/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG PYTHON_VERSION="3.10.10" FROM python:${PYTHON_VERSION}-slim-buster as base # @@ -10,12 +11,15 @@ FROM python:${PYTHON_VERSION}-slim-buster as base LABEL maintainer=mguidon -RUN set -eux && \ +RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux && \ apt-get update && \ - apt-get install -y gosu && \ - rm -rf /var/lib/apt/lists/* && \ + apt-get install -y --no-install-recommends \ + gosu \ + && apt-get clean -y \ # verify that the binary works - gosu nobody true + && gosu nobody true # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) @@ -63,19 +67,21 @@ FROM base as build ENV SC_BUILD_TARGET build -RUN apt-get update \ +RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux \ + && apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ - git \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + git # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" -RUN pip --no-cache-dir install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools @@ -84,7 +90,9 @@ WORKDIR /build # install only base 3rd party dependencies COPY --chown=scu:scu services/storage/requirements/_base.txt requirements_base.txt -RUN pip --no-cache-dir --quiet install -r requirements_base.txt +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ + --requirement requirements_base.txt # -------------------------- ------------------------------- @@ -106,7 +114,10 @@ COPY --chown=scu:scu services/storage /build/services/storage WORKDIR /build/services/storage -RUN pip --no-cache-dir --quiet install -r requirements/prod.txt +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ + --requirement requirements/prod.txt \ + && pip list -v # -------------------------- ------------------------------- @@ -125,13 +136,16 @@ ENV PYTHONOPTIMIZE=TRUE WORKDIR /home/scu +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu # bring installed package without build tools COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} # copy docker entrypoint and boot scripts COPY --chown=scu:scu services/storage/docker services/storage/docker RUN chmod +x services/storage/docker/*.sh - +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu HEALTHCHECK --interval=10s \ --timeout=15s \ --start-period=5s \ diff --git a/services/web/Dockerfile b/services/web/Dockerfile index 4c850908ea7..7874cf1eae8 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG PYTHON_VERSION="3.10.10" FROM python:${PYTHON_VERSION}-slim-buster as base # @@ -12,13 +13,15 @@ FROM python:${PYTHON_VERSION}-slim-buster as base LABEL maintainer=pcrespov -RUN set -eux && \ +RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ - gosu && \ - rm -rf /var/lib/apt/lists/* && \ + gosu \ + && apt-get clean -y \ # verify that the binary works - gosu nobody true + && gosu nobody true # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) @@ -66,19 +69,21 @@ FROM base as build ENV SC_BUILD_TARGET build -RUN apt-get update \ +RUN --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=baseapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux \ + && apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ - libmagic1 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + libmagic1 # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN python -m venv "${VIRTUAL_ENV}" -RUN pip --no-cache-dir install --upgrade \ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools @@ -87,7 +92,9 @@ WORKDIR /build # install only base 3rd party dependencies COPY --chown=scu:scu services/web/server/requirements/_base.txt requirements_base.txt -RUN pip --no-cache-dir --quiet install -r requirements_base.txt +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ + --requirement requirements_base.txt # --------------------------Prod-depends-only stage ------------------- # This stage is for production only dependencies that get partially wiped out afterwards (final docker image concerns) @@ -105,7 +112,10 @@ COPY --chown=scu:scu services/web/server /build/services/web/server WORKDIR /build/services/web/server -RUN pip --no-cache-dir --quiet install -r requirements/prod.txt +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install \ + --requirement requirements/prod.txt \ + && pip list -v # --------------------------Production stage ------------------- # Final cleanup up to reduce image size and startup setup @@ -116,17 +126,23 @@ RUN pip --no-cache-dir --quiet install -r requirements/prod.txt # FROM base as production -ARG BUILD_DATE -ARG VCS_URL -ARG VCS_REF +ENV SC_BUILD_TARGET=production \ + SC_BOOT_MODE=production + +ENV PYTHONOPTIMIZE=TRUE WORKDIR /home/scu +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu # bring installed package without build tools COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} # libmagic is requried by the exporter -RUN apt-get update \ +RUN --mount=type=cache,id=webprodcache,target=/var/cache/apt,mode=0755,sharing=locked \ + --mount=type=cache,id=webprodapt,target=/var/lib/apt,mode=0755,sharing=locked \ + set -eux \ + && apt-get update \ && apt-get install -y --no-install-recommends \ libmagic1 \ && apt-get clean \ @@ -146,14 +162,16 @@ HEALTHCHECK --interval=10s \ --retries=5 \ CMD ["python3", "/home/scu/services/web/server/docker/healthcheck.py", "http://localhost:8080/v0/health"] +# no need to rebuild everything if just these changed +ARG BUILD_DATE +ARG VCS_URL +ARG VCS_REF -ENV SC_BUILD_TARGET=production \ - SC_BOOT_MODE=production \ - SC_BUILD_DATE=${BUILD_DATE} \ +ENV SC_BUILD_DATE=${BUILD_DATE} \ SC_VCS_URL=${VCS_URL} \ SC_VCS_REF=${VCS_REF} -ENV PYTHONOPTIMIZE=TRUE + ENTRYPOINT [ "services/web/server/docker/entrypoint.sh" ] CMD ["services/web/server/docker/boot.sh"] diff --git a/tests/e2e/docker/Makefile b/tests/e2e/docker/Makefile index e2ebb8d430f..8cf22754e53 100644 --- a/tests/e2e/docker/Makefile +++ b/tests/e2e/docker/Makefile @@ -14,7 +14,7 @@ help: ## help on rule's targets .PHONY: build build: ## builds a node image able to run puppeteer - docker buildx build --build-arg BUILD_DATE=$(BUILD_DATE) --build-arg VCS_URL=$(VCS_URL) --build-arg VCS_REF=$(VCS_REF) --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) .; + docker buildx build --load --build-arg BUILD_DATE=$(BUILD_DATE) --build-arg VCS_URL=$(VCS_URL) --build-arg VCS_REF=$(VCS_REF) --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) .; .PHONY: shell shell: guard-URL guard-USER guard-PASSWORD ## runs bash in the image to allow manual testing diff --git a/tests/performance/Makefile b/tests/performance/Makefile index c57ece03810..2c6e098de8f 100644 --- a/tests/performance/Makefile +++ b/tests/performance/Makefile @@ -28,7 +28,7 @@ __check_defined = \ .PHONY: build build: ## builds distributed osparc locust docker image - docker buildx build --build-arg LOCUST_VERSION=$(LOCUST_VERSION) --tag itisfoundation/locust:$(LOCUST_VERSION) --tag local/locust:latest . + docker buildx build --load --build-arg LOCUST_VERSION=$(LOCUST_VERSION) --tag itisfoundation/locust:$(LOCUST_VERSION) --tag local/locust:latest . .PHONY: push push: