From 90d6bf4e16563f4a82cd204eac386f339b47a017 Mon Sep 17 00:00:00 2001 From: Gary Pennington Date: Thu, 24 Nov 2022 14:21:04 +0000 Subject: [PATCH] improve router docker debug image (#2142) fixes: #2135 fixes: #2145 This PR repurposes the -debug image to make it the basis for a memory tracking image which we can use for investigating router memory issues via heaptrack. (https://github.com/KDE/heaptrack) The PR is a *breaking* change because it now automatically starts the router under the control of heaptrack. Technically, it's not really a breaking change and it's certainly not an API change, but I really want to draw people's attention to the fact that the debug image will now execute a lot slower than the non-debug image and use a lot more memory (to track memory with...). I've updated the docker documentation to show how to mount a local directory to store the heaptrack data. I haven't updated the kubernetes docs, because we don't go into that level of detail and we assume that a kubernetes devops person would know how to allocate and mount a PVC. --- NEXT_CHANGELOG.md | 11 +++++ dockerfiles/Dockerfile.router | 33 +++++++++++-- dockerfiles/diy/build_docker_image.sh | 17 ++++++- dockerfiles/diy/dockerfiles/Dockerfile.repo | 46 +++++++++++++++--- docs/source/containerization/docker.mdx | 53 ++++++++++++++------- docs/source/containerization/kubernetes.mdx | 2 +- docs/source/containerization/overview.mdx | 6 +-- 7 files changed, 135 insertions(+), 33 deletions(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 5655e1f346..4509648200 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -25,6 +25,17 @@ By [@USERNAME](https://github.com/USERNAME) in https://github.com/apollographql/ # [x.x.x] (unreleased) - 2022-mm-dd ## ❗ BREAKING ❗ + +### Router debug Docker images now run under the control of heaptrack ([Issue #2135](https://github.com/apollographql/router/pull/2142)) + +From the next release, our debug Docker image will invoke the router under the control of heaptrack. We are making this change to make it simple for users to investigate potential memory issues with the router. + +Do not run debug images in performance sensitive contexts. The tracking of memory allocations will significantly impact performance. In general, the debug image should only be used in consultation with Apollo engineering and support. + +Look at our documentation for examples of how to use the image in either Docker or Kubernetes. + +By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/2142 + ### Fix naming inconsistency of telemetry.metrics.common.attributes.router ([Issue #2076](https://github.com/apollographql/router/issues/2076)) Mirroring the rest of the config `router` should be `supergraph` diff --git a/dockerfiles/Dockerfile.router b/dockerfiles/Dockerfile.router index 40d475ef3d..f8b634cde0 100644 --- a/dockerfiles/Dockerfile.router +++ b/dockerfiles/Dockerfile.router @@ -1,6 +1,7 @@ FROM debian:bullseye-slim ARG ROUTER_RELEASE=latest +ARG DEBUG_IMAGE=false WORKDIR /dist @@ -8,8 +9,17 @@ WORKDIR /dist RUN \ apt-get update -y \ && apt-get install -y \ - curl \ - && rm -rf /var/lib/apt/lists/* + curl + +# If debug image, install heaptrack and make a data directory +RUN \ + if [ "${DEBUG_IMAGE}" = "true" ]; then \ + apt-get install -y heaptrack && \ + mkdir data; \ + fi + +# Clean up apt lists +RUN rm -rf /var/lib/apt/lists/* # Run the Router downloader which puts Router into current working directory RUN curl -sSL https://router.apollo.dev/download/nix/${ROUTER_RELEASE}/ | sh @@ -25,5 +35,20 @@ LABEL org.opencontainers.image.source="https://github.com/apollographql/router" ENV APOLLO_ROUTER_CONFIG_PATH="/dist/config/router.yaml" -# Default executable is the router -ENTRYPOINT ["/dist/router"] +# Create a wrapper script to run the router, use exec to ensure signals are handled correctly +RUN \ + echo '#!/usr/bin/env bash \ +\nset -e \ +\n \ +\nif [ -f "/usr/bin/heaptrack" ]; then \ +\n exec heaptrack -o /dist/data/router_heaptrack /dist/router "$@" \ +\nelse \ +\n exec /dist/router "$@" \ +\nfi \ +' > /dist/router_wrapper.sh + +# Make sure we can run our wrapper +RUN chmod 755 /dist/router_wrapper.sh + +# Default executable is the wrapper script +ENTRYPOINT ["/dist/router_wrapper.sh"] diff --git a/dockerfiles/diy/build_docker_image.sh b/dockerfiles/diy/build_docker_image.sh index 208f9b756b..41458ae52c 100755 --- a/dockerfiles/diy/build_docker_image.sh +++ b/dockerfiles/diy/build_docker_image.sh @@ -15,14 +15,18 @@ # Note: This utility makes assumptions about the existence of files relative # to the directory where it is executed. To work correctly you must # execute in the "repo"/dockerfiles/diy directory. +# Note: A debug image is an image where heaptrack is installed. The router +# is still a release build router, but all memory is being tracked +# under heaptrack. (https://github.com/KDE/heaptrack) ### ### # Terminate with a nice usage message ### usage () { - printf "Usage: build_docker_image.sh [-b [-r ]] []\n" + printf "Usage: build_docker_image.sh [-b [-r ]] [-d] []\n" printf "\t-b build docker image from the default repo, if not present build from a released version\n" + printf "\t-d build debug image, router will run under control of heaptrack\n" printf "\t-r build docker image from a specified repo, only valid with -b flag\n" printf "\t a valid release. If [-b] is specified, this is optional\n" printf "\tExample 1: Building HEAD from the repo\n" @@ -35,6 +39,8 @@ usage () { printf "\t\tbuild_docker_image.sh -b 7f7d223f42af34fad35b898d976bc07d0f5440c5\n" printf "\tExample 5: Building tag v0.9.1 from the released version\n" printf "\t\tbuild_docker_image.sh v0.9.1\n" + printf "\tExample 6: Building a debug image with tag v0.9.1 from the released version\n" + printf "\t\tbuild_docker_image.sh -d v0.9.1\n" exit 2 } @@ -56,13 +62,14 @@ terminate () { # If no ROUTER_VERSION specified, we are building HEAD from a repo ROUTER_VERSION= BUILD_IMAGE=false +DEBUG_IMAGE=false DEFAULT_REPO="https://github.com/apollographql/router.git" GIT_REPO= ### # Process Command Line ### -if ! args=$(getopt bhr: "$@"); then +if ! args=$(getopt bdhr: "$@"); then usage fi @@ -79,6 +86,10 @@ while :; do BUILD_IMAGE=true shift ;; + -d) + DEBUG_IMAGE=true + shift + ;; -r) GIT_REPO="${2}" shift; shift @@ -143,6 +154,7 @@ if [ "${BUILD_IMAGE}" = true ]; then git checkout "${ROUTER_VERSION}" > /dev/null 2>&1 || terminate "Couldn't checkout ${ROUTER_VERSION}" # Build our docker images docker build -q -t "router:${ROUTER_VERSION}" \ + --build-arg DEBUG_IMAGE="${DEBUG_IMAGE}" \ --build-arg ROUTER_VERSION="${ROUTER_VERSION}" \ --no-cache -f ../Dockerfile.repo . \ || terminate "Couldn't build router image" @@ -150,6 +162,7 @@ else # Let the user know what we are going to do echo "Building image: ${ROUTER_VERSION}" from released version"" docker build -q -t "router:${ROUTER_VERSION}" \ + --build-arg DEBUG_IMAGE="${DEBUG_IMAGE}" \ --build-arg ROUTER_RELEASE="${ROUTER_VERSION}" \ --no-cache -f Dockerfile.router . \ || terminate "Couldn't build router image" diff --git a/dockerfiles/diy/dockerfiles/Dockerfile.repo b/dockerfiles/diy/dockerfiles/Dockerfile.repo index 3e0a5b9664..4203c3ca12 100644 --- a/dockerfiles/diy/dockerfiles/Dockerfile.repo +++ b/dockerfiles/diy/dockerfiles/Dockerfile.repo @@ -28,18 +28,52 @@ RUN mkdir -p /dist/config && \ # Copy configuration for docker image COPY dockerfiles/router.yaml /dist/config +# Build our final image FROM debian:bullseye-slim -# Set labels for our image -LABEL org.opencontainers.image.authors="Apollo Graph, Inc. https://github.com/apollographql/router" -LABEL org.opencontainers.image.source="https://github.com/apollographql/router" +ARG DEBUG_IMAGE=false + +WORKDIR /dist + +# Install curl +RUN \ + apt-get update -y \ + && apt-get install -y \ + curl # Copy in the required files from our build image COPY --from=build --chown=root:root /dist /dist -WORKDIR /dist +# If debug image, install heaptrack and make a data directory +RUN \ + if [ "${DEBUG_IMAGE}" = "true" ]; then \ + apt-get install -y heaptrack && \ + mkdir data; \ + fi + +# Clean up apt lists +RUN rm -rf /var/lib/apt/lists/* + +# Set labels for our image +LABEL org.opencontainers.image.authors="Apollo Graph, Inc. https://github.com/apollographql/router" +LABEL org.opencontainers.image.source="https://github.com/apollographql/router" ENV APOLLO_ROUTER_CONFIG_PATH="/dist/config/router.yaml" -# Default executable is the router -ENTRYPOINT ["/dist/router"] +# Create a wrapper script to run the router, use exec to ensure signals are handled correctly +RUN \ + echo '#!/usr/bin/env bash \ +\nset -e \ +\n \ +\nif [ -f "/usr/bin/heaptrack" ]; then \ +\n exec heaptrack -o /dist/data/router_heaptrack /dist/router "$@" \ +\nelse \ +\n exec /dist/router "$@" \ +\nfi \ +' > /dist/router_wrapper.sh + +# Make sure we can run our wrapper +RUN chmod 755 /dist/router_wrapper.sh + +# Default executable is the wrapper script +ENTRYPOINT ["/dist/router_wrapper.sh"] diff --git a/docs/source/containerization/docker.mdx b/docs/source/containerization/docker.mdx index f56764921e..530246f380 100644 --- a/docs/source/containerization/docker.mdx +++ b/docs/source/containerization/docker.mdx @@ -48,7 +48,7 @@ docker run -p 4000:4000 \ ## Debugging your container -If you run the debug image, then it's easy to debug your container: +It's easy to debug your container by changing the `entrypoint` ```bash docker run -p 4000:4000 \ @@ -58,18 +58,31 @@ docker run -p 4000:4000 \ --rm \ --interactive \ --tty \ - --entrypoint=sh - ghcr.io/apollographql/router:-debug -/dist # pwd + --entrypoint=bash \ + ghcr.io/apollographql/router: +dist# pwd /dist -/dist # ls -LICENSE config router -README.md licenses.html schema -/dist # exit +dist# ls +config router schema +dist# exit +exit ``` In this case, we've added interactive and tty flags and changed the entrypoint of the image to be a shell. +## Running the debug container to investigate memory issues + +```bash +docker run -p 4000:4000 \ + --env APOLLO_GRAPH_REF="" \ + --env APOLLO_KEY="" \ + --mount "type=bind,source=/data,target=/dist/data" + --rm \ + ghcr.io/apollographql/router:-debug +``` + +The router will run under the control of heaptrack. The heaptrack output will be saved to the /data directory. The output can be analyzed directly using `heaptrack_gui` or `heaptrack_print` or shared with Apollo support staff. + ## Specifying the Supergraph If we don't want to use uplink to retrieve our subgraph, we can manually specify the details. @@ -91,16 +104,22 @@ In the `dockerfiles/diy` directory, we now provide a script, `build_docker_image ```bash % ./build_docker_image.sh -h -Usage: build_docker_image.sh [-b] [] - -b build docker image from a repo, if not present build from a released tarball +Usage: build_docker_image.sh [-b [-r ]] [-d] [] + -b build docker image from the default repo, if not present build from a released version + -d build debug image, router will run under control of heaptrack + -r build docker image from a specified repo, only valid with -b flag a valid release. If [-b] is specified, this is optional Example 1: Building HEAD from the repo build_docker_image.sh -b - Example 2: Building tag from the repo - build_docker_image.sh -b v1.4.0 - Example 3: Building commit hash from the repo - build_docker_image.sh -b 1c220d35acf9ad2537b8edc58c498390b6701d3d - Example 4: Building tag v1.4.0 from the released tarball - build_docker_image.sh v1.4.0 + Example 2: Building HEAD from a different repo + build_docker_image.sh -b -r /Users/anon/dev/router + Example 3: Building tag from the repo + build_docker_image.sh -b v0.9.1 + Example 4: Building commit hash from the repo + build_docker_image.sh -b 7f7d223f42af34fad35b898d976bc07d0f5440c5 + Example 5: Building tag v0.9.1 from the released version + build_docker_image.sh v0.9.1 + Example 6: Building a debug image with tag v0.9.1 from the released version + build_docker_image.sh -d v0.9.1 ``` -Note: The script has to be run from the `dockerfiles/diy` directory because it makes assumptions about the relative availability of various files. The example uses [distroless images](https://github.com/GoogleContainerTools/distroless) for the final image build. Feel free to modify the script to use images which better suit your own needs. +Note: The script has to be run from the `dockerfiles/diy` directory because it makes assumptions about the relative availability of various files. The example uses [debian:bullseye-slim image](https://hub.docker.com/_/debian/) for the final image build. Feel free to modify the script to use images which better suit your own needs, but be careful if using the [-d] flag since it makes the assumption that there is a `heaptrack` package available to install. diff --git a/docs/source/containerization/kubernetes.mdx b/docs/source/containerization/kubernetes.mdx index 080ec037cf..3be252c9e0 100644 --- a/docs/source/containerization/kubernetes.mdx +++ b/docs/source/containerization/kubernetes.mdx @@ -29,7 +29,7 @@ router docker images. You can use helm to install charts from an OCI registry as follows: ```bash -helm install --set router.configuration.telemetry.metrics.prometheus.enabled=true --set managedFederation.apiKey="REDACTED" --set managedFederation.graphRef="REDACTED" --create-namespace --namespace router-deploy router-test oci://ghcr.io/apollographql/helm-charts/router --version 1.0.0-rc.4 --values router/values.yaml +helm install --set router.configuration.telemetry.metrics.prometheus.enabled=true --set managedFederation.apiKey="REDACTED" --set managedFederation.graphRef="REDACTED" --create-namespace --namespace router-deploy router-test oci://ghcr.io/apollographql/helm-charts/router --version 1.0.0-rc.8 --values router/values.yaml ``` For more details about using helm with OCI based registries, see [here](https://helm.sh/docs/topics/registries/) diff --git a/docs/source/containerization/overview.mdx b/docs/source/containerization/overview.mdx index 236a64a229..0b13cdbbb2 100644 --- a/docs/source/containerization/overview.mdx +++ b/docs/source/containerization/overview.mdx @@ -6,11 +6,11 @@ import ElasticNotice from '../../shared/elastic-notice.mdx'; > More details about Apollo Router images are available [on GitHub](https://github.com/apollographql/router/pkgs/container/router). -Each release of the Apollo Router includes both default and debug images. Each image for a release contains the same build. Debug images additionally contain a "busybox" to help with container debugging. +Each release of the Apollo Router includes both default and debug images. Each image for a release contains the same build. Debug images have helpful debugging utilities installed and run the router under the control of heaptrack (https://github.com/KDE/heaptrack). -Here's a basic example of running a router image with Docker (make sure to replace `` with whichever version you want to use, such as `v0.1.0-preview.1`): +Here's a basic example of running a router image with Docker (make sure to replace `` with whichever version you want to use, such as `v1.4.0`): ```bash title="Docker" docker run -p 4000:4000 \ @@ -35,4 +35,4 @@ For examples of using router images in specific environments, see the guides for ## Image build -Apollo Router images are based on [distroless](https://github.com/GoogleContainerTools/distroless), which is designed to provide constrained, secure, and small images. +Apollo Router images are based on [debian:bullseye-slim](https://hub.docker.com/_/debian/), which is designed to provide constrained, secure, and small images.