From 94ebf5a6fcabb4b14ef65d65478fa4f9220beef6 Mon Sep 17 00:00:00 2001 From: Kate Osborn <50597707+kate-osborn@users.noreply.github.com> Date: Thu, 4 Jan 2024 11:50:27 -0700 Subject: [PATCH] Add ephemeral debugging guide (#1202) Problem: No documentation on how to debug NGF in Kubernetes Solution: Add a developer guide with step-by-step instructions on how to inject an ephemeral dlv debug container into NGF while it's running in a cluster. Includes makefile targets to make the process easier and a dockerfile for the dlv debug image. --- Makefile | 52 +++++++++++++- conformance/Makefile | 2 +- debug/Dockerfile | 12 ++++ docs/developer/debugging.md | 127 +++++++++++++++++++++++++++++++++++ docs/developer/quickstart.md | 23 ++++++- 5 files changed, 211 insertions(+), 5 deletions(-) create mode 100644 debug/Dockerfile create mode 100644 docs/developer/debugging.md diff --git a/Makefile b/Makefile index 800c43820f..6caa2af686 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,8 @@ NGINX_CONF_DIR = internal/mode/static/nginx/conf NJS_DIR = internal/mode/static/nginx/modules/src NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=nginx-repo.crt --secret id=nginx-repo.key,src=nginx-repo.key BUILD_AGENT=local +GW_API_VERSION = 1.0.0 +INSTALL_WEBHOOK = false # go build flags - should not be overridden by the user GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE} @@ -147,13 +149,59 @@ njs-unit-test: ## Run unit tests for the njs httpmatches module lint-helm: ## Run the helm chart linter helm lint $(CHART_DIR) +.PHONY: load-images +load-images: ## Load NGF and NGINX images on configured kind cluster. + kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG) + +.PHONY: load-images-with-plus +load-images-with-plus: ## Load NGF and NGINX Plus images on configured kind cluster. + kind load docker-image $(PREFIX):$(TAG) $(NGINX_PLUS_PREFIX):$(TAG) + +.PHONY: install-ngf-local-build +install-ngf-local-build: build-images load-images helm-install-local ## Install NGF from local build on configured kind cluster. + +.PHONY: install-ngf-local-build-with-plus +install-ngf-local-build-with-plus: build-images-with-plus load-images-with-plus helm-install-local-with-plus ## Install NGF with NGINX Plus from local build on configured kind cluster. + +.PHONY: helm-install-local +helm-install-local: ## Helm install NGF on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build. + ./conformance/scripts/install-gateway.sh $(GW_API_VERSION) $(INSTALL_WEBHOOK) + helm install dev ./deploy/helm-chart --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never -n nginx-gateway + +.PHONY: helm-install-local-with-plus +helm-install-local-with-plus: ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus. + ./conformance/scripts/install-gateway.sh $(GW_API_VERSION) $(INSTALL_WEBHOOK) + helm install dev ./deploy/helm-chart --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginx.plus=true -n nginx-gateway + +# Debug Targets .PHONY: debug-build debug-build: GO_LINKER_FLAGS=$(GO_LINKER_FlAGS_VARS) debug-build: ADDITIONAL_GO_BUILD_FLAGS=-gcflags "all=-N -l" debug-build: build ## Build binary with debug info, symbols, and no optimizations -.PHONY: build-ngf-debug-image -build-ngf-debug-image: debug-build build-ngf-image ## Build NGF image with debug binary +.PHONY: debug-build-dlv-image +debug-build-dlv-image: check-for-docker ## Build the dlv debugger image. + docker build --platform linux/$(GOARCH) -f debug/Dockerfile -t dlv-debug:edge . + +.PHONY: debug-build-images +debug-build-images: debug-build build-ngf-image build-nginx-image debug-build-dlv-image ## Build all images used in debugging. + +.PHONY: debug-build-images-with-plus +debug-build-images-with-plus: debug-build build-ngf-image build-nginx-plus-image debug-build-dlv-image ## Build all images with NGINX plus used in debugging. + +.PHONY: debug-load-images +debug-load-images: load-images ## Load all images used in debugging to kind cluster. + kind load docker-image dlv-debug:edge + +.PHONY: debug-load-images-with-plus +debug-load-images-with-plus: load-images-with-plus ## Load all images with NGINX Plus used in debugging to kind cluster. + kind load docker-image dlv-debug:edge + +.PHONY: debug-install-local-build +debug-install-local-build: debug-build-images debug-load-images helm-install-local ## Install NGF from local build using debug NGF binary on configured kind cluster. + +.PHONY: debug-install-local-build-with-plus +debug-install-local-build-with-plus: debug-build-images-with-plus debug-load-images-with-plus helm-install-local-with-plus ## Install NGF with NGINX Plus from local build using debug NGF binary on configured kind cluster. .PHONY: dev-all dev-all: deps fmt njs-fmt vet lint unit-test njs-unit-test ## Run all the development checks diff --git a/conformance/Makefile b/conformance/Makefile index 27819aefc3..fdd21f972d 100644 --- a/conformance/Makefile +++ b/conformance/Makefile @@ -44,7 +44,7 @@ build-images: ## Build NGF and nginx images .PHONY: load-images load-images: ## Load NGF and NGINX images on configured kind cluster - kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG) + cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) load-images .PHONY: prepare-ngf-dependencies prepare-ngf-dependencies: update-ngf-manifest ## Install NGF dependencies on configured kind cluster diff --git a/debug/Dockerfile b/debug/Dockerfile new file mode 100644 index 0000000000..b9d36763a6 --- /dev/null +++ b/debug/Dockerfile @@ -0,0 +1,12 @@ +# syntax=docker/dockerfile:1.5 +# This Dockerfile builds an image with the dlv debugger. See the debugging guide in the developer docs for details +# on how to use it. +FROM golang:1.21-alpine AS builder + +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +FROM alpine:latest + +COPY --from=builder /go/bin/dlv /usr/bin/ + +CMD ["sh"] diff --git a/docs/developer/debugging.md b/docs/developer/debugging.md new file mode 100644 index 0000000000..e142a808ab --- /dev/null +++ b/docs/developer/debugging.md @@ -0,0 +1,127 @@ +# Debugging + +## Debugging NGF Remotely in Kubernetes + +This section will walk you through how to attach an ephemeral [dlv](https://github.com/go-delve/delve) debugger +container to NGF while it's running in Kubernetes. This will allow you to remotely debug NGF running in Kubernetes +using your IDE. + +- Create a `kind` cluster: + + ```console + make create-kind-cluster + ``` + +- Set GOARCH environment variable: + + The [Makefile](/Makefile) uses the GOARCH variable to build the binary and container images. The default value of GOARCH is `amd64`. + + If you are deploying NGINX Gateway Fabric on a kind cluster, and the architecture of your machine is not `amd64`, you will want to set the GOARCH variable to the architecture of your local machine. You can find the value of GOARCH by running `go env`. Export the GOARCH variable in your `~/.zshrc` or `~/.bashrc`. + + ```console + echo "export GOARCH=< Your architecture (e.g. arm64 or amd64) >" >> ~/.bashrc + source ~/.bashrc + ``` + + or for zsh: + + ```console + echo "export GOARCH=< Your architecture (e.g. arm64 or amd64) >" >> ~/.zshrc + source ~/.zshrc + ``` + +- Build debug images and install NGF on your kind cluster: + + - **For NGINX OSS:** + + ```console + make GOARCH=$GOARCH debug-install-local-build + ``` + + - **For NGINX Plus:** + + ```console + make GOARCH=$GOARCH debug-install-local-build-with-plus + ``` + + > Note: The default value of GOARCH in the [Makefile](/Makefile) is `amd64`. If you try and debug an amd64 container on an ARM machine you will see the following error in the dlv container logs: `could not attach to pid : function not implemented`. + > This is a known issue and the only workaround is to create an arm64 image by specifying `GOARCH=arm64` the above commands. + > For more information, see this [issue](https://github.com/docker/for-mac/issues/5191) + +- Start kubectl proxy in the background: + + ```console + kubectl proxy & + ``` + +- Save the NGF Pod name: + + ```console + POD_NAME= + ``` + +- Run the following curl command to create an ephemeral debug container: + + ```console + curl --location --request PATCH 127.0.0.1:8001/api/v1/namespaces/nginx-gateway/pods/$POD_NAME/ephemeralcontainers \ + --header 'Content-Type: application/strategic-merge-patch+json' \ + --data '{ + "spec": + { + "ephemeralContainers": + [ + { + "name": "dlv", + "command": [ + "/bin/sh", + "-c", + "PID=$(pgrep -f /usr/bin/gateway) && dlv attach $PID --headless --listen 127.0.0.1:40000 --api-version=2 --accept-multiclient --only-same-user=false" + ], + "image": "dlv-debug:edge", + "imagePullPolicy": "Never", + "targetContainerName": "nginx-gateway", + "stdin": true, + "tty": true, + "securityContext": { + "capabilities": { + "add": [ + "SYS_PTRACE" + ] + }, + "runAsNonRoot":false + } + } + ] + } + }' + ``` + +- Verify that the dlv API server is running: + + ```console + kubectl logs -n nginx-gateway $POD_NAME -c dlv + ``` + + you should see the following log: + + ```text + API server listening at: 127.0.0.1:40000 + ``` + +- Kill the kubectl proxy process: + + ```console + kill + ``` + +- Port-forward the dlv API server port on the NGF Pod: + + ```console + kubectl port-forward -n nginx-gateway $POD_NAME 40000 + ``` + +- Connect to the remote dlv API server through your IDE: + - [jetbrains instructions](https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html) + - [vscode instructions](https://github.com/golang/vscode-go/blob/master/docs/debugging.md) + +- Debug! diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index 28bcbf6aa9..53c0ef0f28 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -11,6 +11,7 @@ Follow these steps to set up your development environment. 1. Install: - [Go](https://golang.org/doc/install) v1.21.0+ - [Docker](https://docs.docker.com/get-docker/) v18.09+ + - [Kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) - [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/) - [Helm](https://helm.sh/docs/intro/quickstart/#install-helm) - [git](https://git-scm.com/) @@ -50,12 +51,30 @@ Follow these steps to set up your development environment. ## Build the Binary and Images +### Setting GOARCH + +The [Makefile](/Makefile) uses the GOARCH variable to build the binary and container images. The default value of GOARCH is `amd64`. + +If you are deploying NGINX Gateway Fabric on a kind cluster, and the architecture of your machine is not `amd64`, you will want to set the GOARCH variable to the architecture of your local machine. You can find the value of GOARCH by running `go env`. Export the GOARCH variable in your `~/.zshrc` or `~/.bashrc`. + +```shell +echo "export GOARCH=< Your architecture (e.g. arm64 or amd64) >" >> ~/.bashrc +source ~/.bashrc +``` + +or for zsh: + +```shell +echo "export GOARCH=< Your architecture (e.g. arm64 or amd64) >" >> ~/.zshrc +source ~/.zshrc +``` + ### Build the Binary To build the binary, run the make build command from the project's root directory: ```makefile -make build +make GOARCH=$GOARCH build ``` This command will build the binary and output it to the `/build/.out` directory. @@ -65,7 +84,7 @@ This command will build the binary and output it to the `/build/.out` directory. To build the NGINX Gateway Fabric and NGINX container images from source run the following make command: ```makefile -make TAG=$(whoami) build-images +make GOARCH=$GOARCH TAG=$(whoami) build-images ``` This will build the docker images `nginx-gateway-fabric:` and `nginx-gateway-fabric/nginx:`.