Skip to content

Commit

Permalink
Support Horizontal Scaling (#1048)
Browse files Browse the repository at this point in the history 
Problem: NKG cannot be scaled horizontally because all replicas will write statuses to 
the Gateway API resources.

Solution: Add leader election to the status updater so that only one replica of NKG 
will write statuses to the Gateway API resources. Leader election is enabled by 
default but can be disabled via a cli arg --leader-election-disable. The lock name 
used for leader election can be configured via the cli arg --leader-election-lock-name.
  • Loading branch information
@kate-osborn
kate-osborn authored Sep 18, 2023
1 parent 1d44e2b commit 43c1100
Show file tree
Hide file tree
Showing 26 changed files with 852 additions and 226 deletions.
11 changes: 6 additions & 5 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ TAG ?= $(VERSION:v%=%)## The tag of the image. For example, 0.3.0
TARGET ?= local## The target of the build. Possible values: local and container
KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfig
OUT_DIR ?= $(shell pwd)/build/out## The folder where the binary will be stored
ARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64
GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64
GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin
override HELM_TEMPLATE_COMMON_ARGS += --set creator=template --set nameOverride=nginx-gateway## The common options for the Helm template command.
override HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE += --set service.create=false## The options to be passed to the full Helm templating command only.
override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR)
Expand All @@ -35,11 +36,11 @@ build-images: build-nkg-image build-nginx-image ## Build the NKG and nginx docke

.PHONY: build-nkg-image
build-nkg-image: check-for-docker build ## Build the NKG docker image
docker build --platform linux/$(ARCH) --target $(strip $(TARGET)) -f build/Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) .
docker build --platform linux/$(GOARCH) --target $(strip $(TARGET)) -f build/Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) .

.PHONY: build-nginx-image
build-nginx-image: check-for-docker ## Build the custom nginx image
docker build --platform linux/$(ARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f build/Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) .
docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f build/Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) .

.PHONY: check-for-docker
check-for-docker: ## Check if Docker is installed
Expand All @@ -49,13 +50,13 @@ check-for-docker: ## Check if Docker is installed
build: ## Build the binary
ifeq (${TARGET},local)
@go version || (code=$$?; printf "\033[0;31mError\033[0m: unable to build locally\n"; exit $$code)
CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -trimpath -a -ldflags "$(GO_LINKER_FLAGS)" $(ADDITIONAL_GO_BUILD_FLAGS) -o $(OUT_DIR)/gateway github.com/nginxinc/nginx-kubernetes-gateway/cmd/gateway
CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -trimpath -a -ldflags "$(GO_LINKER_FLAGS)" $(ADDITIONAL_GO_BUILD_FLAGS) -o $(OUT_DIR)/gateway github.com/nginxinc/nginx-kubernetes-gateway/cmd/gateway
endif

.PHONY: build-goreleaser
build-goreleaser: ## Build the binary using GoReleaser
@goreleaser -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with GoReleaser. Follow the docs to install it https://goreleaser.com/install\n"; exit $$code)
GOOS=linux GOPATH=$(shell go env GOPATH) GOARCH=$(ARCH) goreleaser build --clean --snapshot --single-target
GOOS=linux GOPATH=$(shell go env GOPATH) GOARCH=$(GOARCH) goreleaser build --clean --snapshot --single-target

.PHONY: generate
generate: ## Run go generate
Expand Down
117 changes: 80 additions & 37 deletions cmd/gateway/commands.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ const (
gatewayCtrlNameFlag = "gateway-ctlr-name"
gatewayCtrlNameUsageFmt = `The name of the Gateway controller. ` +
`The controller name must be of the form: DOMAIN/PATH. The controller's domain is '%s'`
gatewayFlag = "gateway"
)

var (
Expand All @@ -38,25 +37,6 @@ var (
gatewayClassName = stringValidatingValue{
validator: validateResourceName,
}

// Backing values for static subcommand cli flags.
updateGCStatus bool
disableMetrics bool
metricsSecure bool
disableHealth bool

metricsListenPort = intValidatingValue{
validator: validatePort,
value: 9113,
}
healthListenPort = intValidatingValue{
validator: validatePort,
value: 8081,
}
gateway = namespacedNameValue{}
configName = stringValidatingValue{
validator: validateResourceName,
}
)

func createRootCommand() *cobra.Command {
Expand Down Expand Up @@ -85,6 +65,46 @@ func createRootCommand() *cobra.Command {
}

func createStaticModeCommand() *cobra.Command {
// flag names
const (
gatewayFlag = "gateway"
configFlag = "config"
updateGCStatusFlag = "update-gatewayclass-status"
metricsDisableFlag = "metrics-disable"
metricsSecureFlag = "metrics-secure-serving"
metricsPortFlag = "metrics-port"
healthDisableFlag = "health-disable"
healthPortFlag = "health-port"
leaderElectionDisableFlag = "leader-election-disable"
leaderElectionLockNameFlag = "leader-election-lock-name"
)

// flag values
var (
updateGCStatus bool
gateway = namespacedNameValue{}
configName = stringValidatingValue{
validator: validateResourceName,
}
disableMetrics bool
metricsSecure bool
metricsListenPort = intValidatingValue{
validator: validatePort,
value: 9113,
}
disableHealth bool
healthListenPort = intValidatingValue{
validator: validatePort,
value: 8081,
}

disableLeaderElection bool
leaderElectionLockName = stringValidatingValue{
validator: validateResourceName,
value: "nginx-gateway-leader-election-lock",
}
)

cmd := &cobra.Command{
Use: "static-mode",
Short: "Configure NGINX in the scope of a single Gateway resource",
Expand All @@ -109,23 +129,21 @@ func createStaticModeCommand() *cobra.Command {
return fmt.Errorf("error validating POD_IP environment variable: %w", err)
}

namespace := os.Getenv("MY_NAMESPACE")
namespace := os.Getenv("POD_NAMESPACE")
if namespace == "" {
return errors.New("MY_NAMESPACE environment variable must be set")
return errors.New("POD_NAMESPACE environment variable must be set")
}

podName := os.Getenv("POD_NAME")
if podName == "" {
return errors.New("POD_NAME environment variable must be set")
}

var gwNsName *types.NamespacedName
if cmd.Flags().Changed(gatewayFlag) {
gwNsName = &gateway.value
}

metricsConfig := config.MetricsConfig{}
if !disableMetrics {
metricsConfig.Enabled = true
metricsConfig.Port = metricsListenPort.value
metricsConfig.Secure = metricsSecure
}

conf := config.Config{
GatewayCtlrName: gatewayCtlrName.value,
ConfigName: configName.String(),
Expand All @@ -136,11 +154,20 @@ func createStaticModeCommand() *cobra.Command {
Namespace: namespace,
GatewayNsName: gwNsName,
UpdateGatewayClassStatus: updateGCStatus,
MetricsConfig: metricsConfig,
HealthConfig: config.HealthConfig{
Enabled: !disableHealth,
Port: healthListenPort.value,
},
MetricsConfig: config.MetricsConfig{
Enabled: !disableMetrics,
Port: metricsListenPort.value,
Secure: metricsSecure,
},
LeaderElection: config.LeaderElection{
Enabled: !disableLeaderElection,
LockName: leaderElectionLockName.String(),
Identity: podName,
},
}

if err := static.StartManager(conf); err != nil {
Expand All @@ -163,53 +190,69 @@ func createStaticModeCommand() *cobra.Command {

cmd.Flags().VarP(
&configName,
"config",
configFlag,
"c",
`The name of the NginxGateway resource to be used for this controller's dynamic configuration.`+
` Lives in the same Namespace as the controller.`,
)

cmd.Flags().BoolVar(
&updateGCStatus,
"update-gatewayclass-status",
updateGCStatusFlag,
true,
"Update the status of the GatewayClass resource.",
)

cmd.Flags().BoolVar(
&disableMetrics,
"metrics-disable",
metricsDisableFlag,
false,
"Disable exposing metrics in the Prometheus format.",
)

cmd.Flags().Var(
&metricsListenPort,
"metrics-port",
metricsPortFlag,
"Set the port where the metrics are exposed. Format: [1024 - 65535]",
)

cmd.Flags().BoolVar(
&metricsSecure,
"metrics-secure-serving",
metricsSecureFlag,
false,
"Enable serving metrics via https. By default metrics are served via http."+
" Please note that this endpoint will be secured with a self-signed certificate.",
)

cmd.Flags().BoolVar(
&disableHealth,
"health-disable",
healthDisableFlag,
false,
"Disable running the health probe server.",
)

cmd.Flags().Var(
&healthListenPort,
"health-port",
healthPortFlag,
"Set the port where the health probe server is exposed. Format: [1024 - 65535]",
)

cmd.Flags().BoolVar(
&disableLeaderElection,
leaderElectionDisableFlag,
false,
"Disable leader election. Leader election is used to avoid multiple replicas of the NGINX Kubernetes Gateway"+
" reporting the status of the Gateway API resources. If disabled, "+
"all replicas of NGINX Kubernetes Gateway will update the statuses of the Gateway API resources.",
)

cmd.Flags().Var(
&leaderElectionLockName,
leaderElectionLockNameFlag,
"The name of the leader election lock. "+
"A Lease object with this name will be created in the same Namespace as the controller.",
)

return cmd
}

Expand Down
18 changes: 18 additions & 0 deletions cmd/gateway/commands_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
"--metrics-secure-serving",
"--health-port=8081",
"--health-disable",
"--leader-election-lock-name=my-lock",
"--leader-election-disable=false",
},
wantErr: false,
},
Expand Down Expand Up @@ -243,6 +245,22 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
expectedErrPrefix: `invalid argument "999" for "--health-disable" flag: strconv.ParseBool:` +
` parsing "999": invalid syntax`,
},
{
name: "leader-election-lock-name is set to invalid string",
args: []string{
"--leader-election-lock-name=!@#$",
},
wantErr: true,
expectedErrPrefix: `invalid argument "!@#$" for "--leader-election-lock-name" flag: invalid format`,
},
{
name: "leader-election-disable is set to empty string",
args: []string{
"--leader-election-disable=",
},
wantErr: true,
expectedErrPrefix: `invalid argument "" for "--leader-election-disable" flag: strconv.ParseBool`,
},
}

for _, test := range tests {
Expand Down
8 changes: 6 additions & 2 deletions conformance/provisioner/static-deployment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ metadata:
app.kubernetes.io/instance: nginx-gateway
app.kubernetes.io/version: "edge"
spec:
# We only support a single replica for now
replicas: 1
selector:
matchLabels:
Expand All @@ -30,15 +29,20 @@ spec:
- --config=nginx-gateway-config
- --metrics-disable
- --health-port=8081
- --leader-election-lock-name=nginx-gateway-leader-election
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: MY_NAMESPACE
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
image: ghcr.io/nginxinc/nginx-kubernetes-gateway:edge
imagePullPolicy: Always
name: nginx-gateway
Expand Down
Loading

0 comments on commit 43c1100

Please sign in to comment.