Integrate NGINX prometheus exporter and enable metrics server (#999)

* Integrate prometheus exporter and enable metrics server
nginxinc · Aug 30, 2023 · 37490ef · 37490ef
1 parent 44cefeb
commit 37490ef
Show file tree

Hide file tree

Showing 30 changed files with 636 additions and 106 deletions.
diff --git a/Makefile b/Makefile
@@ -135,7 +135,7 @@ build-nkg-debug-image: debug-build build-nkg-image ## Build NKG image with debug
 generate-manifests: ## Generate manifests using Helm.
 	cp $(CHART_DIR)/crds/* $(MANIFEST_DIR)/crds/
 	helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-gateway.yaml
-	helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/deployment.yaml > conformance/provisioner/static-deployment.yaml
+	helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set metrics.disable=true -n nginx-gateway -s templates/deployment.yaml > conformance/provisioner/static-deployment.yaml
 	helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer.yaml
 	helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.annotations.'service\.beta\.kubernetes\.io\/aws-load-balancer-type'="nlb" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer-aws-nlb.yaml
 	helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.type=NodePort --set service.externalTrafficPolicy="" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/nodeport.yaml

diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go
@@ -24,6 +24,7 @@ const (
 	gatewayCtrlNameFlag     = "gateway-ctlr-name"
 	gatewayCtrlNameUsageFmt = `The name of the Gateway controller. ` +
 		`The controller name must be of the form: DOMAIN/PATH. The controller's domain is '%s'`
+	gatewayFlag = "gateway"
 )
 
 var (
@@ -36,58 +37,21 @@ var (
 	gatewayClassName = stringValidatingValue{
 		validator: validateResourceName,
 	}
-)
 
-// stringValidatingValue is a string flag value with custom validation logic.
-// it implements the pflag.Value interface.
-type stringValidatingValue struct {
-	validator func(v string) error
-	value     string
-}
+	// Backing values for static subcommand cli flags.
+	updateGCStatus bool
+	disableMetrics bool
+	metricsSecure  bool
 
-func (v *stringValidatingValue) String() string {
-	return v.value
-}
-
-func (v *stringValidatingValue) Set(param string) error {
-	if err := v.validator(param); err != nil {
-		return err
+	metricsListenPort = intValidatingValue{
+		validator: validatePort,
+		value:     9113,
 	}
-	v.value = param
-	return nil
-}
-
-func (v *stringValidatingValue) Type() string {
-	return "string"
-}
-
-// namespacedNameValue is a string flag value that represents a namespaced name.
-// it implements the pflag.Value interface.
-type namespacedNameValue struct {
-	value types.NamespacedName
-}
-
-func (v *namespacedNameValue) String() string {
-	if (v.value == types.NamespacedName{}) {
-		// if we don't do that, the default value in the help message will be printed as "/"
-		return ""
-	}
-	return v.value.String()
-}
-
-func (v *namespacedNameValue) Set(param string) error {
-	nsname, err := parseNamespacedResourceName(param)
-	if err != nil {
-		return err
+	gateway    = namespacedNameValue{}
+	configName = stringValidatingValue{
+		validator: validateResourceName,
 	}
-
-	v.value = nsname
-	return nil
-}
-
-func (v *namespacedNameValue) Type() string {
-	return "string"
-}
+)
 
 func createRootCommand() *cobra.Command {
 	rootCmd := &cobra.Command{
@@ -115,15 +79,6 @@ func createRootCommand() *cobra.Command {
 }
 
 func createStaticModeCommand() *cobra.Command {
-	const gatewayFlag = "gateway"
-
-	// flag values
-	gateway := namespacedNameValue{}
-	var updateGCStatus bool
-	configName := stringValidatingValue{
-		validator: validateResourceName,
-	}
-
 	cmd := &cobra.Command{
 		Use:   "static-mode",
 		Short: "Configure NGINX in the scope of a single Gateway resource",
@@ -153,6 +108,13 @@ func createStaticModeCommand() *cobra.Command {
 				gwNsName = &gateway.value
 			}
 
+			metricsConfig := config.MetricsConfig{}
+			if !disableMetrics {
+				metricsConfig.Enabled = true
+				metricsConfig.Port = metricsListenPort.value
+				metricsConfig.Secure = metricsSecure
+			}
+
 			conf := config.Config{
 				GatewayCtlrName:          gatewayCtlrName.value,
 				ConfigName:               configName.String(),
@@ -163,6 +125,7 @@ func createStaticModeCommand() *cobra.Command {
 				Namespace:                namespace,
 				GatewayNsName:            gwNsName,
 				UpdateGatewayClassStatus: updateGCStatus,
+				MetricsConfig:            metricsConfig,
 			}
 
 			if err := static.StartManager(conf); err != nil {
@@ -198,6 +161,27 @@ func createStaticModeCommand() *cobra.Command {
 		"Update the status of the GatewayClass resource.",
 	)
 
+	cmd.Flags().BoolVar(
+		&disableMetrics,
+		"metrics-disable",
+		false,
+		"Disable exposing metrics in the Prometheus format.",
+	)
+
+	cmd.Flags().Var(
+		&metricsListenPort,
+		"metrics-port",
+		"Set the port where the metrics are exposed. Format: [1023 - 65535]",
+	)
+
+	cmd.Flags().BoolVar(
+		&metricsSecure,
+		"metrics-secure-serving",
+		false,
+		"Enable serving metrics via https. By default metrics are served via http."+
+			" Please note that this endpoint will be secured with a self-signed certificate.",
+	)
+
 	return cmd
 }
 

diff --git a/cmd/gateway/commands_test.go b/cmd/gateway/commands_test.go
@@ -118,6 +118,9 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
 				"--gateway=nginx-gateway/nginx",
 				"--config=nginx-gateway-config",
 				"--update-gatewayclass-status=true",
+				"--metrics-port=9114",
+				"--metrics-disable",
+				"--metrics-secure-serving",
 			},
 			wantErr: false,
 		},
@@ -175,6 +178,42 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
 			wantErr:           true,
 			expectedErrPrefix: `invalid argument "invalid" for "--update-gatewayclass-status" flag: strconv.ParseBool`,
 		},
+		{
+			name: "metrics-port is invalid type",
+			args: []string{
+				"--metrics-port=invalid", // not an int
+			},
+			wantErr: true,
+			expectedErrPrefix: `invalid argument "invalid" for "--metrics-port" flag: failed to parse int value:` +
+				` strconv.ParseInt: parsing "invalid": invalid syntax`,
+		},
+		{
+			name: "metrics-port is outside of range",
+			args: []string{
+				"--metrics-port=999", // outside of range
+			},
+			wantErr: true,
+			expectedErrPrefix: `invalid argument "999" for "--metrics-port" flag:` +
+				` port outside of valid port range [1024 - 65535]: 999`,
+		},
+		{
+			name: "metrics-disable is not a bool",
+			args: []string{
+				"--metrics-disable=999", // not a bool
+			},
+			wantErr: true,
+			expectedErrPrefix: `invalid argument "999" for "--metrics-disable" flag: strconv.ParseBool:` +
+				` parsing "999": invalid syntax`,
+		},
+		{
+			name: "metrics-secure-serving is not a bool",
+			args: []string{
+				"--metrics-secure-serving=999", // not a bool
+			},
+			wantErr: true,
+			expectedErrPrefix: `invalid argument "999" for "--metrics-secure-serving" flag: strconv.ParseBool:` +
+				` parsing "999": invalid syntax`,
+		},
 	}
 
 	for _, test := range tests {

diff --git a/cmd/gateway/validating_types.go b/cmd/gateway/validating_types.go
@@ -0,0 +1,86 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// stringValidatingValue is a string flag value with custom validation logic.
+// it implements the pflag.Value interface.
+type stringValidatingValue struct {
+	validator func(v string) error
+	value     string
+}
+
+func (v *stringValidatingValue) String() string {
+	return v.value
+}
+
+func (v *stringValidatingValue) Set(param string) error {
+	if err := v.validator(param); err != nil {
+		return err
+	}
+	v.value = param
+	return nil
+}
+
+func (v *stringValidatingValue) Type() string {
+	return "string"
+}
+
+type intValidatingValue struct {
+	validator func(v int) error
+	value     int
+}
+
+func (v *intValidatingValue) String() string {
+	return strconv.Itoa(v.value)
+}
+
+func (v *intValidatingValue) Set(param string) error {
+	intVal, err := strconv.ParseInt(param, 10, 32)
+	if err != nil {
+		return fmt.Errorf("failed to parse int value: %w", err)
+	}
+
+	if err := v.validator(int(intVal)); err != nil {
+		return err
+	}
+
+	v.value = int(intVal)
+	return nil
+}
+
+func (v *intValidatingValue) Type() string {
+	return "int"
+}
+
+// namespacedNameValue is a string flag value that represents a namespaced name.
+// it implements the pflag.Value interface.
+type namespacedNameValue struct {
+	value types.NamespacedName
+}
+
+func (v *namespacedNameValue) String() string {
+	if (v.value == types.NamespacedName{}) {
+		// if we don't do that, the default value in the help message will be printed as "/"
+		return ""
+	}
+	return v.value.String()
+}
+
+func (v *namespacedNameValue) Set(param string) error {
+	nsname, err := parseNamespacedResourceName(param)
+	if err != nil {
+		return err
+	}
+
+	v.value = nsname
+	return nil
+}
+
+func (v *namespacedNameValue) Type() string {
+	return "string"
+}
diff --git a/cmd/gateway/validation.go b/cmd/gateway/validation.go
@@ -99,3 +99,11 @@ func validateIP(ip string) error {
 
 	return nil
 }
+
+// validatePort makes sure a given port is inside the valid port range for its usage
+func validatePort(port int) error {
+	if port < 1024 || port > 65535 {
+		return fmt.Errorf("port outside of valid port range [1024 - 65535]: %v", port)
+	}
+	return nil
+}
diff --git a/cmd/gateway/validation_test.go b/cmd/gateway/validation_test.go
@@ -294,3 +294,40 @@ func TestValidateIP(t *testing.T) {
 		})
 	}
 }
+
+func TestValidatePort(t *testing.T) {
+	tests := []struct {
+		name   string
+		port   int
+		expErr bool
+	}{
+		{
+			name:   "port under minimum allowed value",
+			port:   1023,
+			expErr: true,
+		},
+		{
+			name:   "port over maximum allowed value",
+			port:   65536,
+			expErr: true,
+		},
+		{
+			name:   "valid port",
+			port:   9113,
+			expErr: false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewGomegaWithT(t)
+
+			err := validatePort(tc.port)
+			if !tc.expErr {
+				g.Expect(err).ToNot(HaveOccurred())
+			} else {
+				g.Expect(err).To(HaveOccurred())
+			}
+		})
+	}
+}
diff --git a/conformance/provisioner/static-deployment.yaml b/conformance/provisioner/static-deployment.yaml
@@ -28,6 +28,7 @@ spec:
         - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
         - --gatewayclass=nginx
         - --config=nginx-gateway-config
+        - --metrics-disable
         env:
         - name: POD_IP
           valueFrom:

diff --git a/deploy/helm-chart/README.md b/deploy/helm-chart/README.md
@@ -142,3 +142,6 @@ The following tables lists the configurable parameters of the NGINX Kubernetes G
 | `service.externalTrafficPolicy`      | The `externalTrafficPolicy` of the service. The value `Local` preserves the client source IP.                                                                                                                | Local                                                                                                           |
 | `service.annotations`                | The `annotations` of the NGINX Kubernetes Gateway service.                                                                                                                                                   | {}                                                                                                              |
 | `service.ports`                      | A list of ports to expose through the NGINX Kubernetes Gateway service. Update it to match the listener ports from your Gateway resource. Follows the conventional Kubernetes yaml syntax for service ports. | [ port: 80, targetPort: 80, protocol: TCP, name: http; port: 443, targetPort: 443, protocol: TCP, name: https ] |
+| `metrics.disable`                    | Disable exposing metrics in the Prometheus format.                                                                                                                                                   |false                                                                                                              |
+| `metrics.port`                       | Set the port where the Prometheus metrics are exposed. Format: [1024 - 65535]                                                                                                                                  |9113                                                                                                              |
+| `metrics.secure`                     | Enable serving metrics via https. By default metrics are served via http. Please note that this endpoint will be secured with a self-signed certificate.                                                                                                              |false                                                                                                              |