diff --git a/build/Dockerfile b/build/Dockerfile index c1b7a6a29b..103e491899 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -14,7 +14,7 @@ FROM golang:1.23 AS ca-certs-provider FROM scratch AS common # CA certs are needed for telemetry report so that NGF can verify the server's certificate. COPY --from=ca-certs-provider --link /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -USER 102:1001 +USER 101:1001 ARG BUILD_AGENT ENV BUILD_AGENT=${BUILD_AGENT} ENTRYPOINT [ "/usr/bin/gateway" ] diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index 9fdd20bca7..4ff5cade5b 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -268,7 +268,6 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `nginx.image.tag` | | string | `"edge"` | | `nginx.lifecycle` | The lifecycle of the nginx container. | object | `{}` | | `nginx.plus` | Is NGINX Plus image being used | bool | `false` | -| `nginx.securityContext.allowPrivilegeEscalation` | Some environments may need this set to true in order for the control plane to successfully reload NGINX. | bool | `false` | | `nginx.usage.caSecretName` | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` | | `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` | | `nginx.usage.endpoint` | The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com | string | `""` | diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml index 9ee1be4254..830fe1b391 100644 --- a/charts/nginx-gateway-fabric/templates/clusterrole.yaml +++ b/charts/nginx-gateway-fabric/templates/clusterrole.yaml @@ -11,6 +11,7 @@ rules: - namespaces - services - secrets + - pods {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }} - configmaps {{- end }} @@ -18,28 +19,13 @@ rules: - get - list - watch -{{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }} -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get -{{- end }} -{{- if .Values.nginx.plus }} -- apiGroups: - - apps - resources: - - replicasets - verbs: - list -{{- end }} {{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }} - apiGroups: - "" diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml index 35468e682b..7a254a3bf6 100644 --- a/charts/nginx-gateway-fabric/templates/deployment.yaml +++ b/charts/nginx-gateway-fabric/templates/deployment.yaml @@ -139,8 +139,9 @@ spec: capabilities: drop: - ALL + allowPrivilegeEscalation: false readOnlyRootFilesystem: true - runAsUser: 102 + runAsUser: 101 runAsGroup: 1001 {{- with .Values.nginxGateway.extraVolumeMounts -}} {{ toYaml . | nindent 8 }} diff --git a/charts/nginx-gateway-fabric/templates/scc.yaml b/charts/nginx-gateway-fabric/templates/scc.yaml index e58389a8ec..6ab7dc92c1 100644 --- a/charts/nginx-gateway-fabric/templates/scc.yaml +++ b/charts/nginx-gateway-fabric/templates/scc.yaml @@ -1,9 +1,10 @@ +# TODO(sberman): will need an SCC for nginx ServiceAccounts as well. {{- if .Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints" }} kind: SecurityContextConstraints apiVersion: security.openshift.io/v1 metadata: name: {{ include "nginx-gateway.scc-name" . }} -allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }} +allowPrivilegeEscalation: false allowHostDirVolumePlugin: false allowHostIPC: false allowHostNetwork: false @@ -14,7 +15,7 @@ readOnlyRootFilesystem: true runAsUser: type: MustRunAsRange uidRangeMin: 101 - uidRangeMax: 102 + uidRangeMax: 101 fsGroup: type: MustRunAs ranges: @@ -29,16 +30,8 @@ seLinuxContext: type: MustRunAs seccompProfiles: - runtime/default -volumes: -- emptyDir -- secret -- configMap -- projected users: - {{ printf "system:serviceaccount:%s:%s" .Release.Namespace (include "nginx-gateway.serviceAccountName" .) }} -allowedCapabilities: -- NET_BIND_SERVICE -- KILL requiredDropCapabilities: - ALL {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml b/charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml index 80aba1c868..6e85efffeb 100644 --- a/charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml +++ b/charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml @@ -15,5 +15,29 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics + {{- if .Values.nginx.plus }} + - api-action + {{- end }} log: level: debug + collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: 9113 diff --git a/charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml b/charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml index 55c9ee5970..bb04bf46eb 100644 --- a/charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml +++ b/charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml @@ -13,15 +13,16 @@ spec: labels: app.kubernetes.io/name: tmp-nginx-deployment app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + {{- if .Values.metrics.enable }} + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.port }}" + {{- if .Values.metrics.secure }} + prometheus.io/scheme: "https" + {{- end }} + {{- end }} spec: initContainers: - - name: sleep # wait for a bit for control plane to be ready - image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }} - imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }} - command: - - /usr/bin/gateway - - sleep - - --duration=15s - name: init image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }} imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }} @@ -29,14 +30,20 @@ spec: - /usr/bin/gateway - initialize - --source + - /agent/nginx-agent.conf + - --destination + - /etc/nginx-agent + - --source - /includes/main.conf + - --destination + - /etc/nginx/main-includes {{- if .Values.nginx.plus }} - --source - /includes/mgmt.conf - --nginx-plus - {{- end }} - --destination - /etc/nginx/main-includes + {{- end }} env: - name: POD_UID valueFrom: @@ -49,9 +56,13 @@ spec: drop: - ALL readOnlyRootFilesystem: true - runAsUser: 102 + runAsUser: 101 runAsGroup: 1001 volumeMounts: + - name: nginx-agent-config + mountPath: /agent + - name: nginx-agent + mountPath: /etc/nginx-agent - name: nginx-includes-bootstrap mountPath: /includes - name: nginx-main-includes @@ -69,10 +80,11 @@ spec: name: http - containerPort: 443 name: https + - name: metrics + containerPort: {{ .Values.metrics.port }} securityContext: seccompProfile: type: RuntimeDefault - allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }} capabilities: add: - NET_BIND_SERVICE @@ -84,6 +96,8 @@ spec: volumeMounts: - name: nginx-agent mountPath: /etc/nginx-agent + - name: nginx-agent-log + mountPath: /var/log/nginx-agent - name: nginx-conf mountPath: /etc/nginx/conf.d - name: nginx-stream-conf @@ -140,8 +154,12 @@ spec: {{- end }} volumes: - name: nginx-agent + emptyDir: {} + - name: nginx-agent-config configMap: name: nginx-agent-config + - name: nginx-agent-log + emptyDir: {} - name: nginx-conf emptyDir: {} - name: nginx-stream-conf diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 651fea311c..9a3a7d9a8f 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -259,20 +259,6 @@ "title": "plus", "type": "boolean" }, - "securityContext": { - "properties": { - "allowPrivilegeEscalation": { - "default": false, - "description": "Some environments may need this set to true in order for the control plane to successfully reload NGINX.", - "required": [], - "title": "allowPrivilegeEscalation", - "type": "boolean" - } - }, - "required": [], - "title": "securityContext", - "type": "object" - }, "usage": { "description": "Configuration for NGINX Plus usage reporting.", "properties": { diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 4168c3d66e..e71cbb5724 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -131,10 +131,6 @@ nginx: # @schema pullPolicy: Always - securityContext: - # -- Some environments may need this set to true in order for the control plane to successfully reload NGINX. - allowPrivilegeEscalation: false - # -- Is NGINX Plus image being used plus: false diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go index aaa18a21ac..fdb80497dc 100644 --- a/cmd/gateway/commands.go +++ b/cmd/gateway/commands.go @@ -519,14 +519,14 @@ func createInitializeCommand() *cobra.Command { // flag values var srcFiles []string - var dest string + var destDirs []string var plus bool cmd := &cobra.Command{ Use: "initialize", Short: "Write initial configuration files", RunE: func(_ *cobra.Command, _ []string) error { - if err := validateCopyArgs(srcFiles, dest); err != nil { + if err := validateCopyArgs(srcFiles, destDirs); err != nil { return err } @@ -546,7 +546,7 @@ func createInitializeCommand() *cobra.Command { logger.Info( "Starting init container", "source filenames to copy", srcFiles, - "destination directory", dest, + "destination directories", destDirs, "nginx-plus", plus, ) @@ -558,16 +558,21 @@ func createInitializeCommand() *cobra.Command { Logger: logger.WithName("deployCtxCollector"), }) + files := make([]fileToCopy, 0, len(srcFiles)) + for i, src := range srcFiles { + files = append(files, fileToCopy{ + destDirName: destDirs[i], + srcFileName: src, + }) + } + return initialize(initializeConfig{ fileManager: file.NewStdLibOSFileManager(), fileGenerator: ngxConfig.NewGeneratorImpl(plus, nil, logger.WithName("generator")), logger: logger, plus: plus, collector: dcc, - copy: copyFiles{ - srcFileNames: srcFiles, - destDirName: dest, - }, + copy: files, }) }, } @@ -579,11 +584,11 @@ func createInitializeCommand() *cobra.Command { "The source files to be copied", ) - cmd.Flags().StringVar( - &dest, + cmd.Flags().StringSliceVar( + &destDirs, destFlag, - "", - "The destination directory for the source files to be copied to", + []string{}, + "The destination directories for the source files at the same array index to be copied to", ) cmd.Flags().BoolVar( diff --git a/cmd/gateway/initialize.go b/cmd/gateway/initialize.go index 4ce6fc0495..0adc9a0b51 100644 --- a/cmd/gateway/initialize.go +++ b/cmd/gateway/initialize.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "os" "path/filepath" "time" @@ -17,9 +18,9 @@ const ( collectDeployCtxTimeout = 10 * time.Second ) -type copyFiles struct { - destDirName string - srcFileNames []string +type fileToCopy struct { + destDirName string + srcFileName string } type initializeConfig struct { @@ -27,13 +28,13 @@ type initializeConfig struct { fileManager file.OSFileManager fileGenerator config.Generator logger logr.Logger - copy copyFiles + copy []fileToCopy plus bool } func initialize(cfg initializeConfig) error { - for _, src := range cfg.copy.srcFileNames { - if err := copyFile(cfg.fileManager, src, cfg.copy.destDirName); err != nil { + for _, f := range cfg.copy { + if err := copyFile(cfg.fileManager, f.srcFileName, f.destDirName); err != nil { return err } } @@ -58,7 +59,7 @@ func initialize(cfg initializeConfig) error { return fmt.Errorf("failed to generate deployment context file: %w", err) } - if err := file.Write(cfg.fileManager, depCtxFile); err != nil { + if err := file.Write(cfg.fileManager, file.Convert(depCtxFile)); err != nil { return fmt.Errorf("failed to write deployment context file: %w", err) } @@ -84,5 +85,9 @@ func copyFile(osFileManager file.OSFileManager, src, dest string) error { return fmt.Errorf("error copying file contents: %w", err) } + if err := osFileManager.Chmod(destFile, os.FileMode(file.RegularFileModeInt)); err != nil { + return fmt.Errorf("error setting file permissions: %w", err) + } + return nil } diff --git a/cmd/gateway/initialize_test.go b/cmd/gateway/initialize_test.go index 4276ff9a24..96b6b30f02 100644 --- a/cmd/gateway/initialize_test.go +++ b/cmd/gateway/initialize_test.go @@ -28,9 +28,15 @@ func TestInitialize_OSS(t *testing.T) { ic := initializeConfig{ fileManager: fakeFileMgr, logger: zap.New(), - copy: copyFiles{ - destDirName: "destDir", - srcFileNames: []string{"src1", "src2"}, + copy: []fileToCopy{ + { + destDirName: "destDir", + srcFileName: "src1", + }, + { + destDirName: "destDir2", + srcFileName: "src2", + }, }, plus: false, } @@ -56,9 +62,15 @@ func TestInitialize_OSS_Error(t *testing.T) { ic := initializeConfig{ fileManager: fakeFileMgr, logger: zap.New(), - copy: copyFiles{ - destDirName: "destDir", - srcFileNames: []string{"src1", "src2"}, + copy: []fileToCopy{ + { + destDirName: "destDir", + srcFileName: "src1", + }, + { + destDirName: "destDir2", + srcFileName: "src2", + }, }, plus: false, } @@ -114,9 +126,15 @@ func TestInitialize_Plus(t *testing.T) { logger: zap.New(), collector: fakeCollector, fileGenerator: fakeGenerator, - copy: copyFiles{ - destDirName: "destDir", - srcFileNames: []string{"src1", "src2"}, + copy: []fileToCopy{ + { + destDirName: "destDir", + srcFileName: "src1", + }, + { + destDirName: "destDir2", + srcFileName: "src2", + }, }, plus: true, } @@ -133,7 +151,7 @@ func TestInitialize_Plus(t *testing.T) { g.Expect(fakeGenerator.GenerateDeploymentContextArgsForCall(0)).To(Equal(test.depCtx)) g.Expect(fakeCollector.CollectCallCount()).To(Equal(1)) g.Expect(fakeFileMgr.WriteCallCount()).To(Equal(1)) - g.Expect(fakeFileMgr.ChmodCallCount()).To(Equal(1)) + g.Expect(fakeFileMgr.ChmodCallCount()).To(Equal(3)) }) } } @@ -161,6 +179,7 @@ func TestCopyFileErrors(t *testing.T) { openErr := errors.New("open error") createErr := errors.New("create error") copyErr := errors.New("copy error") + chmodErr := errors.New("chmod error") tests := []struct { fileMgr *filefakes.FakeOSFileManager @@ -194,6 +213,15 @@ func TestCopyFileErrors(t *testing.T) { }, expErr: copyErr, }, + { + name: "can't set permissions", + fileMgr: &filefakes.FakeOSFileManager{ + ChmodStub: func(_ *os.File, _ os.FileMode) error { + return chmodErr + }, + }, + expErr: chmodErr, + }, } for _, test := range tests { diff --git a/cmd/gateway/validation.go b/cmd/gateway/validation.go index ac6e04fa28..b6e0cc84f3 100644 --- a/cmd/gateway/validation.go +++ b/cmd/gateway/validation.go @@ -206,12 +206,15 @@ func ensureNoPortCollisions(ports ...int) error { return nil } -// validateCopyArgs ensures that arguments to the sleep command are set. -func validateCopyArgs(srcFiles []string, dest string) error { +// validateCopyArgs ensures that arguments to the initialize command are set. +func validateCopyArgs(srcFiles []string, destDirs []string) error { + if len(srcFiles) != len(destDirs) { + return errors.New("source and destination must have the same number of elements") + } if len(srcFiles) == 0 { return errors.New("source must not be empty") } - if len(dest) == 0 { + if len(destDirs) == 0 { return errors.New("destination must not be empty") } diff --git a/cmd/gateway/validation_test.go b/cmd/gateway/validation_test.go index 1774f13619..59db6fc57c 100644 --- a/cmd/gateway/validation_test.go +++ b/cmd/gateway/validation_test.go @@ -554,33 +554,39 @@ func TestEnsureNoPortCollisions(t *testing.T) { g.Expect(ensureNoPortCollisions(9113, 9113)).ToNot(Succeed()) } -func TestValidateSleepArgs(t *testing.T) { +func TestValidateInitializeArgs(t *testing.T) { t.Parallel() tests := []struct { name string - dest string + destDirs []string srcFiles []string expErr bool }{ { name: "valid values", - dest: "/dest/file", + destDirs: []string{"/dest/"}, srcFiles: []string{"/src/file"}, expErr: false, }, { name: "invalid dest", - dest: "", + destDirs: []string{}, srcFiles: []string{"/src/file"}, expErr: true, }, { name: "invalid src", - dest: "/dest/file", + destDirs: []string{"/dest/"}, srcFiles: []string{}, expErr: true, }, + { + name: "different lengths", + destDirs: []string{"/dest/"}, + srcFiles: []string{"src1", "src2"}, + expErr: true, + }, } for _, tc := range tests { @@ -588,7 +594,7 @@ func TestValidateSleepArgs(t *testing.T) { t.Parallel() g := NewWithT(t) - err := validateCopyArgs(tc.srcFiles, tc.dest) + err := validateCopyArgs(tc.srcFiles, tc.destDirs) if !tc.expErr { g.Expect(err).ToNot(HaveOccurred()) } else { diff --git a/config/tests/static-deployment.yaml b/config/tests/static-deployment.yaml index 8e581ff569..35fb3d8ad2 100644 --- a/config/tests/static-deployment.yaml +++ b/config/tests/static-deployment.yaml @@ -69,8 +69,9 @@ spec: capabilities: drop: - ALL + allowPrivilegeEscalation: false readOnlyRootFilesystem: true - runAsUser: 102 + runAsUser: 101 runAsGroup: 1001 terminationGracePeriodSeconds: 30 serviceAccountName: nginx-gateway diff --git a/deploy/aws-nlb/deploy.yaml b/deploy/aws-nlb/deploy.yaml index 295cb42d07..7e300804d0 100644 --- a/deploy/aws-nlb/deploy.yaml +++ b/deploy/aws-nlb/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -157,8 +153,29 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug + collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: 9113 kind: ConfigMap metadata: name: nginx-agent-config @@ -293,12 +310,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -319,6 +337,9 @@ spec: app.kubernetes.io/name: tmp-nginx-deployment template: metadata: + annotations: + prometheus.io/port: "9113" + prometheus.io/scrape: "true" labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: tmp-nginx-deployment @@ -332,8 +353,9 @@ spec: name: http - containerPort: 443 name: https + - containerPort: 9113 + name: metrics securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE @@ -347,6 +369,8 @@ spec: volumeMounts: - mountPath: /etc/nginx-agent name: nginx-agent + - mountPath: /var/log/nginx-agent + name: nginx-agent-log - mountPath: /etc/nginx/conf.d name: nginx-conf - mountPath: /etc/nginx/stream-conf.d @@ -362,17 +386,14 @@ spec: - mountPath: /etc/nginx/includes name: nginx-includes initContainers: - - command: - - /usr/bin/gateway - - sleep - - --duration=15s - image: ghcr.io/nginxinc/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: sleep - command: - /usr/bin/gateway - initialize - --source + - /agent/nginx-agent.conf + - --destination + - /etc/nginx-agent + - --source - /includes/main.conf - --destination - /etc/nginx/main-includes @@ -390,10 +411,14 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: + - mountPath: /agent + name: nginx-agent-config + - mountPath: /etc/nginx-agent + name: nginx-agent - mountPath: /includes name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes @@ -404,9 +429,13 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 volumes: + - emptyDir: {} + name: nginx-agent - configMap: name: nginx-agent-config - name: nginx-agent + name: nginx-agent-config + - emptyDir: {} + name: nginx-agent-log - emptyDir: {} name: nginx-conf - emptyDir: {} diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index 45121bf502..50569a6d03 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -157,8 +153,29 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug + collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: 9113 kind: ConfigMap metadata: name: nginx-agent-config @@ -290,12 +307,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault nodeSelector: @@ -318,6 +336,9 @@ spec: app.kubernetes.io/name: tmp-nginx-deployment template: metadata: + annotations: + prometheus.io/port: "9113" + prometheus.io/scrape: "true" labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: tmp-nginx-deployment @@ -331,8 +352,9 @@ spec: name: http - containerPort: 443 name: https + - containerPort: 9113 + name: metrics securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE @@ -346,6 +368,8 @@ spec: volumeMounts: - mountPath: /etc/nginx-agent name: nginx-agent + - mountPath: /var/log/nginx-agent + name: nginx-agent-log - mountPath: /etc/nginx/conf.d name: nginx-conf - mountPath: /etc/nginx/stream-conf.d @@ -361,17 +385,14 @@ spec: - mountPath: /etc/nginx/includes name: nginx-includes initContainers: - - command: - - /usr/bin/gateway - - sleep - - --duration=15s - image: ghcr.io/nginxinc/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: sleep - command: - /usr/bin/gateway - initialize - --source + - /agent/nginx-agent.conf + - --destination + - /etc/nginx-agent + - --source - /includes/main.conf - --destination - /etc/nginx/main-includes @@ -389,10 +410,14 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: + - mountPath: /agent + name: nginx-agent-config + - mountPath: /etc/nginx-agent + name: nginx-agent - mountPath: /includes name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes @@ -405,9 +430,13 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 volumes: + - emptyDir: {} + name: nginx-agent - configMap: name: nginx-agent-config - name: nginx-agent + name: nginx-agent-config + - emptyDir: {} + name: nginx-agent-log - emptyDir: {} name: nginx-conf - emptyDir: {} diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index 2e1a53f3e1..deb7132f39 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -157,8 +153,29 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug + collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: 9113 kind: ConfigMap metadata: name: nginx-agent-config @@ -290,12 +307,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -316,6 +334,9 @@ spec: app.kubernetes.io/name: tmp-nginx-deployment template: metadata: + annotations: + prometheus.io/port: "9113" + prometheus.io/scrape: "true" labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: tmp-nginx-deployment @@ -329,8 +350,9 @@ spec: name: http - containerPort: 443 name: https + - containerPort: 9113 + name: metrics securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE @@ -344,6 +366,8 @@ spec: volumeMounts: - mountPath: /etc/nginx-agent name: nginx-agent + - mountPath: /var/log/nginx-agent + name: nginx-agent-log - mountPath: /etc/nginx/conf.d name: nginx-conf - mountPath: /etc/nginx/stream-conf.d @@ -359,17 +383,14 @@ spec: - mountPath: /etc/nginx/includes name: nginx-includes initContainers: - - command: - - /usr/bin/gateway - - sleep - - --duration=15s - image: ghcr.io/nginxinc/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: sleep - command: - /usr/bin/gateway - initialize - --source + - /agent/nginx-agent.conf + - --destination + - /etc/nginx-agent + - --source - /includes/main.conf - --destination - /etc/nginx/main-includes @@ -387,10 +408,14 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: + - mountPath: /agent + name: nginx-agent-config + - mountPath: /etc/nginx-agent + name: nginx-agent - mountPath: /includes name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes @@ -401,9 +426,13 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 volumes: + - emptyDir: {} + name: nginx-agent - configMap: name: nginx-agent-config - name: nginx-agent + name: nginx-agent-config + - emptyDir: {} + name: nginx-agent-log - emptyDir: {} name: nginx-conf - emptyDir: {} diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index f846d0ca76..396242796a 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -30,28 +30,18 @@ rules: - namespaces - services - secrets + - pods - configmaps verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get -- apiGroups: - - apps - resources: - - replicasets - verbs: - list - apiGroups: - "" @@ -170,8 +160,30 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics + - api-action log: level: debug + collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: 9113 kind: ConfigMap metadata: name: nginx-agent-config @@ -311,12 +323,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -337,6 +350,9 @@ spec: app.kubernetes.io/name: tmp-nginx-deployment template: metadata: + annotations: + prometheus.io/port: "9113" + prometheus.io/scrape: "true" labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: tmp-nginx-deployment @@ -350,8 +366,9 @@ spec: name: http - containerPort: 443 name: https + - containerPort: 9113 + name: metrics securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE @@ -365,6 +382,8 @@ spec: volumeMounts: - mountPath: /etc/nginx-agent name: nginx-agent + - mountPath: /var/log/nginx-agent + name: nginx-agent-log - mountPath: /etc/nginx/conf.d name: nginx-conf - mountPath: /etc/nginx/stream-conf.d @@ -385,18 +404,17 @@ spec: name: nginx-plus-license subPath: license.jwt initContainers: - - command: - - /usr/bin/gateway - - sleep - - --duration=15s - image: ghcr.io/nginxinc/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: sleep - command: - /usr/bin/gateway - initialize - --source + - /agent/nginx-agent.conf + - --destination + - /etc/nginx-agent + - --source - /includes/main.conf + - --destination + - /etc/nginx/main-includes - --source - /includes/mgmt.conf - --nginx-plus @@ -416,10 +434,14 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: + - mountPath: /agent + name: nginx-agent-config + - mountPath: /etc/nginx-agent + name: nginx-agent - mountPath: /includes name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes @@ -430,9 +452,13 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 volumes: + - emptyDir: {} + name: nginx-agent - configMap: name: nginx-agent-config - name: nginx-agent + name: nginx-agent-config + - emptyDir: {} + name: nginx-agent-log - emptyDir: {} name: nginx-conf - emptyDir: {} diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index 68bd273be1..0607fb4b9c 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -28,23 +28,19 @@ rules: - namespaces - services - secrets + - pods - configmaps verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -162,8 +158,29 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug + collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: 9113 kind: ConfigMap metadata: name: nginx-agent-config @@ -296,12 +313,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -322,6 +340,9 @@ spec: app.kubernetes.io/name: tmp-nginx-deployment template: metadata: + annotations: + prometheus.io/port: "9113" + prometheus.io/scrape: "true" labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: tmp-nginx-deployment @@ -335,8 +356,9 @@ spec: name: http - containerPort: 443 name: https + - containerPort: 9113 + name: metrics securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE @@ -350,6 +372,8 @@ spec: volumeMounts: - mountPath: /etc/nginx-agent name: nginx-agent + - mountPath: /var/log/nginx-agent + name: nginx-agent-log - mountPath: /etc/nginx/conf.d name: nginx-conf - mountPath: /etc/nginx/stream-conf.d @@ -365,17 +389,14 @@ spec: - mountPath: /etc/nginx/includes name: nginx-includes initContainers: - - command: - - /usr/bin/gateway - - sleep - - --duration=15s - image: ghcr.io/nginxinc/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: sleep - command: - /usr/bin/gateway - initialize - --source + - /agent/nginx-agent.conf + - --destination + - /etc/nginx-agent + - --source - /includes/main.conf - --destination - /etc/nginx/main-includes @@ -393,10 +414,14 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: + - mountPath: /agent + name: nginx-agent-config + - mountPath: /etc/nginx-agent + name: nginx-agent - mountPath: /includes name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes @@ -407,9 +432,13 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 volumes: + - emptyDir: {} + name: nginx-agent - configMap: name: nginx-agent-config - name: nginx-agent + name: nginx-agent-config + - emptyDir: {} + name: nginx-agent-log - emptyDir: {} name: nginx-conf - emptyDir: {} diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index adb6593de8..838da5efd7 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -30,27 +30,17 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get -- apiGroups: - - apps - resources: - - replicasets - verbs: - list - apiGroups: - "" @@ -165,8 +155,30 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics + - api-action log: level: debug + collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: 9113 kind: ConfigMap metadata: name: nginx-agent-config @@ -305,12 +317,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -331,6 +344,9 @@ spec: app.kubernetes.io/name: tmp-nginx-deployment template: metadata: + annotations: + prometheus.io/port: "9113" + prometheus.io/scrape: "true" labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: tmp-nginx-deployment @@ -344,8 +360,9 @@ spec: name: http - containerPort: 443 name: https + - containerPort: 9113 + name: metrics securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE @@ -359,6 +376,8 @@ spec: volumeMounts: - mountPath: /etc/nginx-agent name: nginx-agent + - mountPath: /var/log/nginx-agent + name: nginx-agent-log - mountPath: /etc/nginx/conf.d name: nginx-conf - mountPath: /etc/nginx/stream-conf.d @@ -379,18 +398,17 @@ spec: name: nginx-plus-license subPath: license.jwt initContainers: - - command: - - /usr/bin/gateway - - sleep - - --duration=15s - image: ghcr.io/nginxinc/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: sleep - command: - /usr/bin/gateway - initialize - --source + - /agent/nginx-agent.conf + - --destination + - /etc/nginx-agent + - --source - /includes/main.conf + - --destination + - /etc/nginx/main-includes - --source - /includes/mgmt.conf - --nginx-plus @@ -410,10 +428,14 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: + - mountPath: /agent + name: nginx-agent-config + - mountPath: /etc/nginx-agent + name: nginx-agent - mountPath: /includes name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes @@ -424,9 +446,13 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 volumes: + - emptyDir: {} + name: nginx-agent - configMap: name: nginx-agent-config - name: nginx-agent + name: nginx-agent-config + - emptyDir: {} + name: nginx-agent-log - emptyDir: {} name: nginx-conf - emptyDir: {} diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index af66c8fafc..57df7721da 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -157,8 +153,29 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug + collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: 9113 kind: ConfigMap metadata: name: nginx-agent-config @@ -290,12 +307,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -316,6 +334,9 @@ spec: app.kubernetes.io/name: tmp-nginx-deployment template: metadata: + annotations: + prometheus.io/port: "9113" + prometheus.io/scrape: "true" labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: tmp-nginx-deployment @@ -329,8 +350,9 @@ spec: name: http - containerPort: 443 name: https + - containerPort: 9113 + name: metrics securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE @@ -344,6 +366,8 @@ spec: volumeMounts: - mountPath: /etc/nginx-agent name: nginx-agent + - mountPath: /var/log/nginx-agent + name: nginx-agent-log - mountPath: /etc/nginx/conf.d name: nginx-conf - mountPath: /etc/nginx/stream-conf.d @@ -359,17 +383,14 @@ spec: - mountPath: /etc/nginx/includes name: nginx-includes initContainers: - - command: - - /usr/bin/gateway - - sleep - - --duration=15s - image: ghcr.io/nginxinc/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: sleep - command: - /usr/bin/gateway - initialize - --source + - /agent/nginx-agent.conf + - --destination + - /etc/nginx-agent + - --source - /includes/main.conf - --destination - /etc/nginx/main-includes @@ -387,10 +408,14 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: + - mountPath: /agent + name: nginx-agent-config + - mountPath: /etc/nginx-agent + name: nginx-agent - mountPath: /includes name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes @@ -401,9 +426,13 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 volumes: + - emptyDir: {} + name: nginx-agent - configMap: name: nginx-agent-config - name: nginx-agent + name: nginx-agent-config + - emptyDir: {} + name: nginx-agent-log - emptyDir: {} name: nginx-conf - emptyDir: {} diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index a3bc1b01e9..fe8c1b3127 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -165,8 +161,29 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug + collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: 9113 kind: ConfigMap metadata: name: nginx-agent-config @@ -298,12 +315,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -324,6 +342,9 @@ spec: app.kubernetes.io/name: tmp-nginx-deployment template: metadata: + annotations: + prometheus.io/port: "9113" + prometheus.io/scrape: "true" labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: tmp-nginx-deployment @@ -337,8 +358,9 @@ spec: name: http - containerPort: 443 name: https + - containerPort: 9113 + name: metrics securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE @@ -352,6 +374,8 @@ spec: volumeMounts: - mountPath: /etc/nginx-agent name: nginx-agent + - mountPath: /var/log/nginx-agent + name: nginx-agent-log - mountPath: /etc/nginx/conf.d name: nginx-conf - mountPath: /etc/nginx/stream-conf.d @@ -367,17 +391,14 @@ spec: - mountPath: /etc/nginx/includes name: nginx-includes initContainers: - - command: - - /usr/bin/gateway - - sleep - - --duration=15s - image: ghcr.io/nginxinc/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: sleep - command: - /usr/bin/gateway - initialize - --source + - /agent/nginx-agent.conf + - --destination + - /etc/nginx-agent + - --source - /includes/main.conf - --destination - /etc/nginx/main-includes @@ -395,10 +416,14 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: + - mountPath: /agent + name: nginx-agent-config + - mountPath: /etc/nginx-agent + name: nginx-agent - mountPath: /includes name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes @@ -409,9 +434,13 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 volumes: + - emptyDir: {} + name: nginx-agent - configMap: name: nginx-agent-config - name: nginx-agent + name: nginx-agent-config + - emptyDir: {} + name: nginx-agent-log - emptyDir: {} name: nginx-conf - emptyDir: {} @@ -461,9 +490,6 @@ allowHostPID: false allowHostPorts: false allowPrivilegeEscalation: false allowPrivilegedContainer: false -allowedCapabilities: -- NET_BIND_SERVICE -- KILL apiVersion: security.openshift.io/v1 fsGroup: ranges: @@ -478,7 +504,7 @@ requiredDropCapabilities: - ALL runAsUser: type: MustRunAsRange - uidRangeMax: 102 + uidRangeMax: 101 uidRangeMin: 101 seLinuxContext: type: MustRunAs @@ -491,8 +517,3 @@ supplementalGroups: type: MustRunAs users: - system:serviceaccount:nginx-gateway:nginx-gateway -volumes: -- emptyDir -- secret -- configMap -- projected diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index 6278c799f2..928c0d7ba6 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -30,27 +30,17 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get -- apiGroups: - - apps - resources: - - replicasets - verbs: - list - apiGroups: - "" @@ -167,8 +157,30 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics + - api-action log: level: debug + collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: 9113 kind: ConfigMap metadata: name: nginx-agent-config @@ -308,12 +320,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -334,6 +347,9 @@ spec: app.kubernetes.io/name: tmp-nginx-deployment template: metadata: + annotations: + prometheus.io/port: "9113" + prometheus.io/scrape: "true" labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: tmp-nginx-deployment @@ -347,8 +363,9 @@ spec: name: http - containerPort: 443 name: https + - containerPort: 9113 + name: metrics securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE @@ -362,6 +379,8 @@ spec: volumeMounts: - mountPath: /etc/nginx-agent name: nginx-agent + - mountPath: /var/log/nginx-agent + name: nginx-agent-log - mountPath: /etc/nginx/conf.d name: nginx-conf - mountPath: /etc/nginx/stream-conf.d @@ -382,18 +401,17 @@ spec: name: nginx-plus-license subPath: license.jwt initContainers: - - command: - - /usr/bin/gateway - - sleep - - --duration=15s - image: ghcr.io/nginxinc/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: sleep - command: - /usr/bin/gateway - initialize - --source + - /agent/nginx-agent.conf + - --destination + - /etc/nginx-agent + - --source - /includes/main.conf + - --destination + - /etc/nginx/main-includes - --source - /includes/mgmt.conf - --nginx-plus @@ -413,10 +431,14 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: + - mountPath: /agent + name: nginx-agent-config + - mountPath: /etc/nginx-agent + name: nginx-agent - mountPath: /includes name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes @@ -427,9 +449,13 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 volumes: + - emptyDir: {} + name: nginx-agent - configMap: name: nginx-agent-config - name: nginx-agent + name: nginx-agent-config + - emptyDir: {} + name: nginx-agent-log - emptyDir: {} name: nginx-conf - emptyDir: {} diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml index b4d01ca6f6..e253d8e5dd 100644 --- a/deploy/snippets-filters/deploy.yaml +++ b/deploy/snippets-filters/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -159,8 +155,29 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug + collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: 9113 kind: ConfigMap metadata: name: nginx-agent-config @@ -293,12 +310,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -319,6 +337,9 @@ spec: app.kubernetes.io/name: tmp-nginx-deployment template: metadata: + annotations: + prometheus.io/port: "9113" + prometheus.io/scrape: "true" labels: app.kubernetes.io/instance: nginx-gateway app.kubernetes.io/name: tmp-nginx-deployment @@ -332,8 +353,9 @@ spec: name: http - containerPort: 443 name: https + - containerPort: 9113 + name: metrics securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE @@ -347,6 +369,8 @@ spec: volumeMounts: - mountPath: /etc/nginx-agent name: nginx-agent + - mountPath: /var/log/nginx-agent + name: nginx-agent-log - mountPath: /etc/nginx/conf.d name: nginx-conf - mountPath: /etc/nginx/stream-conf.d @@ -362,17 +386,14 @@ spec: - mountPath: /etc/nginx/includes name: nginx-includes initContainers: - - command: - - /usr/bin/gateway - - sleep - - --duration=15s - image: ghcr.io/nginxinc/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: sleep - command: - /usr/bin/gateway - initialize - --source + - /agent/nginx-agent.conf + - --destination + - /etc/nginx-agent + - --source - /includes/main.conf - --destination - /etc/nginx/main-includes @@ -390,10 +411,14 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: + - mountPath: /agent + name: nginx-agent-config + - mountPath: /etc/nginx-agent + name: nginx-agent - mountPath: /includes name: nginx-includes-bootstrap - mountPath: /etc/nginx/main-includes @@ -404,9 +429,13 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 volumes: + - emptyDir: {} + name: nginx-agent - configMap: name: nginx-agent-config - name: nginx-agent + name: nginx-agent-config + - emptyDir: {} + name: nginx-agent-log - emptyDir: {} name: nginx-conf - emptyDir: {} diff --git a/go.mod b/go.mod index 01b9604df8..1989f8b2b1 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,9 @@ require ( github.com/go-kit/log v0.2.1 github.com/go-logr/logr v1.4.2 github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.6.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 - github.com/nginx/agent/v3 v3.0.0-20241220140549-28adb688a8b4 + github.com/nginx/agent/v3 v3.0.0-20250120091728-0f0c0e2478aa github.com/nginxinc/telemetry-exporter v0.1.2 github.com/onsi/ginkgo/v2 v2.22.1 github.com/onsi/gomega v1.36.1 @@ -19,12 +20,13 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 go.uber.org/zap v1.27.0 google.golang.org/grpc v1.69.2 + google.golang.org/protobuf v1.35.2 k8s.io/api v0.32.0 k8s.io/apiextensions-apiserver v0.32.0 k8s.io/apimachinery v0.32.0 k8s.io/client-go v0.32.0 k8s.io/klog/v2 v2.130.1 - sigs.k8s.io/controller-runtime v0.19.3 + sigs.k8s.io/controller-runtime v0.20.0 sigs.k8s.io/gateway-api v1.2.1 ) @@ -36,6 +38,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -46,10 +49,10 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect - github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -70,7 +73,6 @@ require ( go.opentelemetry.io/otel/trace v1.33.0 // indirect go.opentelemetry.io/proto/otlp v1.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e // indirect golang.org/x/mod v0.22.0 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect @@ -83,7 +85,6 @@ require ( gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/protobuf v1.35.2 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index b88cbe43f8..fc7b8b0e36 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,46 @@ buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1 h1:2IGhRovxlsOIQgx2ekZWo4wTPAYpck41+18ICxs37is= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1/go.mod h1:Tgn5bgL220vkFOI0KPStlcClPeOJzAv4uT+V8JXGUnw= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= +github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= @@ -30,6 +54,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -43,6 +69,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -58,6 +86,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -74,30 +104,60 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI= +github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlzz2rTuu18Ra7M8sU= github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nginx/agent/v3 v3.0.0-20241220140549-28adb688a8b4 h1:Tn0SOlxq9uaJuqc6DUGZGYszrtHAHOaLnhbBWMzK1Bs= -github.com/nginx/agent/v3 v3.0.0-20241220140549-28adb688a8b4/go.mod h1:HDi/Je5AKCe5by/hWs2jbzUqi3BN4K32hMD2/hWN5G8= +github.com/nginx/agent/v3 v3.0.0-20250120091728-0f0c0e2478aa h1:PvNHtYSv/glSxDkovCHJsDlNFHkvzoH2wAr6WtSNYcM= +github.com/nginx/agent/v3 v3.0.0-20250120091728-0f0c0e2478aa/go.mod h1:HDi/Je5AKCe5by/hWs2jbzUqi3BN4K32hMD2/hWN5G8= +github.com/nginxinc/nginx-plus-go-client/v2 v2.0.1 h1:5VVK38bnELMDWnwfF6dSv57ResXh9AUzeDa72ENj94o= +github.com/nginxinc/nginx-plus-go-client/v2 v2.0.1/go.mod h1:He+1izxYxVVO5/C9ZTukwOpvkAx5eS19nRQgKXDhX5I= github.com/nginxinc/telemetry-exporter v0.1.2 h1:97vUGhQYgQ2KEsXKCBmr5gqfuujJCKPHwdg5HKoANUs= github.com/nginxinc/telemetry-exporter v0.1.2/go.mod h1:eKa/Ceh9irmyZ1xV2QxBIxduIyVC5RlmtiWwcTlHuMg= github.com/onsi/ginkgo/v2 v2.22.1 h1:QW7tbJAUDyVDVOM5dFa7qaybo+CRfR7bemlQUN6Z8aM= github.com/onsi/ginkgo/v2 v2.22.1/go.mod h1:S6aTpoRsSq2cZOd+pssHAlKW/Q/jZt6cPrPlnj4a1xM= github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= @@ -109,22 +169,52 @@ github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoG github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= +github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= +github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo= +github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= @@ -150,6 +240,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -211,6 +303,8 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= @@ -229,8 +323,8 @@ k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJ k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= -sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= +sigs.k8s.io/controller-runtime v0.20.0 h1:jjkMo29xEXH+02Md9qaVXfEIaMESSpy3TBWPrsfQkQs= +sigs.k8s.io/controller-runtime v0.20.0/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU= sigs.k8s.io/gateway-api v1.2.1 h1:fZZ/+RyRb+Y5tGkwxFKuYuSRQHu9dZtbjenblleOLHM= sigs.k8s.io/gateway-api v1.2.1/go.mod h1:EpNfEXNjiYfUJypf0eZ0P5iXA9ekSGWaS1WgPaM42X0= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= diff --git a/internal/framework/file/file.go b/internal/framework/file/file.go index 0a6fb491a4..82ecea73cd 100644 --- a/internal/framework/file/file.go +++ b/internal/framework/file/file.go @@ -5,15 +5,21 @@ import ( "fmt" "io" "os" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" ) //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate const ( - // regularFileMode defines the default file mode for regular files. - regularFileMode = 0o644 - // secretFileMode defines the default file mode for files with secrets. - secretFileMode = 0o640 + // RegularFileModeInt defines the default file mode for regular files as an integer. + RegularFileModeInt = 0o644 + // RegularFileMode defines the default file mode for regular files. + RegularFileMode = "0644" + // secretFileMode defines the default file mode for files with secrets as an integer. + secretFileModeInt = 0o640 + // SecretFileMode defines the default file mode for files with secrets. + SecretFileMode = "0640" ) // Type is the type of File. @@ -78,14 +84,14 @@ func Write(fileMgr OSFileManager, file File) error { switch file.Type { case TypeRegular: - if err := fileMgr.Chmod(f, regularFileMode); err != nil { + if err := fileMgr.Chmod(f, RegularFileModeInt); err != nil { resultErr = fmt.Errorf( - "failed to set file mode to %#o for %q: %w", regularFileMode, file.Path, err) + "failed to set file mode to %#o for %q: %w", RegularFileModeInt, file.Path, err) return resultErr } case TypeSecret: - if err := fileMgr.Chmod(f, secretFileMode); err != nil { - resultErr = fmt.Errorf("failed to set file mode to %#o for %q: %w", secretFileMode, file.Path, err) + if err := fileMgr.Chmod(f, secretFileModeInt); err != nil { + resultErr = fmt.Errorf("failed to set file mode to %#o for %q: %w", secretFileModeInt, file.Path, err) return resultErr } default: @@ -105,3 +111,24 @@ func ensureType(fileType Type) { panic(fmt.Sprintf("unknown file type %d", fileType)) } } + +// Convert an agent File to an internal File type. +func Convert(agentFile agent.File) File { + if agentFile.Meta == nil { + return File{} + } + + var t Type + switch agentFile.Meta.Permissions { + case RegularFileMode: + t = TypeRegular + case SecretFileMode: + t = TypeSecret + } + + return File{ + Content: agentFile.Contents, + Path: agentFile.Meta.Name, + Type: t, + } +} diff --git a/internal/framework/file/file_test.go b/internal/framework/file/file_test.go index d5b52b48a6..00f58f4809 100644 --- a/internal/framework/file/file_test.go +++ b/internal/framework/file/file_test.go @@ -5,11 +5,13 @@ import ( "os" "path/filepath" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file/filefakes" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" ) var _ = Describe("Write files", Ordered, func() { @@ -152,4 +154,35 @@ var _ = Describe("Write files", Ordered, func() { ), ) }) + + It("converts agent files to internal files", func() { + agentFile := agent.File{ + Contents: []byte("file contents"), + Meta: &pb.FileMeta{ + Name: "regular-file", + Permissions: file.RegularFileMode, + }, + } + expFile := file.File{ + Path: "regular-file", + Content: []byte("file contents"), + Type: file.TypeRegular, + } + + secretAgentFile := agent.File{ + Contents: []byte("secret contents"), + Meta: &pb.FileMeta{ + Name: "secret-file", + Permissions: file.SecretFileMode, + }, + } + expSecretFile := file.File{ + Path: "secret-file", + Content: []byte("secret contents"), + Type: file.TypeSecret, + } + + Expect(file.Convert(agentFile)).To(Equal(expFile)) + Expect(file.Convert(secretAgentFile)).To(Equal(expSecretFile)) + }) }) diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index a076884811..afb72a27aa 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -2,6 +2,7 @@ package static import ( "context" + "errors" "fmt" "sync" "time" @@ -21,6 +22,7 @@ import ( ngfConfig "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/licensing" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" ngxConfig "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" @@ -35,6 +37,7 @@ type handlerMetricsCollector interface { // eventHandlerConfig holds configuration parameters for eventHandlerImpl. type eventHandlerConfig struct { + ctx context.Context // nginxUpdater updates nginx configuration using the NGINX agent. nginxUpdater agent.NginxUpdater // metricsCollector collects metrics for this controller. @@ -59,6 +62,12 @@ type eventHandlerConfig struct { deployCtxCollector licensing.Collector // graphBuiltHealthChecker sets the health of the Pod to Ready once we've built our initial graph. graphBuiltHealthChecker *graphBuiltHealthChecker + // statusQueue contains updates when the handler should write statuses. + statusQueue *status.Queue + // nginxDeployments contains a map of all nginx Deployments, and data about them. + nginxDeployments *agent.DeploymentStore + // logger is the logger for the event handler. + logger logr.Logger // gatewayPodConfig contains information about this Pod. gatewayPodConfig ngfConfig.GatewayPodConfig // controlConfigNSName is the NamespacedName of the NginxGateway config for this controller. @@ -102,8 +111,6 @@ type eventHandlerImpl struct { // objectFilters contains all created objectFilters, with the key being a filterKey objectFilters map[filterKey]objectFilter - latestReloadResult status.NginxReloadResult - cfg eventHandlerConfig lock sync.Mutex @@ -137,6 +144,8 @@ func newEventHandlerImpl(cfg eventHandlerConfig) *eventHandlerImpl { }, } + go handler.waitForStatusUpdates(cfg.ctx) + return handler } @@ -164,7 +173,22 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log h.cfg.graphBuiltHealthChecker.setAsReady() } - var err error + // TODO(sberman): hardcode this deployment name until we support provisioning data planes + // If no deployments exist, we should just return without doing anything. + deploymentName := types.NamespacedName{ + Name: "tmp-nginx-deployment", + Namespace: h.cfg.gatewayPodConfig.Namespace, + } + + // TODO(sberman): if nginx Deployment is scaled down, we should remove the pod from the ConnectionsTracker + // and Deployment. + // If fully deleted, then delete the deployment from the Store + var configApplied bool + deployment := h.cfg.nginxDeployments.GetOrStore(deploymentName, broadcast.NewDeploymentBroadcaster(ctx)) + if deployment == nil { + panic("expected deployment, got nil") + } + switch changeType { case state.NoChange: logger.Info("Handling events didn't result into NGINX configuration changes") @@ -180,11 +204,13 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log h.setLatestConfiguration(&cfg) + deployment.Lock.Lock() if h.cfg.plus { - h.cfg.nginxUpdater.UpdateUpstreamServers() + configApplied = h.cfg.nginxUpdater.UpdateUpstreamServers(deployment, cfg) } else { - err = h.updateNginxConf(cfg) + configApplied = h.updateNginxConf(deployment, cfg) } + deployment.Lock.Unlock() case state.ClusterStateChange: h.version++ cfg := dataplane.BuildConfiguration(ctx, gr, h.cfg.serviceResolver, h.version) @@ -196,26 +222,53 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log h.setLatestConfiguration(&cfg) - err = h.updateNginxConf(cfg) + deployment.Lock.Lock() + configApplied = h.updateNginxConf(deployment, cfg) + deployment.Lock.Unlock() } - var nginxReloadRes status.NginxReloadResult - if err != nil { - logger.Error(err, "Failed to update NGINX configuration") - nginxReloadRes.Error = err - } else { - logger.Info("NGINX configuration was successfully updated") + configErr := deployment.GetLatestConfigError() + upstreamErr := deployment.GetLatestUpstreamError() + err := errors.Join(configErr, upstreamErr) + + if configApplied || err != nil { + obj := &status.QueueObject{ + Error: err, + Deployment: deploymentName, + } + h.cfg.statusQueue.Enqueue(obj) } +} - h.latestReloadResult = nginxReloadRes +func (h *eventHandlerImpl) waitForStatusUpdates(ctx context.Context) { + for { + item := h.cfg.statusQueue.Dequeue(ctx) + if item == nil { + return + } + + var nginxReloadRes graph.NginxReloadResult + switch { + case item.Error != nil: + h.cfg.logger.Error(item.Error, "Failed to update NGINX configuration") + nginxReloadRes.Error = item.Error + default: + h.cfg.logger.Info("NGINX configuration was successfully updated") + } - h.updateStatuses(ctx, logger, gr) + // TODO(sberman): once we support multiple Gateways, we'll have to get + // the correct Graph for the Deployment contained in the update message + gr := h.cfg.processor.GetLatestGraph() + gr.LatestReloadResult = nginxReloadRes + + h.updateStatuses(ctx, gr) + } } -func (h *eventHandlerImpl) updateStatuses(ctx context.Context, logger logr.Logger, gr *graph.Graph) { +func (h *eventHandlerImpl) updateStatuses(ctx context.Context, gr *graph.Graph) { gwAddresses, err := getGatewayAddresses(ctx, h.cfg.k8sClient, nil, h.cfg.gatewayPodConfig) if err != nil { - logger.Error(err, "Setting GatewayStatusAddress to Pod IP Address") + h.cfg.logger.Error(err, "Setting GatewayStatusAddress to Pod IP Address") } transitionTime := metav1.Now() @@ -228,7 +281,7 @@ func (h *eventHandlerImpl) updateStatuses(ctx context.Context, logger logr.Logge gr.L4Routes, gr.Routes, transitionTime, - h.latestReloadResult, + gr.LatestReloadResult, h.cfg.gatewayCtlrName, ) @@ -260,7 +313,7 @@ func (h *eventHandlerImpl) updateStatuses(ctx context.Context, logger logr.Logge gr.IgnoredGateways, transitionTime, gwAddresses, - h.latestReloadResult, + gr.LatestReloadResult, ) h.cfg.statusUpdater.UpdateGroup(ctx, groupGateways, gwReqs...) } @@ -295,19 +348,19 @@ func (h *eventHandlerImpl) parseAndCaptureEvent(ctx context.Context, logger logr } // updateNginxConf updates nginx conf files and reloads nginx. -// -//nolint:unparam // temporarily returning only nil -func (h *eventHandlerImpl) updateNginxConf(conf dataplane.Configuration) error { +func (h *eventHandlerImpl) updateNginxConf( + deployment *agent.Deployment, + conf dataplane.Configuration, +) bool { files := h.cfg.generator.Generate(conf) - - h.cfg.nginxUpdater.UpdateConfig(len(files)) + applied := h.cfg.nginxUpdater.UpdateConfig(deployment, files) // If using NGINX Plus, update upstream servers using the API. if h.cfg.plus { - h.cfg.nginxUpdater.UpdateUpstreamServers() + h.cfg.nginxUpdater.UpdateUpstreamServers(deployment, conf) } - return nil + return applied } // updateControlPlaneAndSetStatus updates the control plane configuration and then sets the status @@ -423,6 +476,8 @@ func (h *eventHandlerImpl) GetLatestConfiguration() *dataplane.Configuration { } // setLatestConfiguration sets the latest configuration. +// TODO(sberman): once we support multiple Gateways, this will likely have to be a map +// of all configurations. func (h *eventHandlerImpl) setLatestConfiguration(cfg *dataplane.Configuration) { h.lock.Lock() defer h.lock.Unlock() @@ -482,7 +537,7 @@ func (h *eventHandlerImpl) nginxGatewayServiceUpsert(ctx context.Context, logger gr.IgnoredGateways, transitionTime, gwAddresses, - h.latestReloadResult, + gr.LatestReloadResult, ) h.cfg.statusUpdater.UpdateGroup(ctx, groupGateways, gatewayStatuses...) } @@ -508,7 +563,7 @@ func (h *eventHandlerImpl) nginxGatewayServiceDelete( gr.IgnoredGateways, transitionTime, gwAddresses, - h.latestReloadResult, + gr.LatestReloadResult, ) h.cfg.statusUpdater.UpdateGroup(ctx, groupGateways, gatewayStatuses...) } diff --git a/internal/mode/static/handler_test.go b/internal/mode/static/handler_test.go index ce185565c6..2842e29c9d 100644 --- a/internal/mode/static/handler_test.go +++ b/internal/mode/static/handler_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/zap" @@ -19,18 +20,20 @@ import ( ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/events" - "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/status/statusfakes" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/licensing/licensingfakes" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/metrics/collectors" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/agentfakes" + agentgrpcfakes "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/grpcfakes" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/configfakes" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/statefakes" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/status" ) var _ = Describe("eventHandler", func() { @@ -42,9 +45,12 @@ var _ = Describe("eventHandler", func() { fakeStatusUpdater *statusfakes.FakeGroupUpdater fakeEventRecorder *record.FakeRecorder fakeK8sClient client.WithWatch + queue *status.Queue namespace = "nginx-gateway" configName = "nginx-gateway-config" zapLogLevelSetter zapLogLevelSetter + ctx context.Context + cancel context.CancelFunc ) const nginxGatewayServiceName = "nginx-gateway" @@ -58,17 +64,20 @@ var _ = Describe("eventHandler", func() { } } - expectReconfig := func(expectedConf dataplane.Configuration, expectedFiles []file.File) { + expectReconfig := func(expectedConf dataplane.Configuration, expectedFiles []agent.File) { Expect(fakeProcessor.ProcessCallCount()).Should(Equal(1)) Expect(fakeGenerator.GenerateCallCount()).Should(Equal(1)) Expect(fakeGenerator.GenerateArgsForCall(0)).Should(Equal(expectedConf)) Expect(fakeNginxUpdater.UpdateConfigCallCount()).Should(Equal(1)) - lenFiles := fakeNginxUpdater.UpdateConfigArgsForCall(0) - Expect(expectedFiles).To(HaveLen(lenFiles)) + _, files := fakeNginxUpdater.UpdateConfigArgsForCall(0) + Expect(expectedFiles).To(Equal(files)) - Expect(fakeStatusUpdater.UpdateGroupCallCount()).Should(Equal(2)) + Eventually( + func() int { + return fakeStatusUpdater.UpdateGroupCallCount() + }).Should(Equal(2)) _, name, reqs := fakeStatusUpdater.UpdateGroupArgsForCall(0) Expect(name).To(Equal(groupAllExceptGateways)) Expect(reqs).To(BeEmpty()) @@ -79,19 +88,25 @@ var _ = Describe("eventHandler", func() { } BeforeEach(func() { + ctx, cancel = context.WithCancel(context.Background()) //nolint:fatcontext // ignore for test + fakeProcessor = &statefakes.FakeChangeProcessor{} fakeProcessor.ProcessReturns(state.NoChange, &graph.Graph{}) + fakeProcessor.GetLatestGraphReturns(&graph.Graph{}) fakeGenerator = &configfakes.FakeGenerator{} fakeNginxUpdater = &agentfakes.FakeNginxUpdater{} + fakeNginxUpdater.UpdateConfigReturns(true) fakeStatusUpdater = &statusfakes.FakeGroupUpdater{} fakeEventRecorder = record.NewFakeRecorder(1) zapLogLevelSetter = newZapLogLevelSetter(zap.NewAtomicLevel()) fakeK8sClient = fake.NewFakeClient() + queue = status.NewQueue() // Needed because handler checks the service from the API on every HandleEventBatch Expect(fakeK8sClient.Create(context.Background(), createService(nginxGatewayServiceName))).To(Succeed()) handler = newEventHandlerImpl(eventHandlerConfig{ + ctx: ctx, k8sClient: fakeK8sClient, processor: fakeProcessor, generator: fakeGenerator, @@ -101,6 +116,8 @@ var _ = Describe("eventHandler", func() { eventRecorder: fakeEventRecorder, deployCtxCollector: &licensingfakes.FakeCollector{}, graphBuiltHealthChecker: newGraphBuiltHealthChecker(), + statusQueue: queue, + nginxDeployments: agent.NewDeploymentStore(&agentgrpcfakes.FakeConnectionsTracker{}), controlConfigNSName: types.NamespacedName{Namespace: namespace, Name: configName}, gatewayPodConfig: config.GatewayPodConfig{ ServiceName: "nginx-gateway", @@ -112,11 +129,16 @@ var _ = Describe("eventHandler", func() { Expect(handler.cfg.graphBuiltHealthChecker.ready).To(BeFalse()) }) + AfterEach(func() { + cancel() + }) + Describe("Process the Gateway API resources events", func() { - fakeCfgFiles := []file.File{ + fakeCfgFiles := []agent.File{ { - Type: file.TypeRegular, - Path: "test.conf", + Meta: &pb.FileMeta{ + Name: "test.conf", + }, }, } @@ -211,7 +233,7 @@ var _ = Describe("eventHandler", func() { }, } - fakeProcessor.ProcessReturns(state.ClusterStateChange, &graph.Graph{ + gr := &graph.Graph{ GatewayClass: &graph.GatewayClass{ Source: gc, Valid: true, @@ -219,7 +241,10 @@ var _ = Describe("eventHandler", func() { IgnoredGatewayClasses: map[types.NamespacedName]*gatewayv1.GatewayClass{ client.ObjectKeyFromObject(ignoredGC): ignoredGC, }, - }) + } + + fakeProcessor.ProcessReturns(state.ClusterStateChange, gr) + fakeProcessor.GetLatestGraphReturns(gr) e := &events.UpsertEvent{ Resource: &gatewayv1.HTTPRoute{}, // any supported is OK @@ -234,7 +259,10 @@ var _ = Describe("eventHandler", func() { handler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) - Expect(fakeStatusUpdater.UpdateGroupCallCount()).To(Equal(2)) + Eventually( + func() int { + return fakeStatusUpdater.UpdateGroupCallCount() + }).Should(Equal(2)) _, name, reqs := fakeStatusUpdater.UpdateGroupArgsForCall(0) Expect(name).To(Equal(groupAllExceptGateways)) @@ -439,6 +467,22 @@ var _ = Describe("eventHandler", func() { }) }) + It("should update status when receiving a queue event", func() { + obj := &status.QueueObject{ + Deployment: types.NamespacedName{}, + Error: errors.New("status error"), + } + queue.Enqueue(obj) + + Eventually( + func() int { + return fakeStatusUpdater.UpdateGroupCallCount() + }).Should(Equal(2)) + + gr := handler.cfg.processor.GetLatestGraph() + Expect(gr.LatestReloadResult.Error.Error()).To(Equal("status error")) + }) + It("should set the health checker status properly", func() { e := &events.UpsertEvent{Resource: &gatewayv1.HTTPRoute{}} batch := []interface{}{e} @@ -531,6 +575,17 @@ var _ = Describe("getDeploymentContext", func() { }) When("nginx plus is true", func() { + var ctx context.Context + var cancel context.CancelFunc + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.Background()) //nolint:fatcontext + }) + + AfterEach(func() { + cancel() + }) + It("returns deployment context", func() { expDepCtx := dataplane.DeploymentContext{ Integration: "ngf", @@ -540,7 +595,9 @@ var _ = Describe("getDeploymentContext", func() { } handler := newEventHandlerImpl(eventHandlerConfig{ - plus: true, + ctx: ctx, + statusQueue: status.NewQueue(), + plus: true, deployCtxCollector: &licensingfakes.FakeCollector{ CollectStub: func(_ context.Context) (dataplane.DeploymentContext, error) { return expDepCtx, nil @@ -556,7 +613,9 @@ var _ = Describe("getDeploymentContext", func() { expErr := errors.New("collect error") handler := newEventHandlerImpl(eventHandlerConfig{ - plus: true, + ctx: ctx, + statusQueue: status.NewQueue(), + plus: true, deployCtxCollector: &licensingfakes.FakeCollector{ CollectStub: func(_ context.Context) (dataplane.DeploymentContext, error) { return dataplane.DeploymentContext{}, expErr diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index b6da949c52..97901b9259 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -44,7 +44,7 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/kinds" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/runnables" - "github.com/nginxinc/nginx-gateway-fabric/internal/framework/status" + frameworkStatus "github.com/nginxinc/nginx-gateway-fabric/internal/framework/status" ngftypes "github.com/nginxinc/nginx-gateway-fabric/internal/framework/types" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/licensing" @@ -61,6 +61,7 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/validation" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/status" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/telemetry" ) @@ -158,7 +159,7 @@ func StartManager(cfg config.Config) error { handlerCollector, ok := handlerCollector.(prometheus.Collector) if !ok { - return fmt.Errorf("handlerCollector is not a prometheus.Collector: %w", status.ErrFailedAssert) + return fmt.Errorf("handlerCollector is not a prometheus.Collector: %w", frameworkStatus.ErrFailedAssert) } metrics.Registry.MustRegister( @@ -167,19 +168,25 @@ func StartManager(cfg config.Config) error { ) } - statusUpdater := status.NewUpdater( + statusUpdater := frameworkStatus.NewUpdater( mgr.GetClient(), cfg.Logger.WithName("statusUpdater"), ) - groupStatusUpdater := status.NewLeaderAwareGroupUpdater(statusUpdater) + groupStatusUpdater := frameworkStatus.NewLeaderAwareGroupUpdater(statusUpdater) deployCtxCollector := licensing.NewDeploymentContextCollector(licensing.DeploymentContextCollectorConfig{ K8sClientReader: mgr.GetAPIReader(), PodUID: cfg.GatewayPodConfig.UID, Logger: cfg.Logger.WithName("deployCtxCollector"), }) - nginxUpdater := agent.NewNginxUpdater(cfg.Logger.WithName("nginxUpdater"), cfg.Plus) + statusQueue := status.NewQueue() + nginxUpdater := agent.NewNginxUpdater( + cfg.Logger.WithName("nginxUpdater"), + mgr.GetAPIReader(), + statusQueue, + cfg.Plus, + ) grpcServer := agentgrpc.NewServer( cfg.Logger.WithName("agentGRPCServer"), @@ -194,8 +201,8 @@ func StartManager(cfg config.Config) error { return fmt.Errorf("cannot register grpc server: %w", err) } - // TODO(sberman): event handler loop should wait on a channel until the grpc server has started eventHandler := newEventHandlerImpl(eventHandlerConfig{ + ctx: ctx, nginxUpdater: nginxUpdater, metricsCollector: handlerCollector, statusUpdater: groupStatusUpdater, @@ -208,6 +215,7 @@ func StartManager(cfg config.Config) error { ), k8sClient: mgr.GetClient(), k8sReader: mgr.GetAPIReader(), + logger: cfg.Logger.WithName("eventHandler"), logLevelSetter: logLevelSetter, eventRecorder: recorder, deployCtxCollector: deployCtxCollector, @@ -217,6 +225,8 @@ func StartManager(cfg config.Config) error { gatewayCtlrName: cfg.GatewayCtlrName, updateGatewayClassStatus: cfg.UpdateGatewayClassStatus, plus: cfg.Plus, + statusQueue: statusQueue, + nginxDeployments: nginxUpdater.NginxDeployments, }) objects, objectLists := prepareFirstEventBatchPreparerArgs(cfg) @@ -505,6 +515,7 @@ func registerControllers( objectType: &ngfAPI.NginxGateway{}, options: []controller.Option{ controller.WithNamespacedNameFilter(filter.CreateSingleResourceFilter(controlConfigNSName)), + controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}), }, }) if err := setInitialConfig( diff --git a/internal/mode/static/nginx/agent/agent.go b/internal/mode/static/nginx/agent/agent.go index 1ce5d21b0b..83aef4f99d 100644 --- a/internal/mode/static/nginx/agent/agent.go +++ b/internal/mode/static/nginx/agent/agent.go @@ -1,48 +1,256 @@ package agent import ( + "context" + "errors" + "fmt" + "time" + "github.com/go-logr/logr" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + "google.golang.org/protobuf/types/known/structpb" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" + agentgrpc "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/status" ) +const retryUpstreamTimeout = 5 * time.Second + //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate //counterfeiter:generate . NginxUpdater // NginxUpdater is an interface for updating NGINX using the NGINX agent. type NginxUpdater interface { - UpdateConfig(int) - UpdateUpstreamServers() + UpdateConfig(deployment *Deployment, files []File) bool + UpdateUpstreamServers(deployment *Deployment, conf dataplane.Configuration) bool } // NginxUpdaterImpl implements the NginxUpdater interface. type NginxUpdaterImpl struct { - CommandService *commandService - FileService *fileService - logger logr.Logger - plus bool + CommandService *commandService + FileService *fileService + NginxDeployments *DeploymentStore + logger logr.Logger + plus bool + retryTimeout time.Duration } // NewNginxUpdater returns a new NginxUpdaterImpl instance. -func NewNginxUpdater(logger logr.Logger, plus bool) *NginxUpdaterImpl { +func NewNginxUpdater( + logger logr.Logger, + reader client.Reader, + statusQueue *status.Queue, + plus bool, +) *NginxUpdaterImpl { + connTracker := agentgrpc.NewConnectionsTracker() + nginxDeployments := NewDeploymentStore(connTracker) + + commandService := newCommandService( + logger.WithName("commandService"), + reader, + nginxDeployments, + connTracker, + statusQueue, + ) + fileService := newFileService(logger.WithName("fileService"), nginxDeployments, connTracker) + return &NginxUpdaterImpl{ - logger: logger, - plus: plus, - CommandService: newCommandService(logger.WithName("commandService")), - FileService: newFileService(logger.WithName("fileService")), + logger: logger, + plus: plus, + NginxDeployments: nginxDeployments, + CommandService: commandService, + FileService: fileService, + retryTimeout: retryUpstreamTimeout, } } // UpdateConfig sends the nginx configuration to the agent. -func (n *NginxUpdaterImpl) UpdateConfig(files int) { - n.logger.Info("Sending nginx configuration to agent", "numFiles", files) +// Returns whether the configuration was sent to any agents. +// +// The flow of events is as follows: +// - Set the configuration files on the deployment. +// - Broadcast the message containing file metadata to all pods (subscriptions) for the deployment. +// - Agent receives a ConfigApplyRequest with the list of file metadata. +// - Agent calls GetFile for each file in the list, which we send back to the agent. +// - Agent updates nginx, and responds with a DataPlaneResponse. +// - Subscriber responds back to the broadcaster to inform that the transaction is complete. +// - If any errors occurred, they are set on the deployment for the handler to use in the status update. +func (n *NginxUpdaterImpl) UpdateConfig( + deployment *Deployment, + files []File, +) bool { + n.logger.Info("Sending nginx configuration to agent") + + // reset the latest error to nil now that we're applying new config + deployment.SetLatestConfigError(nil) + + msg := deployment.SetFiles(files) + applied := deployment.GetBroadcaster().Send(msg) + + latestStatus := deployment.GetConfigurationStatus() + if latestStatus != nil { + deployment.SetLatestConfigError(latestStatus) + } + + return applied } // UpdateUpstreamServers sends an APIRequest to the agent to update upstream servers using the NGINX Plus API. // Only applicable when using NGINX Plus. -func (n *NginxUpdaterImpl) UpdateUpstreamServers() { +// Returns whether the configuration was sent to any agents. +func (n *NginxUpdaterImpl) UpdateUpstreamServers( + deployment *Deployment, + conf dataplane.Configuration, +) bool { if !n.plus { - return + return false + } + + broadcaster := deployment.GetBroadcaster() + + // reset the latest error to nil now that we're applying new config + deployment.SetLatestUpstreamError(nil) + + // TODO(sberman): optimize this by only sending updates that are necessary. + // Call GetUpstreams first (will need Subscribers to send responses back), and + // then determine which upstreams actually need to be updated. + + var errs []error + var applied bool + actions := make([]*pb.NGINXPlusAction, 0, len(conf.Upstreams)+len(conf.StreamUpstreams)) + for _, upstream := range conf.Upstreams { + action := &pb.NGINXPlusAction{ + Action: &pb.NGINXPlusAction_UpdateHttpUpstreamServers{ + UpdateHttpUpstreamServers: buildHTTPUpstreamServers(upstream), + }, + } + actions = append(actions, action) + + msg := broadcast.NginxAgentMessage{ + Type: broadcast.APIRequest, + NGINXPlusAction: action, + } + + requestApplied, err := n.sendRequest(broadcaster, msg, deployment) + if err != nil { + errs = append(errs, fmt.Errorf( + "couldn't update upstream %q via the API: %w", upstream.Name, deployment.GetConfigurationStatus())) + } + applied = applied || requestApplied + } + + for _, upstream := range conf.StreamUpstreams { + action := &pb.NGINXPlusAction{ + Action: &pb.NGINXPlusAction_UpdateStreamServers{ + UpdateStreamServers: buildStreamUpstreamServers(upstream), + }, + } + actions = append(actions, action) + + msg := broadcast.NginxAgentMessage{ + Type: broadcast.APIRequest, + NGINXPlusAction: action, + } + + requestApplied, err := n.sendRequest(broadcaster, msg, deployment) + if err != nil { + errs = append(errs, fmt.Errorf( + "couldn't update upstream %q via the API: %w", upstream.Name, deployment.GetConfigurationStatus())) + } + applied = applied || requestApplied + } + + if len(errs) != 0 { + deployment.SetLatestUpstreamError(errors.Join(errs...)) + } else if applied { + n.logger.Info("Updated upstream servers using NGINX Plus API") + } + + // Store the most recent actions on the deployment so any new subscribers can apply them when first connecting. + deployment.SetNGINXPlusActions(actions) + + return applied +} + +func buildHTTPUpstreamServers(upstream dataplane.Upstream) *pb.UpdateHTTPUpstreamServers { + return &pb.UpdateHTTPUpstreamServers{ + HttpUpstreamName: upstream.Name, + Servers: buildUpstreamServers(upstream), + } +} + +func buildStreamUpstreamServers(upstream dataplane.Upstream) *pb.UpdateStreamServers { + return &pb.UpdateStreamServers{ + UpstreamStreamName: upstream.Name, + Servers: buildUpstreamServers(upstream), + } +} + +func buildUpstreamServers(upstream dataplane.Upstream) []*structpb.Struct { + servers := make([]*structpb.Struct, 0, len(upstream.Endpoints)) + + for _, endpoint := range upstream.Endpoints { + port, format := getPortAndIPFormat(endpoint) + value := fmt.Sprintf(format, endpoint.Address, port) + + server := &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "server": structpb.NewStringValue(value), + }, + } + + servers = append(servers, server) + } + + return servers +} + +func (n *NginxUpdaterImpl) sendRequest( + broadcaster broadcast.Broadcaster, + msg broadcast.NginxAgentMessage, + deployment *Deployment, +) (bool, error) { + // retry the API update request because sometimes nginx isn't quite ready after the config apply reload + ctx, cancel := context.WithTimeout(context.Background(), n.retryTimeout) + defer cancel() + + var applied bool + if err := wait.PollUntilContextCancel( + ctx, + 500*time.Millisecond, + true, // poll immediately + func(_ context.Context) (bool, error) { + applied = broadcaster.Send(msg) + if statusErr := deployment.GetConfigurationStatus(); statusErr != nil { + return false, nil //nolint:nilerr // will get error once done polling + } + + return true, nil + }, + ); err != nil { + return applied, err + } + + return applied, nil +} + +func getPortAndIPFormat(ep resolver.Endpoint) (string, string) { + var port string + + if ep.Port != 0 { + port = fmt.Sprintf(":%d", ep.Port) + } + + format := "%s%s" + if ep.IPv6 { + format = "[%s]%s" } - n.logger.Info("Updating upstream servers using NGINX Plus API") + return port, format } diff --git a/internal/mode/static/nginx/agent/agent_test.go b/internal/mode/static/nginx/agent/agent_test.go new file mode 100644 index 0000000000..bd318953bb --- /dev/null +++ b/internal/mode/static/nginx/agent/agent_test.go @@ -0,0 +1,307 @@ +package agent + +import ( + "errors" + "fmt" + "testing" + + "github.com/go-logr/logr" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + . "github.com/onsi/gomega" + "google.golang.org/protobuf/types/known/structpb" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast/broadcastfakes" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/status" +) + +func TestUpdateConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + configApplied bool + expErr bool + }{ + { + name: "success", + configApplied: true, + expErr: false, + }, + { + name: "error returned from agent", + configApplied: true, + expErr: true, + }, + { + name: "configuration not applied", + configApplied: false, + expErr: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + fakeBroadcaster := &broadcastfakes.FakeBroadcaster{} + fakeBroadcaster.SendReturns(test.configApplied) + + plus := false + updater := NewNginxUpdater(logr.Discard(), fake.NewFakeClient(), &status.Queue{}, plus) + deployment := &Deployment{ + broadcaster: fakeBroadcaster, + podStatuses: make(map[string]error), + } + + file := File{ + Meta: &pb.FileMeta{ + Name: "test.conf", + Hash: "12345", + }, + Contents: []byte("test content"), + } + + testErr := errors.New("test error") + if test.expErr { + deployment.SetPodErrorStatus("pod1", testErr) + } + + applied := updater.UpdateConfig(deployment, []File{file}) + + g.Expect(applied).To(Equal(test.configApplied)) + g.Expect(deployment.GetFile(file.Meta.Name, file.Meta.Hash)).To(Equal(file.Contents)) + + if test.expErr { + g.Expect(deployment.GetLatestConfigError()).To(Equal(testErr)) + // ensure that the error is cleared after the next config is applied + deployment.SetPodErrorStatus("pod1", nil) + updater.UpdateConfig(deployment, []File{file}) + g.Expect(deployment.GetLatestConfigError()).ToNot(HaveOccurred()) + } else { + g.Expect(deployment.GetLatestConfigError()).ToNot(HaveOccurred()) + } + }) + } +} + +func TestUpdateUpstreamServers(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + buildUpstreams bool + plus bool + configApplied bool + expErr bool + }{ + { + name: "success", + plus: true, + buildUpstreams: true, + configApplied: true, + expErr: false, + }, + { + name: "no upstreams to apply", + plus: true, + buildUpstreams: false, + configApplied: false, + expErr: false, + }, + { + name: "not running nginx plus", + plus: false, + configApplied: false, + expErr: false, + }, + { + name: "error returned from agent", + plus: true, + buildUpstreams: true, + configApplied: true, + expErr: true, + }, + { + name: "configuration not applied", + plus: true, + buildUpstreams: true, + configApplied: false, + expErr: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + fakeBroadcaster := &broadcastfakes.FakeBroadcaster{} + fakeBroadcaster.SendReturns(test.configApplied) + + updater := NewNginxUpdater(logr.Discard(), fake.NewFakeClient(), &status.Queue{}, test.plus) + updater.retryTimeout = 0 + + deployment := &Deployment{ + broadcaster: fakeBroadcaster, + podStatuses: make(map[string]error), + } + + testErr := errors.New("test error") + if test.expErr { + deployment.SetPodErrorStatus("pod1", testErr) + } + + var conf dataplane.Configuration + if test.buildUpstreams { + conf = dataplane.Configuration{ + Upstreams: []dataplane.Upstream{ + { + Name: "test-upstream", + Endpoints: []resolver.Endpoint{ + { + Address: "1.2.3.4", + Port: 8080, + }, + }, + }, + }, + StreamUpstreams: []dataplane.Upstream{ + { + Name: "test-stream-upstream", + Endpoints: []resolver.Endpoint{ + { + Address: "5.6.7.8", + }, + }, + }, + }, + } + } + + applied := updater.UpdateUpstreamServers(deployment, conf) + g.Expect(applied).To(Equal(test.configApplied)) + + expActions := make([]*pb.NGINXPlusAction, 0) + if test.buildUpstreams { + expActions = []*pb.NGINXPlusAction{ + { + Action: &pb.NGINXPlusAction_UpdateHttpUpstreamServers{ + UpdateHttpUpstreamServers: &pb.UpdateHTTPUpstreamServers{ + HttpUpstreamName: "test-upstream", + Servers: []*structpb.Struct{ + { + Fields: map[string]*structpb.Value{ + "server": structpb.NewStringValue("1.2.3.4:8080"), + }, + }, + }, + }, + }, + }, + { + Action: &pb.NGINXPlusAction_UpdateStreamServers{ + UpdateStreamServers: &pb.UpdateStreamServers{ + UpstreamStreamName: "test-stream-upstream", + Servers: []*structpb.Struct{ + { + Fields: map[string]*structpb.Value{ + "server": structpb.NewStringValue("5.6.7.8"), + }, + }, + }, + }, + }, + }, + } + } + + if !test.plus { + g.Expect(deployment.GetNGINXPlusActions()).To(BeNil()) + } else { + g.Expect(deployment.GetNGINXPlusActions()).To(Equal(expActions)) + } + + if test.expErr { + expErr := errors.Join( + fmt.Errorf("couldn't update upstream \"test-upstream\" via the API: %w", testErr), + fmt.Errorf("couldn't update upstream \"test-stream-upstream\" via the API: %w", testErr), + ) + + g.Expect(deployment.GetLatestUpstreamError()).To(Equal(expErr)) + // ensure that the error is cleared after the next config is applied + deployment.SetPodErrorStatus("pod1", nil) + updater.UpdateUpstreamServers(deployment, conf) + g.Expect(deployment.GetLatestUpstreamError()).ToNot(HaveOccurred()) + } else { + g.Expect(deployment.GetLatestUpstreamError()).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGetPortAndIPFormat(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + expPort string + expFormat string + endpoint resolver.Endpoint + }{ + { + name: "IPv4 with port", + endpoint: resolver.Endpoint{ + Address: "1.2.3.4", + Port: 8080, + IPv6: false, + }, + expPort: ":8080", + expFormat: "%s%s", + }, + { + name: "IPv4 without port", + endpoint: resolver.Endpoint{ + Address: "1.2.3.4", + Port: 0, + IPv6: false, + }, + expPort: "", + expFormat: "%s%s", + }, + { + name: "IPv6 with port", + endpoint: resolver.Endpoint{ + Address: "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + Port: 8080, + IPv6: true, + }, + expPort: ":8080", + expFormat: "[%s]%s", + }, + { + name: "IPv6 without port", + endpoint: resolver.Endpoint{ + Address: "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + Port: 0, + IPv6: true, + }, + expPort: "", + expFormat: "[%s]%s", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + port, format := getPortAndIPFormat(test.endpoint) + g.Expect(port).To(Equal(test.expPort)) + g.Expect(format).To(Equal(test.expFormat)) + }) + } +} diff --git a/internal/mode/static/nginx/agent/agentfakes/fake_nginx_updater.go b/internal/mode/static/nginx/agent/agentfakes/fake_nginx_updater.go index 0133814151..606fa67532 100644 --- a/internal/mode/static/nginx/agent/agentfakes/fake_nginx_updater.go +++ b/internal/mode/static/nginx/agent/agentfakes/fake_nginx_updater.go @@ -5,33 +5,61 @@ import ( "sync" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" ) type FakeNginxUpdater struct { - UpdateConfigStub func(int) + UpdateConfigStub func(*agent.Deployment, []agent.File) bool updateConfigMutex sync.RWMutex updateConfigArgsForCall []struct { - arg1 int + arg1 *agent.Deployment + arg2 []agent.File } - UpdateUpstreamServersStub func() + updateConfigReturns struct { + result1 bool + } + updateConfigReturnsOnCall map[int]struct { + result1 bool + } + UpdateUpstreamServersStub func(*agent.Deployment, dataplane.Configuration) bool updateUpstreamServersMutex sync.RWMutex updateUpstreamServersArgsForCall []struct { + arg1 *agent.Deployment + arg2 dataplane.Configuration + } + updateUpstreamServersReturns struct { + result1 bool + } + updateUpstreamServersReturnsOnCall map[int]struct { + result1 bool } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } -func (fake *FakeNginxUpdater) UpdateConfig(arg1 int) { +func (fake *FakeNginxUpdater) UpdateConfig(arg1 *agent.Deployment, arg2 []agent.File) bool { + var arg2Copy []agent.File + if arg2 != nil { + arg2Copy = make([]agent.File, len(arg2)) + copy(arg2Copy, arg2) + } fake.updateConfigMutex.Lock() + ret, specificReturn := fake.updateConfigReturnsOnCall[len(fake.updateConfigArgsForCall)] fake.updateConfigArgsForCall = append(fake.updateConfigArgsForCall, struct { - arg1 int - }{arg1}) + arg1 *agent.Deployment + arg2 []agent.File + }{arg1, arg2Copy}) stub := fake.UpdateConfigStub - fake.recordInvocation("UpdateConfig", []interface{}{arg1}) + fakeReturns := fake.updateConfigReturns + fake.recordInvocation("UpdateConfig", []interface{}{arg1, arg2Copy}) fake.updateConfigMutex.Unlock() if stub != nil { - fake.UpdateConfigStub(arg1) + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 } + return fakeReturns.result1 } func (fake *FakeNginxUpdater) UpdateConfigCallCount() int { @@ -40,29 +68,60 @@ func (fake *FakeNginxUpdater) UpdateConfigCallCount() int { return len(fake.updateConfigArgsForCall) } -func (fake *FakeNginxUpdater) UpdateConfigCalls(stub func(int)) { +func (fake *FakeNginxUpdater) UpdateConfigCalls(stub func(*agent.Deployment, []agent.File) bool) { fake.updateConfigMutex.Lock() defer fake.updateConfigMutex.Unlock() fake.UpdateConfigStub = stub } -func (fake *FakeNginxUpdater) UpdateConfigArgsForCall(i int) int { +func (fake *FakeNginxUpdater) UpdateConfigArgsForCall(i int) (*agent.Deployment, []agent.File) { fake.updateConfigMutex.RLock() defer fake.updateConfigMutex.RUnlock() argsForCall := fake.updateConfigArgsForCall[i] - return argsForCall.arg1 + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeNginxUpdater) UpdateConfigReturns(result1 bool) { + fake.updateConfigMutex.Lock() + defer fake.updateConfigMutex.Unlock() + fake.UpdateConfigStub = nil + fake.updateConfigReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeNginxUpdater) UpdateConfigReturnsOnCall(i int, result1 bool) { + fake.updateConfigMutex.Lock() + defer fake.updateConfigMutex.Unlock() + fake.UpdateConfigStub = nil + if fake.updateConfigReturnsOnCall == nil { + fake.updateConfigReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.updateConfigReturnsOnCall[i] = struct { + result1 bool + }{result1} } -func (fake *FakeNginxUpdater) UpdateUpstreamServers() { +func (fake *FakeNginxUpdater) UpdateUpstreamServers(arg1 *agent.Deployment, arg2 dataplane.Configuration) bool { fake.updateUpstreamServersMutex.Lock() + ret, specificReturn := fake.updateUpstreamServersReturnsOnCall[len(fake.updateUpstreamServersArgsForCall)] fake.updateUpstreamServersArgsForCall = append(fake.updateUpstreamServersArgsForCall, struct { - }{}) + arg1 *agent.Deployment + arg2 dataplane.Configuration + }{arg1, arg2}) stub := fake.UpdateUpstreamServersStub - fake.recordInvocation("UpdateUpstreamServers", []interface{}{}) + fakeReturns := fake.updateUpstreamServersReturns + fake.recordInvocation("UpdateUpstreamServers", []interface{}{arg1, arg2}) fake.updateUpstreamServersMutex.Unlock() if stub != nil { - fake.UpdateUpstreamServersStub() + return stub(arg1, arg2) } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 } func (fake *FakeNginxUpdater) UpdateUpstreamServersCallCount() int { @@ -71,12 +130,42 @@ func (fake *FakeNginxUpdater) UpdateUpstreamServersCallCount() int { return len(fake.updateUpstreamServersArgsForCall) } -func (fake *FakeNginxUpdater) UpdateUpstreamServersCalls(stub func()) { +func (fake *FakeNginxUpdater) UpdateUpstreamServersCalls(stub func(*agent.Deployment, dataplane.Configuration) bool) { fake.updateUpstreamServersMutex.Lock() defer fake.updateUpstreamServersMutex.Unlock() fake.UpdateUpstreamServersStub = stub } +func (fake *FakeNginxUpdater) UpdateUpstreamServersArgsForCall(i int) (*agent.Deployment, dataplane.Configuration) { + fake.updateUpstreamServersMutex.RLock() + defer fake.updateUpstreamServersMutex.RUnlock() + argsForCall := fake.updateUpstreamServersArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeNginxUpdater) UpdateUpstreamServersReturns(result1 bool) { + fake.updateUpstreamServersMutex.Lock() + defer fake.updateUpstreamServersMutex.Unlock() + fake.UpdateUpstreamServersStub = nil + fake.updateUpstreamServersReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeNginxUpdater) UpdateUpstreamServersReturnsOnCall(i int, result1 bool) { + fake.updateUpstreamServersMutex.Lock() + defer fake.updateUpstreamServersMutex.Unlock() + fake.UpdateUpstreamServersStub = nil + if fake.updateUpstreamServersReturnsOnCall == nil { + fake.updateUpstreamServersReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.updateUpstreamServersReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeNginxUpdater) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() diff --git a/internal/mode/static/nginx/agent/broadcast/broadcast.go b/internal/mode/static/nginx/agent/broadcast/broadcast.go new file mode 100644 index 0000000000..2543399555 --- /dev/null +++ b/internal/mode/static/nginx/agent/broadcast/broadcast.go @@ -0,0 +1,157 @@ +package broadcast + +import ( + "context" + "sync" + + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + "k8s.io/apimachinery/pkg/util/uuid" +) + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate + +//counterfeiter:generate . Broadcaster + +// Broadcaster defines an interface for consumers to subscribe to File updates. +type Broadcaster interface { + Subscribe() SubscriberChannels + Send(NginxAgentMessage) bool + CancelSubscription(string) +} + +// SubscriberChannels are the channels sent to the subscriber to listen and respond on. +// The ID is used for map lookup to delete a subscriber when it's gone. +type SubscriberChannels struct { + ListenCh <-chan NginxAgentMessage + ResponseCh chan<- struct{} + ID string +} + +// storedChannels are the same channels used in the SubscriberChannels, but reverse direction. +// These are used to store the channels for the broadcaster to send and listen on, +// and can be looked up in the map using the same ID. +type storedChannels struct { + listenCh chan<- NginxAgentMessage + responseCh <-chan struct{} + id string +} + +// DeploymentBroadcaster sends out a signal when an nginx Deployment has updated +// configuration files. The signal is received by any agent Subscription that cares +// about this Deployment. The agent Subscription will then send a response of whether or not +// the configuration was successfully applied. +type DeploymentBroadcaster struct { + publishCh chan NginxAgentMessage + subCh chan storedChannels + unsubCh chan string + listeners map[string]storedChannels + doneCh chan struct{} +} + +// NewDeploymentBroadcaster returns a new instance of a DeploymentBroadcaster. +func NewDeploymentBroadcaster(ctx context.Context) *DeploymentBroadcaster { + broadcaster := &DeploymentBroadcaster{ + listeners: make(map[string]storedChannels), + publishCh: make(chan NginxAgentMessage), + subCh: make(chan storedChannels), + unsubCh: make(chan string), + doneCh: make(chan struct{}), + } + go broadcaster.run(ctx) + + return broadcaster +} + +// Subscribe allows a listener to subscribe to broadcast messages. It returns the channel +// to listen on for messages, as well as a channel to respond on. +func (b *DeploymentBroadcaster) Subscribe() SubscriberChannels { + listenCh := make(chan NginxAgentMessage) + responseCh := make(chan struct{}) + id := string(uuid.NewUUID()) + + subscriberChans := SubscriberChannels{ + ID: id, + ListenCh: listenCh, + ResponseCh: responseCh, + } + storedChans := storedChannels{ + id: id, + listenCh: listenCh, + responseCh: responseCh, + } + + b.subCh <- storedChans + return subscriberChans +} + +// Send the message to all listeners. Wait for all listeners to respond. +// Returns true if there were listeners that received the message. +func (b *DeploymentBroadcaster) Send(message NginxAgentMessage) bool { + b.publishCh <- message + <-b.doneCh + + return len(b.listeners) > 0 +} + +// CancelSubscription removes a Subscriber from the channel list. +func (b *DeploymentBroadcaster) CancelSubscription(id string) { + b.unsubCh <- id +} + +// run starts the broadcaster loop. It handles the following events: +// - if context is canceled, return. +// - if receiving a new subscriber, add it to the subscriber list. +// - if receiving a canceled subscription, remove it from the subscriber list. +// - if receiving a message to publish, send it to all subscribers. +func (b *DeploymentBroadcaster) run(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case channels := <-b.subCh: + b.listeners[channels.id] = channels + case id := <-b.unsubCh: + delete(b.listeners, id) + case msg := <-b.publishCh: + var wg sync.WaitGroup + wg.Add(len(b.listeners)) + + for _, channels := range b.listeners { + go func() { + defer wg.Done() + + // send message and wait for it to be read + channels.listenCh <- msg + // wait for response + <-channels.responseCh + }() + } + wg.Wait() + + b.doneCh <- struct{}{} + } + } +} + +// MessageType is the type of message to be sent. +type MessageType int + +const ( + // ConfigApplyRequest sends files to update nginx configuration. + ConfigApplyRequest MessageType = iota + // APIRequest sends an NGINX Plus API request to update configuration. + APIRequest +) + +// NginxAgentMessage is sent to all subscribers to send to the nginx agents for either a ConfigApplyRequest +// or an APIActionRequest. +type NginxAgentMessage struct { + // ConfigVersion is the hashed configuration version of the included files. + ConfigVersion string + // NGINXPlusAction is an NGINX Plus API action to be sent. + NGINXPlusAction *pb.NGINXPlusAction + // FileOverviews contain the overviews of all files to be sent. + FileOverviews []*pb.File + // Type defines the type of message to be sent. + Type MessageType +} diff --git a/internal/mode/static/nginx/agent/broadcast/broadcast_test.go b/internal/mode/static/nginx/agent/broadcast/broadcast_test.go new file mode 100644 index 0000000000..d27d723547 --- /dev/null +++ b/internal/mode/static/nginx/agent/broadcast/broadcast_test.go @@ -0,0 +1,108 @@ +package broadcast_test + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" +) + +func TestSubscribe(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + broadcaster := broadcast.NewDeploymentBroadcaster(ctx) + + subscriber := broadcaster.Subscribe() + g.Expect(subscriber.ID).NotTo(BeEmpty()) + + message := broadcast.NginxAgentMessage{ + ConfigVersion: "v1", + Type: broadcast.ConfigApplyRequest, + } + + go func() { + result := broadcaster.Send(message) + g.Expect(result).To(BeTrue()) + }() + + g.Eventually(subscriber.ListenCh).Should(Receive(Equal(message))) +} + +func TestSubscribe_MultipleListeners(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + broadcaster := broadcast.NewDeploymentBroadcaster(ctx) + + subscriber1 := broadcaster.Subscribe() + subscriber2 := broadcaster.Subscribe() + + message := broadcast.NginxAgentMessage{ + ConfigVersion: "v1", + Type: broadcast.ConfigApplyRequest, + } + + go func() { + result := broadcaster.Send(message) + g.Expect(result).To(BeTrue()) + }() + + g.Eventually(subscriber1.ListenCh).Should(Receive(Equal(message))) + g.Eventually(subscriber2.ListenCh).Should(Receive(Equal(message))) + + subscriber1.ResponseCh <- struct{}{} + subscriber2.ResponseCh <- struct{}{} +} + +func TestSubscribe_NoListeners(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + broadcaster := broadcast.NewDeploymentBroadcaster(ctx) + + message := broadcast.NginxAgentMessage{ + ConfigVersion: "v1", + Type: broadcast.ConfigApplyRequest, + } + + result := broadcaster.Send(message) + g.Expect(result).To(BeFalse()) +} + +func TestCancelSubscription(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + broadcaster := broadcast.NewDeploymentBroadcaster(ctx) + + subscriber := broadcaster.Subscribe() + + broadcaster.CancelSubscription(subscriber.ID) + + message := broadcast.NginxAgentMessage{ + ConfigVersion: "v1", + Type: broadcast.ConfigApplyRequest, + } + + go func() { + result := broadcaster.Send(message) + g.Expect(result).To(BeFalse()) + }() + + g.Consistently(subscriber.ListenCh).ShouldNot(Receive()) +} diff --git a/internal/mode/static/nginx/agent/broadcast/broadcastfakes/fake_broadcaster.go b/internal/mode/static/nginx/agent/broadcast/broadcastfakes/fake_broadcaster.go new file mode 100644 index 0000000000..95421dc5d6 --- /dev/null +++ b/internal/mode/static/nginx/agent/broadcast/broadcastfakes/fake_broadcaster.go @@ -0,0 +1,215 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package broadcastfakes + +import ( + "sync" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" +) + +type FakeBroadcaster struct { + CancelSubscriptionStub func(string) + cancelSubscriptionMutex sync.RWMutex + cancelSubscriptionArgsForCall []struct { + arg1 string + } + SendStub func(broadcast.NginxAgentMessage) bool + sendMutex sync.RWMutex + sendArgsForCall []struct { + arg1 broadcast.NginxAgentMessage + } + sendReturns struct { + result1 bool + } + sendReturnsOnCall map[int]struct { + result1 bool + } + SubscribeStub func() broadcast.SubscriberChannels + subscribeMutex sync.RWMutex + subscribeArgsForCall []struct { + } + subscribeReturns struct { + result1 broadcast.SubscriberChannels + } + subscribeReturnsOnCall map[int]struct { + result1 broadcast.SubscriberChannels + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeBroadcaster) CancelSubscription(arg1 string) { + fake.cancelSubscriptionMutex.Lock() + fake.cancelSubscriptionArgsForCall = append(fake.cancelSubscriptionArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.CancelSubscriptionStub + fake.recordInvocation("CancelSubscription", []interface{}{arg1}) + fake.cancelSubscriptionMutex.Unlock() + if stub != nil { + fake.CancelSubscriptionStub(arg1) + } +} + +func (fake *FakeBroadcaster) CancelSubscriptionCallCount() int { + fake.cancelSubscriptionMutex.RLock() + defer fake.cancelSubscriptionMutex.RUnlock() + return len(fake.cancelSubscriptionArgsForCall) +} + +func (fake *FakeBroadcaster) CancelSubscriptionCalls(stub func(string)) { + fake.cancelSubscriptionMutex.Lock() + defer fake.cancelSubscriptionMutex.Unlock() + fake.CancelSubscriptionStub = stub +} + +func (fake *FakeBroadcaster) CancelSubscriptionArgsForCall(i int) string { + fake.cancelSubscriptionMutex.RLock() + defer fake.cancelSubscriptionMutex.RUnlock() + argsForCall := fake.cancelSubscriptionArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeBroadcaster) Send(arg1 broadcast.NginxAgentMessage) bool { + fake.sendMutex.Lock() + ret, specificReturn := fake.sendReturnsOnCall[len(fake.sendArgsForCall)] + fake.sendArgsForCall = append(fake.sendArgsForCall, struct { + arg1 broadcast.NginxAgentMessage + }{arg1}) + stub := fake.SendStub + fakeReturns := fake.sendReturns + fake.recordInvocation("Send", []interface{}{arg1}) + fake.sendMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeBroadcaster) SendCallCount() int { + fake.sendMutex.RLock() + defer fake.sendMutex.RUnlock() + return len(fake.sendArgsForCall) +} + +func (fake *FakeBroadcaster) SendCalls(stub func(broadcast.NginxAgentMessage) bool) { + fake.sendMutex.Lock() + defer fake.sendMutex.Unlock() + fake.SendStub = stub +} + +func (fake *FakeBroadcaster) SendArgsForCall(i int) broadcast.NginxAgentMessage { + fake.sendMutex.RLock() + defer fake.sendMutex.RUnlock() + argsForCall := fake.sendArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeBroadcaster) SendReturns(result1 bool) { + fake.sendMutex.Lock() + defer fake.sendMutex.Unlock() + fake.SendStub = nil + fake.sendReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeBroadcaster) SendReturnsOnCall(i int, result1 bool) { + fake.sendMutex.Lock() + defer fake.sendMutex.Unlock() + fake.SendStub = nil + if fake.sendReturnsOnCall == nil { + fake.sendReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.sendReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + +func (fake *FakeBroadcaster) Subscribe() broadcast.SubscriberChannels { + fake.subscribeMutex.Lock() + ret, specificReturn := fake.subscribeReturnsOnCall[len(fake.subscribeArgsForCall)] + fake.subscribeArgsForCall = append(fake.subscribeArgsForCall, struct { + }{}) + stub := fake.SubscribeStub + fakeReturns := fake.subscribeReturns + fake.recordInvocation("Subscribe", []interface{}{}) + fake.subscribeMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeBroadcaster) SubscribeCallCount() int { + fake.subscribeMutex.RLock() + defer fake.subscribeMutex.RUnlock() + return len(fake.subscribeArgsForCall) +} + +func (fake *FakeBroadcaster) SubscribeCalls(stub func() broadcast.SubscriberChannels) { + fake.subscribeMutex.Lock() + defer fake.subscribeMutex.Unlock() + fake.SubscribeStub = stub +} + +func (fake *FakeBroadcaster) SubscribeReturns(result1 broadcast.SubscriberChannels) { + fake.subscribeMutex.Lock() + defer fake.subscribeMutex.Unlock() + fake.SubscribeStub = nil + fake.subscribeReturns = struct { + result1 broadcast.SubscriberChannels + }{result1} +} + +func (fake *FakeBroadcaster) SubscribeReturnsOnCall(i int, result1 broadcast.SubscriberChannels) { + fake.subscribeMutex.Lock() + defer fake.subscribeMutex.Unlock() + fake.SubscribeStub = nil + if fake.subscribeReturnsOnCall == nil { + fake.subscribeReturnsOnCall = make(map[int]struct { + result1 broadcast.SubscriberChannels + }) + } + fake.subscribeReturnsOnCall[i] = struct { + result1 broadcast.SubscriberChannels + }{result1} +} + +func (fake *FakeBroadcaster) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.cancelSubscriptionMutex.RLock() + defer fake.cancelSubscriptionMutex.RUnlock() + fake.sendMutex.RLock() + defer fake.sendMutex.RUnlock() + fake.subscribeMutex.RLock() + defer fake.subscribeMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeBroadcaster) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ broadcast.Broadcaster = new(FakeBroadcaster) diff --git a/internal/mode/static/nginx/agent/broadcast/doc.go b/internal/mode/static/nginx/agent/broadcast/doc.go new file mode 100644 index 0000000000..3640dcfa5e --- /dev/null +++ b/internal/mode/static/nginx/agent/broadcast/doc.go @@ -0,0 +1,5 @@ +/* +Package broadcast contains the functions for creating a broadcaster to send updates to consumers. +It is used to send nginx configuration for an nginx Deployment to all pod subscribers for that Deployment. +*/ +package broadcast diff --git a/internal/mode/static/nginx/agent/command.go b/internal/mode/static/nginx/agent/command.go index 9eabd8680e..96f2036c64 100644 --- a/internal/mode/static/nginx/agent/command.go +++ b/internal/mode/static/nginx/agent/command.go @@ -4,28 +4,59 @@ import ( "context" "errors" "fmt" + "io" + "strings" "time" "github.com/go-logr/logr" + "github.com/google/uuid" pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + grpcStatus "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" agentgrpc "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" grpcContext "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/context" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/messenger" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/status" ) +const connectionWaitTimeout = 30 * time.Second + // commandService handles the connection and subscription to the data plane agent. type commandService struct { pb.CommandServiceServer - connTracker *agentgrpc.ConnectionsTracker + nginxDeployments *DeploymentStore + statusQueue *status.Queue + connTracker agentgrpc.ConnectionsTracker + k8sReader client.Reader // TODO(sberman): all logs are at Info level right now. Adjust appropriately. - logger logr.Logger + logger logr.Logger + connectionTimeout time.Duration } -func newCommandService(logger logr.Logger) *commandService { +func newCommandService( + logger logr.Logger, + reader client.Reader, + depStore *DeploymentStore, + connTracker agentgrpc.ConnectionsTracker, + statusQueue *status.Queue, +) *commandService { return &commandService{ - logger: logger, - connTracker: agentgrpc.NewConnectionsTracker(), + connectionTimeout: connectionWaitTimeout, + k8sReader: reader, + logger: logger, + connTracker: connTracker, + nginxDeployments: depStore, + statusQueue: statusQueue, } } @@ -34,6 +65,8 @@ func (cs *commandService) Register(server *grpc.Server) { } // CreateConnection registers a data plane agent with the control plane. +// The nginx InstanceID could be empty if the agent hasn't discovered its nginx instance yet. +// Once discovered, the agent will send an UpdateDataPlaneStatus request with the nginx InstanceID set. func (cs *commandService) CreateConnection( ctx context.Context, req *pb.CreateConnectionRequest, @@ -47,10 +80,28 @@ func (cs *commandService) CreateConnection( return nil, agentgrpc.ErrStatusInvalidConnection } - podName := req.GetResource().GetContainerInfo().GetHostname() - + resource := req.GetResource() + podName := resource.GetContainerInfo().GetHostname() cs.logger.Info(fmt.Sprintf("Creating connection for nginx pod: %s", podName)) - cs.connTracker.Track(gi.IPAddress, podName) + + owner, err := cs.getPodOwner(podName) + if err != nil { + response := &pb.CreateConnectionResponse{ + Response: &pb.CommandResponse{ + Status: pb.CommandResponse_COMMAND_STATUS_ERROR, + Message: "error getting pod owner", + Error: err.Error(), + }, + } + return response, grpcStatus.Errorf(codes.Internal, "error getting pod owner %s", err.Error()) + } + + conn := agentgrpc.Connection{ + Parent: owner, + PodName: podName, + InstanceID: getNginxInstanceID(resource.GetInstances()), + } + cs.connTracker.Track(gi.IPAddress, conn) return &pb.CreateConnectionResponse{ Response: &pb.CommandResponse{ @@ -60,6 +111,13 @@ func (cs *commandService) CreateConnection( } // Subscribe is a decoupled communication mechanism between the data plane agent and control plane. +// The series of events are as follows: +// - Wait for the agent to register its nginx instance with the control plane. +// - Grab the most recent deployment configuration for itself, and attempt to apply it. +// - Subscribe to any future updates from the NginxUpdater and start a loop to listen for those updates. +// If any connection or unrecoverable errors occur, return and agent should re-establish a subscription. +// If errors occur with applying the config, log and put those errors into the status queue to be written +// to the Gateway status. func (cs *commandService) Subscribe(in pb.CommandService_SubscribeServer) error { ctx := in.Context() @@ -68,73 +126,349 @@ func (cs *commandService) Subscribe(in pb.CommandService_SubscribeServer) error return agentgrpc.ErrStatusInvalidConnection } - cs.logger.Info(fmt.Sprintf("Received subscribe request from %q", gi.IPAddress)) - - go cs.listenForDataPlaneResponse(ctx, in) - - // wait for the agent to report itself - podName, err := cs.waitForConnection(ctx, gi) + // wait for the agent to report itself and nginx + conn, deployment, err := cs.waitForConnection(ctx, gi) if err != nil { cs.logger.Error(err, "error waiting for connection") return err } - cs.logger.Info(fmt.Sprintf("Handling subscription for %s/%s", podName, gi.IPAddress)) + cs.logger.Info(fmt.Sprintf("Successfully connected to nginx agent %s", conn.PodName)) + + msgr := messenger.New(in) + go msgr.Run(ctx) + + // apply current config before starting event loop + deployment.Lock.RLock() + if err := cs.setInitialConfig(ctx, deployment, conn, msgr); err != nil { + deployment.Lock.RUnlock() + + return err + } + + // subscribe to the deployment broadcaster to get file updates + broadcaster := deployment.GetBroadcaster() + channels := broadcaster.Subscribe() + defer broadcaster.CancelSubscription(channels.ID) + deployment.Lock.RUnlock() + for { select { case <-ctx.Done(): - return ctx.Err() - case <-time.After(1 * time.Minute): - dummyRequest := &pb.ManagementPlaneRequest{ - Request: &pb.ManagementPlaneRequest_HealthRequest{ - HealthRequest: &pb.HealthRequest{}, - }, + return grpcStatus.Error(codes.Canceled, context.Cause(ctx).Error()) + case msg := <-channels.ListenCh: + var req *pb.ManagementPlaneRequest + switch msg.Type { + case broadcast.ConfigApplyRequest: + req = buildRequest(msg.FileOverviews, conn.InstanceID, msg.ConfigVersion) + case broadcast.APIRequest: + req = buildPlusAPIRequest(msg.NGINXPlusAction, conn.InstanceID) + default: + panic(fmt.Sprintf("unknown request type %d", msg.Type)) } - if err := in.Send(dummyRequest); err != nil { // TODO(sberman): will likely need retry logic + + if err := msgr.Send(ctx, req); err != nil { cs.logger.Error(err, "error sending request to agent") + deployment.SetPodErrorStatus(conn.PodName, err) + channels.ResponseCh <- struct{}{} + + return grpcStatus.Error(codes.Internal, err.Error()) + } + case err = <-msgr.Errors(): + cs.logger.Error(err, "connection error") + if errors.Is(err, io.EOF) { + return grpcStatus.Error(codes.Aborted, err.Error()) } + return grpcStatus.Error(codes.Internal, err.Error()) + case msg := <-msgr.Messages(): + res := msg.GetCommandResponse() + if res.GetStatus() != pb.CommandResponse_COMMAND_STATUS_OK { + err := fmt.Errorf("bad response from agent: msg: %s; error: %s", res.GetMessage(), res.GetError()) + deployment.SetPodErrorStatus(conn.PodName, err) + } else { + deployment.SetPodErrorStatus(conn.PodName, nil) + } + channels.ResponseCh <- struct{}{} } } } // TODO(sberman): current issue: when control plane restarts, agent doesn't re-establish a CreateConnection call, // so this fails. -func (cs *commandService) waitForConnection(ctx context.Context, gi grpcContext.GrpcInfo) (string, error) { - var podName string +func (cs *commandService) waitForConnection( + ctx context.Context, + gi grpcContext.GrpcInfo, +) (*agentgrpc.Connection, *Deployment, error) { ticker := time.NewTicker(time.Second) defer ticker.Stop() - timer := time.NewTimer(30 * time.Second) + timer := time.NewTimer(cs.connectionTimeout) defer timer.Stop() + agentConnectErr := errors.New("timed out waiting for agent to register nginx") + deploymentStoreErr := errors.New("timed out waiting for nginx deployment to be added to store") + + var err error for { select { case <-ctx.Done(): - return "", ctx.Err() + return nil, nil, ctx.Err() case <-timer.C: - return "", errors.New("timed out waiting for agent connection") + return nil, nil, err case <-ticker.C: - if podName = cs.connTracker.GetConnection(gi.IPAddress); podName != "" { - return podName, nil + if conn, ok := cs.connTracker.ConnectionIsReady(gi.IPAddress); ok { + // connection has been established, now ensure that the deployment exists in the store + if deployment := cs.nginxDeployments.Get(conn.Parent); deployment != nil { + return &conn, deployment, nil + } + err = deploymentStoreErr + continue } + err = agentConnectErr } } } -func (cs *commandService) listenForDataPlaneResponse(ctx context.Context, in pb.CommandService_SubscribeServer) { +// setInitialConfig gets the initial configuration for this connection and applies it. +// The caller MUST lock the deployment before calling this. +func (cs *commandService) setInitialConfig( + ctx context.Context, + deployment *Deployment, + conn *agentgrpc.Connection, + msgr messenger.Messenger, +) error { + fileOverviews, configVersion := deployment.GetFileOverviews() + if err := msgr.Send(ctx, buildRequest(fileOverviews, conn.InstanceID, configVersion)); err != nil { + cs.logAndSendErrorStatus(deployment, conn, err) + + return grpcStatus.Error(codes.Internal, err.Error()) + } + + applyErr, connErr := cs.waitForInitialConfigApply(ctx, msgr) + if connErr != nil { + cs.logger.Error(connErr, "error setting initial configuration") + + return connErr + } + + errs := []error{applyErr} + for _, action := range deployment.GetNGINXPlusActions() { + // retry the API update request because sometimes nginx isn't quite ready after the config apply reload + timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + if err := wait.PollUntilContextCancel( + timeoutCtx, + 500*time.Millisecond, + true, // poll immediately + func(ctx context.Context) (bool, error) { + if err := msgr.Send(ctx, buildPlusAPIRequest(action, conn.InstanceID)); err != nil { + cs.logAndSendErrorStatus(deployment, conn, err) + + return false, grpcStatus.Error(codes.Internal, err.Error()) + } + + upstreamApplyErr, connErr := cs.waitForInitialConfigApply(ctx, msgr) + if connErr != nil { + cs.logger.Error(connErr, "error setting initial configuration") + + return false, connErr + } + + if upstreamApplyErr != nil { + return false, nil //nolint:nilerr // this error is collected at the end + } + return true, nil + }, + ); err != nil { + if strings.Contains(err.Error(), "bad response from agent") { + errs = append(errs, err) + } else { + cancel() + return err + } + } + cancel() + } + // send the status (error or nil) to the status queue + cs.logAndSendErrorStatus(deployment, conn, errors.Join(errs...)) + + return nil +} + +// waitForInitialConfigApply waits for the nginx agent to respond after a Subscriber attempts +// to apply its initial config. +// Two errors are returned +// - applyErr is an error applying the configuration +// - connectionErr is an error with the connection or sending the configuration +// The caller treats a connectionErr as unrecoverable, while the applyErr is used +// to set the status on the Gateway resources. +func (cs *commandService) waitForInitialConfigApply( + ctx context.Context, + msgr messenger.Messenger, +) (applyErr error, connectionErr error) { for { select { case <-ctx.Done(): - return - default: - dataPlaneResponse, err := in.Recv() - cs.logger.Info(fmt.Sprintf("Received data plane response: %v", dataPlaneResponse)) - if err != nil { - cs.logger.Error(err, "failed to receive data plane response") - return + return nil, grpcStatus.Error(codes.Canceled, context.Cause(ctx).Error()) + case err := <-msgr.Errors(): + if errors.Is(err, io.EOF) { + return nil, grpcStatus.Error(codes.Aborted, err.Error()) } + return nil, grpcStatus.Error(codes.Internal, err.Error()) + case msg := <-msgr.Messages(): + res := msg.GetCommandResponse() + if res.GetStatus() != pb.CommandResponse_COMMAND_STATUS_OK { + applyErr := fmt.Errorf("bad response from agent: msg: %s; error: %s", res.GetMessage(), res.GetError()) + return applyErr, nil + } + + return applyErr, connectionErr + } + } +} + +// logAndSendErrorStatus logs an error, sets it on the Deployment object for that Pod, and then sends +// the full Deployment error status to the status queue. This ensures that any other Pod errors that already +// exist on the Deployment are not overwritten. +// If the error is nil, then we just enqueue the nil value and don't log it, which indicates success. +func (cs *commandService) logAndSendErrorStatus(deployment *Deployment, conn *agentgrpc.Connection, err error) { + if err != nil { + cs.logger.Error(err, "error sending request to agent") + } else { + cs.logger.Info(fmt.Sprintf("Successfully configured nginx for new subscription %q", conn.PodName)) + } + deployment.SetPodErrorStatus(conn.PodName, err) + + queueObj := &status.QueueObject{ + Deployment: conn.Parent, + Error: deployment.GetConfigurationStatus(), + } + cs.statusQueue.Enqueue(queueObj) +} + +func buildRequest(fileOverviews []*pb.File, instanceID, version string) *pb.ManagementPlaneRequest { + return &pb.ManagementPlaneRequest{ + MessageMeta: &pb.MessageMeta{ + MessageId: uuid.NewString(), + CorrelationId: uuid.NewString(), + Timestamp: timestamppb.Now(), + }, + Request: &pb.ManagementPlaneRequest_ConfigApplyRequest{ + ConfigApplyRequest: &pb.ConfigApplyRequest{ + Overview: &pb.FileOverview{ + Files: fileOverviews, + ConfigVersion: &pb.ConfigVersion{ + InstanceId: instanceID, + Version: version, + }, + }, + }, + }, + } +} + +func buildPlusAPIRequest(action *pb.NGINXPlusAction, instanceID string) *pb.ManagementPlaneRequest { + return &pb.ManagementPlaneRequest{ + MessageMeta: &pb.MessageMeta{ + MessageId: uuid.NewString(), + CorrelationId: uuid.NewString(), + Timestamp: timestamppb.Now(), + }, + Request: &pb.ManagementPlaneRequest_ActionRequest{ + ActionRequest: &pb.APIActionRequest{ + InstanceId: instanceID, + Action: &pb.APIActionRequest_NginxPlusAction{ + NginxPlusAction: action, + }, + }, + }, + } +} + +func (cs *commandService) getPodOwner(podName string) (types.NamespacedName, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + var pods v1.PodList + listOpts := &client.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{"metadata.name": podName}), + } + if err := cs.k8sReader.List(ctx, &pods, listOpts); err != nil { + return types.NamespacedName{}, fmt.Errorf("error listing pods: %w", err) + } + + if len(pods.Items) == 0 { + return types.NamespacedName{}, fmt.Errorf("no pods found with name %q", podName) + } + + if len(pods.Items) > 1 { + return types.NamespacedName{}, fmt.Errorf("should only be one pod with name %q", podName) + } + pod := pods.Items[0] + + podOwnerRefs := pod.GetOwnerReferences() + if len(podOwnerRefs) != 1 { + return types.NamespacedName{}, fmt.Errorf("expected one owner reference of the nginx Pod, got %d", len(podOwnerRefs)) + } + + if podOwnerRefs[0].Kind != "ReplicaSet" { + err := fmt.Errorf("expected pod owner reference to be ReplicaSet, got %s", podOwnerRefs[0].Kind) + return types.NamespacedName{}, err + } + + var replicaSet appsv1.ReplicaSet + if err := cs.k8sReader.Get( + ctx, + types.NamespacedName{Namespace: pod.Namespace, Name: podOwnerRefs[0].Name}, + &replicaSet, + ); err != nil { + return types.NamespacedName{}, fmt.Errorf("failed to get nginx Pod's ReplicaSet: %w", err) + } + + replicaOwnerRefs := replicaSet.GetOwnerReferences() + if len(replicaOwnerRefs) != 1 { + err := fmt.Errorf("expected one owner reference of the nginx ReplicaSet, got %d", len(replicaOwnerRefs)) + return types.NamespacedName{}, err + } + + return types.NamespacedName{Namespace: pod.Namespace, Name: replicaOwnerRefs[0].Name}, nil +} + +// UpdateDataPlaneStatus is called by agent on startup and upon any change in agent metadata, +// instance metadata, or configurations. InstanceID may not be set on an initial CreateConnection, +// and will instead be set on a call to UpdateDataPlaneStatus once the agent discovers its nginx instance. +func (cs *commandService) UpdateDataPlaneStatus( + ctx context.Context, + req *pb.UpdateDataPlaneStatusRequest, +) (*pb.UpdateDataPlaneStatusResponse, error) { + if req == nil { + return nil, errors.New("empty UpdateDataPlaneStatus request") + } + + gi, ok := grpcContext.GrpcInfoFromContext(ctx) + if !ok { + return nil, agentgrpc.ErrStatusInvalidConnection + } + + instanceID := getNginxInstanceID(req.GetResource().GetInstances()) + if instanceID == "" { + return nil, grpcStatus.Errorf(codes.InvalidArgument, "request does not contain nginx instanceID") + } + + cs.connTracker.SetInstanceID(gi.IPAddress, instanceID) + + return &pb.UpdateDataPlaneStatusResponse{}, nil +} + +func getNginxInstanceID(instances []*pb.Instance) string { + for _, instance := range instances { + instanceType := instance.GetInstanceMeta().GetInstanceType() + if instanceType == pb.InstanceMeta_INSTANCE_TYPE_NGINX || + instanceType == pb.InstanceMeta_INSTANCE_TYPE_NGINX_PLUS { + return instance.GetInstanceMeta().GetInstanceId() } } + + return "" } // UpdateDataPlaneHealth includes full health information about the data plane as reported by the agent. @@ -145,13 +479,3 @@ func (cs *commandService) UpdateDataPlaneHealth( ) (*pb.UpdateDataPlaneHealthResponse, error) { return &pb.UpdateDataPlaneHealthResponse{}, nil } - -// UpdateDataPlaneStatus is called by agent on startup and upon any change in agent metadata, -// instance metadata, or configurations. Since directly changing nginx configuration on the instance -// is not supported, this is a no-op for NGF. -func (cs *commandService) UpdateDataPlaneStatus( - _ context.Context, - _ *pb.UpdateDataPlaneStatusRequest, -) (*pb.UpdateDataPlaneStatusResponse, error) { - return &pb.UpdateDataPlaneStatusResponse{}, nil -} diff --git a/internal/mode/static/nginx/agent/command_test.go b/internal/mode/static/nginx/agent/command_test.go new file mode 100644 index 0000000000..897650a26d --- /dev/null +++ b/internal/mode/static/nginx/agent/command_test.go @@ -0,0 +1,905 @@ +package agent + +import ( + "context" + "errors" + "io" + "testing" + "time" + + "github.com/go-logr/logr" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + . "github.com/onsi/gomega" + "google.golang.org/grpc" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast/broadcastfakes" + agentgrpc "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" + grpcContext "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/context" + agentgrpcfakes "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/grpcfakes" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/messenger/messengerfakes" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/status" +) + +type mockSubscribeServer struct { + grpc.ServerStream + ctx context.Context + recvChan chan *pb.DataPlaneResponse + sendChan chan *pb.ManagementPlaneRequest +} + +func newMockSubscribeServer(ctx context.Context) *mockSubscribeServer { + return &mockSubscribeServer{ + ctx: ctx, + recvChan: make(chan *pb.DataPlaneResponse, 1), + sendChan: make(chan *pb.ManagementPlaneRequest, 1), + } +} + +func (m *mockSubscribeServer) Send(msg *pb.ManagementPlaneRequest) error { + m.sendChan <- msg + return nil +} + +func (m *mockSubscribeServer) Recv() (*pb.DataPlaneResponse, error) { + req, ok := <-m.recvChan + if !ok { + return nil, io.EOF + } + return req, nil +} + +func (m *mockSubscribeServer) Context() context.Context { + return m.ctx +} + +func createFakeK8sClient(initObjs ...runtime.Object) (client.Client, error) { + fakeClient := fake.NewFakeClient(initObjs...) + if err := fake.AddIndex(fakeClient, &v1.Pod{}, "metadata.name", func(obj client.Object) []string { + return []string{obj.GetName()} + }); err != nil { + return nil, err + } + + return fakeClient, nil +} + +func createGrpcContext() context.Context { + return grpcContext.NewGrpcContext(context.Background(), grpcContext.GrpcInfo{ + IPAddress: "127.0.0.1", + }) +} + +func createGrpcContextWithCancel() (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + + return grpcContext.NewGrpcContext(ctx, grpcContext.GrpcInfo{ + IPAddress: "127.0.0.1", + }), cancel +} + +func TestCreateConnection(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + request *pb.CreateConnectionRequest + response *pb.CreateConnectionResponse + ctx context.Context + errString string + }{ + { + name: "successfully tracks a connection", + ctx: createGrpcContext(), + request: &pb.CreateConnectionRequest{ + Resource: &pb.Resource{ + Info: &pb.Resource_ContainerInfo{ + ContainerInfo: &pb.ContainerInfo{ + Hostname: "nginx-pod", + }, + }, + Instances: []*pb.Instance{ + { + InstanceMeta: &pb.InstanceMeta{ + InstanceId: "nginx-id", + InstanceType: pb.InstanceMeta_INSTANCE_TYPE_NGINX, + }, + }, + }, + }, + }, + response: &pb.CreateConnectionResponse{ + Response: &pb.CommandResponse{ + Status: pb.CommandResponse_COMMAND_STATUS_OK, + }, + }, + }, + { + name: "request is nil", + request: nil, + response: nil, + errString: "empty connection request", + }, + { + name: "context is missing data", + ctx: context.Background(), + request: &pb.CreateConnectionRequest{}, + response: nil, + errString: agentgrpc.ErrStatusInvalidConnection.Error(), + }, + { + name: "error getting pod owner", + ctx: createGrpcContext(), + request: &pb.CreateConnectionRequest{ + Resource: &pb.Resource{ + Info: &pb.Resource_ContainerInfo{ + ContainerInfo: &pb.ContainerInfo{ + Hostname: "nginx-pod", + }, + }, + }, + }, + response: &pb.CreateConnectionResponse{ + Response: &pb.CommandResponse{ + Status: pb.CommandResponse_COMMAND_STATUS_ERROR, + Message: "error getting pod owner", + Error: "no pods found with name \"nginx-pod\"", + }, + }, + errString: "error getting pod owner", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + connTracker := agentgrpcfakes.FakeConnectionsTracker{} + + var objs []runtime.Object + if test.errString == "" { + pod := &v1.PodList{ + Items: []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-pod", + Namespace: "test", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "nginx-replicaset", + }, + }, + }, + }, + }, + } + + replicaSet := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-replicaset", + Namespace: "test", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Deployment", + Name: "nginx-deployment", + }, + }, + }, + } + + objs = []runtime.Object{pod, replicaSet} + } + + fakeClient, err := createFakeK8sClient(objs...) + g.Expect(err).ToNot(HaveOccurred()) + + cs := newCommandService( + logr.Discard(), + fakeClient, + NewDeploymentStore(&connTracker), + &connTracker, + status.NewQueue(), + ) + + resp, err := cs.CreateConnection(test.ctx, test.request) + g.Expect(resp).To(Equal(test.response)) + + if test.errString != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(test.errString)) + + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(connTracker.TrackCallCount()).To(Equal(1)) + + expConn := agentgrpc.Connection{ + Parent: types.NamespacedName{Namespace: "test", Name: "nginx-deployment"}, + PodName: "nginx-pod", + InstanceID: "nginx-id", + } + + key, conn := connTracker.TrackArgsForCall(0) + g.Expect(key).To(Equal("127.0.0.1")) + g.Expect(conn).To(Equal(expConn)) + }) + } +} + +func ensureFileWasSent( + g *WithT, + server *mockSubscribeServer, + expFile *pb.File, +) { + var req *pb.ManagementPlaneRequest + g.Eventually(func() *pb.ManagementPlaneRequest { + req = <-server.sendChan + return req + }).ShouldNot(BeNil()) + + g.Expect(req.GetConfigApplyRequest()).ToNot(BeNil()) + overview := req.GetConfigApplyRequest().GetOverview() + g.Expect(overview).ToNot(BeNil()) + g.Expect(overview.Files).To(ContainElement(expFile)) +} + +func ensureAPIRequestWasSent( + g *WithT, + server *mockSubscribeServer, + expAction *pb.NGINXPlusAction, +) { + var req *pb.ManagementPlaneRequest + g.Eventually(func() *pb.ManagementPlaneRequest { + req = <-server.sendChan + return req + }).ShouldNot(BeNil()) + + g.Expect(req.GetActionRequest()).ToNot(BeNil()) + action := req.GetActionRequest().GetNginxPlusAction() + g.Expect(action).To(Equal(expAction)) +} + +func verifyResponse( + g *WithT, + server *mockSubscribeServer, + responseCh chan struct{}, +) { + server.recvChan <- &pb.DataPlaneResponse{ + CommandResponse: &pb.CommandResponse{ + Status: pb.CommandResponse_COMMAND_STATUS_OK, + }, + } + + g.Eventually(func() struct{} { + return <-responseCh + }).Should(Equal(struct{}{})) +} + +func TestSubscribe(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + connTracker := agentgrpcfakes.FakeConnectionsTracker{} + conn := agentgrpc.Connection{ + Parent: types.NamespacedName{Namespace: "test", Name: "nginx-deployment"}, + PodName: "nginx-pod", + InstanceID: "nginx-id", + } + connTracker.ConnectionIsReadyReturns(conn, true) + + cs := newCommandService( + logr.Discard(), + fake.NewFakeClient(), + NewDeploymentStore(&connTracker), + &connTracker, + status.NewQueue(), + ) + + broadcaster := &broadcastfakes.FakeBroadcaster{} + responseCh := make(chan struct{}) + listenCh := make(chan broadcast.NginxAgentMessage, 2) + subChannels := broadcast.SubscriberChannels{ + ListenCh: listenCh, + ResponseCh: responseCh, + } + broadcaster.SubscribeReturns(subChannels) + + // set the initial files and actions to be applied by the Subscription + deployment := cs.nginxDeployments.GetOrStore(conn.Parent, broadcaster) + files := []File{ + { + Meta: &pb.FileMeta{ + Name: "nginx.conf", + Hash: "12345", + }, + Contents: []byte("file contents"), + }, + } + deployment.SetFiles(files) + + initialAction := &pb.NGINXPlusAction{ + Action: &pb.NGINXPlusAction_UpdateHttpUpstreamServers{}, + } + deployment.SetNGINXPlusActions([]*pb.NGINXPlusAction{initialAction}) + + ctx, cancel := createGrpcContextWithCancel() + defer cancel() + + mockServer := newMockSubscribeServer(ctx) + + // put the requests on the listenCh for the Subscription loop to pick up + loopFile := &pb.File{ + FileMeta: &pb.FileMeta{ + Name: "some-other.conf", + Hash: "56789", + }, + } + listenCh <- broadcast.NginxAgentMessage{ + Type: broadcast.ConfigApplyRequest, + FileOverviews: []*pb.File{loopFile}, + } + + loopAction := &pb.NGINXPlusAction{ + Action: &pb.NGINXPlusAction_UpdateStreamServers{}, + } + listenCh <- broadcast.NginxAgentMessage{ + Type: broadcast.APIRequest, + NGINXPlusAction: loopAction, + } + + // start the Subscriber + errCh := make(chan error) + go func() { + errCh <- cs.Subscribe(mockServer) + }() + + // ensure that the initial config file was sent when the Subscription connected + expFile := &pb.File{ + FileMeta: &pb.FileMeta{ + Name: "nginx.conf", + Hash: "12345", + }, + } + ensureFileWasSent(g, mockServer, expFile) + mockServer.recvChan <- &pb.DataPlaneResponse{ + CommandResponse: &pb.CommandResponse{ + Status: pb.CommandResponse_COMMAND_STATUS_OK, + }, + } + + // ensure that the initial API request was sent when the Subscription connected + ensureAPIRequestWasSent(g, mockServer, initialAction) + mockServer.recvChan <- &pb.DataPlaneResponse{ + CommandResponse: &pb.CommandResponse{ + Status: pb.CommandResponse_COMMAND_STATUS_OK, + }, + } + + g.Eventually(func() string { + obj := cs.statusQueue.Dequeue(ctx) + return obj.Deployment.Name + }).Should(Equal("nginx-deployment")) + + // ensure the second file was sent in the loop + ensureFileWasSent(g, mockServer, loopFile) + verifyResponse(g, mockServer, responseCh) + + // ensure the second action was sent in the loop + ensureAPIRequestWasSent(g, mockServer, loopAction) + verifyResponse(g, mockServer, responseCh) + + cancel() + + g.Eventually(func() error { + return <-errCh + }).Should(MatchError(ContainSubstring("context canceled"))) +} + +func TestSubscribe_Errors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + setup func( + cs *commandService, + ct *agentgrpcfakes.FakeConnectionsTracker, + ) + ctx context.Context + errString string + }{ + { + name: "context is missing data", + ctx: context.Background(), + errString: agentgrpc.ErrStatusInvalidConnection.Error(), + }, + { + name: "error waiting for connection; not connected", + setup: func( + cs *commandService, + _ *agentgrpcfakes.FakeConnectionsTracker, + ) { + cs.connectionTimeout = 1100 * time.Millisecond + }, + errString: "timed out waiting for agent to register nginx", + }, + { + name: "error waiting for connection; deployment not tracked", + setup: func( + cs *commandService, + ct *agentgrpcfakes.FakeConnectionsTracker, + ) { + ct.ConnectionIsReadyReturns(agentgrpc.Connection{}, true) + cs.connectionTimeout = 1100 * time.Millisecond + }, + errString: "timed out waiting for nginx deployment to be added to store", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + connTracker := agentgrpcfakes.FakeConnectionsTracker{} + + cs := newCommandService( + logr.Discard(), + fake.NewFakeClient(), + NewDeploymentStore(&connTracker), + &connTracker, + status.NewQueue(), + ) + + if test.setup != nil { + test.setup(cs, &connTracker) + } + + var ctx context.Context + var cancel context.CancelFunc + + if test.ctx != nil { + ctx = test.ctx + } else { + ctx, cancel = createGrpcContextWithCancel() + defer cancel() + } + + mockServer := newMockSubscribeServer(ctx) + + // start the Subscriber + errCh := make(chan error) + go func() { + errCh <- cs.Subscribe(mockServer) + }() + + g.Eventually(func() error { + err := <-errCh + g.Expect(err).To(HaveOccurred()) + return err + }).Should(MatchError(ContainSubstring(test.errString))) + }) + } +} + +func TestSetInitialConfig_Errors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + setup func(msgr *messengerfakes.FakeMessenger, deployment *Deployment) + errString string + }{ + { + name: "error sending initial config", + setup: func(msgr *messengerfakes.FakeMessenger, _ *Deployment) { + msgr.SendReturns(errors.New("send error")) + }, + errString: "send error", + }, + { + name: "error waiting for initial config apply", + setup: func(msgr *messengerfakes.FakeMessenger, _ *Deployment) { + errCh := make(chan error, 1) + msgr.ErrorsReturns(errCh) + errCh <- errors.New("apply error") + }, + errString: "apply error", + }, + { + name: "error sending initial API request", + setup: func(msgr *messengerfakes.FakeMessenger, deployment *Deployment) { + deployment.SetNGINXPlusActions([]*pb.NGINXPlusAction{ + { + Action: &pb.NGINXPlusAction_UpdateHttpUpstreamServers{}, + }, + }) + msgCh := make(chan *pb.DataPlaneResponse, 1) + msgr.MessagesReturns(msgCh) + msgCh <- &pb.DataPlaneResponse{ + CommandResponse: &pb.CommandResponse{ + Status: pb.CommandResponse_COMMAND_STATUS_OK, + }, + } + + msgr.SendReturnsOnCall(1, errors.New("api send error")) + }, + errString: "api send error", + }, + { + name: "error waiting for initial API request apply", + setup: func(msgr *messengerfakes.FakeMessenger, deployment *Deployment) { + deployment.SetNGINXPlusActions([]*pb.NGINXPlusAction{ + { + Action: &pb.NGINXPlusAction_UpdateHttpUpstreamServers{}, + }, + }) + msgCh := make(chan *pb.DataPlaneResponse, 1) + msgr.MessagesReturns(msgCh) + msgCh <- &pb.DataPlaneResponse{ + CommandResponse: &pb.CommandResponse{ + Status: pb.CommandResponse_COMMAND_STATUS_OK, + }, + } + + errCh := make(chan error, 1) + msgr.ErrorsReturns(errCh) + errCh <- errors.New("api apply error") + }, + errString: "api apply error", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + connTracker := agentgrpcfakes.FakeConnectionsTracker{} + msgr := &messengerfakes.FakeMessenger{} + + cs := newCommandService( + logr.Discard(), + fake.NewFakeClient(), + NewDeploymentStore(&connTracker), + &connTracker, + status.NewQueue(), + ) + + conn := &agentgrpc.Connection{ + Parent: types.NamespacedName{Namespace: "test", Name: "nginx-deployment"}, + PodName: "nginx-pod", + InstanceID: "nginx-id", + } + + broadcaster := &broadcastfakes.FakeBroadcaster{} + deployment := cs.nginxDeployments.GetOrStore(conn.Parent, broadcaster) + + if test.setup != nil { + test.setup(msgr, deployment) + } + + err := cs.setInitialConfig(context.Background(), deployment, conn, msgr) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(test.errString)) + }) + } +} + +func TestGetPodOwner(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + podName string + podList *v1.PodList + replicaSet *appsv1.ReplicaSet + errString string + expected types.NamespacedName + }{ + { + name: "successfully gets pod owner", + podName: "nginx-pod", + podList: &v1.PodList{ + Items: []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-pod", + Namespace: "test", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "nginx-replicaset", + }, + }, + }, + }, + }, + }, + replicaSet: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-replicaset", + Namespace: "test", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Deployment", + Name: "nginx-deployment", + }, + }, + }, + }, + expected: types.NamespacedName{ + Namespace: "test", + Name: "nginx-deployment", + }, + }, + { + name: "error listing pods", + podName: "nginx-pod", + podList: &v1.PodList{}, + replicaSet: &appsv1.ReplicaSet{}, + errString: "no pods found", + }, + { + name: "multiple pods with same name", + podName: "nginx-pod", + podList: &v1.PodList{ + Items: []v1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "nginx-pod"}}, + {ObjectMeta: metav1.ObjectMeta{Namespace: "test2", Name: "nginx-pod"}}, + }, + }, + replicaSet: &appsv1.ReplicaSet{}, + errString: "should only be one pod with name", + }, + { + name: "pod owner reference is not ReplicaSet", + podName: "nginx-pod", + podList: &v1.PodList{ + Items: []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-pod", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Owner", + Name: "nginx-owner", + }, + }, + }, + }, + }, + }, + replicaSet: &appsv1.ReplicaSet{}, + errString: "expected pod owner reference to be ReplicaSet", + }, + { + name: "pod has multiple owners", + podName: "nginx-pod", + podList: &v1.PodList{ + Items: []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-pod", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "nginx-replicaset", + }, + { + Kind: "ReplicaSet", + Name: "nginx-replicaset2", + }, + }, + }, + }, + }, + }, + replicaSet: &appsv1.ReplicaSet{}, + errString: "expected one owner reference of the nginx Pod", + }, + { + name: "replicaSet has multiple owners", + podName: "nginx-pod", + podList: &v1.PodList{ + Items: []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-pod", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "nginx-replicaset", + }, + }, + }, + }, + }, + }, + replicaSet: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-replicaset", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Deployment", + Name: "nginx-deployment", + }, + { + Kind: "Deployment", + Name: "nginx-deployment2", + }, + }, + }, + }, + errString: "expected one owner reference of the nginx ReplicaSet", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + fakeClient, err := createFakeK8sClient(test.podList, test.replicaSet) + g.Expect(err).ToNot(HaveOccurred()) + + cs := newCommandService( + logr.Discard(), + fakeClient, + NewDeploymentStore(nil), + nil, + status.NewQueue(), + ) + + owner, err := cs.getPodOwner(test.podName) + + if test.errString != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(test.errString)) + g.Expect(owner).To(Equal(types.NamespacedName{})) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(owner).To(Equal(test.expected)) + }) + } +} + +func TestUpdateDataPlaneStatus(t *testing.T) { + t.Parallel() + + tests := []struct { + request *pb.UpdateDataPlaneStatusRequest + response *pb.UpdateDataPlaneStatusResponse + ctx context.Context + errString string + expID string + name string + }{ + { + name: "successfully sets the status", + ctx: createGrpcContext(), + request: &pb.UpdateDataPlaneStatusRequest{ + Resource: &pb.Resource{ + Instances: []*pb.Instance{ + { + InstanceMeta: &pb.InstanceMeta{ + InstanceId: "nginx-id", + InstanceType: pb.InstanceMeta_INSTANCE_TYPE_NGINX, + }, + }, + }, + }, + }, + expID: "nginx-id", + response: &pb.UpdateDataPlaneStatusResponse{}, + }, + { + name: "successfully sets the status using plus", + ctx: createGrpcContext(), + request: &pb.UpdateDataPlaneStatusRequest{ + Resource: &pb.Resource{ + Instances: []*pb.Instance{ + { + InstanceMeta: &pb.InstanceMeta{ + InstanceId: "nginx-plus-id", + InstanceType: pb.InstanceMeta_INSTANCE_TYPE_NGINX_PLUS, + }, + }, + }, + }, + }, + expID: "nginx-plus-id", + response: &pb.UpdateDataPlaneStatusResponse{}, + }, + { + name: "request is nil", + request: nil, + response: nil, + errString: "empty UpdateDataPlaneStatus request", + }, + { + name: "context is missing data", + ctx: context.Background(), + request: &pb.UpdateDataPlaneStatusRequest{}, + response: nil, + errString: agentgrpc.ErrStatusInvalidConnection.Error(), + }, + { + name: "request does not contain ID", + ctx: createGrpcContext(), + request: &pb.UpdateDataPlaneStatusRequest{}, + response: nil, + errString: "request does not contain nginx instanceID", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + connTracker := agentgrpcfakes.FakeConnectionsTracker{} + + cs := newCommandService( + logr.Discard(), + fake.NewFakeClient(), + NewDeploymentStore(&connTracker), + &connTracker, + status.NewQueue(), + ) + + resp, err := cs.UpdateDataPlaneStatus(test.ctx, test.request) + + if test.errString != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(test.errString)) + g.Expect(resp).To(BeNil()) + + g.Expect(connTracker.SetInstanceIDCallCount()).To(Equal(0)) + + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(resp).To(Equal(test.response)) + + g.Expect(connTracker.SetInstanceIDCallCount()).To(Equal(1)) + + key, id := connTracker.SetInstanceIDArgsForCall(0) + g.Expect(key).To(Equal("127.0.0.1")) + g.Expect(id).To(Equal(test.expID)) + }) + } +} + +func TestUpdateDataPlaneHealth(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + connTracker := agentgrpcfakes.FakeConnectionsTracker{} + + cs := newCommandService( + logr.Discard(), + fake.NewFakeClient(), + NewDeploymentStore(&connTracker), + &connTracker, + status.NewQueue(), + ) + + resp, err := cs.UpdateDataPlaneHealth(context.Background(), &pb.UpdateDataPlaneHealthRequest{}) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(resp).To(Equal(&pb.UpdateDataPlaneHealthResponse{})) +} diff --git a/internal/mode/static/nginx/agent/deployment.go b/internal/mode/static/nginx/agent/deployment.go new file mode 100644 index 0000000000..eb71abdfed --- /dev/null +++ b/internal/mode/static/nginx/agent/deployment.go @@ -0,0 +1,245 @@ +package agent + +import ( + "errors" + "fmt" + "sync" + + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + filesHelper "github.com/nginx/agent/v3/pkg/files" + "k8s.io/apimachinery/pkg/types" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" + agentgrpc "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" +) + +// ignoreFiles is a list of static or base files that live in the +// nginx container that should not be touched by the agent. Any files +// that we add directly into the container should be added here. +var ignoreFiles = []string{ + "/etc/nginx/nginx.conf", + "/etc/nginx/mime.types", + "/etc/nginx/grpc-error-locations.conf", + "/etc/nginx/grpc-error-pages.conf", + "/usr/share/nginx/html/50x.html", + "/usr/share/nginx/html/dashboard.html", + "/usr/share/nginx/html/index.html", + "/usr/share/nginx/html/nginx-modules-reference.pdf", +} + +const fileMode = "0644" + +// Deployment represents an nginx Deployment. It contains its own nginx configuration files, +// a broadcaster for sending those files to all of its pods that are subscribed, and errors +// that may have occurred while applying configuration. +type Deployment struct { + // podStatuses is a map of all Pods for this Deployment and the most recent error + // (or nil if successful) that occurred on a config call to the nginx agent. + podStatuses map[string]error + + broadcaster broadcast.Broadcaster + + configVersion string + // error that is set if a ConfigApply call failed for a Pod. This is needed + // because if subsequent upstream API calls are made within the same update event, + // and are successful, the previous error would be lost in the podStatuses map. + // It's used to preserve the error for when we write status after fully updating nginx. + latestConfigError error + // error that is set when at least one upstream API call failed for a Pod. + // This is needed because subsequent API calls within the same update event could succeed, + // and therefore the previous error would be lost in the podStatuses map. It's used to preserve + // the error for when we write status after fully updating nginx. + latestUpstreamError error + + nginxPlusActions []*pb.NGINXPlusAction + fileOverviews []*pb.File + files []File + + Lock sync.RWMutex +} + +// newDeployment returns a new deployment object. +func newDeployment(broadcaster broadcast.Broadcaster) *Deployment { + return &Deployment{ + broadcaster: broadcaster, + podStatuses: make(map[string]error), + } +} + +// GetBroadcaster returns the deployment's broadcaster. +func (d *Deployment) GetBroadcaster() broadcast.Broadcaster { + return d.broadcaster +} + +// GetFileOverviews returns the current list of fileOverviews and configVersion for the deployment. +func (d *Deployment) GetFileOverviews() ([]*pb.File, string) { + d.Lock.RLock() + defer d.Lock.RUnlock() + + return d.fileOverviews, d.configVersion +} + +// GetNGINXPlusActions returns the current NGINX Plus API Actions for the deployment. +func (d *Deployment) GetNGINXPlusActions() []*pb.NGINXPlusAction { + d.Lock.RLock() + defer d.Lock.RUnlock() + + return d.nginxPlusActions +} + +// GetLatestConfigError gets the latest config apply error for the deployment. +func (d *Deployment) GetLatestConfigError() error { + d.Lock.RLock() + defer d.Lock.RUnlock() + + return d.latestConfigError +} + +// GetLatestUpstreamError gets the latest upstream update error for the deployment. +func (d *Deployment) GetLatestUpstreamError() error { + d.Lock.RLock() + defer d.Lock.RUnlock() + + return d.latestUpstreamError +} + +/* +The following functions for the Deployment object are UNLOCKED, meaning that they are unsafe. +Callers of these functions MUST ensure the lock is set before calling. + +These functions are called as part of the ConfigApply or APIRequest processes. These entire processes +are locked by the caller, hence why the functions themselves do not set the locks. +*/ + +// GetFile gets the requested file for the deployment and returns its contents. +// Caller MUST lock the deployment before calling this function. +func (d *Deployment) GetFile(name, hash string) []byte { + for _, file := range d.files { + if name == file.Meta.GetName() && hash == file.Meta.GetHash() { + return file.Contents + } + } + + return nil +} + +// SetFiles updates the nginx files and fileOverviews for the deployment and returns the message to send. +// Caller MUST lock the deployment before calling this function. +func (d *Deployment) SetFiles(files []File) broadcast.NginxAgentMessage { + d.files = files + + fileOverviews := make([]*pb.File, 0, len(files)) + for _, file := range files { + fileOverviews = append(fileOverviews, &pb.File{FileMeta: file.Meta}) + } + + // add ignored files to the overview as 'unmanaged' so agent doesn't touch them + for _, f := range ignoreFiles { + meta := &pb.FileMeta{ + Name: f, + Permissions: fileMode, + } + + fileOverviews = append(fileOverviews, &pb.File{ + FileMeta: meta, + Unmanaged: true, + }) + } + + d.configVersion = filesHelper.GenerateConfigVersion(fileOverviews) + d.fileOverviews = fileOverviews + + return broadcast.NginxAgentMessage{ + Type: broadcast.ConfigApplyRequest, + FileOverviews: fileOverviews, + ConfigVersion: d.configVersion, + } +} + +// SetNGINXPlusActions updates the deployment's latest NGINX Plus Actions to perform if using NGINX Plus. +// Used by a Subscriber when it first connects. +// Caller MUST lock the deployment before calling this function. +func (d *Deployment) SetNGINXPlusActions(actions []*pb.NGINXPlusAction) { + d.nginxPlusActions = actions +} + +// SetPodErrorStatus sets the error status of a Pod in this Deployment if applying the config failed. +func (d *Deployment) SetPodErrorStatus(pod string, err error) { + d.podStatuses[pod] = err +} + +// SetLatestConfigError sets the latest config apply error for the deployment. +// Caller MUST lock the deployment before calling this function. +func (d *Deployment) SetLatestConfigError(err error) { + d.latestConfigError = err +} + +// SetLatestUpstreamError sets the latest upstream update error for the deployment. +// Caller MUST lock the deployment before calling this function. +func (d *Deployment) SetLatestUpstreamError(err error) { + d.latestUpstreamError = err +} + +// GetConfigurationStatus returns the current config status for this Deployment. It combines +// the most recent errors (if they exist) for all Pods in the Deployment into a single error. +// Caller MUST lock the deployment before calling this function. +func (d *Deployment) GetConfigurationStatus() error { + errs := make([]error, 0, len(d.podStatuses)) + for _, err := range d.podStatuses { + errs = append(errs, err) + } + + if len(errs) == 1 { + return errs[0] + } + + return errors.Join(errs...) +} + +// DeploymentStore holds a map of all Deployments. +type DeploymentStore struct { + connTracker agentgrpc.ConnectionsTracker + deployments sync.Map +} + +// NewDeploymentStore returns a new instance of a DeploymentStore. +func NewDeploymentStore(connTracker agentgrpc.ConnectionsTracker) *DeploymentStore { + return &DeploymentStore{ + connTracker: connTracker, + } +} + +// Get returns the desired deployment from the store. +func (d *DeploymentStore) Get(nsName types.NamespacedName) *Deployment { + val, ok := d.deployments.Load(nsName) + if !ok { + return nil + } + + deployment, ok := val.(*Deployment) + if !ok { + panic(fmt.Sprintf("expected Deployment, got type %T", val)) + } + + return deployment +} + +// GetOrStore returns the existing value for the key if present. +// Otherwise, it stores and returns the given value. +func (d *DeploymentStore) GetOrStore(nsName types.NamespacedName, broadcaster broadcast.Broadcaster) *Deployment { + if deployment := d.Get(nsName); deployment != nil { + return deployment + } + + deployment := newDeployment(broadcaster) + d.deployments.Store(nsName, deployment) + + return deployment +} + +// Remove cleans up any connections that are tracked for this deployment, and then removes +// the deployment from the store. +func (d *DeploymentStore) Remove(nsName types.NamespacedName) { + d.connTracker.UntrackConnectionsForParent(nsName) + d.deployments.Delete(nsName) +} diff --git a/internal/mode/static/nginx/agent/deployment_test.go b/internal/mode/static/nginx/agent/deployment_test.go new file mode 100644 index 0000000000..a27464df2d --- /dev/null +++ b/internal/mode/static/nginx/agent/deployment_test.go @@ -0,0 +1,136 @@ +package agent + +import ( + "errors" + "testing" + + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast/broadcastfakes" + agentgrpcfakes "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/grpcfakes" +) + +func TestNewDeployment(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + deployment := newDeployment(&broadcastfakes.FakeBroadcaster{}) + g.Expect(deployment).ToNot(BeNil()) + + g.Expect(deployment.GetBroadcaster()).ToNot(BeNil()) + g.Expect(deployment.GetFileOverviews()).To(BeEmpty()) + g.Expect(deployment.GetNGINXPlusActions()).To(BeEmpty()) + g.Expect(deployment.GetLatestConfigError()).ToNot(HaveOccurred()) + g.Expect(deployment.GetLatestUpstreamError()).ToNot(HaveOccurred()) +} + +func TestSetAndGetFiles(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + deployment := newDeployment(&broadcastfakes.FakeBroadcaster{}) + + files := []File{ + { + Meta: &pb.FileMeta{ + Name: "test.conf", + Hash: "12345", + }, + Contents: []byte("test content"), + }, + } + + msg := deployment.SetFiles(files) + fileOverviews, configVersion := deployment.GetFileOverviews() + + g.Expect(msg.Type).To(Equal(broadcast.ConfigApplyRequest)) + g.Expect(msg.ConfigVersion).To(Equal(configVersion)) + g.Expect(msg.FileOverviews).To(HaveLen(9)) // 1 file + 8 ignored files + g.Expect(fileOverviews).To(Equal(msg.FileOverviews)) + + file := deployment.GetFile("test.conf", "12345") + g.Expect(file).To(Equal([]byte("test content"))) + + g.Expect(deployment.GetFile("invalid", "12345")).To(BeNil()) + g.Expect(deployment.GetFile("test.conf", "invalid")).To(BeNil()) +} + +func TestSetNGINXPlusActions(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + deployment := newDeployment(&broadcastfakes.FakeBroadcaster{}) + + actions := []*pb.NGINXPlusAction{ + { + Action: &pb.NGINXPlusAction_UpdateHttpUpstreamServers{}, + }, + { + Action: &pb.NGINXPlusAction_UpdateStreamServers{}, + }, + } + + deployment.SetNGINXPlusActions(actions) + g.Expect(deployment.GetNGINXPlusActions()).To(Equal(actions)) +} + +func TestSetPodErrorStatus(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + deployment := newDeployment(&broadcastfakes.FakeBroadcaster{}) + + err := errors.New("test error") + err2 := errors.New("test error 2") + deployment.SetPodErrorStatus("test-pod", err) + deployment.SetPodErrorStatus("test-pod2", err2) + + g.Expect(deployment.GetConfigurationStatus()).To(MatchError(ContainSubstring("test error"))) + g.Expect(deployment.GetConfigurationStatus()).To(MatchError(ContainSubstring("test error 2"))) +} + +func TestSetLatestConfigError(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + deployment := newDeployment(&broadcastfakes.FakeBroadcaster{}) + + err := errors.New("test error") + deployment.SetLatestConfigError(err) + g.Expect(deployment.GetLatestConfigError()).To(MatchError(err)) +} + +func TestSetLatestUpstreamError(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + deployment := newDeployment(&broadcastfakes.FakeBroadcaster{}) + + err := errors.New("test error") + deployment.SetLatestUpstreamError(err) + g.Expect(deployment.GetLatestUpstreamError()).To(MatchError(err)) +} + +func TestDeploymentStore(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + store := NewDeploymentStore(&agentgrpcfakes.FakeConnectionsTracker{}) + + nsName := types.NamespacedName{Namespace: "default", Name: "test-deployment"} + + deployment := store.GetOrStore(nsName, &broadcastfakes.FakeBroadcaster{}) + g.Expect(deployment).ToNot(BeNil()) + + fetchedDeployment := store.Get(nsName) + g.Expect(fetchedDeployment).To(Equal(deployment)) + + deployment = store.GetOrStore(nsName, &broadcastfakes.FakeBroadcaster{}) + g.Expect(fetchedDeployment).To(Equal(deployment)) + + store.Remove(nsName) + g.Expect(store.Get(nsName)).To(BeNil()) +} diff --git a/internal/mode/static/nginx/agent/file.go b/internal/mode/static/nginx/agent/file.go index 296e1705ee..549beca17a 100644 --- a/internal/mode/static/nginx/agent/file.go +++ b/internal/mode/static/nginx/agent/file.go @@ -2,51 +2,91 @@ package agent import ( "context" - "fmt" "github.com/go-logr/logr" pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + agentgrpc "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" + grpcContext "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/context" ) +// File is an nginx configuration file that the nginx agent gets from the control plane +// after a ConfigApplyRequest. +type File struct { + Meta *pb.FileMeta + Contents []byte +} + // fileService handles file management between the control plane and the agent. type fileService struct { pb.FileServiceServer + nginxDeployments *DeploymentStore + connTracker agentgrpc.ConnectionsTracker // TODO(sberman): all logs are at Info level right now. Adjust appropriately. logger logr.Logger } -func newFileService(logger logr.Logger) *fileService { - return &fileService{logger: logger} +func newFileService( + logger logr.Logger, + depStore *DeploymentStore, + connTracker agentgrpc.ConnectionsTracker, +) *fileService { + return &fileService{ + logger: logger, + nginxDeployments: depStore, + connTracker: connTracker, + } } func (fs *fileService) Register(server *grpc.Server) { pb.RegisterFileServiceServer(server, fs) } -// GetOverview gets the overview of files for a particular configuration version of an instance. -// Agent calls this if it's missing an overview when a ConfigApplyRequest is called by the control plane. -func (fs *fileService) GetOverview( - _ context.Context, - _ *pb.GetOverviewRequest, -) (*pb.GetOverviewResponse, error) { - fs.logger.Info("Get overview request") - - return &pb.GetOverviewResponse{ - Overview: &pb.FileOverview{}, - }, nil -} - // GetFile is called by the agent when it needs to download a file for a ConfigApplyRequest. func (fs *fileService) GetFile( - _ context.Context, + ctx context.Context, req *pb.GetFileRequest, ) (*pb.GetFileResponse, error) { filename := req.GetFileMeta().GetName() hash := req.GetFileMeta().GetHash() - fs.logger.Info(fmt.Sprintf("Getting file: %s, %s", filename, hash)) - return &pb.GetFileResponse{}, nil + gi, ok := grpcContext.GrpcInfoFromContext(ctx) + if !ok { + return nil, agentgrpc.ErrStatusInvalidConnection + } + + conn := fs.connTracker.GetConnection(gi.IPAddress) + if conn.PodName == "" { + return nil, status.Errorf(codes.NotFound, "connection not found") + } + + deployment := fs.nginxDeployments.Get(conn.Parent) + if deployment == nil { + return nil, status.Errorf(codes.NotFound, "deployment not found in store") + } + + contents := deployment.GetFile(filename, hash) + if len(contents) == 0 { + return nil, status.Errorf(codes.NotFound, "file not found") + } + + return &pb.GetFileResponse{ + Contents: &pb.FileContents{ + Contents: contents, + }, + }, nil +} + +// GetOverview gets the overview of files for a particular configuration version of an instance. +// At the moment it doesn't appear to be used by the agent. +func (fs *fileService) GetOverview( + _ context.Context, + _ *pb.GetOverviewRequest, +) (*pb.GetOverviewResponse, error) { + return &pb.GetOverviewResponse{}, nil } // UpdateOverview is called by agent on startup and whenever any files change on the instance. diff --git a/internal/mode/static/nginx/agent/file_test.go b/internal/mode/static/nginx/agent/file_test.go new file mode 100644 index 0000000000..514d05dcbf --- /dev/null +++ b/internal/mode/static/nginx/agent/file_test.go @@ -0,0 +1,210 @@ +package agent + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + . "github.com/onsi/gomega" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/types" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast/broadcastfakes" + agentgrpc "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" + grpcContext "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/context" + agentgrpcfakes "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/grpcfakes" +) + +func TestGetFile(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + deploymentName := types.NamespacedName{Name: "nginx-deployment", Namespace: "default"} + + connTracker := &agentgrpcfakes.FakeConnectionsTracker{} + conn := agentgrpc.Connection{ + PodName: "nginx-pod", + InstanceID: "12345", + Parent: deploymentName, + } + connTracker.GetConnectionReturns(conn) + + depStore := NewDeploymentStore(connTracker) + dep := depStore.GetOrStore(deploymentName, &broadcastfakes.FakeBroadcaster{}) + + fileMeta := &pb.FileMeta{ + Name: "test.conf", + Hash: "some-hash", + } + contents := []byte("test contents") + + dep.files = []File{ + { + Meta: fileMeta, + Contents: contents, + }, + } + + fs := newFileService(logr.Discard(), depStore, connTracker) + + ctx := grpcContext.NewGrpcContext(context.Background(), grpcContext.GrpcInfo{ + IPAddress: "127.0.0.1", + }) + + req := &pb.GetFileRequest{ + FileMeta: fileMeta, + } + + resp, err := fs.GetFile(ctx, req) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(resp).ToNot(BeNil()) + g.Expect(resp.GetContents()).ToNot(BeNil()) + g.Expect(resp.GetContents().GetContents()).To(Equal(contents)) +} + +func TestGetFile_InvalidConnection(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + fs := newFileService(logr.Discard(), nil, nil) + + req := &pb.GetFileRequest{ + FileMeta: &pb.FileMeta{ + Name: "test.conf", + Hash: "some-hash", + }, + } + + resp, err := fs.GetFile(context.Background(), req) + + g.Expect(err).To(Equal(agentgrpc.ErrStatusInvalidConnection)) + g.Expect(resp).To(BeNil()) +} + +func TestGetFile_ConnectionNotFound(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + fs := newFileService(logr.Discard(), nil, &agentgrpcfakes.FakeConnectionsTracker{}) + + req := &pb.GetFileRequest{ + FileMeta: &pb.FileMeta{ + Name: "test.conf", + Hash: "some-hash", + }, + } + + ctx := grpcContext.NewGrpcContext(context.Background(), grpcContext.GrpcInfo{ + IPAddress: "127.0.0.1", + }) + + resp, err := fs.GetFile(ctx, req) + + g.Expect(err).To(Equal(status.Errorf(codes.NotFound, "connection not found"))) + g.Expect(resp).To(BeNil()) +} + +func TestGetFile_DeploymentNotFound(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + deploymentName := types.NamespacedName{Name: "nginx-deployment", Namespace: "default"} + + connTracker := &agentgrpcfakes.FakeConnectionsTracker{} + conn := agentgrpc.Connection{ + PodName: "nginx-pod", + InstanceID: "12345", + Parent: deploymentName, + } + connTracker.GetConnectionReturns(conn) + + fs := newFileService(logr.Discard(), NewDeploymentStore(connTracker), connTracker) + + req := &pb.GetFileRequest{ + FileMeta: &pb.FileMeta{ + Name: "test.conf", + Hash: "some-hash", + }, + } + + ctx := grpcContext.NewGrpcContext(context.Background(), grpcContext.GrpcInfo{ + IPAddress: "127.0.0.1", + }) + + resp, err := fs.GetFile(ctx, req) + + g.Expect(err).To(Equal(status.Errorf(codes.NotFound, "deployment not found in store"))) + g.Expect(resp).To(BeNil()) +} + +func TestGetFile_FileNotFound(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + deploymentName := types.NamespacedName{Name: "nginx-deployment", Namespace: "default"} + + connTracker := &agentgrpcfakes.FakeConnectionsTracker{} + conn := agentgrpc.Connection{ + PodName: "nginx-pod", + InstanceID: "12345", + Parent: deploymentName, + } + connTracker.GetConnectionReturns(conn) + + depStore := NewDeploymentStore(connTracker) + depStore.GetOrStore(deploymentName, &broadcastfakes.FakeBroadcaster{}) + + fs := newFileService(logr.Discard(), depStore, connTracker) + + req := &pb.GetFileRequest{ + FileMeta: &pb.FileMeta{ + Name: "test.conf", + Hash: "some-hash", + }, + } + + ctx := grpcContext.NewGrpcContext(context.Background(), grpcContext.GrpcInfo{ + IPAddress: "127.0.0.1", + }) + + resp, err := fs.GetFile(ctx, req) + + g.Expect(err).To(Equal(status.Errorf(codes.NotFound, "file not found"))) + g.Expect(resp).To(BeNil()) +} + +func TestGetOverview(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + fs := newFileService(logr.Discard(), nil, nil) + resp, err := fs.GetOverview(context.Background(), &pb.GetOverviewRequest{}) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(resp).To(Equal(&pb.GetOverviewResponse{})) +} + +func TestUpdateOverview(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + fs := newFileService(logr.Discard(), nil, nil) + resp, err := fs.UpdateOverview(context.Background(), &pb.UpdateOverviewRequest{}) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(resp).To(Equal(&pb.UpdateOverviewResponse{})) +} + +func TestUpdateFile(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + fs := newFileService(logr.Discard(), nil, nil) + resp, err := fs.UpdateFile(context.Background(), &pb.UpdateFileRequest{}) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(resp).To(Equal(&pb.UpdateFileResponse{})) +} diff --git a/internal/mode/static/nginx/agent/grpc/connections.go b/internal/mode/static/nginx/agent/grpc/connections.go index af99b84002..534d3729c6 100644 --- a/internal/mode/static/nginx/agent/grpc/connections.go +++ b/internal/mode/static/nginx/agent/grpc/connections.go @@ -2,23 +2,43 @@ package grpc import ( "sync" + + "k8s.io/apimachinery/pkg/types" ) -// ConnectionsTracker keeps track of all connections between the control plane and nginx agents. -type ConnectionsTracker struct { - // connections contains a map of all IP addresses that have connected and their associated pod names. - // TODO(sberman): we'll likely need to create a channel for each connection that can be stored in this map. - // Then the Subscription listens on the channel for its connection, while the nginxUpdater sends the config - // for the pod over that channel. - connections map[string]string +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate + +//counterfeiter:generate . ConnectionsTracker + +// ConnectionsTracker defines an interface to track all connections between the control plane +// and nginx agents. +type ConnectionsTracker interface { + Track(key string, conn Connection) + GetConnection(key string) Connection + ConnectionIsReady(key string) (Connection, bool) + SetInstanceID(key, id string) + UntrackConnectionsForParent(parent types.NamespacedName) +} + +// Connection contains the data about a single nginx agent connection. +type Connection struct { + PodName string + InstanceID string + Parent types.NamespacedName +} + +// AgentConnectionsTracker keeps track of all connections between the control plane and nginx agents. +type AgentConnectionsTracker struct { + // connections contains a map of all IP addresses that have connected and their connection info. + connections map[string]Connection - lock sync.Mutex + lock sync.RWMutex } -// NewConnectionsTracker returns a new ConnectionsTracker instance. -func NewConnectionsTracker() *ConnectionsTracker { - return &ConnectionsTracker{ - connections: make(map[string]string), +// NewConnectionsTracker returns a new AgentConnectionsTracker instance. +func NewConnectionsTracker() ConnectionsTracker { + return &AgentConnectionsTracker{ + connections: make(map[string]Connection), } } @@ -26,25 +46,50 @@ func NewConnectionsTracker() *ConnectionsTracker { // TODO(sberman): we need to handle the case when the token expires (once we support the token). // This likely involves setting a callback to cancel a context when the token expires, which triggers // the connection to be removed from the tracking list. -func (c *ConnectionsTracker) Track(address, hostname string) { +func (c *AgentConnectionsTracker) Track(key string, conn Connection) { c.lock.Lock() defer c.lock.Unlock() - c.connections[address] = hostname + c.connections[key] = conn } -// GetConnections returns all connections that are currently tracked. -func (c *ConnectionsTracker) GetConnections() map[string]string { +// GetConnection returns the requested connection. +func (c *AgentConnectionsTracker) GetConnection(key string) Connection { + c.lock.RLock() + defer c.lock.RUnlock() + + return c.connections[key] +} + +// ConnectionIsReady returns if the connection is ready to be used. In other words, agent +// has registered itself and an nginx instance with the control plane. +func (c *AgentConnectionsTracker) ConnectionIsReady(key string) (Connection, bool) { + c.lock.RLock() + defer c.lock.RUnlock() + + conn, ok := c.connections[key] + return conn, ok && conn.InstanceID != "" +} + +// SetInstanceID sets the nginx instanceID for a connection. +func (c *AgentConnectionsTracker) SetInstanceID(key, id string) { c.lock.Lock() defer c.lock.Unlock() - return c.connections + if conn, ok := c.connections[key]; ok { + conn.InstanceID = id + c.connections[key] = conn + } } -// GetConnection returns the hostname of the requested connection. -func (c *ConnectionsTracker) GetConnection(address string) string { +// UntrackConnectionsForParent removes all Connections that reference the specified parent. +func (c *AgentConnectionsTracker) UntrackConnectionsForParent(parent types.NamespacedName) { c.lock.Lock() defer c.lock.Unlock() - return c.connections[address] + for key, conn := range c.connections { + if conn.Parent == parent { + delete(c.connections, key) + } + } } diff --git a/internal/mode/static/nginx/agent/grpc/connections_test.go b/internal/mode/static/nginx/agent/grpc/connections_test.go new file mode 100644 index 0000000000..c13dfa8bb0 --- /dev/null +++ b/internal/mode/static/nginx/agent/grpc/connections_test.go @@ -0,0 +1,108 @@ +package grpc_test + +import ( + "testing" + + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" + + agentgrpc "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" +) + +func TestGetConnection(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + tracker := agentgrpc.NewConnectionsTracker() + + conn := agentgrpc.Connection{ + PodName: "pod1", + InstanceID: "instance1", + Parent: types.NamespacedName{Namespace: "default", Name: "parent1"}, + } + tracker.Track("key1", conn) + + trackedConn := tracker.GetConnection("key1") + g.Expect(trackedConn).To(Equal(conn)) + + nonExistent := tracker.GetConnection("nonexistent") + g.Expect(nonExistent).To(Equal(agentgrpc.Connection{})) +} + +func TestConnectionIsReady(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + tracker := agentgrpc.NewConnectionsTracker() + + conn := agentgrpc.Connection{ + PodName: "pod1", + InstanceID: "instance1", + Parent: types.NamespacedName{Namespace: "default", Name: "parent1"}, + } + tracker.Track("key1", conn) + + trackedConn, ready := tracker.ConnectionIsReady("key1") + g.Expect(ready).To(BeTrue()) + g.Expect(trackedConn).To(Equal(conn)) +} + +func TestConnectionIsNotReady(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + tracker := agentgrpc.NewConnectionsTracker() + + conn := agentgrpc.Connection{ + PodName: "pod1", + Parent: types.NamespacedName{Namespace: "default", Name: "parent1"}, + } + tracker.Track("key1", conn) + + _, ready := tracker.ConnectionIsReady("key1") + g.Expect(ready).To(BeFalse()) +} + +func TestSetInstanceID(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + tracker := agentgrpc.NewConnectionsTracker() + conn := agentgrpc.Connection{ + PodName: "pod1", + Parent: types.NamespacedName{Namespace: "default", Name: "parent1"}, + } + tracker.Track("key1", conn) + + _, ready := tracker.ConnectionIsReady("key1") + g.Expect(ready).To(BeFalse()) + + tracker.SetInstanceID("key1", "instance1") + + trackedConn, ready := tracker.ConnectionIsReady("key1") + g.Expect(ready).To(BeTrue()) + g.Expect(trackedConn.InstanceID).To(Equal("instance1")) +} + +func TestUntrackConnectionsForParent(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + tracker := agentgrpc.NewConnectionsTracker() + + parent := types.NamespacedName{Namespace: "default", Name: "parent1"} + conn1 := agentgrpc.Connection{PodName: "pod1", InstanceID: "instance1", Parent: parent} + conn2 := agentgrpc.Connection{PodName: "pod2", InstanceID: "instance2", Parent: parent} + + parent2 := types.NamespacedName{Namespace: "default", Name: "parent2"} + conn3 := agentgrpc.Connection{PodName: "pod3", InstanceID: "instance3", Parent: parent2} + + tracker.Track("key1", conn1) + tracker.Track("key2", conn2) + tracker.Track("key3", conn3) + + tracker.UntrackConnectionsForParent(parent) + g.Expect(tracker.GetConnection("key1")).To(Equal(agentgrpc.Connection{})) + g.Expect(tracker.GetConnection("key2")).To(Equal(agentgrpc.Connection{})) + g.Expect(tracker.GetConnection("key3")).To(Equal(conn3)) +} diff --git a/internal/mode/static/nginx/agent/grpc/context/context_test.go b/internal/mode/static/nginx/agent/grpc/context/context_test.go new file mode 100644 index 0000000000..64de596bc2 --- /dev/null +++ b/internal/mode/static/nginx/agent/grpc/context/context_test.go @@ -0,0 +1,31 @@ +package context_test + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + + grpcContext "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/context" +) + +func TestGrpcInfoInContext(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + grpcInfo := grpcContext.GrpcInfo{IPAddress: "192.168.1.1"} + + newCtx := grpcContext.NewGrpcContext(context.Background(), grpcInfo) + info, ok := grpcContext.GrpcInfoFromContext(newCtx) + g.Expect(ok).To(BeTrue()) + g.Expect(info).To(Equal(grpcInfo)) +} + +func TestGrpcInfoNotInContext(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + info, ok := grpcContext.GrpcInfoFromContext(context.Background()) + g.Expect(ok).To(BeFalse()) + g.Expect(info).To(Equal(grpcContext.GrpcInfo{})) +} diff --git a/internal/mode/static/nginx/agent/grpc/grpc.go b/internal/mode/static/nginx/agent/grpc/grpc.go index 0bac99f1b8..d26e90eb68 100644 --- a/internal/mode/static/nginx/agent/grpc/grpc.go +++ b/internal/mode/static/nginx/agent/grpc/grpc.go @@ -17,7 +17,7 @@ import ( ) const ( - keepAliveTime = 10 * time.Second + keepAliveTime = 15 * time.Second keepAliveTimeout = 10 * time.Second ) diff --git a/internal/mode/static/nginx/agent/grpc/grpcfakes/fake_connections_tracker.go b/internal/mode/static/nginx/agent/grpc/grpcfakes/fake_connections_tracker.go new file mode 100644 index 0000000000..6824df1a87 --- /dev/null +++ b/internal/mode/static/nginx/agent/grpc/grpcfakes/fake_connections_tracker.go @@ -0,0 +1,312 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package grpcfakes + +import ( + "sync" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" + "k8s.io/apimachinery/pkg/types" +) + +type FakeConnectionsTracker struct { + ConnectionIsReadyStub func(string) (grpc.Connection, bool) + connectionIsReadyMutex sync.RWMutex + connectionIsReadyArgsForCall []struct { + arg1 string + } + connectionIsReadyReturns struct { + result1 grpc.Connection + result2 bool + } + connectionIsReadyReturnsOnCall map[int]struct { + result1 grpc.Connection + result2 bool + } + GetConnectionStub func(string) grpc.Connection + getConnectionMutex sync.RWMutex + getConnectionArgsForCall []struct { + arg1 string + } + getConnectionReturns struct { + result1 grpc.Connection + } + getConnectionReturnsOnCall map[int]struct { + result1 grpc.Connection + } + SetInstanceIDStub func(string, string) + setInstanceIDMutex sync.RWMutex + setInstanceIDArgsForCall []struct { + arg1 string + arg2 string + } + TrackStub func(string, grpc.Connection) + trackMutex sync.RWMutex + trackArgsForCall []struct { + arg1 string + arg2 grpc.Connection + } + UntrackConnectionsForParentStub func(types.NamespacedName) + untrackConnectionsForParentMutex sync.RWMutex + untrackConnectionsForParentArgsForCall []struct { + arg1 types.NamespacedName + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeConnectionsTracker) ConnectionIsReady(arg1 string) (grpc.Connection, bool) { + fake.connectionIsReadyMutex.Lock() + ret, specificReturn := fake.connectionIsReadyReturnsOnCall[len(fake.connectionIsReadyArgsForCall)] + fake.connectionIsReadyArgsForCall = append(fake.connectionIsReadyArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.ConnectionIsReadyStub + fakeReturns := fake.connectionIsReadyReturns + fake.recordInvocation("ConnectionIsReady", []interface{}{arg1}) + fake.connectionIsReadyMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeConnectionsTracker) ConnectionIsReadyCallCount() int { + fake.connectionIsReadyMutex.RLock() + defer fake.connectionIsReadyMutex.RUnlock() + return len(fake.connectionIsReadyArgsForCall) +} + +func (fake *FakeConnectionsTracker) ConnectionIsReadyCalls(stub func(string) (grpc.Connection, bool)) { + fake.connectionIsReadyMutex.Lock() + defer fake.connectionIsReadyMutex.Unlock() + fake.ConnectionIsReadyStub = stub +} + +func (fake *FakeConnectionsTracker) ConnectionIsReadyArgsForCall(i int) string { + fake.connectionIsReadyMutex.RLock() + defer fake.connectionIsReadyMutex.RUnlock() + argsForCall := fake.connectionIsReadyArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeConnectionsTracker) ConnectionIsReadyReturns(result1 grpc.Connection, result2 bool) { + fake.connectionIsReadyMutex.Lock() + defer fake.connectionIsReadyMutex.Unlock() + fake.ConnectionIsReadyStub = nil + fake.connectionIsReadyReturns = struct { + result1 grpc.Connection + result2 bool + }{result1, result2} +} + +func (fake *FakeConnectionsTracker) ConnectionIsReadyReturnsOnCall(i int, result1 grpc.Connection, result2 bool) { + fake.connectionIsReadyMutex.Lock() + defer fake.connectionIsReadyMutex.Unlock() + fake.ConnectionIsReadyStub = nil + if fake.connectionIsReadyReturnsOnCall == nil { + fake.connectionIsReadyReturnsOnCall = make(map[int]struct { + result1 grpc.Connection + result2 bool + }) + } + fake.connectionIsReadyReturnsOnCall[i] = struct { + result1 grpc.Connection + result2 bool + }{result1, result2} +} + +func (fake *FakeConnectionsTracker) GetConnection(arg1 string) grpc.Connection { + fake.getConnectionMutex.Lock() + ret, specificReturn := fake.getConnectionReturnsOnCall[len(fake.getConnectionArgsForCall)] + fake.getConnectionArgsForCall = append(fake.getConnectionArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.GetConnectionStub + fakeReturns := fake.getConnectionReturns + fake.recordInvocation("GetConnection", []interface{}{arg1}) + fake.getConnectionMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeConnectionsTracker) GetConnectionCallCount() int { + fake.getConnectionMutex.RLock() + defer fake.getConnectionMutex.RUnlock() + return len(fake.getConnectionArgsForCall) +} + +func (fake *FakeConnectionsTracker) GetConnectionCalls(stub func(string) grpc.Connection) { + fake.getConnectionMutex.Lock() + defer fake.getConnectionMutex.Unlock() + fake.GetConnectionStub = stub +} + +func (fake *FakeConnectionsTracker) GetConnectionArgsForCall(i int) string { + fake.getConnectionMutex.RLock() + defer fake.getConnectionMutex.RUnlock() + argsForCall := fake.getConnectionArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeConnectionsTracker) GetConnectionReturns(result1 grpc.Connection) { + fake.getConnectionMutex.Lock() + defer fake.getConnectionMutex.Unlock() + fake.GetConnectionStub = nil + fake.getConnectionReturns = struct { + result1 grpc.Connection + }{result1} +} + +func (fake *FakeConnectionsTracker) GetConnectionReturnsOnCall(i int, result1 grpc.Connection) { + fake.getConnectionMutex.Lock() + defer fake.getConnectionMutex.Unlock() + fake.GetConnectionStub = nil + if fake.getConnectionReturnsOnCall == nil { + fake.getConnectionReturnsOnCall = make(map[int]struct { + result1 grpc.Connection + }) + } + fake.getConnectionReturnsOnCall[i] = struct { + result1 grpc.Connection + }{result1} +} + +func (fake *FakeConnectionsTracker) SetInstanceID(arg1 string, arg2 string) { + fake.setInstanceIDMutex.Lock() + fake.setInstanceIDArgsForCall = append(fake.setInstanceIDArgsForCall, struct { + arg1 string + arg2 string + }{arg1, arg2}) + stub := fake.SetInstanceIDStub + fake.recordInvocation("SetInstanceID", []interface{}{arg1, arg2}) + fake.setInstanceIDMutex.Unlock() + if stub != nil { + fake.SetInstanceIDStub(arg1, arg2) + } +} + +func (fake *FakeConnectionsTracker) SetInstanceIDCallCount() int { + fake.setInstanceIDMutex.RLock() + defer fake.setInstanceIDMutex.RUnlock() + return len(fake.setInstanceIDArgsForCall) +} + +func (fake *FakeConnectionsTracker) SetInstanceIDCalls(stub func(string, string)) { + fake.setInstanceIDMutex.Lock() + defer fake.setInstanceIDMutex.Unlock() + fake.SetInstanceIDStub = stub +} + +func (fake *FakeConnectionsTracker) SetInstanceIDArgsForCall(i int) (string, string) { + fake.setInstanceIDMutex.RLock() + defer fake.setInstanceIDMutex.RUnlock() + argsForCall := fake.setInstanceIDArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeConnectionsTracker) Track(arg1 string, arg2 grpc.Connection) { + fake.trackMutex.Lock() + fake.trackArgsForCall = append(fake.trackArgsForCall, struct { + arg1 string + arg2 grpc.Connection + }{arg1, arg2}) + stub := fake.TrackStub + fake.recordInvocation("Track", []interface{}{arg1, arg2}) + fake.trackMutex.Unlock() + if stub != nil { + fake.TrackStub(arg1, arg2) + } +} + +func (fake *FakeConnectionsTracker) TrackCallCount() int { + fake.trackMutex.RLock() + defer fake.trackMutex.RUnlock() + return len(fake.trackArgsForCall) +} + +func (fake *FakeConnectionsTracker) TrackCalls(stub func(string, grpc.Connection)) { + fake.trackMutex.Lock() + defer fake.trackMutex.Unlock() + fake.TrackStub = stub +} + +func (fake *FakeConnectionsTracker) TrackArgsForCall(i int) (string, grpc.Connection) { + fake.trackMutex.RLock() + defer fake.trackMutex.RUnlock() + argsForCall := fake.trackArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeConnectionsTracker) UntrackConnectionsForParent(arg1 types.NamespacedName) { + fake.untrackConnectionsForParentMutex.Lock() + fake.untrackConnectionsForParentArgsForCall = append(fake.untrackConnectionsForParentArgsForCall, struct { + arg1 types.NamespacedName + }{arg1}) + stub := fake.UntrackConnectionsForParentStub + fake.recordInvocation("UntrackConnectionsForParent", []interface{}{arg1}) + fake.untrackConnectionsForParentMutex.Unlock() + if stub != nil { + fake.UntrackConnectionsForParentStub(arg1) + } +} + +func (fake *FakeConnectionsTracker) UntrackConnectionsForParentCallCount() int { + fake.untrackConnectionsForParentMutex.RLock() + defer fake.untrackConnectionsForParentMutex.RUnlock() + return len(fake.untrackConnectionsForParentArgsForCall) +} + +func (fake *FakeConnectionsTracker) UntrackConnectionsForParentCalls(stub func(types.NamespacedName)) { + fake.untrackConnectionsForParentMutex.Lock() + defer fake.untrackConnectionsForParentMutex.Unlock() + fake.UntrackConnectionsForParentStub = stub +} + +func (fake *FakeConnectionsTracker) UntrackConnectionsForParentArgsForCall(i int) types.NamespacedName { + fake.untrackConnectionsForParentMutex.RLock() + defer fake.untrackConnectionsForParentMutex.RUnlock() + argsForCall := fake.untrackConnectionsForParentArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeConnectionsTracker) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.connectionIsReadyMutex.RLock() + defer fake.connectionIsReadyMutex.RUnlock() + fake.getConnectionMutex.RLock() + defer fake.getConnectionMutex.RUnlock() + fake.setInstanceIDMutex.RLock() + defer fake.setInstanceIDMutex.RUnlock() + fake.trackMutex.RLock() + defer fake.trackMutex.RUnlock() + fake.untrackConnectionsForParentMutex.RLock() + defer fake.untrackConnectionsForParentMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeConnectionsTracker) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ grpc.ConnectionsTracker = new(FakeConnectionsTracker) diff --git a/internal/mode/static/nginx/agent/grpc/messenger/doc.go b/internal/mode/static/nginx/agent/grpc/messenger/doc.go new file mode 100644 index 0000000000..60150e4ad8 --- /dev/null +++ b/internal/mode/static/nginx/agent/grpc/messenger/doc.go @@ -0,0 +1,4 @@ +/* +Package messenger provides a wrapper around a gRPC stream with the nginx agent. +*/ +package messenger diff --git a/internal/mode/static/nginx/agent/grpc/messenger/messenger.go b/internal/mode/static/nginx/agent/grpc/messenger/messenger.go new file mode 100644 index 0000000000..dde16c74f3 --- /dev/null +++ b/internal/mode/static/nginx/agent/grpc/messenger/messenger.go @@ -0,0 +1,111 @@ +package messenger + +import ( + "context" + "errors" + + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" +) + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate + +//counterfeiter:generate . Messenger + +// Messenger is a wrapper around a gRPC stream with the nginx agent. +type Messenger interface { + Run(context.Context) + Send(context.Context, *pb.ManagementPlaneRequest) error + Messages() <-chan *pb.DataPlaneResponse + Errors() <-chan error +} + +// NginxAgentMessenger is the implementation of the Messenger interface. +type NginxAgentMessenger struct { + incoming chan *pb.ManagementPlaneRequest + outgoing chan *pb.DataPlaneResponse + errorCh chan error + server pb.CommandService_SubscribeServer +} + +// New returns a new Messenger instance. +func New(server pb.CommandService_SubscribeServer) Messenger { + return &NginxAgentMessenger{ + incoming: make(chan *pb.ManagementPlaneRequest), + outgoing: make(chan *pb.DataPlaneResponse), + errorCh: make(chan error), + server: server, + } +} + +// Run starts the Messenger to listen for any Send() or Recv() events over the stream. +func (m *NginxAgentMessenger) Run(ctx context.Context) { + go m.handleRecv(ctx) + m.handleSend(ctx) +} + +// Send a message, will return error if the context is Done. +func (m *NginxAgentMessenger) Send(ctx context.Context, msg *pb.ManagementPlaneRequest) error { + select { + case <-ctx.Done(): + return ctx.Err() + case m.incoming <- msg: + } + return nil +} + +func (m *NginxAgentMessenger) handleSend(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case msg := <-m.incoming: + err := m.server.Send(msg) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(ctx.Err(), context.Canceled) { + return + } + m.errorCh <- err + + return + } + } + } +} + +// Messages returns the data plane response channel. +func (m *NginxAgentMessenger) Messages() <-chan *pb.DataPlaneResponse { + return m.outgoing +} + +// Errors returns the error channel. +func (m *NginxAgentMessenger) Errors() <-chan error { + return m.errorCh +} + +// handleRecv handles an incoming message from the nginx agent. +// It blocks until Recv returns. The result from the Recv is either going to Error or Messages channel. +func (m *NginxAgentMessenger) handleRecv(ctx context.Context) { + for { + msg, err := m.server.Recv() + if err != nil { + select { + case <-ctx.Done(): + return + case m.errorCh <- err: + } + return + } + + if msg == nil { + // close the outgoing channel to signal no more messages to be sent + close(m.outgoing) + return + } + + select { + case <-ctx.Done(): + return + case m.outgoing <- msg: + } + } +} diff --git a/internal/mode/static/nginx/agent/grpc/messenger/messenger_test.go b/internal/mode/static/nginx/agent/grpc/messenger/messenger_test.go new file mode 100644 index 0000000000..9cc6da41bc --- /dev/null +++ b/internal/mode/static/nginx/agent/grpc/messenger/messenger_test.go @@ -0,0 +1,125 @@ +package messenger_test + +import ( + "context" + "errors" + "testing" + + v1 "github.com/nginx/agent/v3/api/grpc/mpi/v1" + . "github.com/onsi/gomega" + "google.golang.org/grpc" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/messenger" +) + +type mockServer struct { + grpc.ServerStream + sendChan chan *v1.ManagementPlaneRequest + recvChan chan *v1.DataPlaneResponse +} + +func (m *mockServer) Send(msg *v1.ManagementPlaneRequest) error { + m.sendChan <- msg + return nil +} + +func (m *mockServer) Recv() (*v1.DataPlaneResponse, error) { + msg, ok := <-m.recvChan + if !ok { + return nil, errors.New("channel closed") + } + return msg, nil +} + +type mockErrorServer struct { + grpc.ServerStream + sendChan chan *v1.ManagementPlaneRequest + recvChan chan *v1.DataPlaneResponse +} + +func (m *mockErrorServer) Send(_ *v1.ManagementPlaneRequest) error { + return errors.New("error sending to server") +} + +func (m *mockErrorServer) Recv() (*v1.DataPlaneResponse, error) { + <-m.recvChan + return nil, errors.New("error received from server") +} + +func createServer() *mockServer { + return &mockServer{ + sendChan: make(chan *v1.ManagementPlaneRequest, 1), + recvChan: make(chan *v1.DataPlaneResponse, 1), + } +} + +func createErrorServer() *mockErrorServer { + return &mockErrorServer{ + sendChan: make(chan *v1.ManagementPlaneRequest, 1), + recvChan: make(chan *v1.DataPlaneResponse, 1), + } +} + +func TestSend(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + server := createServer() + msgr := messenger.New(server) + + go msgr.Run(ctx) + + msg := &v1.ManagementPlaneRequest{ + MessageMeta: &v1.MessageMeta{ + MessageId: "test", + }, + } + g.Expect(msgr.Send(ctx, msg)).To(Succeed()) + + g.Eventually(server.sendChan).Should(Receive(Equal(msg))) + + cancel() + + g.Expect(msgr.Send(ctx, &v1.ManagementPlaneRequest{})).ToNot(Succeed()) +} + +func TestMessages(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + server := createServer() + msgr := messenger.New(server) + + go msgr.Run(ctx) + + msg := &v1.DataPlaneResponse{InstanceId: "test"} + server.recvChan <- msg + + g.Eventually(msgr.Messages()).Should(Receive(Equal(msg))) +} + +func TestErrors(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + server := createErrorServer() + msgr := messenger.New(server) + + go msgr.Run(ctx) + + g.Expect(msgr.Send(ctx, &v1.ManagementPlaneRequest{})).To(Succeed()) + g.Eventually(msgr.Errors()).Should(Receive(MatchError("error sending to server"))) + + server.recvChan <- &v1.DataPlaneResponse{} + + g.Eventually(msgr.Errors()).Should(Receive(MatchError("error received from server"))) +} diff --git a/internal/mode/static/nginx/agent/grpc/messenger/messengerfakes/fake_messenger.go b/internal/mode/static/nginx/agent/grpc/messenger/messengerfakes/fake_messenger.go new file mode 100644 index 0000000000..1c73578270 --- /dev/null +++ b/internal/mode/static/nginx/agent/grpc/messenger/messengerfakes/fake_messenger.go @@ -0,0 +1,284 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package messengerfakes + +import ( + "context" + "sync" + + v1 "github.com/nginx/agent/v3/api/grpc/mpi/v1" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/messenger" +) + +type FakeMessenger struct { + ErrorsStub func() <-chan error + errorsMutex sync.RWMutex + errorsArgsForCall []struct { + } + errorsReturns struct { + result1 <-chan error + } + errorsReturnsOnCall map[int]struct { + result1 <-chan error + } + MessagesStub func() <-chan *v1.DataPlaneResponse + messagesMutex sync.RWMutex + messagesArgsForCall []struct { + } + messagesReturns struct { + result1 <-chan *v1.DataPlaneResponse + } + messagesReturnsOnCall map[int]struct { + result1 <-chan *v1.DataPlaneResponse + } + RunStub func(context.Context) + runMutex sync.RWMutex + runArgsForCall []struct { + arg1 context.Context + } + SendStub func(context.Context, *v1.ManagementPlaneRequest) error + sendMutex sync.RWMutex + sendArgsForCall []struct { + arg1 context.Context + arg2 *v1.ManagementPlaneRequest + } + sendReturns struct { + result1 error + } + sendReturnsOnCall map[int]struct { + result1 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeMessenger) Errors() <-chan error { + fake.errorsMutex.Lock() + ret, specificReturn := fake.errorsReturnsOnCall[len(fake.errorsArgsForCall)] + fake.errorsArgsForCall = append(fake.errorsArgsForCall, struct { + }{}) + stub := fake.ErrorsStub + fakeReturns := fake.errorsReturns + fake.recordInvocation("Errors", []interface{}{}) + fake.errorsMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMessenger) ErrorsCallCount() int { + fake.errorsMutex.RLock() + defer fake.errorsMutex.RUnlock() + return len(fake.errorsArgsForCall) +} + +func (fake *FakeMessenger) ErrorsCalls(stub func() <-chan error) { + fake.errorsMutex.Lock() + defer fake.errorsMutex.Unlock() + fake.ErrorsStub = stub +} + +func (fake *FakeMessenger) ErrorsReturns(result1 <-chan error) { + fake.errorsMutex.Lock() + defer fake.errorsMutex.Unlock() + fake.ErrorsStub = nil + fake.errorsReturns = struct { + result1 <-chan error + }{result1} +} + +func (fake *FakeMessenger) ErrorsReturnsOnCall(i int, result1 <-chan error) { + fake.errorsMutex.Lock() + defer fake.errorsMutex.Unlock() + fake.ErrorsStub = nil + if fake.errorsReturnsOnCall == nil { + fake.errorsReturnsOnCall = make(map[int]struct { + result1 <-chan error + }) + } + fake.errorsReturnsOnCall[i] = struct { + result1 <-chan error + }{result1} +} + +func (fake *FakeMessenger) Messages() <-chan *v1.DataPlaneResponse { + fake.messagesMutex.Lock() + ret, specificReturn := fake.messagesReturnsOnCall[len(fake.messagesArgsForCall)] + fake.messagesArgsForCall = append(fake.messagesArgsForCall, struct { + }{}) + stub := fake.MessagesStub + fakeReturns := fake.messagesReturns + fake.recordInvocation("Messages", []interface{}{}) + fake.messagesMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMessenger) MessagesCallCount() int { + fake.messagesMutex.RLock() + defer fake.messagesMutex.RUnlock() + return len(fake.messagesArgsForCall) +} + +func (fake *FakeMessenger) MessagesCalls(stub func() <-chan *v1.DataPlaneResponse) { + fake.messagesMutex.Lock() + defer fake.messagesMutex.Unlock() + fake.MessagesStub = stub +} + +func (fake *FakeMessenger) MessagesReturns(result1 <-chan *v1.DataPlaneResponse) { + fake.messagesMutex.Lock() + defer fake.messagesMutex.Unlock() + fake.MessagesStub = nil + fake.messagesReturns = struct { + result1 <-chan *v1.DataPlaneResponse + }{result1} +} + +func (fake *FakeMessenger) MessagesReturnsOnCall(i int, result1 <-chan *v1.DataPlaneResponse) { + fake.messagesMutex.Lock() + defer fake.messagesMutex.Unlock() + fake.MessagesStub = nil + if fake.messagesReturnsOnCall == nil { + fake.messagesReturnsOnCall = make(map[int]struct { + result1 <-chan *v1.DataPlaneResponse + }) + } + fake.messagesReturnsOnCall[i] = struct { + result1 <-chan *v1.DataPlaneResponse + }{result1} +} + +func (fake *FakeMessenger) Run(arg1 context.Context) { + fake.runMutex.Lock() + fake.runArgsForCall = append(fake.runArgsForCall, struct { + arg1 context.Context + }{arg1}) + stub := fake.RunStub + fake.recordInvocation("Run", []interface{}{arg1}) + fake.runMutex.Unlock() + if stub != nil { + fake.RunStub(arg1) + } +} + +func (fake *FakeMessenger) RunCallCount() int { + fake.runMutex.RLock() + defer fake.runMutex.RUnlock() + return len(fake.runArgsForCall) +} + +func (fake *FakeMessenger) RunCalls(stub func(context.Context)) { + fake.runMutex.Lock() + defer fake.runMutex.Unlock() + fake.RunStub = stub +} + +func (fake *FakeMessenger) RunArgsForCall(i int) context.Context { + fake.runMutex.RLock() + defer fake.runMutex.RUnlock() + argsForCall := fake.runArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeMessenger) Send(arg1 context.Context, arg2 *v1.ManagementPlaneRequest) error { + fake.sendMutex.Lock() + ret, specificReturn := fake.sendReturnsOnCall[len(fake.sendArgsForCall)] + fake.sendArgsForCall = append(fake.sendArgsForCall, struct { + arg1 context.Context + arg2 *v1.ManagementPlaneRequest + }{arg1, arg2}) + stub := fake.SendStub + fakeReturns := fake.sendReturns + fake.recordInvocation("Send", []interface{}{arg1, arg2}) + fake.sendMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMessenger) SendCallCount() int { + fake.sendMutex.RLock() + defer fake.sendMutex.RUnlock() + return len(fake.sendArgsForCall) +} + +func (fake *FakeMessenger) SendCalls(stub func(context.Context, *v1.ManagementPlaneRequest) error) { + fake.sendMutex.Lock() + defer fake.sendMutex.Unlock() + fake.SendStub = stub +} + +func (fake *FakeMessenger) SendArgsForCall(i int) (context.Context, *v1.ManagementPlaneRequest) { + fake.sendMutex.RLock() + defer fake.sendMutex.RUnlock() + argsForCall := fake.sendArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeMessenger) SendReturns(result1 error) { + fake.sendMutex.Lock() + defer fake.sendMutex.Unlock() + fake.SendStub = nil + fake.sendReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeMessenger) SendReturnsOnCall(i int, result1 error) { + fake.sendMutex.Lock() + defer fake.sendMutex.Unlock() + fake.SendStub = nil + if fake.sendReturnsOnCall == nil { + fake.sendReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.sendReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeMessenger) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.errorsMutex.RLock() + defer fake.errorsMutex.RUnlock() + fake.messagesMutex.RLock() + defer fake.messagesMutex.RUnlock() + fake.runMutex.RLock() + defer fake.runMutex.RUnlock() + fake.sendMutex.RLock() + defer fake.sendMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeMessenger) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ messenger.Messenger = new(FakeMessenger) diff --git a/internal/mode/static/nginx/conf/nginx-plus.conf b/internal/mode/static/nginx/conf/nginx-plus.conf index 17ac6de4e3..e119a9b454 100644 --- a/internal/mode/static/nginx/conf/nginx-plus.conf +++ b/internal/mode/static/nginx/conf/nginx-plus.conf @@ -28,6 +28,15 @@ http { server_tokens off; + server { + listen unix:/var/run/nginx/nginx-plus-api.sock; + access_log off; + + location /api { + api write=on; + } + } + server { listen 127.0.0.1:8765; root /usr/share/nginx/html; @@ -42,15 +51,6 @@ http { api write=off; } } - - server { - listen unix:/var/run/nginx/nginx-plus-api.sock; - access_log off; - - location /api { - api write=on; - } - } } stream { diff --git a/internal/mode/static/nginx/config/configfakes/fake_generator.go b/internal/mode/static/nginx/config/configfakes/fake_generator.go index 9746df5178..f5475a16b2 100644 --- a/internal/mode/static/nginx/config/configfakes/fake_generator.go +++ b/internal/mode/static/nginx/config/configfakes/fake_generator.go @@ -4,41 +4,41 @@ package configfakes import ( "sync" - "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" ) type FakeGenerator struct { - GenerateStub func(dataplane.Configuration) []file.File + GenerateStub func(dataplane.Configuration) []agent.File generateMutex sync.RWMutex generateArgsForCall []struct { arg1 dataplane.Configuration } generateReturns struct { - result1 []file.File + result1 []agent.File } generateReturnsOnCall map[int]struct { - result1 []file.File + result1 []agent.File } - GenerateDeploymentContextStub func(dataplane.DeploymentContext) (file.File, error) + GenerateDeploymentContextStub func(dataplane.DeploymentContext) (agent.File, error) generateDeploymentContextMutex sync.RWMutex generateDeploymentContextArgsForCall []struct { arg1 dataplane.DeploymentContext } generateDeploymentContextReturns struct { - result1 file.File + result1 agent.File result2 error } generateDeploymentContextReturnsOnCall map[int]struct { - result1 file.File + result1 agent.File result2 error } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } -func (fake *FakeGenerator) Generate(arg1 dataplane.Configuration) []file.File { +func (fake *FakeGenerator) Generate(arg1 dataplane.Configuration) []agent.File { fake.generateMutex.Lock() ret, specificReturn := fake.generateReturnsOnCall[len(fake.generateArgsForCall)] fake.generateArgsForCall = append(fake.generateArgsForCall, struct { @@ -63,7 +63,7 @@ func (fake *FakeGenerator) GenerateCallCount() int { return len(fake.generateArgsForCall) } -func (fake *FakeGenerator) GenerateCalls(stub func(dataplane.Configuration) []file.File) { +func (fake *FakeGenerator) GenerateCalls(stub func(dataplane.Configuration) []agent.File) { fake.generateMutex.Lock() defer fake.generateMutex.Unlock() fake.GenerateStub = stub @@ -76,30 +76,30 @@ func (fake *FakeGenerator) GenerateArgsForCall(i int) dataplane.Configuration { return argsForCall.arg1 } -func (fake *FakeGenerator) GenerateReturns(result1 []file.File) { +func (fake *FakeGenerator) GenerateReturns(result1 []agent.File) { fake.generateMutex.Lock() defer fake.generateMutex.Unlock() fake.GenerateStub = nil fake.generateReturns = struct { - result1 []file.File + result1 []agent.File }{result1} } -func (fake *FakeGenerator) GenerateReturnsOnCall(i int, result1 []file.File) { +func (fake *FakeGenerator) GenerateReturnsOnCall(i int, result1 []agent.File) { fake.generateMutex.Lock() defer fake.generateMutex.Unlock() fake.GenerateStub = nil if fake.generateReturnsOnCall == nil { fake.generateReturnsOnCall = make(map[int]struct { - result1 []file.File + result1 []agent.File }) } fake.generateReturnsOnCall[i] = struct { - result1 []file.File + result1 []agent.File }{result1} } -func (fake *FakeGenerator) GenerateDeploymentContext(arg1 dataplane.DeploymentContext) (file.File, error) { +func (fake *FakeGenerator) GenerateDeploymentContext(arg1 dataplane.DeploymentContext) (agent.File, error) { fake.generateDeploymentContextMutex.Lock() ret, specificReturn := fake.generateDeploymentContextReturnsOnCall[len(fake.generateDeploymentContextArgsForCall)] fake.generateDeploymentContextArgsForCall = append(fake.generateDeploymentContextArgsForCall, struct { @@ -124,7 +124,7 @@ func (fake *FakeGenerator) GenerateDeploymentContextCallCount() int { return len(fake.generateDeploymentContextArgsForCall) } -func (fake *FakeGenerator) GenerateDeploymentContextCalls(stub func(dataplane.DeploymentContext) (file.File, error)) { +func (fake *FakeGenerator) GenerateDeploymentContextCalls(stub func(dataplane.DeploymentContext) (agent.File, error)) { fake.generateDeploymentContextMutex.Lock() defer fake.generateDeploymentContextMutex.Unlock() fake.GenerateDeploymentContextStub = stub @@ -137,28 +137,28 @@ func (fake *FakeGenerator) GenerateDeploymentContextArgsForCall(i int) dataplane return argsForCall.arg1 } -func (fake *FakeGenerator) GenerateDeploymentContextReturns(result1 file.File, result2 error) { +func (fake *FakeGenerator) GenerateDeploymentContextReturns(result1 agent.File, result2 error) { fake.generateDeploymentContextMutex.Lock() defer fake.generateDeploymentContextMutex.Unlock() fake.GenerateDeploymentContextStub = nil fake.generateDeploymentContextReturns = struct { - result1 file.File + result1 agent.File result2 error }{result1, result2} } -func (fake *FakeGenerator) GenerateDeploymentContextReturnsOnCall(i int, result1 file.File, result2 error) { +func (fake *FakeGenerator) GenerateDeploymentContextReturnsOnCall(i int, result1 agent.File, result2 error) { fake.generateDeploymentContextMutex.Lock() defer fake.generateDeploymentContextMutex.Unlock() fake.GenerateDeploymentContextStub = nil if fake.generateDeploymentContextReturnsOnCall == nil { fake.generateDeploymentContextReturnsOnCall = make(map[int]struct { - result1 file.File + result1 agent.File result2 error }) } fake.generateDeploymentContextReturnsOnCall[i] = struct { - result1 file.File + result1 agent.File result2 error }{result1, result2} } diff --git a/internal/mode/static/nginx/config/generator.go b/internal/mode/static/nginx/config/generator.go index df43d0fb5c..e55f40b061 100644 --- a/internal/mode/static/nginx/config/generator.go +++ b/internal/mode/static/nginx/config/generator.go @@ -6,9 +6,12 @@ import ( "path/filepath" "github.com/go-logr/logr" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + filesHelper "github.com/nginx/agent/v3/pkg/files" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" ngfConfig "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/http" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/clientsettings" @@ -61,9 +64,9 @@ const ( // This interface is used for testing purposes only. type Generator interface { // Generate generates NGINX configuration files from internal representation. - Generate(configuration dataplane.Configuration) []file.File + Generate(configuration dataplane.Configuration) []agent.File // GenerateDeploymentContext generates the deployment context used for N+ licensing. - GenerateDeploymentContext(depCtx dataplane.DeploymentContext) (file.File, error) + GenerateDeploymentContext(depCtx dataplane.DeploymentContext) (agent.File, error) } // GeneratorImpl is an implementation of Generator. @@ -103,8 +106,8 @@ type executeFunc func(configuration dataplane.Configuration) []executeResult // It is the responsibility of the caller to validate the configuration before calling this function. // In case of invalid configuration, NGINX will fail to reload or could be configured with malicious configuration. // To validate, use the validators from the validation package. -func (g GeneratorImpl) Generate(conf dataplane.Configuration) []file.File { - files := make([]file.File, 0) +func (g GeneratorImpl) Generate(conf dataplane.Configuration) []agent.File { + files := make([]agent.File, 0) for id, pair := range conf.SSLKeyPairs { files = append(files, generatePEM(id, pair.Cert, pair.Key)) @@ -126,16 +129,19 @@ func (g GeneratorImpl) Generate(conf dataplane.Configuration) []file.File { // GenerateDeploymentContext generates the deployment_ctx.json file needed for N+ licensing. // It's exported since it's used by the init container process. -func (g GeneratorImpl) GenerateDeploymentContext(depCtx dataplane.DeploymentContext) (file.File, error) { +func (g GeneratorImpl) GenerateDeploymentContext(depCtx dataplane.DeploymentContext) (agent.File, error) { depCtxBytes, err := json.Marshal(depCtx) if err != nil { - return file.File{}, fmt.Errorf("error building deployment context for mgmt block: %w", err) + return agent.File{}, fmt.Errorf("error building deployment context for mgmt block: %w", err) } - deploymentCtxFile := file.File{ - Content: depCtxBytes, - Path: mainIncludesFolder + "/deployment_ctx.json", - Type: file.TypeRegular, + deploymentCtxFile := agent.File{ + Meta: &pb.FileMeta{ + Name: mainIncludesFolder + "/deployment_ctx.json", + Hash: filesHelper.GenerateHash(depCtxBytes), + Permissions: file.RegularFileMode, + }, + Contents: depCtxBytes, } return deploymentCtxFile, nil @@ -144,7 +150,7 @@ func (g GeneratorImpl) GenerateDeploymentContext(depCtx dataplane.DeploymentCont func (g GeneratorImpl) executeConfigTemplates( conf dataplane.Configuration, generator policies.Generator, -) []file.File { +) []agent.File { fileBytes := make(map[string][]byte) httpUpstreams := g.createUpstreams(conf.Upstreams, upstreamsettings.NewProcessor()) @@ -157,17 +163,20 @@ func (g GeneratorImpl) executeConfigTemplates( } } - var mgmtFiles []file.File + var mgmtFiles []agent.File if g.plus { mgmtFiles = g.generateMgmtFiles(conf) } - files := make([]file.File, 0, len(fileBytes)+len(mgmtFiles)) + files := make([]agent.File, 0, len(fileBytes)+len(mgmtFiles)) for fp, bytes := range fileBytes { - files = append(files, file.File{ - Path: fp, - Content: bytes, - Type: file.TypeRegular, + files = append(files, agent.File{ + Meta: &pb.FileMeta{ + Name: fp, + Hash: filesHelper.GenerateHash(bytes), + Permissions: file.RegularFileMode, + }, + Contents: bytes, }) } files = append(files, mgmtFiles...) @@ -194,16 +203,19 @@ func (g GeneratorImpl) getExecuteFuncs( } } -func generatePEM(id dataplane.SSLKeyPairID, cert []byte, key []byte) file.File { +func generatePEM(id dataplane.SSLKeyPairID, cert []byte, key []byte) agent.File { c := make([]byte, 0, len(cert)+len(key)+1) c = append(c, cert...) c = append(c, '\n') c = append(c, key...) - return file.File{ - Content: c, - Path: generatePEMFileName(id), - Type: file.TypeSecret, + return agent.File{ + Meta: &pb.FileMeta{ + Name: generatePEMFileName(id), + Hash: filesHelper.GenerateHash(c), + Permissions: file.SecretFileMode, + }, + Contents: c, } } @@ -211,11 +223,14 @@ func generatePEMFileName(id dataplane.SSLKeyPairID) string { return filepath.Join(secretsFolder, string(id)+".pem") } -func generateCertBundle(id dataplane.CertBundleID, cert []byte) file.File { - return file.File{ - Content: cert, - Path: generateCertBundleFileName(id), - Type: file.TypeRegular, +func generateCertBundle(id dataplane.CertBundleID, cert []byte) agent.File { + return agent.File{ + Meta: &pb.FileMeta{ + Name: generateCertBundleFileName(id), + Hash: filesHelper.GenerateHash(cert), + Permissions: file.SecretFileMode, + }, + Contents: cert, } } diff --git a/internal/mode/static/nginx/config/generator_test.go b/internal/mode/static/nginx/config/generator_test.go index a6f4540adb..14e9171dba 100644 --- a/internal/mode/static/nginx/config/generator_test.go +++ b/internal/mode/static/nginx/config/generator_test.go @@ -4,6 +4,8 @@ import ( "sort" "testing" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + filesHelper "github.com/nginx/agent/v3/pkg/files" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/types" ctlrZap "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -11,6 +13,7 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" ngfConfig "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" @@ -145,7 +148,7 @@ func TestGenerate(t *testing.T) { g.Expect(files).To(HaveLen(16)) arrange := func(i, j int) bool { - return files[i].Path < files[j].Path + return files[i].Meta.Name < files[j].Meta.Name } sort.Slice(files, arrange) @@ -169,9 +172,9 @@ func TestGenerate(t *testing.T) { /etc/nginx/stream-conf.d/stream.conf */ - g.Expect(files[0].Type).To(Equal(file.TypeRegular)) - g.Expect(files[0].Path).To(Equal("/etc/nginx/conf.d/http.conf")) - httpCfg := string(files[0].Content) // converting to string so that on failure gomega prints strings not byte arrays + g.Expect(files[0].Meta.Permissions).To(Equal(file.RegularFileMode)) + g.Expect(files[0].Meta.Name).To(Equal("/etc/nginx/conf.d/http.conf")) + httpCfg := string(files[0].Contents) // converting to string so that on failure gomega prints strings not byte arrays // Note: this only verifies that Generate() returns a byte array with upstream, server, and split_client blocks. // It does not test the correctness of those blocks. That functionality is covered by other tests in this package. g.Expect(httpCfg).To(ContainSubstring("listen 80")) @@ -188,34 +191,34 @@ func TestGenerate(t *testing.T) { g.Expect(httpCfg).To(ContainSubstring("include /etc/nginx/includes/http_snippet1.conf;")) g.Expect(httpCfg).To(ContainSubstring("include /etc/nginx/includes/http_snippet2.conf;")) - g.Expect(files[1].Path).To(Equal("/etc/nginx/conf.d/matches.json")) + g.Expect(files[1].Meta.Name).To(Equal("/etc/nginx/conf.d/matches.json")) - g.Expect(files[1].Type).To(Equal(file.TypeRegular)) + g.Expect(files[1].Meta.Permissions).To(Equal(file.RegularFileMode)) expString := "{}" - g.Expect(string(files[1].Content)).To(Equal(expString)) + g.Expect(string(files[1].Contents)).To(Equal(expString)) // snippet include files // content is not checked in this test. - g.Expect(files[2].Path).To(Equal("/etc/nginx/includes/http_snippet1.conf")) - g.Expect(files[3].Path).To(Equal("/etc/nginx/includes/http_snippet2.conf")) - g.Expect(files[4].Path).To(Equal("/etc/nginx/includes/main_snippet1.conf")) - g.Expect(files[5].Path).To(Equal("/etc/nginx/includes/main_snippet2.conf")) + g.Expect(files[2].Meta.Name).To(Equal("/etc/nginx/includes/http_snippet1.conf")) + g.Expect(files[3].Meta.Name).To(Equal("/etc/nginx/includes/http_snippet2.conf")) + g.Expect(files[4].Meta.Name).To(Equal("/etc/nginx/includes/main_snippet1.conf")) + g.Expect(files[5].Meta.Name).To(Equal("/etc/nginx/includes/main_snippet2.conf")) - g.Expect(files[6].Path).To(Equal("/etc/nginx/main-includes/deployment_ctx.json")) - deploymentCtx := string(files[6].Content) + g.Expect(files[6].Meta.Name).To(Equal("/etc/nginx/main-includes/deployment_ctx.json")) + deploymentCtx := string(files[6].Contents) g.Expect(deploymentCtx).To(ContainSubstring("\"integration\":\"ngf\"")) g.Expect(deploymentCtx).To(ContainSubstring("\"cluster_id\":\"test-uid\"")) g.Expect(deploymentCtx).To(ContainSubstring("\"installation_id\":\"test-uid-replicaSet\"")) g.Expect(deploymentCtx).To(ContainSubstring("\"cluster_node_count\":1")) - g.Expect(files[7].Path).To(Equal("/etc/nginx/main-includes/main.conf")) - mainConfStr := string(files[7].Content) + g.Expect(files[7].Meta.Name).To(Equal("/etc/nginx/main-includes/main.conf")) + mainConfStr := string(files[7].Contents) g.Expect(mainConfStr).To(ContainSubstring("load_module modules/ngx_otel_module.so;")) g.Expect(mainConfStr).To(ContainSubstring("include /etc/nginx/includes/main_snippet1.conf;")) g.Expect(mainConfStr).To(ContainSubstring("include /etc/nginx/includes/main_snippet2.conf;")) - g.Expect(files[8].Path).To(Equal("/etc/nginx/main-includes/mgmt.conf")) - mgmtConf := string(files[8].Content) + g.Expect(files[8].Meta.Name).To(Equal("/etc/nginx/main-includes/mgmt.conf")) + mgmtConf := string(files[8].Contents) g.Expect(mgmtConf).To(ContainSubstring("usage_report endpoint=test-endpoint")) g.Expect(mgmtConf).To(ContainSubstring("license_token /etc/nginx/secrets/license.jwt")) g.Expect(mgmtConf).To(ContainSubstring("deployment_context /etc/nginx/main-includes/deployment_ctx.json")) @@ -223,31 +226,34 @@ func TestGenerate(t *testing.T) { g.Expect(mgmtConf).To(ContainSubstring("ssl_certificate /etc/nginx/secrets/mgmt-tls.crt")) g.Expect(mgmtConf).To(ContainSubstring("ssl_certificate_key /etc/nginx/secrets/mgmt-tls.key")) - g.Expect(files[9].Path).To(Equal("/etc/nginx/secrets/license.jwt")) - g.Expect(string(files[9].Content)).To(Equal("license")) + g.Expect(files[9].Meta.Name).To(Equal("/etc/nginx/secrets/license.jwt")) + g.Expect(string(files[9].Contents)).To(Equal("license")) - g.Expect(files[10].Path).To(Equal("/etc/nginx/secrets/mgmt-ca.crt")) - g.Expect(string(files[10].Content)).To(Equal("ca")) + g.Expect(files[10].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-ca.crt")) + g.Expect(string(files[10].Contents)).To(Equal("ca")) - g.Expect(files[11].Path).To(Equal("/etc/nginx/secrets/mgmt-tls.crt")) - g.Expect(string(files[11].Content)).To(Equal("cert")) + g.Expect(files[11].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-tls.crt")) + g.Expect(string(files[11].Contents)).To(Equal("cert")) - g.Expect(files[12].Path).To(Equal("/etc/nginx/secrets/mgmt-tls.key")) - g.Expect(string(files[12].Content)).To(Equal("key")) + g.Expect(files[12].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-tls.key")) + g.Expect(string(files[12].Contents)).To(Equal("key")) - g.Expect(files[13].Path).To(Equal("/etc/nginx/secrets/test-certbundle.crt")) - certBundle := string(files[13].Content) + g.Expect(files[13].Meta.Name).To(Equal("/etc/nginx/secrets/test-certbundle.crt")) + certBundle := string(files[13].Contents) g.Expect(certBundle).To(Equal("test-cert")) - g.Expect(files[14]).To(Equal(file.File{ - Type: file.TypeSecret, - Path: "/etc/nginx/secrets/test-keypair.pem", - Content: []byte("test-cert\ntest-key"), + g.Expect(files[14]).To(Equal(agent.File{ + Meta: &pb.FileMeta{ + Name: "/etc/nginx/secrets/test-keypair.pem", + Hash: filesHelper.GenerateHash([]byte("test-cert\ntest-key")), + Permissions: file.SecretFileMode, + }, + Contents: []byte("test-cert\ntest-key"), })) - g.Expect(files[15].Path).To(Equal("/etc/nginx/stream-conf.d/stream.conf")) - g.Expect(files[15].Type).To(Equal(file.TypeRegular)) - streamCfg := string(files[15].Content) + g.Expect(files[15].Meta.Name).To(Equal("/etc/nginx/stream-conf.d/stream.conf")) + g.Expect(files[15].Meta.Permissions).To(Equal(file.RegularFileMode)) + streamCfg := string(files[15].Contents) g.Expect(streamCfg).To(ContainSubstring("listen unix:/var/run/nginx/app.example.com-443.sock")) g.Expect(streamCfg).To(ContainSubstring("listen 443")) g.Expect(streamCfg).To(ContainSubstring("app.example.com unix:/var/run/nginx/app.example.com-443.sock")) diff --git a/internal/mode/static/nginx/config/main_config.go b/internal/mode/static/nginx/config/main_config.go index bd6f2256f9..0d75cea1f4 100644 --- a/internal/mode/static/nginx/config/main_config.go +++ b/internal/mode/static/nginx/config/main_config.go @@ -3,8 +3,12 @@ package config import ( gotemplate "text/template" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + filesHelper "github.com/nginx/agent/v3/pkg/files" + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/shared" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" @@ -50,7 +54,7 @@ type mgmtConf struct { // generateMgmtFiles generates the NGINX Plus configuration file for the mgmt block. As part of this, // it writes the secret and deployment context files that are referenced in the mgmt block. -func (g GeneratorImpl) generateMgmtFiles(conf dataplane.Configuration) []file.File { +func (g GeneratorImpl) generateMgmtFiles(conf dataplane.Configuration) []agent.File { if !g.plus { return nil } @@ -60,47 +64,59 @@ func (g GeneratorImpl) generateMgmtFiles(conf dataplane.Configuration) []file.Fi panic("nginx plus token not set in expected map") } - tokenFile := file.File{ - Content: tokenContent, - Path: secretsFolder + "/license.jwt", - Type: file.TypeSecret, + tokenFile := agent.File{ + Meta: &pb.FileMeta{ + Name: secretsFolder + "/license.jwt", + Hash: filesHelper.GenerateHash(tokenContent), + Permissions: file.SecretFileMode, + }, + Contents: tokenContent, } - files := []file.File{tokenFile} + files := []agent.File{tokenFile} cfg := mgmtConf{ Endpoint: g.usageReportConfig.Endpoint, Resolver: g.usageReportConfig.Resolver, - LicenseTokenFile: tokenFile.Path, + LicenseTokenFile: tokenFile.Meta.Name, SkipVerify: g.usageReportConfig.SkipVerify, } if content, ok := conf.AuxiliarySecrets[graph.PlusReportCACertificate]; ok { - caFile := file.File{ - Content: content, - Path: secretsFolder + "/mgmt-ca.crt", - Type: file.TypeSecret, + caFile := agent.File{ + Meta: &pb.FileMeta{ + Name: secretsFolder + "/mgmt-ca.crt", + Hash: filesHelper.GenerateHash(content), + Permissions: file.SecretFileMode, + }, + Contents: content, } - cfg.CACertFile = caFile.Path + cfg.CACertFile = caFile.Meta.Name files = append(files, caFile) } if content, ok := conf.AuxiliarySecrets[graph.PlusReportClientSSLCertificate]; ok { - certFile := file.File{ - Content: content, - Path: secretsFolder + "/mgmt-tls.crt", - Type: file.TypeSecret, + certFile := agent.File{ + Meta: &pb.FileMeta{ + Name: secretsFolder + "/mgmt-tls.crt", + Hash: filesHelper.GenerateHash(content), + Permissions: file.SecretFileMode, + }, + Contents: content, } - cfg.ClientSSLCertFile = certFile.Path + cfg.ClientSSLCertFile = certFile.Meta.Name files = append(files, certFile) } if content, ok := conf.AuxiliarySecrets[graph.PlusReportClientSSLKey]; ok { - keyFile := file.File{ - Content: content, - Path: secretsFolder + "/mgmt-tls.key", - Type: file.TypeSecret, + keyFile := agent.File{ + Meta: &pb.FileMeta{ + Name: secretsFolder + "/mgmt-tls.key", + Hash: filesHelper.GenerateHash(content), + Permissions: file.SecretFileMode, + }, + Contents: content, } - cfg.ClientSSLKeyFile = keyFile.Path + cfg.ClientSSLKeyFile = keyFile.Meta.Name files = append(files, keyFile) } @@ -111,10 +127,14 @@ func (g GeneratorImpl) generateMgmtFiles(conf dataplane.Configuration) []file.Fi files = append(files, deploymentCtxFile) } - mgmtBlockFile := file.File{ - Content: helpers.MustExecuteTemplate(mgmtConfigTemplate, cfg), - Path: mgmtIncludesFile, - Type: file.TypeRegular, + mgmtContents := helpers.MustExecuteTemplate(mgmtConfigTemplate, cfg) + mgmtBlockFile := agent.File{ + Meta: &pb.FileMeta{ + Name: mgmtIncludesFile, + Hash: filesHelper.GenerateHash(mgmtContents), + Permissions: file.RegularFileMode, + }, + Contents: mgmtContents, } return append(files, mgmtBlockFile) diff --git a/internal/mode/static/state/conditions/conditions.go b/internal/mode/static/state/conditions/conditions.go index c97f9efa2d..1d79ed3487 100644 --- a/internal/mode/static/state/conditions/conditions.go +++ b/internal/mode/static/state/conditions/conditions.go @@ -19,7 +19,7 @@ const ( // ListenerMessageFailedNginxReload is a message used with ListenerConditionProgrammed (false) // when nginx fails to reload. ListenerMessageFailedNginxReload = "The Listener is not programmed due to a failure to " + - "reload nginx with the configuration. Please see the nginx container logs for any possible configuration issues." + "reload nginx with the configuration" // RouteReasonBackendRefUnsupportedValue is used with the "ResolvedRefs" condition when one of the // Route rules has a backendRef with an unsupported value. @@ -68,7 +68,7 @@ const ( // GatewayMessageFailedNginxReload is a message used with GatewayConditionProgrammed (false) // when nginx fails to reload. GatewayMessageFailedNginxReload = "The Gateway is not programmed due to a failure to " + - "reload nginx with the configuration. Please see the nginx container logs for any possible configuration issues" + "reload nginx with the configuration" // RouteMessageFailedNginxReload is a message used with RouteReasonGatewayNotProgrammed // when nginx fails to reload. diff --git a/internal/mode/static/state/graph/graph.go b/internal/mode/static/state/graph/graph.go index 0b56ec1018..40a04598a9 100644 --- a/internal/mode/static/state/graph/graph.go +++ b/internal/mode/static/state/graph/graph.go @@ -83,6 +83,14 @@ type Graph struct { SnippetsFilters map[types.NamespacedName]*SnippetsFilter // PlusSecrets holds the secrets related to NGINX Plus licensing. PlusSecrets map[types.NamespacedName][]PlusSecretFile + + LatestReloadResult NginxReloadResult +} + +// NginxReloadResult describes the result of an NGINX reload. +type NginxReloadResult struct { + // Error is the error that occurred during the reload. + Error error } // ProtectedPorts are the ports that may not be configured by a listener with a descriptive name of each port. diff --git a/internal/mode/static/status/prepare_requests.go b/internal/mode/static/status/prepare_requests.go index e0add956a8..f678054aa2 100644 --- a/internal/mode/static/status/prepare_requests.go +++ b/internal/mode/static/status/prepare_requests.go @@ -19,18 +19,12 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" ) -// NginxReloadResult describes the result of an NGINX reload. -type NginxReloadResult struct { - // Error is the error that occurred during the reload. - Error error -} - // PrepareRouteRequests prepares status UpdateRequests for the given Routes. func PrepareRouteRequests( l4routes map[graph.L4RouteKey]*graph.L4Route, routes map[graph.RouteKey]*graph.L7Route, transitionTime metav1.Time, - nginxReloadRes NginxReloadResult, + nginxReloadRes graph.NginxReloadResult, gatewayCtlrName string, ) []frameworkStatus.UpdateRequest { reqs := make([]frameworkStatus.UpdateRequest, 0, len(routes)) @@ -107,7 +101,7 @@ func prepareRouteStatus( gatewayCtlrName string, parentRefs []graph.ParentRef, conds []conditions.Condition, - nginxReloadRes NginxReloadResult, + nginxReloadRes graph.NginxReloadResult, transitionTime metav1.Time, srcGeneration int64, ) v1.RouteStatus { @@ -214,7 +208,7 @@ func PrepareGatewayRequests( ignoredGateways map[types.NamespacedName]*v1.Gateway, transitionTime metav1.Time, gwAddresses []v1.GatewayStatusAddress, - nginxReloadRes NginxReloadResult, + nginxReloadRes graph.NginxReloadResult, ) []frameworkStatus.UpdateRequest { reqs := make([]frameworkStatus.UpdateRequest, 0, 1+len(ignoredGateways)) @@ -240,7 +234,7 @@ func prepareGatewayRequest( gateway *graph.Gateway, transitionTime metav1.Time, gwAddresses []v1.GatewayStatusAddress, - nginxReloadRes NginxReloadResult, + nginxReloadRes graph.NginxReloadResult, ) frameworkStatus.UpdateRequest { if !gateway.Valid { conds := conditions.ConvertConditions( @@ -272,9 +266,10 @@ func prepareGatewayRequest( } if nginxReloadRes.Error != nil { + msg := fmt.Sprintf("%s: %s", staticConds.ListenerMessageFailedNginxReload, nginxReloadRes.Error.Error()) conds = append( conds, - staticConds.NewListenerNotProgrammedInvalid(staticConds.ListenerMessageFailedNginxReload), + staticConds.NewListenerNotProgrammedInvalid(msg), ) } @@ -300,9 +295,10 @@ func prepareGatewayRequest( } if nginxReloadRes.Error != nil { + msg := fmt.Sprintf("%s: %s", staticConds.GatewayMessageFailedNginxReload, nginxReloadRes.Error.Error()) gwConds = append( gwConds, - staticConds.NewGatewayNotProgrammedInvalid(staticConds.GatewayMessageFailedNginxReload), + staticConds.NewGatewayNotProgrammedInvalid(msg), ) } diff --git a/internal/mode/static/status/prepare_requests_test.go b/internal/mode/static/status/prepare_requests_test.go index d52b43e7a8..03b9cd69bc 100644 --- a/internal/mode/static/status/prepare_requests_test.go +++ b/internal/mode/static/status/prepare_requests_test.go @@ -3,6 +3,7 @@ package status import ( "context" "errors" + "fmt" "testing" . "github.com/onsi/gomega" @@ -274,7 +275,7 @@ func TestBuildHTTPRouteStatuses(t *testing.T) { map[graph.L4RouteKey]*graph.L4Route{}, routes, transitionTime, - NginxReloadResult{}, + graph.NginxReloadResult{}, gatewayCtlrName, ) @@ -353,7 +354,7 @@ func TestBuildGRPCRouteStatuses(t *testing.T) { map[graph.L4RouteKey]*graph.L4Route{}, routes, transitionTime, - NginxReloadResult{}, + graph.NginxReloadResult{}, gatewayCtlrName, ) @@ -430,7 +431,7 @@ func TestBuildTLSRouteStatuses(t *testing.T) { routes, map[graph.RouteKey]*graph.L7Route{}, transitionTime, - NginxReloadResult{}, + graph.NginxReloadResult{}, gatewayCtlrName, ) @@ -534,7 +535,7 @@ func TestBuildRouteStatusesNginxErr(t *testing.T) { map[graph.L4RouteKey]*graph.L4Route{}, routes, transitionTime, - NginxReloadResult{Error: errors.New("test error")}, + graph.NginxReloadResult{Error: errors.New("test error")}, gatewayCtlrName, ) @@ -740,7 +741,7 @@ func TestBuildGatewayStatuses(t *testing.T) { routeKey := graph.RouteKey{NamespacedName: types.NamespacedName{Namespace: "test", Name: "hr-1"}} tests := []struct { - nginxReloadRes NginxReloadResult + nginxReloadRes graph.NginxReloadResult gateway *graph.Gateway ignoredGateways map[types.NamespacedName]*v1.Gateway expected map[types.NamespacedName]v1.GatewayStatus @@ -1087,7 +1088,7 @@ func TestBuildGatewayStatuses(t *testing.T) { ObservedGeneration: 2, LastTransitionTime: transitionTime, Reason: string(v1.GatewayReasonInvalid), - Message: staticConds.GatewayMessageFailedNginxReload, + Message: fmt.Sprintf("%s: test error", staticConds.GatewayMessageFailedNginxReload), }, }, Listeners: []v1.ListenerStatus{ @@ -1125,14 +1126,14 @@ func TestBuildGatewayStatuses(t *testing.T) { ObservedGeneration: 2, LastTransitionTime: transitionTime, Reason: string(v1.ListenerReasonInvalid), - Message: staticConds.ListenerMessageFailedNginxReload, + Message: fmt.Sprintf("%s: test error", staticConds.ListenerMessageFailedNginxReload), }, }, }, }, }, }, - nginxReloadRes: NginxReloadResult{Error: errors.New("test error")}, + nginxReloadRes: graph.NginxReloadResult{Error: errors.New("test error")}, }, } diff --git a/internal/mode/static/status/queue.go b/internal/mode/static/status/queue.go new file mode 100644 index 0000000000..5f31bbec6d --- /dev/null +++ b/internal/mode/static/status/queue.go @@ -0,0 +1,66 @@ +package status + +import ( + "context" + "sync" + + "k8s.io/apimachinery/pkg/types" +) + +// QueueObject is the object to be passed to the queue for status updates. +type QueueObject struct { + Error error + Deployment types.NamespacedName +} + +// Queue represents a queue with unlimited size. +type Queue struct { + notifyCh chan struct{} + items []*QueueObject + + lock sync.Mutex +} + +// NewQueue returns a new Queue object. +func NewQueue() *Queue { + return &Queue{ + items: []*QueueObject{}, + notifyCh: make(chan struct{}, 1), + } +} + +// Enqueue adds an item to the queue and notifies any blocked readers. +func (q *Queue) Enqueue(item *QueueObject) { + q.lock.Lock() + defer q.lock.Unlock() + + q.items = append(q.items, item) + + select { + case q.notifyCh <- struct{}{}: + default: + } +} + +// Dequeue removes and returns the front item from the queue. +// It blocks if the queue is empty or when the context is canceled. +func (q *Queue) Dequeue(ctx context.Context) *QueueObject { + q.lock.Lock() + defer q.lock.Unlock() + + for len(q.items) == 0 { + q.lock.Unlock() + select { + case <-ctx.Done(): + q.lock.Lock() + return nil + case <-q.notifyCh: + q.lock.Lock() + } + } + + front := q.items[0] + q.items = q.items[1:] + + return front +} diff --git a/internal/mode/static/status/queue_test.go b/internal/mode/static/status/queue_test.go new file mode 100644 index 0000000000..0bed3cee62 --- /dev/null +++ b/internal/mode/static/status/queue_test.go @@ -0,0 +1,94 @@ +package status + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" +) + +func TestNewQueue(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + q := NewQueue() + + g.Expect(q).ToNot(BeNil()) + g.Expect(q.items).To(BeEmpty()) +} + +func TestEnqueue(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + q := NewQueue() + item := &QueueObject{ + Error: nil, + Deployment: types.NamespacedName{Namespace: "default", Name: "test-object"}, + } + q.Enqueue(item) + + g.Expect(q.items).To(HaveLen(1)) + g.Expect(q.items[0]).To(Equal(item)) +} + +func TestDequeue(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + q := NewQueue() + item := &QueueObject{ + Error: nil, + Deployment: types.NamespacedName{Namespace: "default", Name: "test-object"}, + } + q.Enqueue(item) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dequeuedItem := q.Dequeue(ctx) + g.Expect(dequeuedItem).To(Equal(item)) + g.Expect(q.items).To(BeEmpty()) +} + +func TestDequeueEmptyQueue(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + q := NewQueue() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + dequeuedItem := q.Dequeue(ctx) + g.Expect(dequeuedItem).To(BeNil()) +} + +func TestDequeueWithMultipleItems(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + q := NewQueue() + item1 := &QueueObject{ + Error: nil, + Deployment: types.NamespacedName{Namespace: "default", Name: "test-object-1"}, + } + item2 := &QueueObject{ + Error: nil, + Deployment: types.NamespacedName{Namespace: "default", Name: "test-object-2"}, + } + q.Enqueue(item1) + q.Enqueue(item2) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dequeuedItem1 := q.Dequeue(ctx) + g.Expect(dequeuedItem1).To(Equal(item1)) + + dequeuedItem2 := q.Dequeue(ctx) + + g.Expect(dequeuedItem2).To(Equal(item2)) + g.Expect(q.items).To(BeEmpty()) +} diff --git a/site/content/how-to/monitoring/troubleshooting.md b/site/content/how-to/monitoring/troubleshooting.md index 5c8cc89e4c..67a698187a 100644 --- a/site/content/how-to/monitoring/troubleshooting.md +++ b/site/content/how-to/monitoring/troubleshooting.md @@ -354,17 +354,6 @@ Events: Normal Started 39s kubelet Started container nginx ``` -##### Insufficient Privileges errors - -Depending on your environment's configuration, the control plane may not have the proper permissions to reload NGINX. The NGINX configuration will not be applied and you will see the following error in the _nginx-gateway_ logs: - -`failed to reload NGINX: failed to send the HUP signal to NGINX main: operation not permitted` - -To **resolve** this issue you will need to set `allowPrivilegeEscalation` to `true`. - -- If using Helm, you can set the `nginxGateway.securityContext.allowPrivilegeEscalation` value. -- If using the manifests directly, you can update this field under the `nginx-gateway` container's `securityContext`. - ##### NGINX Plus failure to start or traffic interruptions Beginning with NGINX Gateway Fabric 1.5.0, NGINX Plus requires a valid JSON Web Token (JWT) to run. If this is not set up properly, or your JWT token has expired, you may see errors in the NGINX logs that look like the following: diff --git a/tests/go.mod b/tests/go.mod index 2ca50a5bc7..268ae2c7c3 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -15,7 +15,7 @@ require ( k8s.io/apiextensions-apiserver v0.32.0 k8s.io/apimachinery v0.32.0 k8s.io/client-go v0.32.0 - sigs.k8s.io/controller-runtime v0.19.3 + sigs.k8s.io/controller-runtime v0.20.0 sigs.k8s.io/gateway-api v1.2.1 sigs.k8s.io/yaml v1.4.0 ) @@ -26,6 +26,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -34,6 +35,7 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect diff --git a/tests/go.sum b/tests/go.sum index 8418d36ab2..856a035bea 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -18,6 +18,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -38,6 +40,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -226,8 +230,8 @@ k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6J k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw= pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= -sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= -sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= +sigs.k8s.io/controller-runtime v0.20.0 h1:jjkMo29xEXH+02Md9qaVXfEIaMESSpy3TBWPrsfQkQs= +sigs.k8s.io/controller-runtime v0.20.0/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU= sigs.k8s.io/gateway-api v1.2.1 h1:fZZ/+RyRb+Y5tGkwxFKuYuSRQHu9dZtbjenblleOLHM= sigs.k8s.io/gateway-api v1.2.1/go.mod h1:EpNfEXNjiYfUJypf0eZ0P5iXA9ekSGWaS1WgPaM42X0= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=