update nginx template for TLS passthrough (#2166)

Update nginx template for TLS passthrough Problem: nginx configuration templates didn't support TLS passthrough Solution: I added a template setup fro stream servers
nginx · Aug 7, 2024 · 8448d09 · 8448d09
1 parent 7b5c6ab
commit 8448d09
Show file tree

Hide file tree

Showing 35 changed files with 945 additions and 76 deletions.
diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml
@@ -129,6 +129,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -166,6 +168,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -200,6 +204,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

diff --git a/config/tests/static-deployment.yaml b/config/tests/static-deployment.yaml
@@ -72,6 +72,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -102,6 +104,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -121,6 +125,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

diff --git a/deploy/aws-nlb/deploy.yaml b/deploy/aws-nlb/deploy.yaml
@@ -246,6 +246,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -276,6 +278,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -295,6 +299,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml
@@ -243,6 +243,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -273,6 +275,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -294,6 +298,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml
@@ -243,6 +243,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -273,6 +275,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -292,6 +296,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml
@@ -256,6 +256,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -286,6 +288,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -305,6 +309,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml
@@ -247,6 +247,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -277,6 +279,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -296,6 +300,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml
@@ -254,6 +254,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -284,6 +286,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -303,6 +307,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml
@@ -243,6 +243,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -273,6 +275,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -292,6 +296,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml
@@ -251,6 +251,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -281,6 +283,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -300,6 +304,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/internal/mode/static/nginx/conf/nginx-plus.conf b/internal/mode/static/nginx/conf/nginx-plus.conf
@@ -54,6 +54,21 @@ http {
   }
 }
 
+stream {
+	variables_hash_bucket_size 512;
+    variables_hash_max_size 1024;
+
+    map_hash_max_size 2048;
+    map_hash_bucket_size 256;
+
+    log_format stream-main '$remote_addr [$time_local] '
+                              '$protocol $status $bytes_sent $bytes_received '
+                              '$session_time "$ssl_preread_server_name"';
+    access_log /dev/stdout stream-main;
+
+	include /etc/nginx/stream-conf.d/*.conf;
+}
+
 mgmt {
   usage_report interval=0s;
 }
diff --git a/internal/mode/static/nginx/conf/nginx.conf b/internal/mode/static/nginx/conf/nginx.conf
@@ -38,3 +38,18 @@ http {
     }
   }
 }
+
+stream {
+	variables_hash_bucket_size 512;
+    variables_hash_max_size 1024;
+
+    map_hash_max_size 2048;
+    map_hash_bucket_size 256;
+
+    log_format stream-main '$remote_addr [$time_local] '
+                              '$protocol $status $bytes_sent $bytes_received '
+                              '$session_time "$ssl_preread_server_name"';
+    access_log /dev/stdout stream-main;
+
+	include /etc/nginx/stream-conf.d/*.conf;
+}
diff --git a/internal/mode/static/nginx/config/base_http_config_template.go b/internal/mode/static/nginx/config/base_http_config_template.go
@@ -2,4 +2,20 @@ package config
 
 const baseHTTPTemplateText = `
 {{- if .HTTP2 }}http2 on;{{ end }}
+
+# Set $gw_api_compliant_host variable to the value of $http_host unless $http_host is empty, then set it to the value
+# of $host. We prefer $http_host because it contains the original value of the host header, which is required by the
+# Gateway API. However, in an HTTP/1.0 request, it's possible that $http_host can be empty. In this case, we will use
+# the value of $host. See http://nginx.org/en/docs/http/ngx_http_core_module.html#var_host.
+map $http_host $gw_api_compliant_host {
+    '' $host;
+    default $http_host;
+}
+
+# Set $connection_header variable to upgrade when the $http_upgrade header is set, otherwise, set it to close. This
+# allows support for websocket connections. See https://nginx.org/en/docs/http/websocket.html.
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
 `
diff --git a/internal/mode/static/nginx/config/base_http_config_test.go b/internal/mode/static/nginx/config/base_http_config_test.go
@@ -47,5 +47,7 @@ func TestExecuteBaseHttp(t *testing.T) {
 		res := executeBaseHTTPConfig(test.conf)
 		g.Expect(res).To(HaveLen(1))
 		g.Expect(test.expCount).To(Equal(strings.Count(string(res[0].data), expSubStr)))
+		g.Expect(strings.Count(string(res[0].data), "map $http_host $gw_api_compliant_host {")).To(Equal(1))
+		g.Expect(strings.Count(string(res[0].data), "map $http_upgrade $connection_upgrade {")).To(Equal(1))
 	}
 }
diff --git a/internal/mode/static/nginx/config/generator.go b/internal/mode/static/nginx/config/generator.go
@@ -17,6 +17,9 @@ const (
 	// httpFolder is the folder where NGINX HTTP configuration files are stored.
 	httpFolder = configFolder + "/conf.d"
 
+	// streamFolder is the folder where NGINX Stream configuration files are stored.
+	streamFolder = configFolder + "/stream-conf.d"
+
 	// modulesIncludesFolder is the folder where the included "load_module" file is stored.
 	modulesIncludesFolder = configFolder + "/module-includes"
 
@@ -29,6 +32,9 @@ const (
 	// httpConfigFile is the path to the configuration file with HTTP configuration.
 	httpConfigFile = httpFolder + "/http.conf"
 
+	// streamConfigFile is the path to the configuration file with Stream configuration.
+	streamConfigFile = streamFolder + "/stream.conf"
+
 	// configVersionFile is the path to the config version configuration file.
 	configVersionFile = httpFolder + "/config-version.conf"
 
@@ -40,7 +46,7 @@ const (
 )
 
 // ConfigFolders is a list of folders where NGINX configuration files are stored.
-var ConfigFolders = []string{httpFolder, secretsFolder, includesFolder, modulesIncludesFolder}
+var ConfigFolders = []string{httpFolder, secretsFolder, includesFolder, modulesIncludesFolder, streamFolder}
 
 // Generator generates NGINX configuration files.
 // This interface is used for testing purposes only.
@@ -157,6 +163,9 @@ func (g GeneratorImpl) getExecuteFuncs() []executeFunc {
 		executeSplitClients,
 		executeMaps,
 		executeTelemetry,
+		executeStreamServers,
+		g.executeStreamUpstreams,
+		executeStreamMaps,
 	}
 }