Support multiple gateways

deploy/manifests/nginx-gateway.yaml

@@ -9,6 +9,13 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: nginx-gateway
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+  - delete
 - apiGroups:
   - ""
   resources:

go.mod

@@ -47,10 +47,12 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/stretchr/testify v1.8.1 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect

go.sum

@@ -278,13 +278,18 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

internal/events/handler.go

@@ -87,10 +87,14 @@ func (h *EventHandlerImpl) HandleEventBatch(ctx context.Context, batch EventBatc
 		h.cfg.Logger.Info("NGINX configuration was successfully updated")
 	}
 
+	// provision or update Services
+
+	// update Gateway-related statuses to include information about Services (IPs)
+
 	h.cfg.StatusUpdater.Update(ctx, statuses)
 }
 
-func (h *EventHandlerImpl) updateNginx(ctx context.Context, conf dataplane.Configuration) error {
+func (h *EventHandlerImpl) updateNginx(ctx context.Context, conf []dataplane.Configuration) error {
 	// Write all secrets (nuke and pave).
 	// This will remove all secrets in the secrets directory before writing the requested secrets.
 	// FIXME(kate-osborn): We may want to rethink this approach in the future and write and remove secrets individually.

internal/manager/manager.go

@@ -127,6 +127,7 @@ func Start(cfg config.Config) error {
 		},
 		EventRecorder: recorder,
 		Scheme:        scheme,
+		Client:        mgr.GetClient(),
 	})
 
 	configGenerator := ngxcfg.NewGeneratorImpl()

internal/nginx/config/generator.go

@@ -10,7 +10,7 @@ import (
 // This interface is used for testing purposes only.
 type Generator interface {
 	// Generate generates NGINX configuration from internal representation.
-	Generate(configuration dataplane.Configuration) []byte
+	Generate(configuration []dataplane.Configuration) []byte
 }
 
 // GeneratorImpl is an implementation of Generator.
@@ -22,13 +22,13 @@ func NewGeneratorImpl() GeneratorImpl {
 }
 
 // executeFunc is a function that generates NGINX configuration from internal representation.
-type executeFunc func(configuration dataplane.Configuration) []byte
+type executeFunc func(configuration []dataplane.Configuration) []byte
 
 // Generate generates NGINX configuration from internal representation.
 // It is the responsibility of the caller to validate the configuration before calling this function.
 // In case of invalid configuration, NGINX will fail to reload or could be configured with malicious configuration.
 // To validate, use the validators from the validation package.
-func (g GeneratorImpl) Generate(conf dataplane.Configuration) []byte {
+func (g GeneratorImpl) Generate(conf []dataplane.Configuration) []byte {
 	var generated []byte
 	for _, execute := range getExecuteFuncs() {
 		generated = append(generated, execute(conf)...)

internal/nginx/config/http/config.go

@@ -2,6 +2,7 @@ package http
 
 // Server holds all configuration for an HTTP server.
 type Server struct {
+	Port          int32
 	SSL           *SSL
 	ServerName    string
 	Locations     []Location

internal/nginx/config/servers.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/nginxinc/nginx-kubernetes-gateway/internal/nginx/config/http"
 	"github.com/nginxinc/nginx-kubernetes-gateway/internal/state/dataplane"
+	"github.com/nginxinc/nginx-kubernetes-gateway/internal/state/graph"
 )
 
 var serversTemplate = gotemplate.Must(gotemplate.New("servers").Parse(serversTemplateText))
@@ -20,53 +21,61 @@ const (
 	rootPath             = "/"
 )
 
-func executeServers(conf dataplane.Configuration) []byte {
-	servers := createServers(conf.HTTPServers, conf.SSLServers)
+func executeServers(confs []dataplane.Configuration) []byte {
+	var servers []http.Server
+
+	for _, conf := range confs {
+		servers = append(servers, createServers(conf.Key, conf.Ports, conf.HTTPServers, conf.SSLServers)...)
+	}
 
 	return execute(serversTemplate, servers)
 }
 
-func createServers(httpServers, sslServers []dataplane.VirtualServer) []http.Server {
+func createServers(key string, ports graph.GatewayPorts, httpServers, sslServers []dataplane.VirtualServer) []http.Server {
+	// user port information to create servers
+
 	servers := make([]http.Server, 0, len(httpServers)+len(sslServers))
 
 	for _, s := range httpServers {
-		servers = append(servers, createServer(s))
+		servers = append(servers, createServer(key, ports.HTTP, s))
 	}
 
 	for _, s := range sslServers {
-		servers = append(servers, createSSLServer(s))
+		servers = append(servers, createSSLServer(key, ports.HTTPS, s))
 	}
 
 	return servers
 }
 
-func createSSLServer(virtualServer dataplane.VirtualServer) http.Server {
+func createSSLServer(key string, httpsPort int32, virtualServer dataplane.VirtualServer) http.Server {
 	if virtualServer.IsDefault {
-		return createDefaultSSLServer()
+		return createDefaultSSLServer(httpsPort)
 	}
 
 	return http.Server{
+		Port:       httpsPort,
 		ServerName: virtualServer.Hostname,
 		SSL: &http.SSL{
 			Certificate:    virtualServer.SSL.CertificatePath,
 			CertificateKey: virtualServer.SSL.CertificatePath,
 		},
-		Locations: createLocations(virtualServer.PathRules, 443),
+		Locations: createLocations(key, virtualServer.PathRules, httpsPort),
 	}
 }
 
-func createServer(virtualServer dataplane.VirtualServer) http.Server {
+func createServer(key string, httpPort int32, virtualServer dataplane.VirtualServer) http.Server {
 	if virtualServer.IsDefault {
-		return createDefaultHTTPServer()
+		return createDefaultHTTPServer(httpPort)
 	}
 
 	return http.Server{
+		Port:       httpPort,
 		ServerName: virtualServer.Hostname,
-		Locations:  createLocations(virtualServer.PathRules, 80),
+		Locations:  createLocations(key, virtualServer.PathRules, httpPort),
 	}
 }
 
-func createLocations(pathRules []dataplane.PathRule, listenerPort int) []http.Location {
+func createLocations(key string, pathRules []dataplane.PathRule, listenerPort int32) []http.Location {
 	lenPathRules := len(pathRules)
 
 	if lenPathRules == 0 {
@@ -126,12 +135,13 @@ func createLocations(pathRules []dataplane.PathRule, listenerPort int) []http.Lo
 
 			// RequestRedirect and proxying are mutually exclusive.
 			if r.Filters.RequestRedirect != nil {
-				loc.Return = createReturnValForRedirectFilter(r.Filters.RequestRedirect, listenerPort)
+				loc.Return = createReturnValForRedirectFilter(r.Filters.RequestRedirect, int(listenerPort))
 				locs = append(locs, loc)
 				continue
 			}
 
 			backendName := backendGroupName(r.BackendGroup)
+			backendName = fmt.Sprintf("%s__%s", key, backendName)
 
 			if backendGroupNeedsSplit(r.BackendGroup) {
 				loc.ProxyPass = createProxyPassForVar(backendName)
@@ -165,12 +175,18 @@ func createLocations(pathRules []dataplane.PathRule, listenerPort int) []http.Lo
 	return locs
 }
 
-func createDefaultSSLServer() http.Server {
-	return http.Server{IsDefaultSSL: true}
+func createDefaultSSLServer(port int32) http.Server {
+	return http.Server{
+		Port:         port,
+		IsDefaultSSL: true,
+	}
 }
 
-func createDefaultHTTPServer() http.Server {
-	return http.Server{IsDefaultHTTP: true}
+func createDefaultHTTPServer(port int32) http.Server {
+	return http.Server{
+		Port:          port,
+		IsDefaultHTTP: true,
+	}
 }
 
 func createReturnValForRedirectFilter(filter *v1beta1.HTTPRequestRedirectFilter, listenerPort int) *http.Return {

internal/nginx/config/servers_template.go

@@ -4,28 +4,31 @@ var serversTemplateText = `
 {{ range $s := . }}
 	{{ if $s.IsDefaultSSL }}
 server {
-	listen 443 ssl default_server;
+	listen {{ $s.Port }} ssl default_server;
 
 	ssl_reject_handshake on;
 }
 	{{ else if $s.IsDefaultHTTP }}
 server {
-	listen 80 default_server;
+	listen {{ $s.Port }} default_server;
 
 	default_type text/html;
 	return 404;
 }
 	{{ else }}
 server {
 		{{ if $s.SSL }}
-	listen 443 ssl;
+	listen {{ $s.Port }} ssl;
 	ssl_certificate {{ $s.SSL.Certificate }};
 	ssl_certificate_key {{ $s.SSL.CertificateKey }};
 
 	if ($ssl_server_name != $host) {
 		return 421;
 	}
+		{{ else }}
+	listen {{ $s.Port }};
 		{{ end }}
+
 
 	server_name {{ $s.ServerName }};
 

internal/nginx/config/split_clients.go

@@ -12,13 +12,17 @@ import (
 
 var splitClientsTemplate = gotemplate.Must(gotemplate.New("split_clients").Parse(splitClientsTemplateText))
 
-func executeSplitClients(conf dataplane.Configuration) []byte {
-	splitClients := createSplitClients(conf.BackendGroups)
+func executeSplitClients(confs []dataplane.Configuration) []byte {
+	var splitClients []http.SplitClient
+
+	for _, conf := range confs {
+		splitClients = append(splitClients, createSplitClients(conf.Key, conf.BackendGroups)...)
+	}
 
 	return execute(splitClientsTemplate, splitClients)
 }
 
-func createSplitClients(backendGroups []graph.BackendGroup) []http.SplitClient {
+func createSplitClients(key string, backendGroups []graph.BackendGroup) []http.SplitClient {
 	numSplits := 0
 	for _, group := range backendGroups {
 		if backendGroupNeedsSplit(group) {
@@ -39,8 +43,10 @@ func createSplitClients(backendGroups []graph.BackendGroup) []http.SplitClient {
 			continue
 		}
 
+		name := fmt.Sprintf("%s__%s", key, group.GroupName())
+
 		splitClients = append(splitClients, http.SplitClient{
-			VariableName:  convertStringToSafeVariableName(group.GroupName()),
+			VariableName:  convertStringToSafeVariableName(name),
 			Distributions: distributions,
 		})
 

internal/nginx/config/upstreams.go

@@ -19,29 +19,33 @@ const (
 	invalidBackendRef = "invalid-backend-ref"
 )
 
-func executeUpstreams(conf dataplane.Configuration) []byte {
-	upstreams := createUpstreams(conf.Upstreams)
+func executeUpstreams(confs []dataplane.Configuration) []byte {
+	var upstreams []http.Upstream
+
+	for _, conf := range confs {
+		upstreams = append(upstreams, createUpstreams(conf.Key, conf.Upstreams)...)
+	}
 
 	return execute(upstreamsTemplate, upstreams)
 }
 
-func createUpstreams(upstreams []dataplane.Upstream) []http.Upstream {
+func createUpstreams(key string, upstreams []dataplane.Upstream) []http.Upstream {
 	// capacity is the number of upstreams + 1 for the invalid backend ref upstream
 	ups := make([]http.Upstream, 0, len(upstreams)+1)
 
 	for _, u := range upstreams {
-		ups = append(ups, createUpstream(u))
+		ups = append(ups, createUpstream(key, u))
 	}
 
-	ups = append(ups, createInvalidBackendRefUpstream())
+	ups = append(ups, createInvalidBackendRefUpstream(key))
 
 	return ups
 }
 
-func createUpstream(up dataplane.Upstream) http.Upstream {
+func createUpstream(key string, up dataplane.Upstream) http.Upstream {
 	if len(up.Endpoints) == 0 {
 		return http.Upstream{
-			Name: up.Name,
+			Name: fmt.Sprintf("%s__%s", key, up.Name),
 			Servers: []http.UpstreamServer{
 				{
 					Address: nginx502Server,
@@ -58,14 +62,14 @@ func createUpstream(up dataplane.Upstream) http.Upstream {
 	}
 
 	return http.Upstream{
-		Name:    up.Name,
+		Name:    fmt.Sprintf("%s__%s", key, up.Name),
 		Servers: upstreamServers,
 	}
 }
 
-func createInvalidBackendRefUpstream() http.Upstream {
+func createInvalidBackendRefUpstream(key string) http.Upstream {
 	return http.Upstream{
-		Name: invalidBackendRef,
+		Name: fmt.Sprintf("%s__%s", key, invalidBackendRef),
 		Servers: []http.UpstreamServer{
 			{
 				Address: nginx500Server,