feat(kuma-cp) XDS, SDS, Bootstrap on one port

app/kuma-cp/cmd/run.go

@@ -16,14 +16,13 @@ import (
 	"github.com/kumahq/kuma/pkg/defaults"
 	"github.com/kumahq/kuma/pkg/diagnostics"
 	"github.com/kumahq/kuma/pkg/dns/components"
+	dp_server "github.com/kumahq/kuma/pkg/dp-server"
 	"github.com/kumahq/kuma/pkg/gc"
 	kds_global "github.com/kumahq/kuma/pkg/kds/global"
 	kds_remote "github.com/kumahq/kuma/pkg/kds/remote"
 	mads_server "github.com/kumahq/kuma/pkg/mads/server"
 	metrics "github.com/kumahq/kuma/pkg/metrics/components"
-	sds_server "github.com/kumahq/kuma/pkg/sds/server"
 	kuma_version "github.com/kumahq/kuma/pkg/version"
-	xds_server "github.com/kumahq/kuma/pkg/xds/server"
 )
 
 var (
@@ -76,14 +75,6 @@ func newRunCmdWithOpts(opts runCmdOpts) *cobra.Command {
 			runLog.Info(fmt.Sprintf("Running in mode `%s`", cfg.Mode))
 			switch cfg.Mode {
 			case config_core.Standalone:
-				if err := sds_server.SetupServer(rt); err != nil {
-					runLog.Error(err, "unable to set up SDS server")
-					return err
-				}
-				if err := xds_server.SetupServer(rt); err != nil {
-					runLog.Error(err, "unable to set up xDS server")
-					return err
-				}
 				if err := mads_server.SetupServer(rt); err != nil {
 					runLog.Error(err, "unable to set up Monitoring Assignment server")
 					return err
@@ -96,15 +87,11 @@ func newRunCmdWithOpts(opts runCmdOpts) *cobra.Command {
 					runLog.Error(err, "unable to set up GC")
 					return err
 				}
-			case config_core.Remote:
-				if err := sds_server.SetupServer(rt); err != nil {
-					runLog.Error(err, "unable to set up SDS server")
-					return err
-				}
-				if err := xds_server.SetupServer(rt); err != nil {
-					runLog.Error(err, "unable to set up xDS server")
+				if err := dp_server.SetupServer(rt); err != nil {
+					runLog.Error(err, "unable to set up DP Server")
 					return err
 				}
+			case config_core.Remote:
 				if err := mads_server.SetupServer(rt); err != nil {
 					runLog.Error(err, "unable to set up Monitoring Assignment server")
 					return err
@@ -121,6 +108,10 @@ func newRunCmdWithOpts(opts runCmdOpts) *cobra.Command {
 					runLog.Error(err, "unable to set up GC")
 					return err
 				}
+				if err := dp_server.SetupServer(rt); err != nil {
+					runLog.Error(err, "unable to set up DP Server")
+					return err
+				}
 			case config_core.Global:
 				if err := kds_global.Setup(rt); err != nil {
 					runLog.Error(err, "unable to set up KDS Global")

mk/kind.mk

@@ -119,8 +119,6 @@ kind/deploy/example-app:
 run/k8s: fmt vet ## Dev: Run Control Plane locally in Kubernetes mode
 	@KUBECONFIG=$(KIND_KUBECONFIG) $(MAKE) crd/upgrade -C pkg/plugins/resources/k8s/native
 	KUBECONFIG=$(KIND_KUBECONFIG) \
-	KUMA_SDS_SERVER_GRPC_PORT=$(SDS_GRPC_PORT) \
-	KUMA_GRPC_PORT=$(CP_GRPC_PORT) \
 	KUMA_ENVIRONMENT=kubernetes \
 	KUMA_STORE_TYPE=kubernetes \
 	KUMA_SDS_SERVER_TLS_CERT_FILE=app/kuma-cp/cmd/testdata/tls.crt \

mk/run.mk

@@ -1,10 +1,5 @@
 GO_RUN := CGO_ENABLED=0 go run $(GOFLAGS) $(LD_FLAGS)
 
-CP_BIND_HOST ?= localhost
-CP_GRPC_PORT ?= 5678
-SDS_GRPC_PORT ?= 5677
-CP_K8S_ADMISSION_PORT ?= 5443
-
 EXAMPLE_DATAPLANE_MESH ?= default
 EXAMPLE_DATAPLANE_NAME ?= example
 ENVOY_ADMIN_PORT ?= 9901
@@ -32,8 +27,6 @@ run/universal/postgres: fmt vet ## Dev: Run Control Plane locally in universal m
 	KUMA_STORE_POSTGRES_TLS_CA_PATH=$(POSTGRES_SSL_ROOT_CERT_PATH) \
 	$(GO_RUN) ./app/kuma-cp/main.go migrate up --log-level=debug
 
-	KUMA_SDS_SERVER_GRPC_PORT=$(SDS_GRPC_PORT) \
-	KUMA_GRPC_PORT=$(CP_GRPC_PORT) \
 	KUMA_ENVIRONMENT=universal \
 	KUMA_STORE_TYPE=postgres \
 	KUMA_STORE_POSTGRES_HOST=localhost \
@@ -65,8 +58,6 @@ config_dump/example/envoy: ## Dev: Dump effective configuration of example Envoy
 
 .PHONY: run/universal/memory
 run/universal/memory: ## Dev: Run Control Plane locally in universal mode with in-memory store
-	KUMA_SDS_SERVER_GRPC_PORT=$(SDS_GRPC_PORT) \
-	KUMA_GRPC_PORT=$(CP_GRPC_PORT) \
 	KUMA_ENVIRONMENT=universal \
 	KUMA_STORE_TYPE=memory \
 	$(GO_RUN) ./app/kuma-cp/main.go run --log-level=debug

pkg/api-server/config_ws_test.go

@@ -77,10 +77,7 @@ var _ = Describe("Config WS", func() {
               "xdsConnectTimeout": "1s",
               "xdsHost": "",
               "xdsPort": 0
-            },
-            "port": 5682,
-            "tlsCertFile": "",
-            "tlsKeyFile": ""
+            }
           },
           "adminServer": {
             "local": {
@@ -217,11 +214,13 @@ var _ = Describe("Config WS", func() {
             }
           },
           "sdsServer": {
-            "grpcPort": 5677,
-            "tlsCertFile": "",
-            "tlsKeyFile": "",
             "dataplaneConfigurationRefreshInterval": "1s"
           },
+          "dpServer": {
+            "port": 5678,
+            "tlsCertFile": "",
+            "tlsKeyFile": ""
+          },
           "store": {
             "kubernetes": {
               "systemNamespace": "kuma-system"
@@ -250,10 +249,7 @@ var _ = Describe("Config WS", func() {
           "xdsServer": {
             "dataplaneConfigurationRefreshInterval": "1s",
             "dataplaneStatusFlushInterval": "1s",
-            "diagnosticsPort": 5680,
-            "grpcPort": 5678,
-            "tlsCertFile": "",
-            "tlsKeyFile": ""
+            "diagnosticsPort": 5680
           },
           "diagnostics": {
             "debugEndpoints": false

pkg/config/app/kuma-cp/config.go

@@ -5,17 +5,17 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/kumahq/kuma/pkg/config/diagnostics"
-	"github.com/kumahq/kuma/pkg/config/multicluster"
-
 	"github.com/kumahq/kuma/pkg/config"
 	admin_server "github.com/kumahq/kuma/pkg/config/admin-server"
 	api_server "github.com/kumahq/kuma/pkg/config/api-server"
 	"github.com/kumahq/kuma/pkg/config/core"
 	"github.com/kumahq/kuma/pkg/config/core/resources/store"
+	"github.com/kumahq/kuma/pkg/config/diagnostics"
 	dns_server "github.com/kumahq/kuma/pkg/config/dns-server"
+	dp_server "github.com/kumahq/kuma/pkg/config/dp-server"
 	gui_server "github.com/kumahq/kuma/pkg/config/gui-server"
 	"github.com/kumahq/kuma/pkg/config/mads"
+	"github.com/kumahq/kuma/pkg/config/multicluster"
 	"github.com/kumahq/kuma/pkg/config/plugins/runtime"
 	"github.com/kumahq/kuma/pkg/config/sds"
 	"github.com/kumahq/kuma/pkg/config/xds"
@@ -124,6 +124,8 @@ type Config struct {
 	DNSServer *dns_server.DNSServerConfig `yaml:"dnsServer,omitempty"`
 	// Diagnostics configuration
 	Diagnostics *diagnostics.DiagnosticsConfig `yaml:"diagnostics,omitempty"`
+	// Dataplane Server configuration
+	DpServer *dp_server.DpServerConfig `yaml:"dpServer"`
 }
 
 func (c *Config) Sanitize() {
@@ -177,6 +179,7 @@ func DefaultConfig() Config {
 		DNSServer:    dns_server.DefaultDNSServerConfig(),
 		Multicluster: multicluster.DefaultMulticlusterConfig(),
 		Diagnostics:  diagnostics.DefaultDiagnosticsConfig(),
+		DpServer:     dp_server.DefaultDpServerConfig(),
 	}
 }
 

pkg/config/app/kuma-cp/kuma-cp.defaults.yaml

@@ -49,12 +49,6 @@ store:
 
 # Configuration of Bootstrap Server, which provides bootstrap config to Dataplanes
 bootstrapServer:
-  # Port of Server that provides bootstrap configuration for dataplanes
-  port: 5682 # ENV: KUMA_BOOTSTRAP_SERVER_PORT
-  # TlsCertFile defines a path to a file with PEM-encoded TLS cert.
-  tlsCertFile: # ENV: KUMA_BOOTSTRAP_SERVER_TLS_CERT_FILE
-  # TlsKeyFile defines a path to a file with PEM-encoded TLS key.
-  tlsKeyFile: # ENV: KUMA_BOOTSTRAP_SERVER_TLS_KEY_FILE
   # Parameters of bootstrap configuration
   params:
     # Address of Envoy Admin
@@ -72,12 +66,6 @@ bootstrapServer:
 
 # Envoy SDS server configuration
 sdsServer:
-  # Port of GRPC server that Envoy connects to
-  grpcPort: 5677 # ENV: KUMA_SDS_SERVER_GRPC_PORT
-  # TlsCertFile defines a path to a file with PEM-encoded TLS cert.
-  tlsCertFile: # ENV: KUMA_SDS_SERVER_TLS_CERT_FILE
-  # TlsKeyFile defines a path to a file with PEM-encoded TLS key.
-  tlsKeyFile: # ENV: KUMA_SDS_SERVER_TLS_KEY_FILE
   # Interval for re-genarting configuration for Dataplanes connected to the Control Plane
   dataplaneConfigurationRefreshInterval: 1s # ENV: KUMA_SDS_SERVER_DATAPLANE_CONFIGURATION_REFRESH_INTERVAL
 
@@ -117,18 +105,12 @@ adminServer:
 
 # Envoy XDS server configuration
 xdsServer:
-  # Port of GRPC server that Envoy connects to
-  grpcPort: 5678 # ENV: KUMA_XDS_SERVER_GRPC_PORT
   # Port of Diagnostic Server for checking health and readiness of the Control Plane
   diagnosticsPort: 5680 # ENV: KUMA_XDS_SERVER_DIAGNOSTICS_PORT
   # Interval for re-genarting configuration for Dataplanes connected to the Control Plane
   dataplaneConfigurationRefreshInterval: 1s # ENV: KUMA_XDS_SERVER_DATAPLANE_CONFIGURATION_REFRESH_INTERVAL
   # Interval for flushing status of Dataplanes connected to the Control Plane
   dataplaneStatusFlushInterval: 1s # ENV: KUMA_XDS_SERVER_DATAPLANE_STATUS_FLUSH_INTERVAL
-  # TlsCertFile defines a path to a file with PEM-encoded TLS cert.
-  tlsCertFile: # ENV: KUMA_XDS_SERVER_TLS_CERT_FILE
-  # TlsKeyFile defines a path to a file with PEM-encoded TLS key.
-  tlsKeyFile: # ENV: KUMA_XDS_SERVER_TLS_KEY_FILE
 
 # API Server configuration
 apiServer:
@@ -279,7 +261,11 @@ general:
   # Control Plane will use this value in configuration generated for dataplanes, in responses to `kumactl`, etc.
   advertisedHostname: localhost # ENV: KUMA_GENERAL_ADVERTISED_HOSTNAME
   # dnsCacheTTL represents duration for how long Kuma CP will cache result of resolving dataplane's domain name
-  dnsCacheTTL: 10s
+  dnsCacheTTL: 10s # ENV: KUMA_GENERAL_DNS_CACHE_TTL
+  # TlsCertFile defines a path to a file with PEM-encoded TLS cert that will be used across all the Kuma Servers.
+  tlsCertFile: # ENV: KUMA_GENERAL_TLS_CERT_FILE
+  # TlsKeyFile defines a path to a file with PEM-encoded TLS key that will be used across all the Kuma Servers.
+  tlsKeyFile: # ENV: KUMA_GENERAL_TLS_KEY_FILE
 
 # Web GUI Server configuration
 guiServer:
@@ -324,3 +310,12 @@ multicluster:
 diagnostics:
   # If true, enables https://golang.org/pkg/net/http/pprof/ debug endpoints
   debugEndpoints: false # ENV: KUMA_DIAGNOSTICS_DEBUG_ENDPOINTS
+
+# Dataplane Server configuration that servers API like Bootstrap/XDS/SDS for the Dataplane.
+dpServer:
+  # Port of the DP Server
+  port: 5678 # ENV: KUMA_DP_SERVER_PORT
+  # TlsCertFile defines a path to a file with PEM-encoded TLS cert. If empty, autoconfigured from general.tlsCertFile
+  tlsCertFile: # ENV: KUMA_DP_SERVER_TLS_CERT_FILE
+  # TlsKeyFile defines a path to a file with PEM-encoded TLS key. If empty, autoconfigured from general.tlsKeyFile
+  tlsKeyFile: # ENV: KUMA_DP_SERVER_TLS_KEY_FILE

pkg/config/dp-server/config.go

@@ -0,0 +1,35 @@
+package dp_server
+
+import (
+	"errors"
+
+	"github.com/kumahq/kuma/pkg/config"
+)
+
+var _ config.Config = &DpServerConfig{}
+
+// Dataplane Server configuration that servers API like Bootstrap/XDS/SDS.
+type DpServerConfig struct {
+	// Port of the DP Server
+	Port int `yaml:"port" envconfig:"kuma_dp_server_port"`
+	// TlsCertFile defines a path to a file with PEM-encoded TLS cert. If empty, autoconfigured from general.tlsCertFile
+	TlsCertFile string `yaml:"tlsCertFile" envconfig:"kuma_dp_server_tls_cert_file"`
+	// TlsKeyFile defines a path to a file with PEM-encoded TLS key. If empty, autoconfigured from general.tlsKeyFile
+	TlsKeyFile string `yaml:"tlsKeyFile" envconfig:"kuma_dp_server_tls_key_file"`
+}
+
+func (a *DpServerConfig) Sanitize() {
+}
+
+func (a *DpServerConfig) Validate() error {
+	if a.Port < 0 {
+		return errors.New("Port cannot be negative")
+	}
+	return nil
+}
+
+func DefaultDpServerConfig() *DpServerConfig {
+	return &DpServerConfig{
+		Port: 5678,
+	}
+}

pkg/config/loader_test.go

@@ -63,10 +63,8 @@ var _ = Describe("Config loader", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			// then
-			Expect(cfg.XdsServer.GrpcPort).To(Equal(5000))
 			Expect(cfg.XdsServer.DiagnosticsPort).To(Equal(5003))
 
-			Expect(cfg.BootstrapServer.Port).To(Equal(uint32(5004)))
 			Expect(cfg.BootstrapServer.Params.AdminPort).To(Equal(uint32(1234)))
 			Expect(cfg.BootstrapServer.Params.XdsHost).To(Equal("kuma-control-plane"))
 			Expect(cfg.BootstrapServer.Params.XdsPort).To(Equal(uint32(4321)))
@@ -160,10 +158,8 @@ store:
     enabled: false
     expirationTime: 3s
 xdsServer:
-  grpcPort: 5000
   diagnosticsPort: 5003
 bootstrapServer:
-  port: 5004
   params:
     adminPort: 1234
     xdsHost: kuma-control-plane
@@ -238,9 +234,7 @@ diagnostics:
 		}),
 		Entry("from env variables", testCase{
 			envVars: map[string]string{
-				"KUMA_XDS_SERVER_GRPC_PORT":                                      "5000",
 				"KUMA_XDS_SERVER_DIAGNOSTICS_PORT":                               "5003",
-				"KUMA_BOOTSTRAP_SERVER_PORT":                                     "5004",
 				"KUMA_BOOTSTRAP_SERVER_PARAMS_ADMIN_PORT":                        "1234",
 				"KUMA_BOOTSTRAP_SERVER_PARAMS_XDS_HOST":                          "kuma-control-plane",
 				"KUMA_BOOTSTRAP_SERVER_PARAMS_XDS_PORT":                          "4321",

pkg/config/sds/config.go

@@ -3,27 +3,17 @@ package sds
 import (
 	"time"
 
-	"github.com/pkg/errors"
-
 	"github.com/kumahq/kuma/pkg/config"
 )
 
 func DefaultSdsServerConfig() *SdsServerConfig {
 	return &SdsServerConfig{
-		GrpcPort:                              5677,
 		DataplaneConfigurationRefreshInterval: 1 * time.Second,
 	}
 }
 
 // Envoy SDS server configuration
 type SdsServerConfig struct {
-	// Port of GRPC server that Envoy connects to
-	GrpcPort int `yaml:"grpcPort" envconfig:"kuma_sds_server_grpc_port"`
-	// TlsCertFile defines a path to a file with PEM-encoded TLS cert.
-	TlsCertFile string `yaml:"tlsCertFile" envconfig:"kuma_sds_server_tls_cert_file"`
-	// TlsKeyFile defines a path to a file with PEM-encoded TLS key.
-	TlsKeyFile string `yaml:"tlsKeyFile" envconfig:"kuma_sds_server_tls_key_file"`
-
 	// Interval for re-genarting configuration for Dataplanes connected to the Control Plane
 	DataplaneConfigurationRefreshInterval time.Duration `yaml:"dataplaneConfigurationRefreshInterval" envconfig:"kuma_sds_server_dataplane_configuration_refresh_interval"`
 }
@@ -34,14 +24,5 @@ func (c *SdsServerConfig) Sanitize() {
 }
 
 func (c *SdsServerConfig) Validate() error {
-	if c.GrpcPort < 0 {
-		return errors.New("GrpcPort cannot be negative")
-	}
-	if c.TlsCertFile == "" && c.TlsKeyFile != "" {
-		return errors.New("TlsCertFile cannot be empty if TlsKeyFile has been set")
-	}
-	if c.TlsKeyFile == "" && c.TlsCertFile != "" {
-		return errors.New("TlsKeyFile cannot be empty if TlsCertFile has been set")
-	}
 	return nil
 }

pkg/config/xds/bootstrap/config.go

@@ -12,24 +12,15 @@ import (
 var _ config.Config = &BootstrapServerConfig{}
 
 type BootstrapServerConfig struct {
-	// Port of Server that provides bootstrap configuration for dataplanes
-	Port uint32 `yaml:"port" envconfig:"kuma_bootstrap_server_port"`
 	// Parameters of bootstrap configuration
 	Params *BootstrapParamsConfig `yaml:"params"`
-	// TlsCertFile defines a path to a file with PEM-encoded TLS cert.
-	TlsCertFile string `yaml:"tlsCertFile" envconfig:"kuma_bootstrap_server_tls_cert_file"`
-	// TlsKeyFile defines a path to a file with PEM-encoded TLS key.
-	TlsKeyFile string `yaml:"tlsKeyFile" envconfig:"kuma_bootstrap_server_tls_key_file"`
 }
 
 func (b *BootstrapServerConfig) Sanitize() {
 	b.Params.Sanitize()
 }
 
 func (b *BootstrapServerConfig) Validate() error {
-	if b.Port > 65535 {
-		return errors.New("Port must be in the range [0, 65535]")
-	}
 	if err := b.Params.Validate(); err != nil {
 		return errors.Wrap(err, "Params validation failed")
 	}
@@ -38,7 +29,6 @@ func (b *BootstrapServerConfig) Validate() error {
 
 func DefaultBootstrapServerConfig() *BootstrapServerConfig {
 	return &BootstrapServerConfig{
-		Port:   5682,
 		Params: DefaultBootstrapParamsConfig(),
 	}
 }