feat(kuma-dp) override bootstrap version (#1412)

Signed-off-by: Jakub Dyszkiewicz <[email protected]>

app/kuma-dp/cmd/run.go

@@ -145,6 +145,7 @@ func newRunCmd() *cobra.Command {
 	cmd.PersistentFlags().StringVar(&cfg.ControlPlane.URL, "cp-address", cfg.ControlPlane.URL, "URL of the Control Plane Dataplane Server. Example: https://localhost:5678")
 	cmd.PersistentFlags().StringVar(&cfg.ControlPlane.CaCertFile, "ca-cert-file", cfg.ControlPlane.CaCert, "Path to CA cert by which connection to the Control Plane will be verified if HTTPS is used")
 	cmd.PersistentFlags().StringVar(&cfg.DataplaneRuntime.BinaryPath, "binary-path", cfg.DataplaneRuntime.BinaryPath, "Binary path of Envoy executable")
+	cmd.PersistentFlags().StringVar(&cfg.Dataplane.BootstrapVersion, "bootstrap-version", cfg.Dataplane.BootstrapVersion, "Bootstrap version (and API version) of xDS config. If empty, default version defined in Kuma CP will be used. (ex. '2', '3')")
 	cmd.PersistentFlags().StringVar(&cfg.DataplaneRuntime.ConfigDir, "config-dir", cfg.DataplaneRuntime.ConfigDir, "Directory in which Envoy config will be generated")
 	cmd.PersistentFlags().StringVar(&cfg.DataplaneRuntime.TokenPath, "dataplane-token-file", cfg.DataplaneRuntime.TokenPath, "Path to a file with dataplane token (use 'kumactl generate dataplane-token' to get one)")
 	cmd.PersistentFlags().StringVar(&cfg.DataplaneRuntime.Token, "dataplane-token", cfg.DataplaneRuntime.Token, "Dataplane Token")

app/kuma-dp/cmd/run_test.go

@@ -14,6 +14,7 @@ import (
 	"syscall"
 
 	"github.com/kumahq/kuma/pkg/core/resources/model/rest"
+	"github.com/kumahq/kuma/pkg/xds/bootstrap/types"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
@@ -33,10 +34,10 @@ var _ = Describe("run", func() {
 	BeforeEach(func() {
 		backupSetupSignalHandler = core.SetupSignalHandler
 		backupBootstrapGenerator = bootstrapGenerator
-		bootstrapGenerator = func(_ string, cfg kumadp.Config, _ *rest.Resource, version envoy.EnvoyVersion) ([]byte, error) {
+		bootstrapGenerator = func(_ string, cfg kumadp.Config, _ *rest.Resource, _ types.BootstrapVersion, version envoy.EnvoyVersion) ([]byte, types.BootstrapVersion, error) {
 			respBytes, err := ioutil.ReadFile(filepath.Join("testdata", "bootstrap-config.golden.yaml"))
 			Expect(err).ToNot(HaveOccurred())
-			return respBytes, nil
+			return respBytes, "", nil
 		}
 	})
 	AfterEach(func() {

app/kuma-dp/pkg/dataplane/envoy/envoy.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/kumahq/kuma/pkg/core/resources/model/rest"
+	"github.com/kumahq/kuma/pkg/xds/bootstrap/types"
 
 	"github.com/pkg/errors"
 
@@ -24,7 +25,7 @@ var (
 	runLog = core.Log.WithName("kuma-dp").WithName("run").WithName("envoy")
 )
 
-type BootstrapConfigFactoryFunc func(url string, cfg kuma_dp.Config, dp *rest.Resource, ev EnvoyVersion) ([]byte, error)
+type BootstrapConfigFactoryFunc func(url string, cfg kuma_dp.Config, dp *rest.Resource, bootstrapVersion types.BootstrapVersion, ev EnvoyVersion) ([]byte, types.BootstrapVersion, error)
 
 type Opts struct {
 	Config    kuma_dp.Config
@@ -107,7 +108,7 @@ func (e *Envoy) Start(stop <-chan struct{}) error {
 	}
 	runLog.Info("fetched Envoy version", "version", envoyVersion)
 	runLog.Info("generating bootstrap configuration")
-	bootstrapConfig, err := e.opts.Generator(e.opts.Config.ControlPlane.URL, e.opts.Config, e.opts.Dataplane, *envoyVersion)
+	bootstrapConfig, version, err := e.opts.Generator(e.opts.Config.ControlPlane.URL, e.opts.Config, e.opts.Dataplane, types.BootstrapVersion(e.opts.Config.Dataplane.BootstrapVersion), *envoyVersion)
 	if err != nil {
 		return errors.Errorf("Failed to generate Envoy bootstrap config. %v", err)
 	}
@@ -140,6 +141,9 @@ func (e *Envoy) Start(stop <-chan struct{}) error {
 		// so, let's turn it off to simplify getting started experience.
 		"--disable-hot-restart",
 	}
+	if version != "" { // version is always send by Kuma CP, but we check empty for backwards compatibility reasons (new Kuma DP connects to old Kuma CP)
+		args = append(args, "--bootstrap-version", string(version))
+	}
 	command := exec.CommandContext(ctx, resolvedPath, args...)
 	command.Stdout = e.opts.Stdout
 	command.Stderr = e.opts.Stderr

app/kuma-dp/pkg/dataplane/envoy/envoy_test.go

@@ -17,6 +17,7 @@ import (
 
 	kuma_dp "github.com/kumahq/kuma/pkg/config/app/kuma-dp"
 	"github.com/kumahq/kuma/pkg/core/resources/model/rest"
+	"github.com/kumahq/kuma/pkg/xds/bootstrap/types"
 )
 
 var _ = Describe("Envoy", func() {
@@ -68,9 +69,9 @@ var _ = Describe("Envoy", func() {
 					ConfigDir:  configDir,
 				},
 			}
-			sampleConfig := func(string, kuma_dp.Config, *rest.Resource, EnvoyVersion) ([]byte, error) {
+			sampleConfig := func(string, kuma_dp.Config, *rest.Resource, types.BootstrapVersion, EnvoyVersion) ([]byte, types.BootstrapVersion, error) {
 				return []byte(`node:
-  id: example`), nil
+  id: example`), types.BootstrapV2, nil
 			}
 			expectedConfigFile := filepath.Join(configDir, "bootstrap.yaml")
 
@@ -112,7 +113,7 @@ var _ = Describe("Envoy", func() {
 			// then
 			Expect(err).ToNot(HaveOccurred())
 			// and
-			Expect(strings.TrimSpace(buf.String())).To(Equal(fmt.Sprintf("-c %s --drain-time-s 15 --disable-hot-restart", expectedConfigFile)))
+			Expect(strings.TrimSpace(buf.String())).To(Equal(fmt.Sprintf("-c %s --drain-time-s 15 --disable-hot-restart --bootstrap-version 2", expectedConfigFile)))
 
 			By("verifying the contents Envoy config file")
 			// when
@@ -136,8 +137,8 @@ var _ = Describe("Envoy", func() {
 					ConfigDir:  configDir,
 				},
 			}
-			sampleConfig := func(string, kuma_dp.Config, *rest.Resource, EnvoyVersion) ([]byte, error) {
-				return nil, nil
+			sampleConfig := func(string, kuma_dp.Config, *rest.Resource, types.BootstrapVersion, EnvoyVersion) ([]byte, types.BootstrapVersion, error) {
+				return nil, "", nil
 			}
 
 			By("starting a mock dataplane")
@@ -178,8 +179,8 @@ var _ = Describe("Envoy", func() {
 					ConfigDir:  configDir,
 				},
 			}
-			sampleConfig := func(string, kuma_dp.Config, *rest.Resource, EnvoyVersion) ([]byte, error) {
-				return nil, nil
+			sampleConfig := func(string, kuma_dp.Config, *rest.Resource, types.BootstrapVersion, EnvoyVersion) ([]byte, types.BootstrapVersion, error) {
+				return nil, "", nil
 			}
 
 			By("starting a mock dataplane")

app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go

@@ -43,17 +43,17 @@ func IsInvalidRequestErr(err error) bool {
 	return strings.HasPrefix(err.Error(), "Invalid request: ")
 }
 
-func (b *remoteBootstrap) Generate(url string, cfg kuma_dp.Config, dp *rest_types.Resource, ev EnvoyVersion) ([]byte, error) {
+func (b *remoteBootstrap) Generate(url string, cfg kuma_dp.Config, dp *rest_types.Resource, bootstrapVersion types.BootstrapVersion, ev EnvoyVersion) ([]byte, types.BootstrapVersion, error) {
 	bootstrapUrl, err := net_url.Parse(url)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 
 	if bootstrapUrl.Scheme == "https" {
 		if cfg.ControlPlane.CaCert != "" {
 			certPool := x509.NewCertPool()
 			if ok := certPool.AppendCertsFromPEM([]byte(cfg.ControlPlane.CaCert)); !ok {
-				return nil, errors.New("could not add certificate")
+				return nil, "", errors.New("could not add certificate")
 			}
 			b.client.Transport = &http.Transport{
 				TLSClientConfig: &tls.Config{
@@ -67,19 +67,21 @@ func (b *remoteBootstrap) Generate(url string, cfg kuma_dp.Config, dp *rest_type
 
 	backoff, err := retry.NewConstant(cfg.ControlPlane.Retry.Backoff)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not create retry backoff")
+		return nil, "", errors.Wrap(err, "could not create retry backoff")
 	}
 	backoff = retry.WithMaxDuration(cfg.ControlPlane.Retry.MaxDuration, backoff)
 	var respBytes []byte
+	var version types.BootstrapVersion
 	err = retry.Do(context.Background(), backoff, func(ctx context.Context) error {
 		log.Info("trying to fetch bootstrap configuration from the Control Plane")
-		respBytes, err = b.requestForBootstrap(bootstrapUrl, cfg, dp, ev)
+		respBytes, version, err = b.requestForBootstrap(bootstrapUrl, cfg, dp, bootstrapVersion, ev)
 		if err == nil {
 			return nil
 		}
 		if IsInvalidRequestErr(err) { // there is no point in retrying invalid request
 			return err
 		}
+
 		switch err {
 		case DpNotFoundErr:
 			log.Info("Dataplane entity is not yet found in the Control Plane. If you are running on Kubernetes, CP is most likely still in the process of converting Pod to Dataplane. Retrying.", "backoff", cfg.ControlPlane.Retry.Backoff)
@@ -89,18 +91,18 @@ func (b *remoteBootstrap) Generate(url string, cfg kuma_dp.Config, dp *rest_type
 		return retry.RetryableError(err)
 	})
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
-	return respBytes, nil
+	return respBytes, version, nil
 }
 
-func (b *remoteBootstrap) requestForBootstrap(url *net_url.URL, cfg kuma_dp.Config, dp *rest_types.Resource, ev EnvoyVersion) ([]byte, error) {
+func (b *remoteBootstrap) requestForBootstrap(url *net_url.URL, cfg kuma_dp.Config, dp *rest_types.Resource, bootstrapVersion types.BootstrapVersion, ev EnvoyVersion) ([]byte, types.BootstrapVersion, error) {
 	url.Path = "/bootstrap"
 	var dataplaneResource string
 	if dp != nil {
 		dpJSON, err := json.Marshal(dp)
 		if err != nil {
-			return nil, err
+			return nil, "", err
 		}
 		dataplaneResource = string(dpJSON)
 	}
@@ -112,6 +114,7 @@ func (b *remoteBootstrap) requestForBootstrap(url *net_url.URL, cfg kuma_dp.Conf
 		AdminPort:          cfg.Dataplane.AdminPort.Lowest(),
 		DataplaneTokenPath: cfg.DataplaneRuntime.TokenPath,
 		DataplaneResource:  dataplaneResource,
+		BootstrapVersion:   bootstrapVersion,
 		Version: types.Version{
 			KumaDp: types.KumaDpVersion{
 				Version:   kuma_version.Build.Version,
@@ -127,32 +130,32 @@ func (b *remoteBootstrap) requestForBootstrap(url *net_url.URL, cfg kuma_dp.Conf
 	}
 	jsonBytes, err := json.Marshal(request)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not marshal request to json")
+		return nil, "", errors.Wrap(err, "could not marshal request to json")
 	}
 	resp, err := b.client.Post(url.String(), "application/json", bytes.NewReader(jsonBytes))
 	if err != nil {
-		return nil, errors.Wrap(err, "request to bootstrap server failed")
+		return nil, "", errors.Wrap(err, "request to bootstrap server failed")
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		bodyBytes, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			return nil, errors.Wrapf(err, "Unable to read the response with status code: %d. Make sure you are using https URL", resp.StatusCode)
+			return nil, "", errors.Wrapf(err, "Unable to read the response with status code: %d. Make sure you are using https URL", resp.StatusCode)
 		}
 		if resp.StatusCode == http.StatusNotFound && len(bodyBytes) == 0 {
-			return nil, DpNotFoundErr
+			return nil, "", DpNotFoundErr
 		}
 		if resp.StatusCode == http.StatusNotFound && string(bodyBytes) == "404: Page Not Found" { // response body of Go HTTP Server when hit for invalid endpoint
-			return nil, errors.New("There is no /bootstrap endpoint for provided CP address. Double check if the address passed to the CP has a DP Server port (5678 by default), not HTTP API (5681 by default)")
+			return nil, "", errors.New("There is no /bootstrap endpoint for provided CP address. Double check if the address passed to the CP has a DP Server port (5678 by default), not HTTP API (5681 by default)")
 		}
 		if resp.StatusCode/100 == 4 {
-			return nil, InvalidRequestErr(string(bodyBytes))
+			return nil, "", InvalidRequestErr(string(bodyBytes))
 		}
-		return nil, errors.Errorf("unexpected status code: %d", resp.StatusCode)
+		return nil, "", errors.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
 	respBytes, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not read the body of the response")
+		return nil, "", errors.Wrap(err, "could not read the body of the response")
 	}
-	return respBytes, nil
+	return respBytes, types.BootstrapVersion(resp.Header.Get(types.BootstrapVersionHeader)), nil
 }

app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	kuma_version "github.com/kumahq/kuma/pkg/version"
+	"github.com/kumahq/kuma/pkg/xds/bootstrap/types"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
@@ -26,6 +27,7 @@ var _ = Describe("Remote Bootstrap", func() {
 
 	type testCase struct {
 		config                   kuma_dp.Config
+		bootstrapVersion         types.BootstrapVersion
 		dataplane                *rest.Resource
 		expectedBootstrapRequest string
 	}
@@ -50,6 +52,7 @@ var _ = Describe("Remote Bootstrap", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(body).To(MatchJSON(given.expectedBootstrapRequest))
 
+			writer.Header().Set(types.BootstrapVersionHeader, string(given.bootstrapVersion))
 			response, err := ioutil.ReadFile(filepath.Join("testdata", "remote-bootstrap-config.golden.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 			_, err = writer.Write(response)
@@ -62,12 +65,13 @@ var _ = Describe("Remote Bootstrap", func() {
 		generator := NewRemoteBootstrapGenerator(http.DefaultClient)
 
 		// when
-		config, err := generator(fmt.Sprintf("http://localhost:%d", port), given.config, given.dataplane, EnvoyVersion{
+		config, version, err := generator(fmt.Sprintf("http://localhost:%d", port), given.config, given.dataplane, given.bootstrapVersion, EnvoyVersion{
 			Build:   "hash/1.15.0/RELEASE",
 			Version: "1.15.0",
 		})
 
 		// then
+		Expect(version).To(Equal(given.bootstrapVersion))
 		Expect(err).ToNot(HaveOccurred())
 		Expect(config).ToNot(BeNil())
 	},
@@ -80,7 +84,8 @@ var _ = Describe("Remote Bootstrap", func() {
 				cfg.DataplaneRuntime.TokenPath = "/tmp/token"
 
 				return testCase{
-					config: cfg,
+					config:           cfg,
+					bootstrapVersion: "2",
 					dataplane: &rest.Resource{
 						Meta: rest.ResourceMeta{
 							Type: "Dataplane",
@@ -106,7 +111,8 @@ var _ = Describe("Remote Bootstrap", func() {
 						  "version": "1.15.0",
 						  "build": "hash/1.15.0/RELEASE"
 						}
-					  }
+					  },
+					  "bootstrapVersion": "2"
 					}`,
 				}
 			}()),
@@ -128,6 +134,7 @@ var _ = Describe("Remote Bootstrap", func() {
 							Name: "sample",
 						},
 					},
+					bootstrapVersion: "3",
 					expectedBootstrapRequest: `
                     {
                       "mesh": "demo",
@@ -146,7 +153,8 @@ var _ = Describe("Remote Bootstrap", func() {
                           "version": "1.15.0",
                           "build": "hash/1.15.0/RELEASE"
                         }
-                      }
+                      },
+                      "bootstrapVersion": "3"
                     }`,
 				}
 			}()),
@@ -184,7 +192,8 @@ var _ = Describe("Remote Bootstrap", func() {
                           "version": "1.15.0",
                           "build": "hash/1.15.0/RELEASE"
                         }
-                      }
+                      },
+                      "bootstrapVersion": ""
                     }`,
 				}
 			}()),
@@ -217,13 +226,13 @@ var _ = Describe("Remote Bootstrap", func() {
 		// when
 		cfg := kuma_dp.DefaultConfig()
 		cfg.ControlPlane.Retry.Backoff = 10 * time.Millisecond
-		_, err = generator(fmt.Sprintf("http://localhost:%d", port), cfg, &rest.Resource{
+		_, _, err = generator(fmt.Sprintf("http://localhost:%d", port), cfg, &rest.Resource{
 			Meta: rest.ResourceMeta{
 				Type: "Dataplane",
 				Mesh: "default",
 				Name: "dp-1",
 			},
-		}, EnvoyVersion{})
+		}, "", EnvoyVersion{})
 
 		// then
 		Expect(err).ToNot(HaveOccurred())
@@ -249,9 +258,9 @@ var _ = Describe("Remote Bootstrap", func() {
 		config := kuma_dp.DefaultConfig()
 		config.ControlPlane.Retry.Backoff = 10 * time.Millisecond
 		config.ControlPlane.Retry.MaxDuration = 100 * time.Millisecond
-		_, err = generator(fmt.Sprintf("http://localhost:%d", port), config, &rest.Resource{
+		_, _, err = generator(fmt.Sprintf("http://localhost:%d", port), config, &rest.Resource{
 			Meta: rest.ResourceMeta{Mesh: "default", Name: "dp-1"},
-		}, EnvoyVersion{})
+		}, "", EnvoyVersion{})
 
 		// then
 		Expect(err).To(MatchError("retryable: Dataplane entity not found. If you are running on Universal please create a Dataplane entity on kuma-cp before starting kuma-dp. If you are running on Kubernetes, please check the kuma-cp logs to determine why the Dataplane entity could not be created by the automatic sidecar injection."))

pkg/config/app/kuma-dp/config.go

@@ -102,6 +102,9 @@ type Dataplane struct {
 	AdminPort config_types.PortRange `yaml:"adminPort,omitempty" envconfig:"kuma_dataplane_admin_port"`
 	// Drain time for listeners.
 	DrainTime time.Duration `yaml:"drainTime,omitempty" envconfig:"kuma_dataplane_drain_time"`
+	// BootstrapVersion defines bootstrap version (and API version) of xDS config.
+	// If empty, default version defined in Kuma CP will be used.
+	BootstrapVersion string `yaml:"bootstrapVersion" envconfig:"kuma_dataplane_bootstrap_version"`
 }
 
 // DataplaneRuntime defines the context in which dataplane (Envoy) runs.

pkg/config/app/kuma-dp/testdata/default-config.golden.yaml

@@ -8,5 +8,6 @@ controlPlane:
 dataplane:
   mesh: default
   drainTime: 30s
+  bootstrapVersion: ""
 dataplaneRuntime:
   binaryPath: envoy

pkg/sds/server/v2/server_test.go

@@ -7,6 +7,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	prometheus_client "github.com/prometheus/client_model/go"
+
 	"github.com/kumahq/kuma/pkg/xds/envoy/tls"
 
 	envoy_api "github.com/envoyproxy/go-control-plane/envoy/api/v2"
@@ -203,9 +205,13 @@ var _ = Describe("SDS Server", func() {
 		expirationSeconds := now.Load().(time.Time).Add(60 * time.Second).Unix()
 		Expect(dpInsight.Spec.MTLS.CertificateExpirationTime.Seconds).To(Equal(expirationSeconds))
 
-		// and metrics are published
-		Expect(test_metrics.FindMetric(metrics, "sds_cert_generation").GetCounter().GetValue()).To(Equal(1.0))
-		Expect(test_metrics.FindMetric(metrics, "sds_generation")).ToNot(BeNil())
+		// and metrics are published (metrics are published async, it does not have to be done before response is sent)
+		Eventually(func() float64 {
+			return test_metrics.FindMetric(metrics, "sds_cert_generation").GetCounter().GetValue()
+		}, "5s").Should(Equal(1.0))
+		Eventually(func() *prometheus_client.Metric {
+			return test_metrics.FindMetric(metrics, "sds_generation")
+		}, "5s").ShouldNot(BeNil())
 
 		close(done)
 	}, 10)