feat(kuma-cp): add metrics and timeouts to CA interface

api/mesh/v1alpha1/mesh.proto

@@ -6,6 +6,7 @@ option go_package = "github.com/kumahq/kuma/api/mesh/v1alpha1";
 
 import "mesh/options.proto";
 import "mesh/v1alpha1/metrics.proto";
+import "google/protobuf/duration.proto";
 import "google/protobuf/wrappers.proto";
 import "google/protobuf/struct.proto";
 
@@ -109,6 +110,9 @@ message CertificateAuthorityBackend {
     }
     // Rotation settings
     Rotation rotation = 1;
+
+    // Timeout on request to CA for DP certificate generation and retrieval
+    google.protobuf.Duration requestTimeout = 2;
   }
 
   // Dataplane certificate settings
@@ -131,6 +135,14 @@ message CertificateAuthorityBackend {
   // Mode defines the behaviour of inbound listeners with regard to traffic
   // encryption
   Mode mode = 5;
+
+  // RootChain defines settings related to CA root certificate chain.
+  message RootChain {
+    // Timeout on request for to CA for root certificate chain.
+    google.protobuf.Duration requestTimeout = 1;
+  }
+
+  RootChain rootChain = 6;
 }
 
 // Networking defines the networking configuration of the mesh

pkg/core/bootstrap/bootstrap.go

@@ -117,7 +117,11 @@ func buildRuntime(appCtx context.Context, cfg kuma_cp.Config) (core_runtime.Runt
 	builder.WithEnvoyAdminClient(envoyAdminClient)
 	builder.WithAPIManager(customization.NewAPIList())
 	builder.WithXDSHooks(&xds_hooks.Hooks{})
-	builder.WithCAProvider(secrets.NewCaProvider(builder.CaManagers()))
+	caProvider, err := secrets.NewCaProvider(builder.CaManagers(), builder.Metrics())
+	if err != nil {
+		return nil, err
+	}
+	builder.WithCAProvider(caProvider)
 	builder.WithDpServer(server.NewDpServer(*cfg.DpServer, builder.Metrics()))
 	builder.WithKDSContext(kds_context.DefaultContext(builder.ResourceManager(), cfg.Multizone.Zone.Name))
 

pkg/plugins/runtime/gateway/suite_test.go

@@ -109,9 +109,12 @@ func MakeGeneratorContext(rt runtime.Runtime, key core_model.ResourceKey) (*xds_
 	cache, err := cla.NewCache(rt.Config().Store.Cache.ExpirationTime, rt.Metrics())
 	Expect(err).To(Succeed())
 
+	idProvider, err := secrets.NewIdentityProvider(rt.CaManagers(), rt.Metrics())
+	Expect(err).To(Succeed())
+
 	secrets, err := secrets.NewSecrets(
-		secrets.NewCaProvider(rt.CaManagers()),
-		secrets.NewIdentityProvider(rt.CaManagers()),
+		rt.CAProvider(),
+		idProvider,
 		rt.Metrics(),
 	)
 	Expect(err).To(Succeed())

pkg/test/runtime/runtime.go

@@ -94,7 +94,11 @@ func BuilderFor(appCtx context.Context, cfg kuma_cp.Config) (*core_runtime.Build
 	builder.WithXDSHooks(&xds_hooks.Hooks{})
 	builder.WithDpServer(server.NewDpServer(*cfg.DpServer, metrics))
 	builder.WithKDSContext(kds_context.DefaultContext(builder.ResourceManager(), cfg.Multizone.Zone.Name))
-	builder.WithCAProvider(secrets.NewCaProvider(builder.CaManagers()))
+	caProvider, err := secrets.NewCaProvider(builder.CaManagers(), metrics)
+	if err != nil {
+		return nil, err
+	}
+	builder.WithCAProvider(caProvider)
 	builder.WithAPIServerAuthenticator(certs.ClientCertAuthenticator)
 	builder.WithAccess(core_runtime.Access{
 		ResourceAccess:       resources_access.NewAdminResourceAccess(builder.Config().Access.Static.AdminResources),

pkg/xds/secrets/ca_provider.go

@@ -2,27 +2,44 @@ package secrets
 
 import (
 	"context"
+	"time"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 
 	core_ca "github.com/kumahq/kuma/pkg/core/ca"
 	core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
 	core_xds "github.com/kumahq/kuma/pkg/core/xds"
+	core_metrics "github.com/kumahq/kuma/pkg/metrics"
 )
 
 type CaProvider interface {
 	// Get returns all PEM encoded CAs, a list of CAs that were used to generate a secret and an error.
 	Get(context.Context, *core_mesh.MeshResource) (*core_xds.CaSecret, []string, error)
 }
 
-func NewCaProvider(caManagers core_ca.Managers) CaProvider {
-	return &meshCaProvider{
-		caManagers: caManagers,
+func NewCaProvider(caManagers core_ca.Managers, metrics core_metrics.Metrics) (CaProvider, error) {
+	latencyMetrics := map[string]*prometheus.SummaryVec{}
+	for backendType := range caManagers {
+		latencyMetrics[backendType] = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+			Name:       "ca_manager_get_root_cert_chain",
+			Help:       "Summary of CA manager get CA root certificate chain latencies",
+			Objectives: core_metrics.DefaultObjectives,
+		}, []string{"backend_name"})
+		if err := metrics.Register(latencyMetrics[backendType]); err != nil {
+			return nil, err
+		}
 	}
+	return &meshCaProvider{
+		caManagers:     caManagers,
+		latencyMetrics: latencyMetrics,
+	}, nil
 }
 
 type meshCaProvider struct {
 	caManagers core_ca.Managers
+	// latencyMetrics maps backend type to backend cert retrieval summary metrics
+	latencyMetrics map[string]*prometheus.SummaryVec
 }
 
 func (s *meshCaProvider) Get(ctx context.Context, mesh *core_mesh.MeshResource) (*core_xds.CaSecret, []string, error) {
@@ -31,12 +48,27 @@ func (s *meshCaProvider) Get(ctx context.Context, mesh *core_mesh.MeshResource)
 		return nil, nil, errors.New("CA backend is nil")
 	}
 
+	var cancel context.CancelFunc
+	timeout := backend.GetRootChain().GetRequestTimeout()
+	if timeout != nil {
+		ctx, cancel = context.WithTimeout(ctx, timeout.AsDuration())
+		defer cancel()
+	}
+
 	caManager, exist := s.caManagers[backend.Type]
 	if !exist {
 		return nil, nil, errors.Errorf("CA manager of type %s not exist", backend.Type)
 	}
 
-	certs, err := caManager.GetRootCert(ctx, mesh.GetMeta().GetName(), backend)
+	var certs [][]byte
+	var err error
+	func() {
+		start := time.Now()
+		defer func() {
+			s.latencyMetrics[backend.Type].WithLabelValues(backend.GetName()).Observe(float64(time.Since(start).Milliseconds()))
+		}()
+		certs, err = caManager.GetRootCert(ctx, mesh.GetMeta().GetName(), backend)
+	}()
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "could not get root certs")
 	}

pkg/xds/secrets/identity_provider.go

@@ -2,13 +2,16 @@ package secrets
 
 import (
 	"context"
+	"time"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 
 	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
 	core_ca "github.com/kumahq/kuma/pkg/core/ca"
 	core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
 	core_xds "github.com/kumahq/kuma/pkg/core/xds"
+	core_metrics "github.com/kumahq/kuma/pkg/metrics"
 )
 
 type Identity struct {
@@ -22,14 +25,29 @@ type IdentityProvider interface {
 	Get(context.Context, Identity, *core_mesh.MeshResource) (*core_xds.IdentitySecret, string, error)
 }
 
-func NewIdentityProvider(caManagers core_ca.Managers) IdentityProvider {
-	return &identityCertProvider{
-		caManagers: caManagers,
+func NewIdentityProvider(caManagers core_ca.Managers, metrics core_metrics.Metrics) (IdentityProvider, error) {
+	latencyMetrics := map[string]*prometheus.SummaryVec{}
+	for backendType := range caManagers {
+		latencyMetrics[backendType] = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+			Name:       "ca_manager_get_cert",
+			Help:       "Summary of CA manager get certificate latencies",
+			Objectives: core_metrics.DefaultObjectives,
+		}, []string{"backend_name"})
+		if err := metrics.Register(latencyMetrics[backendType]); err != nil {
+			return nil, err
+		}
 	}
+
+	return &identityCertProvider{
+		caManagers:     caManagers,
+		latencyMetrics: latencyMetrics,
+	}, nil
 }
 
 type identityCertProvider struct {
 	caManagers core_ca.Managers
+	// latencyMetrics maps backend type to backend cert retrieval summary metrics
+	latencyMetrics map[string]*prometheus.SummaryVec
 }
 
 func (s *identityCertProvider) Get(ctx context.Context, requestor Identity, mesh *core_mesh.MeshResource) (*core_xds.IdentitySecret, string, error) {
@@ -38,12 +56,28 @@ func (s *identityCertProvider) Get(ctx context.Context, requestor Identity, mesh
 		return nil, "", errors.Errorf("CA default backend in mesh %q has to be defined", mesh.GetMeta().GetName())
 	}
 
+	var cancel context.CancelFunc
+	timeout := backend.GetDpCert().GetRequestTimeout()
+	if timeout != nil {
+		ctx, cancel = context.WithTimeout(ctx, timeout.AsDuration())
+		defer cancel()
+	}
+
 	caManager, exist := s.caManagers[backend.Type]
 	if !exist {
 		return nil, "", errors.Errorf("CA manager of type %s not exist", backend.Type)
 	}
 
-	pair, err := caManager.GenerateDataplaneCert(ctx, mesh.GetMeta().GetName(), backend, requestor.Services)
+	var pair core_ca.KeyPair
+	var err error
+	func() {
+		start := time.Now()
+		defer func() {
+			s.latencyMetrics[backend.Type].WithLabelValues(backend.GetName()).Observe(float64(time.Since(start).Milliseconds()))
+		}()
+		pair, err = caManager.GenerateDataplaneCert(ctx, mesh.GetMeta().GetName(), backend, requestor.Services)
+	}()
+
 	if err != nil {
 		return nil, "", errors.Wrapf(err, "could not generate dataplane cert for mesh: %q backend: %q services: %q", mesh.GetMeta().GetName(), backend.Name, requestor.Services)
 	}

pkg/xds/secrets/secrets_test.go

@@ -110,13 +110,15 @@ var _ = Describe("Secrets", func() {
 		err := builtinCaManager.EnsureBackends(context.Background(), "default", newMesh().Spec.Mtls.Backends)
 		Expect(err).ToNot(HaveOccurred())
 
-		caProvider := NewCaProvider(caManagers)
-		identityProvider := NewIdentityProvider(caManagers)
-
 		m, err := core_metrics.NewMetrics("local")
 		Expect(err).ToNot(HaveOccurred())
 		metrics = m
 
+		caProvider, err := NewCaProvider(caManagers, metrics)
+		Expect(err).ToNot(HaveOccurred())
+		identityProvider, err := NewIdentityProvider(caManagers, metrics)
+		Expect(err).ToNot(HaveOccurred())
+
 		secrets, err = NewSecrets(caProvider, identityProvider, metrics)
 		Expect(err).ToNot(HaveOccurred())
 

pkg/xds/server/components.go

@@ -77,9 +77,14 @@ func RegisterXDS(rt core_runtime.Runtime) error {
 		return err
 	}
 
+	idProvider, err := secrets.NewIdentityProvider(rt.CaManagers(), rt.Metrics())
+	if err != nil {
+		return err
+	}
+
 	secrets, err := secrets.NewSecrets(
 		rt.CAProvider(),
-		secrets.NewIdentityProvider(rt.CaManagers()),
+		idProvider,
 		rt.Metrics(),
 	)
 	if err != nil {