Reload TLS certificates

Signed-off-by: Pavol Loffay <[email protected]>
jaegertracing · Aug 21, 2020 · 88e5c65 · 88e5c65
1 parent 3eeedf0
commit 88e5c65
Show file tree

Hide file tree

Showing 37 changed files with 847 additions and 94 deletions.
diff --git a/cmd/agent/app/reporter/client_metrics.go b/cmd/agent/app/reporter/client_metrics.go
@@ -122,10 +122,11 @@ func (r *ClientMetricsReporter) EmitBatch(ctx context.Context, batch *jaeger.Bat
 }
 
 // Close stops background gc goroutine for client stats map.
-func (r *ClientMetricsReporter) Close() {
+func (r *ClientMetricsReporter) Close() error {
 	if r.closed.CAS(false, true) {
 		close(r.shutdown)
 	}
+	return nil
 }
 
 func (r *ClientMetricsReporter) expireClientMetricsLoop() {

diff --git a/cmd/agent/app/reporter/grpc/builder.go b/cmd/agent/app/reporter/grpc/builder.go
@@ -55,7 +55,7 @@ func (b *ConnBuilder) CreateConnection(logger *zap.Logger) (*grpc.ClientConn, er
 	var dialTarget string
 	if b.TLS.Enabled { // user requested a secure connection
 		logger.Info("Agent requested secure grpc connection to collector(s)")
-		tlsConf, err := b.TLS.Config()
+		tlsConf, err := b.TLS.Config(logger)
 		if err != nil {
 			return nil, fmt.Errorf("failed to load TLS config: %w", err)
 		}

diff --git a/cmd/agent/app/reporter/grpc/builder_test.go b/cmd/agent/app/reporter/grpc/builder_test.go
@@ -302,7 +302,7 @@ func TestProxyClientTLS(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			var opts []grpc.ServerOption
 			if test.serverTLS.Enabled {
-				tlsCfg, err := test.serverTLS.Config()
+				tlsCfg, err := test.serverTLS.Config(zap.NewNop())
 				require.NoError(t, err)
 				opts = []grpc.ServerOption{grpc.Creds(credentials.NewTLS(tlsCfg))}
 			}

diff --git a/cmd/agent/app/reporter/grpc/collector_proxy.go b/cmd/agent/app/reporter/grpc/collector_proxy.go
@@ -15,20 +15,24 @@
 package grpc
 
 import (
+	"io"
+
 	"github.com/uber/jaeger-lib/metrics"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 
 	"github.com/jaegertracing/jaeger/cmd/agent/app/configmanager"
 	grpcManager "github.com/jaegertracing/jaeger/cmd/agent/app/configmanager/grpc"
 	"github.com/jaegertracing/jaeger/cmd/agent/app/reporter"
+	"github.com/jaegertracing/jaeger/pkg/multicloser"
 )
 
 // ProxyBuilder holds objects communicating with collector
 type ProxyBuilder struct {
-	reporter *reporter.ClientMetricsReporter
-	manager  configmanager.ClientConfigManager
-	conn     *grpc.ClientConn
+	reporter  *reporter.ClientMetricsReporter
+	manager   configmanager.ClientConfigManager
+	conn      *grpc.ClientConn
+	tlsCloser io.Closer
 }
 
 // NewCollectorProxy creates ProxyBuilder
@@ -46,9 +50,10 @@ func NewCollectorProxy(builder *ConnBuilder, agentTags map[string]string, mFacto
 		MetricsFactory: mFactory,
 	})
 	return &ProxyBuilder{
-		conn:     conn,
-		reporter: r3,
-		manager:  configmanager.WrapWithMetrics(grpcManager.NewConfigManager(conn), grpcMetrics),
+		conn:      conn,
+		reporter:  r3,
+		manager:   configmanager.WrapWithMetrics(grpcManager.NewConfigManager(conn), grpcMetrics),
+		tlsCloser: &builder.TLS,
 	}, nil
 }
 
@@ -69,6 +74,5 @@ func (b ProxyBuilder) GetManager() configmanager.ClientConfigManager {
 
 // Close closes connections used by proxy.
 func (b ProxyBuilder) Close() error {
-	b.reporter.Close()
-	return b.conn.Close()
+	return multicloser.Wrap(b.reporter, b.tlsCloser, b.GetConn()).Close()
 }
diff --git a/cmd/all-in-one/main.go b/cmd/all-in-one/main.go
@@ -165,6 +165,9 @@ by default uses only in-memory database.`,
 						logger.Error("Failed to close span writer", zap.Error(err))
 					}
 				}
+				if err := storageFactory.Close(); err != nil {
+					logger.Error("Failed to close storage factory", zap.Error(err))
+				}
 				tracerCloser.Close()
 			})
 			return nil

diff --git a/cmd/collector/app/collector.go b/cmd/collector/app/collector.go
@@ -16,6 +16,7 @@ package app
 
 import (
 	"context"
+	"io"
 	"net/http"
 	"time"
 
@@ -46,6 +47,7 @@ type Collector struct {
 	hServer    *http.Server
 	zkServer   *http.Server
 	grpcServer *grpc.Server
+	tlsCloser  io.Closer
 }
 
 // CollectorParams to construct a new Jaeger Collector.
@@ -107,6 +109,7 @@ func (c *Collector) Start(builderOpts *CollectorOptions) error {
 		c.hServer = httpServer
 	}
 
+	c.tlsCloser = &builderOpts.TLS
 	if zkServer, err := server.StartZipkinServer(&server.ZipkinServerParams{
 		HostPort:       builderOpts.CollectorZipkinHTTPHostPort,
 		Handler:        c.spanHandlers.ZipkinSpansHandler,
@@ -154,6 +157,10 @@ func (c *Collector) Close() error {
 		c.logger.Error("failed to close span processor.", zap.Error(err))
 	}
 
+	if err := c.tlsCloser.Close(); err != nil {
+		c.logger.Error("failed to close TLS certificate watcher", zap.Error(err))
+	}
+
 	return nil
 }
 

diff --git a/cmd/collector/app/server/grpc.go b/cmd/collector/app/server/grpc.go
@@ -45,7 +45,7 @@ func StartGRPCServer(params *GRPCServerParams) (*grpc.Server, error) {
 
 	if params.TLSConfig.Enabled {
 		// user requested a server with TLS, setup creds
-		tlsCfg, err := params.TLSConfig.Config()
+		tlsCfg, err := params.TLSConfig.Config(params.Logger)
 		if err != nil {
 			return nil, err
 		}

diff --git a/cmd/collector/main.go b/cmd/collector/main.go
@@ -104,6 +104,9 @@ func main() {
 						logger.Error("failed to close span writer", zap.Error(err))
 					}
 				}
+				if err := storageFactory.Close(); err != nil {
+					logger.Error("Failed to close storage factory", zap.Error(err))
+				}
 
 			})
 			return nil

diff --git a/cmd/ingester/app/builder/builder.go b/cmd/ingester/app/builder/builder.go
@@ -58,7 +58,7 @@ func CreateConsumer(logger *zap.Logger, metricsFactory metrics.Factory, spanWrit
 		ProtocolVersion:      options.ProtocolVersion,
 		AuthenticationConfig: options.AuthenticationConfig,
 	}
-	saramaConsumer, err := consumerConfig.NewConsumer()
+	saramaConsumer, err := consumerConfig.NewConsumer(logger)
 	if err != nil {
 		return nil, err
 	}

diff --git a/cmd/ingester/main.go b/cmd/ingester/main.go
@@ -76,6 +76,9 @@ func main() {
 			consumer.Start()
 
 			svc.RunAndThen(func() {
+				if err := options.TLS.Close(); err != nil {
+					logger.Error("Failed to close TLS certificates watcher", zap.Error(err))
+				}
 				if err = consumer.Close(); err != nil {
 					logger.Error("Failed to close consumer", zap.Error(err))
 				}
@@ -85,6 +88,9 @@ func main() {
 						logger.Error("Failed to close span writer", zap.Error(err))
 					}
 				}
+				if err := storageFactory.Close(); err != nil {
+					logger.Error("Failed to close storage factory", zap.Error(err))
+				}
 			})
 			return nil
 		},

diff --git a/cmd/opentelemetry/app/exporter/elasticsearchexporter/exporter.go b/cmd/opentelemetry/app/exporter/elasticsearchexporter/exporter.go
@@ -38,5 +38,8 @@ func new(ctx context.Context, config *Config, params component.ExporterCreatePar
 	}
 	return exporterhelper.NewTraceExporter(
 		config,
-		w.WriteTraces)
+		w.WriteTraces,
+		exporterhelper.WithShutdown(func(ctx context.Context) error {
+			return esCfg.TLS.Close()
+		}))
 }
diff --git a/cmd/opentelemetry/cmd/all-in-one/main.go b/cmd/opentelemetry/cmd/all-in-one/main.go
@@ -45,6 +45,7 @@ import (
 	queryApp "github.com/jaegertracing/jaeger/cmd/query/app"
 	"github.com/jaegertracing/jaeger/cmd/query/app/querysvc"
 	jConfig "github.com/jaegertracing/jaeger/pkg/config"
+	"github.com/jaegertracing/jaeger/pkg/multicloser"
 	"github.com/jaegertracing/jaeger/pkg/version"
 	pluginStorage "github.com/jaegertracing/jaeger/plugin/storage"
 	cassandraStorage "github.com/jaegertracing/jaeger/plugin/storage/cassandra"
@@ -144,17 +145,14 @@ func main() {
 	if exp == nil {
 		svc.ReportFatalError(fmt.Errorf("exporter type for storage %s not found", storageType))
 	}
-	queryServer, tracerCloser, err := startQuery(v, svc.GetLogger(), exp)
+	closer, err := startQuery(v, svc.GetLogger(), exp)
 	if err != nil {
 		svc.ReportFatalError(err)
 	}
 	for state := range svc.GetStateChannel() {
 		if state == service.Closing {
-			if queryServer != nil {
-				queryServer.Close()
-			}
-			if tracerCloser != nil {
-				tracerCloser.Close()
+			if closer != nil {
+				closer.Close()
 			}
 		} else if state == service.Closed {
 			break
@@ -176,18 +174,18 @@ func getStorageExporter(storageType string, exporters map[configmodels.Exporter]
 	return nil
 }
 
-func startQuery(v *viper.Viper, logger *zap.Logger, exporter configmodels.Exporter) (*queryApp.Server, io.Closer, error) {
+func startQuery(v *viper.Viper, logger *zap.Logger, exporter configmodels.Exporter) (io.Closer, error) {
 	storageFactory, err := getFactory(exporter, v, logger)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	spanReader, err := storageFactory.CreateSpanReader()
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	dependencyReader, err := storageFactory.CreateDependencyReader()
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	queryOpts := new(queryApp.QueryOptions).InitFromViper(v, logger)
 	queryServiceOptions := queryOpts.BuildQueryServiceOptions(storageFactory, logger)
@@ -199,12 +197,16 @@ func startQuery(v *viper.Viper, logger *zap.Logger, exporter configmodels.Export
 	tracerCloser := initTracer(logger)
 	server, err := queryApp.NewServer(logger, queryService, queryOpts, opentracing.GlobalTracer())
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	if err := server.Start(); err != nil {
-		return nil, nil, err
+		return nil, err
+	}
+	var storageCloser io.Closer
+	if closer, ok := storageFactory.(io.Closer); ok {
+		storageCloser = closer
 	}
-	return server, tracerCloser, nil
+	return multicloser.Wrap(tracerCloser, server, storageCloser), nil
 }
 
 func getFactory(exporter configmodels.Exporter, v *viper.Viper, logger *zap.Logger) (storage.Factory, error) {

diff --git a/cmd/opentelemetry/go.sum b/cmd/opentelemetry/go.sum
@@ -576,8 +576,6 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-plugin v1.0.1 h1:4OtAfUGbnKC6yS48p0CtMX2oFYtzFZVv6rok3cRWgnE=
 github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY=
-github.com/hashicorp/go-plugin v1.3.0 h1:4d/wJojzvHV1I4i/rrjVaeuyxWrLzDE1mDCyDy8fXS8=
-github.com/hashicorp/go-plugin v1.3.0/go.mod h1:F9eH4LrE/ZsRdbwhfjs9k9HoDUwAHnYtXdgmf1AVNs0=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
@@ -633,8 +631,6 @@ github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0
 github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
 github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
-github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
 github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a h1:GmsqmapfzSJkm28dhRoHz2tLRbJmqhU86IPgBtN3mmk=
 github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s=
 github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYPNLe3d8smommaoQlK7LOA5ESyUJJ+Wf79ZtA7Vp4=
@@ -1195,7 +1191,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1433,7 +1428,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -1461,7 +1455,6 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY
 google.golang.org/genproto v0.0.0-20200603110839-e855014d5736/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
 google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad h1:uAwc13+y0Y8QZLTYhLCu6lHhnG99ecQU5FYTj8zxAng=
 google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@@ -1485,7 +1478,6 @@ google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc/examples v0.0.0-20200728065043-dfc0c05b2da9 h1:f+/+gfZ/tfaHBXXiv1gWRmCej6wlX3mLY4bnLpI99wk=
 google.golang.org/grpc/examples v0.0.0-20200728065043-dfc0c05b2da9/go.mod h1:5j1uub0jRGhRiSghIlrThmBUgcgLXOVJQ/l1getT4uo=
-google.golang.org/grpc/examples v0.0.0-20200819190100-f640ae6a4f43/go.mod h1:wQWkdCkP0Pl3MzFPvfqTNUnXA2eIVY4eakDiKJvniKc=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

diff --git a/cmd/query/app/server.go b/cmd/query/app/server.go
@@ -74,7 +74,7 @@ func createGRPCServer(querySvc *querysvc.QueryService, options *QueryOptions, lo
 	var grpcOpts []grpc.ServerOption
 
 	if options.TLS.Enabled {
-		tlsCfg, err := options.TLS.Config()
+		tlsCfg, err := options.TLS.Config(logger)
 		if err != nil {
 			return nil, err
 		}
@@ -182,8 +182,10 @@ func (s *Server) Start() error {
 }
 
 // Close stops http, GRPC servers and closes the port listener.
-func (s *Server) Close() {
+func (s *Server) Close() error {
+	s.queryOptions.TLS.Close()
 	s.grpcServer.Stop()
 	s.httpServer.Close()
 	s.conn.Close()
+	return nil
 }
diff --git a/cmd/query/main.go b/cmd/query/main.go
@@ -123,6 +123,9 @@ func main() {
 
 			svc.RunAndThen(func() {
 				server.Close()
+				if err := storageFactory.Close(); err != nil {
+					logger.Error("Failed to close storage factory", zap.Error(err))
+				}
 			})
 			return nil
 		},