Merge pull request #2675 from oasislabs/andrej/feature/ephemeral-node…

…-tls-certs

Make node TLS certificates totally ephemeral

.changelog/2098.breaking.md

@@ -0,0 +1,5 @@
+Sentry nodes no longer require TLS certificate file of the upstream node
+
+The `worker.sentry.grpc.upstream.cert` option has been removed.
+Instead, use `worker.sentry.grpc.upstream.id` to specify the
+upstream node's ID.

.changelog/2098.feature.md

@@ -0,0 +1,6 @@
+node: Add automatic TLS certificate rotation support
+
+It is now possible to automatically rotate the node's TLS
+certificates every N epochs by passing the command-line flag
+`worker.registration.rotate_certs`.
+Do not use this option on sentry nodes or IAS proxies.

go/common/accessctl/accessctl_test.go

@@ -49,11 +49,11 @@ func TestSubjectFromCertificate(t *testing.T) {
 	require.NoError(err, "Failed to create a temporary directory")
 	defer os.RemoveAll(dataDir)
 
-	ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory())
+	ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory(), false)
 	require.NoError(err, "Failed to generate a new identity")
-	require.Len(ident.TLSCertificate.Certificate, 1, "The generated identity contains more than 1 certificate in the chain")
+	require.Len(ident.GetTLSCertificate().Certificate, 1, "The generated identity contains more than 1 certificate in the chain")
 
-	x509Cert, err := x509.ParseCertificate(ident.TLSCertificate.Certificate[0])
+	x509Cert, err := x509.ParseCertificate(ident.GetTLSCertificate().Certificate[0])
 	require.NoError(err, "Failed to parse X.509 certificate from TLS certificate")
 
 	sub := SubjectFromX509Certificate(x509Cert)

go/common/grpc/grpc.go

@@ -23,6 +23,7 @@ import (
 	"google.golang.org/grpc/keepalive"
 
 	"github.com/oasislabs/oasis-core/go/common/grpc/auth"
+	"github.com/oasislabs/oasis-core/go/common/identity"
 	"github.com/oasislabs/oasis-core/go/common/logging"
 	"github.com/oasislabs/oasis-core/go/common/service"
 )
@@ -298,8 +299,8 @@ type ServerConfig struct { // nolint: maligned
 	Port uint16
 	// Path is the path for the local server. Leave nil to create a TCP server.
 	Path string
-	// Certificate is the certificate used by the server. Should be nil for local servers.
-	Certificate *tls.Certificate
+	// Identity is the identity of the worker that's running the server.
+	Identity *identity.Identity
 	// InstallWrapper specifies whether intercepting facilities should be enabled on this server,
 	// to enable intercepting RPC calls with a wrapper.
 	InstallWrapper bool
@@ -473,11 +474,14 @@ func NewServer(config *ServerConfig) (*Server, error) {
 		grpc.KeepaliveParams(serverKeepAliveParams),
 		grpc.CustomCodec(&CBORCodec{}),
 	}
-	if config.Certificate != nil {
+	if config.Identity != nil && config.Identity.GetTLSCertificate() != nil {
 		tlsConfig := &tls.Config{
-			Certificates: []tls.Certificate{*config.Certificate},
-			ClientAuth:   clientAuthType,
+			ClientAuth: clientAuthType,
+			GetCertificate: func(ch *tls.ClientHelloInfo) (*tls.Certificate, error) {
+				return config.Identity.GetTLSCertificate(), nil
+			},
 		}
+
 		sOpts = append(sOpts, grpc.Creds(credentials.NewTLS(tlsConfig)))
 	}
 	sOpts = append(sOpts, config.CustomOptions...)

go/common/grpc/policy/policy_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/oasislabs/oasis-core/go/common/grpc/policy"
 	"github.com/oasislabs/oasis-core/go/common/grpc/policy/api"
 	cmnTesting "github.com/oasislabs/oasis-core/go/common/grpc/testing"
+	"github.com/oasislabs/oasis-core/go/common/identity"
 )
 
 var testNs = common.NewTestNamespaceFromSeed([]byte("oasis common grpc policy test ns"), 0)
@@ -60,9 +61,10 @@ func TestAccessPolicy(t *testing.T) {
 	serverConfig := &cmnGrpc.ServerConfig{
 		Name:          host,
 		Port:          port,
-		Certificate:   serverTLSCert,
+		Identity:      &identity.Identity{},
 		CustomOptions: []grpc.ServerOption{grpc.CustomCodec(&cmnGrpc.CBORCodec{})},
 	}
+	serverConfig.Identity.SetTLSCertificate(serverTLSCert)
 	grpcServer, err := cmnGrpc.NewServer(serverConfig)
 	require.NoErrorf(err, "Failed to create a new gRPC server: %v", err)
 

go/common/grpc/proxy/proxy.go

@@ -9,6 +9,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 
@@ -17,21 +18,29 @@ import (
 	"github.com/oasislabs/oasis-core/go/common/logging"
 )
 
-// Handler returns a grpc StreamHandler than can be used
-// to proxy requests to provided client.
-// XXX: potentially the connection should be established in this package,
-// with some sensible defaults e.g. KeepAlive set.
-// We might also want to establish a pool of connections to the upstream.
-func Handler(conn *grpc.ClientConn) grpc.StreamHandler {
+// Dialer should return a gRPC ClientConn that will be used
+// to forward calls to.
+type Dialer func(ctx context.Context) (*grpc.ClientConn, error)
+
+// Handler returns a gRPC StreamHandler than can be used
+// to proxy requests to the client returned by the proxy dialer.
+func Handler(dialer Dialer) grpc.StreamHandler {
 	proxy := &proxy{
 		logger:       logging.GetLogger("grpc/proxy"),
-		upstreamConn: conn,
+		dialer:       dialer,
+		upstreamConn: nil, // Will be dialed on-demand.
 	}
 
 	return grpc.StreamHandler(proxy.handler)
 }
 
 type proxy struct {
+	// This is the dialer callback we use to make new connections to the
+	// upstream server if the connection drops, etc.
+	dialer Dialer
+
+	// This is a cached client connection to the upstream server, so we
+	// don't have to re-dial it on every call.
 	upstreamConn *grpc.ClientConn
 
 	logger *logging.Logger
@@ -66,6 +75,21 @@ func (p *proxy) handler(srv interface{}, stream grpc.ServerStream) error {
 	// Pass subject header upstream.
 	upstreamCtx = metadata.AppendToOutgoingContext(upstreamCtx, policy.ForwardedSubjectMD, sub)
 
+	// Check if upstream connection was disconnected.
+	if p.upstreamConn != nil && p.upstreamConn.GetState() == connectivity.Shutdown {
+		// We need to redial if the connection was shut down.
+		p.upstreamConn = nil
+	}
+
+	// Dial upstream if necessary.
+	if p.upstreamConn == nil {
+		var grr error
+		p.upstreamConn, grr = p.dialer(stream.Context())
+		if grr != nil {
+			return grr
+		}
+	}
+
 	upstreamStream, err := grpc.NewClientStream(
 		upstreamCtx,
 		desc,
@@ -92,7 +116,7 @@ func (p *proxy) handler(srv interface{}, stream grpc.ServerStream) error {
 				// can still be in progress.
 				p.logger.Debug("downstream EOF")
 				if err = upstreamStream.CloseSend(); err != nil {
-					p.logger.Error("failrue closing upstream stream",
+					p.logger.Error("failure closing upstream stream",
 						"err", err,
 					)
 				}
@@ -154,6 +178,7 @@ func (p *proxy) proxyUpstream(downstream grpc.ServerStream, upstream grpc.Client
 func (p *proxy) proxyDownstream(upstream grpc.ClientStream, downstream grpc.ServerStream) <-chan error {
 	errCh := make(chan error, 1)
 	var headerSent bool
+
 	go func() {
 		for {
 			// Wait for stream msg (from upstream).

go/common/grpc/proxy/proxy_test.go

@@ -17,6 +17,7 @@ import (
 	commonGrpc "github.com/oasislabs/oasis-core/go/common/grpc"
 	"github.com/oasislabs/oasis-core/go/common/grpc/auth"
 	cmnTesting "github.com/oasislabs/oasis-core/go/common/grpc/testing"
+	"github.com/oasislabs/oasis-core/go/common/identity"
 )
 
 const (
@@ -58,9 +59,10 @@ func TestGRPCProxy(t *testing.T) {
 	serverConfig := &commonGrpc.ServerConfig{
 		Name:          host,
 		Port:          port,
-		Certificate:   serverTLSCert,
+		Identity:      &identity.Identity{},
 		CustomOptions: []grpc.ServerOption{grpc.CustomCodec(&commonGrpc.CBORCodec{})},
 	}
+	serverConfig.Identity.SetTLSCertificate(serverTLSCert)
 	grpcServer, err := commonGrpc.NewServer(serverConfig)
 	require.NoErrorf(err, "Failed to create a new gRPC server: %v", err)
 
@@ -72,39 +74,47 @@ func TestGRPCProxy(t *testing.T) {
 	err = grpcServer.Start()
 	require.NoErrorf(err, "Failed to start the gRPC server: %v", err)
 
-	// Connect to gRPC server.
 	clientTLSCreds := credentials.NewTLS(&tls.Config{
 		Certificates: []tls.Certificate{*clientTLSCert},
 		RootCAs:      serverCertPool,
 		ServerName:   "oasis-node",
 	})
-	address := fmt.Sprintf("%s:%d", host, port)
-	conn := connectToGrpcServer(ctx, t, address, clientTLSCreds)
-	defer conn.Close()
+
+	// Create upstream dialer.
+	upstreamDialer := func(ctx context.Context) (*grpc.ClientConn, error) {
+		// Connect to gRPC server.
+		address := fmt.Sprintf("%s:%d", host, port)
+		conn := connectToGrpcServer(ctx, t, address, clientTLSCreds)
+		return conn, nil
+	}
 
 	// Create a proxy gRPC server.
 	proxyServerConfig := &commonGrpc.ServerConfig{
-		Name:        host,
-		Port:        port + 1,
-		Certificate: serverTLSCert,
+		Name:     host,
+		Port:     port + 1,
+		Identity: &identity.Identity{},
 		CustomOptions: []grpc.ServerOption{
 			// All unknown requests will be proxied to the grpc server above.
-			grpc.UnknownServiceHandler(Handler(conn)),
+			grpc.UnknownServiceHandler(Handler(upstreamDialer)),
 		},
 	}
+	proxyServerConfig.Identity.SetTLSCertificate(serverTLSCert)
 	proxyGrpcServer, err := commonGrpc.NewServer(proxyServerConfig)
 	require.NoErrorf(err, "Failed to create a proxy gRPC server: %v", err)
 
 	err = proxyGrpcServer.Start()
 	require.NoErrorf(err, "Failed to start the proxy gRPC server: %v", err)
 
 	// Connect to the proxy grpc server.
-	address = fmt.Sprintf("%s:%d", host, port+1)
+	address := fmt.Sprintf("%s:%d", host, port+1)
 	proxyConn := connectToGrpcServer(ctx, t, address, clientTLSCreds)
 	defer proxyConn.Close()
 
 	// Create a new ping client.
-	client := cmnTesting.NewPingClient(conn)
+	upstreamAddress := fmt.Sprintf("%s:%d", host, port)
+	upstreamConn := connectToGrpcServer(ctx, t, upstreamAddress, clientTLSCreds)
+	defer upstreamConn.Close()
+	client := cmnTesting.NewPingClient(upstreamConn)
 	pingQuery := &cmnTesting.PingQuery{}
 	// Test Ping.
 	res, err := client.Ping(ctx, pingQuery)

go/common/grpc/testing/ping.go

@@ -64,14 +64,14 @@ func CreateCertificate(t *testing.T) (*tls.Certificate, *x509.Certificate) {
 	require.NoError(err, "Failed to create a temporary directory")
 	defer os.RemoveAll(dataDir)
 
-	ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory())
+	ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory(), false)
 	require.NoError(err, "Failed to generate a new identity")
-	require.Len(ident.TLSCertificate.Certificate, 1, "The generated identity contains more than 1 TLS certificate in the chain")
+	require.Len(ident.GetTLSCertificate().Certificate, 1, "The generated identity contains more than 1 TLS certificate in the chain")
 
-	x509Cert, err := x509.ParseCertificate(ident.TLSCertificate.Certificate[0])
+	x509Cert, err := x509.ParseCertificate(ident.GetTLSCertificate().Certificate[0])
 	require.NoError(err, "Failed to parse X.509 certificate from TLS certificate")
 
-	return ident.TLSCertificate, x509Cert
+	return ident.GetTLSCertificate(), x509Cert
 }
 
 // PingQuery is the PingServer query.