diff --git a/go/common/accessctl/accessctl_test.go b/go/common/accessctl/accessctl_test.go index 084786a34ed..e270abe2ade 100644 --- a/go/common/accessctl/accessctl_test.go +++ b/go/common/accessctl/accessctl_test.go @@ -49,11 +49,11 @@ func TestSubjectFromCertificate(t *testing.T) { require.NoError(err, "Failed to create a temporary directory") defer os.RemoveAll(dataDir) - ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory()) + ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory(), false) require.NoError(err, "Failed to generate a new identity") - require.Len(ident.TLSCertificate.Certificate, 1, "The generated identity contains more than 1 certificate in the chain") + require.Len(ident.GetTLSCertificate().Certificate, 1, "The generated identity contains more than 1 certificate in the chain") - x509Cert, err := x509.ParseCertificate(ident.TLSCertificate.Certificate[0]) + x509Cert, err := x509.ParseCertificate(ident.GetTLSCertificate().Certificate[0]) require.NoError(err, "Failed to parse X.509 certificate from TLS certificate") sub := SubjectFromX509Certificate(x509Cert) diff --git a/go/common/grpc/grpc.go b/go/common/grpc/grpc.go index f2c49047dd0..bca3abb7d30 100644 --- a/go/common/grpc/grpc.go +++ b/go/common/grpc/grpc.go @@ -23,6 +23,7 @@ import ( "google.golang.org/grpc/keepalive" "github.com/oasislabs/oasis-core/go/common/grpc/auth" + "github.com/oasislabs/oasis-core/go/common/identity" "github.com/oasislabs/oasis-core/go/common/logging" "github.com/oasislabs/oasis-core/go/common/service" ) @@ -298,8 +299,8 @@ type ServerConfig struct { // nolint: maligned Port uint16 // Path is the path for the local server. Leave nil to create a TCP server. Path string - // Certificate is the certificate used by the server. Should be nil for local servers. - Certificate *tls.Certificate + // Identity is the identity of the worker that's running the server. + Identity *identity.Identity // InstallWrapper specifies whether intercepting facilities should be enabled on this server, // to enable intercepting RPC calls with a wrapper. InstallWrapper bool @@ -473,11 +474,14 @@ func NewServer(config *ServerConfig) (*Server, error) { sOpts = append(sOpts, grpc.CustomCodec(&CBORCodec{})) sOpts = append(sOpts, config.CustomOptions...) - if config.Certificate != nil { + if config.Identity != nil && config.Identity.GetTLSCertificate() != nil { tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{*config.Certificate}, - ClientAuth: clientAuthType, + ClientAuth: clientAuthType, + GetCertificate: func(ch *tls.ClientHelloInfo) (*tls.Certificate, error) { + return config.Identity.GetTLSCertificate(), nil + }, } + sOpts = append(sOpts, grpc.Creds(credentials.NewTLS(tlsConfig))) } diff --git a/go/common/grpc/policy/policy_test.go b/go/common/grpc/policy/policy_test.go index 7e4fb740898..d0b79bf9bf8 100644 --- a/go/common/grpc/policy/policy_test.go +++ b/go/common/grpc/policy/policy_test.go @@ -20,6 +20,7 @@ import ( "github.com/oasislabs/oasis-core/go/common/grpc/policy" "github.com/oasislabs/oasis-core/go/common/grpc/policy/api" cmnTesting "github.com/oasislabs/oasis-core/go/common/grpc/testing" + "github.com/oasislabs/oasis-core/go/common/identity" ) var testNs = common.NewTestNamespaceFromSeed([]byte("oasis common grpc policy test ns"), 0) @@ -60,9 +61,10 @@ func TestAccessPolicy(t *testing.T) { serverConfig := &cmnGrpc.ServerConfig{ Name: host, Port: port, - Certificate: serverTLSCert, + Identity: &identity.Identity{}, CustomOptions: []grpc.ServerOption{grpc.CustomCodec(&cmnGrpc.CBORCodec{})}, } + serverConfig.Identity.SetTLSCertificate(serverTLSCert) grpcServer, err := cmnGrpc.NewServer(serverConfig) require.NoErrorf(err, "Failed to create a new gRPC server: %v", err) diff --git a/go/common/grpc/proxy/proxy_test.go b/go/common/grpc/proxy/proxy_test.go index 6bb3ccaae54..b2bf7798259 100644 --- a/go/common/grpc/proxy/proxy_test.go +++ b/go/common/grpc/proxy/proxy_test.go @@ -17,6 +17,7 @@ import ( commonGrpc "github.com/oasislabs/oasis-core/go/common/grpc" "github.com/oasislabs/oasis-core/go/common/grpc/auth" cmnTesting "github.com/oasislabs/oasis-core/go/common/grpc/testing" + "github.com/oasislabs/oasis-core/go/common/identity" ) const ( @@ -58,9 +59,10 @@ func TestGRPCProxy(t *testing.T) { serverConfig := &commonGrpc.ServerConfig{ Name: host, Port: port, - Certificate: serverTLSCert, + Identity: &identity.Identity{}, CustomOptions: []grpc.ServerOption{grpc.CustomCodec(&commonGrpc.CBORCodec{})}, } + serverConfig.Identity.SetTLSCertificate(serverTLSCert) grpcServer, err := commonGrpc.NewServer(serverConfig) require.NoErrorf(err, "Failed to create a new gRPC server: %v", err) @@ -84,14 +86,15 @@ func TestGRPCProxy(t *testing.T) { // Create a proxy gRPC server. proxyServerConfig := &commonGrpc.ServerConfig{ - Name: host, - Port: port + 1, - Certificate: serverTLSCert, + Name: host, + Port: port + 1, + Identity: &identity.Identity{}, CustomOptions: []grpc.ServerOption{ // All unknown requests will be proxied to the grpc server above. grpc.UnknownServiceHandler(Handler(conn)), }, } + proxyServerConfig.Identity.SetTLSCertificate(serverTLSCert) proxyGrpcServer, err := commonGrpc.NewServer(proxyServerConfig) require.NoErrorf(err, "Failed to create a proxy gRPC server: %v", err) diff --git a/go/common/grpc/testing/ping.go b/go/common/grpc/testing/ping.go index 4ad056d6e48..2c9994843dd 100644 --- a/go/common/grpc/testing/ping.go +++ b/go/common/grpc/testing/ping.go @@ -64,14 +64,14 @@ func CreateCertificate(t *testing.T) (*tls.Certificate, *x509.Certificate) { require.NoError(err, "Failed to create a temporary directory") defer os.RemoveAll(dataDir) - ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory()) + ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory(), false) require.NoError(err, "Failed to generate a new identity") - require.Len(ident.TLSCertificate.Certificate, 1, "The generated identity contains more than 1 TLS certificate in the chain") + require.Len(ident.GetTLSCertificate().Certificate, 1, "The generated identity contains more than 1 TLS certificate in the chain") - x509Cert, err := x509.ParseCertificate(ident.TLSCertificate.Certificate[0]) + x509Cert, err := x509.ParseCertificate(ident.GetTLSCertificate().Certificate[0]) require.NoError(err, "Failed to parse X.509 certificate from TLS certificate") - return ident.TLSCertificate, x509Cert + return ident.GetTLSCertificate(), x509Cert } // PingQuery is the PingServer query. diff --git a/go/common/identity/identity.go b/go/common/identity/identity.go index 977afbd7b8e..42a10dafd92 100644 --- a/go/common/identity/identity.go +++ b/go/common/identity/identity.go @@ -6,10 +6,12 @@ import ( "crypto/rand" "crypto/tls" "path/filepath" + "sync" "github.com/oasislabs/oasis-core/go/common/crypto/signature" "github.com/oasislabs/oasis-core/go/common/crypto/signature/signers/memory" tlsCert "github.com/oasislabs/oasis-core/go/common/crypto/tls" + "github.com/oasislabs/oasis-core/go/common/errors" ) const ( @@ -30,8 +32,16 @@ const ( tlsCertFilename = "tls_identity_cert.pem" ) +// ErrCertificateRotationForbidden is returned by RotateCertificates if +// TLS certificate rotation is forbidden. This happens when rotation is +// enabled and an existing TLS certificate was successfully loaded +// (or a new one was generated and persisted to disk). +var ErrCertificateRotationForbidden = errors.New("identity", 1, "identity: TLS certificate rotation forbidden") + // Identity is a node identity. type Identity struct { + sync.RWMutex + // NodeSigner is a node identity key signer. NodeSigner signature.Signer // P2PSigner is a node P2P link key signer. @@ -40,21 +50,85 @@ type Identity struct { ConsensusSigner signature.Signer // TLSSigner is a node TLS certificate signer. TLSSigner signature.Signer + // DoNotRotate flag is true if we mustn't rotate the TLS certificates. + DoNotRotate bool // TLSCertificate is a certificate that can be used for TLS. - TLSCertificate *tls.Certificate + tlsCertificate *tls.Certificate + // NextTLSCertificate is a certificate that can be used for TLS in the next epoch. + nextTLSCertificate *tls.Certificate +} + +// RotateCertificates rotates the TLS certificates. +// This is called on each epoch change. +func (i *Identity) RotateCertificates() error { + if i.DoNotRotate { + return ErrCertificateRotationForbidden + } + + i.Lock() + defer i.Unlock() + + if i.tlsCertificate != nil { + // Use the prepared certificate. + if i.nextTLSCertificate != nil { + i.tlsCertificate = i.nextTLSCertificate + } + + // Generate a new TLS certificate to be used in the next epoch. + var err error + i.nextTLSCertificate, err = tlsCert.Generate(CommonName) + if err != nil { + return err + } + } + + return nil +} + +// GetTLSCertificate returns the current TLS certificate. +func (i *Identity) GetTLSCertificate() *tls.Certificate { + i.RLock() + defer i.RUnlock() + + return i.tlsCertificate +} + +// SetTLSCertificate sets the current TLS certificate. +func (i *Identity) SetTLSCertificate(cert *tls.Certificate) { + i.Lock() + defer i.Unlock() + + i.tlsCertificate = cert +} + +// GetNextTLSCertificate returns the next TLS certificate. +func (i *Identity) GetNextTLSCertificate() *tls.Certificate { + i.RLock() + defer i.RUnlock() + + return i.nextTLSCertificate +} + +// SetNextTLSCertificate sets the next TLS certificate. +func (i *Identity) SetNextTLSCertificate(nextCert *tls.Certificate) { + i.Lock() + defer i.Unlock() + + i.nextTLSCertificate = nextCert } // Load loads an identity. func Load(dataDir string, signerFactory signature.SignerFactory) (*Identity, error) { - return doLoadOrGenerate(dataDir, signerFactory, false) + return doLoadOrGenerate(dataDir, signerFactory, false, false) } // LoadOrGenerate loads or generates an identity. -func LoadOrGenerate(dataDir string, signerFactory signature.SignerFactory) (*Identity, error) { - return doLoadOrGenerate(dataDir, signerFactory, true) +// If persistTLS is true, it saves the generated TLS certificates to disk. +func LoadOrGenerate(dataDir string, signerFactory signature.SignerFactory, persistTLS bool) (*Identity, error) { + return doLoadOrGenerate(dataDir, signerFactory, true, persistTLS) } -func doLoadOrGenerate(dataDir string, signerFactory signature.SignerFactory, shouldGenerate bool) (*Identity, error) { +func doLoadOrGenerate(dataDir string, signerFactory signature.SignerFactory, shouldGenerate bool, persistTLS bool) (*Identity, error) { var signers []signature.Signer for _, v := range []struct { role signature.SignerRole @@ -86,30 +160,51 @@ func doLoadOrGenerate(dataDir string, signerFactory signature.SignerFactory, sho signers = append(signers, signer) } - // TLS certificate. - // - // TODO: The key and cert could probably be made totally ephemeral, as long - // as the registry update takes effect immediately. var ( - cert *tls.Certificate - err error + nextCert *tls.Certificate + dnr bool ) + + // First, check if we can load the TLS certificate from disk. tlsCertPath, tlsKeyPath := TLSCertPaths(dataDir) - if shouldGenerate { - cert, err = tlsCert.LoadOrGenerate(tlsCertPath, tlsKeyPath, CommonName) + cert, err := tlsCert.Load(tlsCertPath, tlsKeyPath) + if err == nil { + // Load successful, ensure that we won't ever rotate the certificates. + dnr = true } else { - cert, err = tlsCert.Load(tlsCertPath, tlsKeyPath) - } - if err != nil { - return nil, err + // Freshly generate TLS certificates. + cert, err = tlsCert.Generate(CommonName) + if err != nil { + return nil, err + } + + if persistTLS { + // Save generated TLS certificate to disk. + err = tlsCert.Save(tlsCertPath, tlsKeyPath, cert) + if err != nil { + return nil, err + } + + // Disable TLS rotation if we're persisting TLS certificates. + dnr = true + } else { + // Not persisting TLS certificate to disk, generate a new + // certificate to be used in the next rotation. + nextCert, err = tlsCert.Generate(CommonName) + if err != nil { + return nil, err + } + } } return &Identity{ - NodeSigner: signers[0], - P2PSigner: signers[1], - ConsensusSigner: signers[2], - TLSSigner: memory.NewFromRuntime(cert.PrivateKey.(ed25519.PrivateKey)), - TLSCertificate: cert, + NodeSigner: signers[0], + P2PSigner: signers[1], + ConsensusSigner: signers[2], + TLSSigner: memory.NewFromRuntime(cert.PrivateKey.(ed25519.PrivateKey)), + DoNotRotate: dnr, + tlsCertificate: cert, + nextTLSCertificate: nextCert, }, nil } diff --git a/go/common/identity/identity_test.go b/go/common/identity/identity_test.go index 796f0bfbdb9..b1755bdf19b 100644 --- a/go/common/identity/identity_test.go +++ b/go/common/identity/identity_test.go @@ -19,16 +19,36 @@ func TestLoadOrGenerate(t *testing.T) { factory := fileSigner.NewFactory(dataDir, signature.SignerNode, signature.SignerP2P, signature.SignerConsensus) // Generate a new identity. - identity, err := LoadOrGenerate(dataDir, factory) + identity, err := LoadOrGenerate(dataDir, factory, true) require.NoError(t, err, "LoadOrGenerate") // Load an existing identity. - identity2, err := LoadOrGenerate(dataDir, factory) + identity2, err := LoadOrGenerate(dataDir, factory, false) require.NoError(t, err, "LoadOrGenerate (2)") require.EqualValues(t, identity.NodeSigner, identity2.NodeSigner) require.EqualValues(t, identity.P2PSigner, identity2.P2PSigner) require.EqualValues(t, identity.ConsensusSigner, identity2.ConsensusSigner) require.EqualValues(t, identity.TLSSigner, identity2.TLSSigner) - // TODO: Check that it always generates a fresh certificate once oasis-core#1541 is done. - require.EqualValues(t, identity.TLSCertificate, identity2.TLSCertificate) + require.EqualValues(t, identity.GetTLSCertificate(), identity2.GetTLSCertificate()) + + dataDir2, err := ioutil.TempDir("", "oasis-identity-test2_") + require.NoError(t, err, "create data dir (2)") + defer os.RemoveAll(dataDir2) + + // Generate a new identity again, this time without persisting TLS certs. + identity3, err := LoadOrGenerate(dataDir2, factory, false) + require.NoError(t, err, "LoadOrGenerate (3)") + + // Load it back. + identity4, err := LoadOrGenerate(dataDir, factory, false) + require.NoError(t, err, "LoadOrGenerate (4)") + require.EqualValues(t, identity3.NodeSigner, identity4.NodeSigner) + require.EqualValues(t, identity3.P2PSigner, identity4.P2PSigner) + require.EqualValues(t, identity3.ConsensusSigner, identity4.ConsensusSigner) + require.NotEqual(t, identity.TLSSigner, identity3.TLSSigner) + require.NotEqual(t, identity2.TLSSigner, identity3.TLSSigner) + require.NotEqual(t, identity3.TLSSigner, identity4.TLSSigner) + require.NotEqual(t, identity.GetTLSCertificate(), identity3.GetTLSCertificate()) + require.NotEqual(t, identity2.GetTLSCertificate(), identity3.GetTLSCertificate()) + require.NotEqual(t, identity3.GetTLSCertificate(), identity4.GetTLSCertificate()) } diff --git a/go/common/node/node.go b/go/common/node/node.go index 5f81e5d83ad..a5b590561d6 100644 --- a/go/common/node/node.go +++ b/go/common/node/node.go @@ -174,17 +174,23 @@ type CommitteeInfo struct { // Certificate is the certificate for establishing TLS connections. Certificate []byte `json:"certificate"` + // NextCertificate is the certificate that will be used for establishing TLS connections in the next epoch. + NextCertificate []byte `json:"next_certificate,omitempty"` + // Addresses is the list of committee addresses at which the node can be reached. Addresses []CommitteeAddress `json:"addresses"` } // Equal compares vs another CommitteeInfo for equality. func (c *CommitteeInfo) Equal(other *CommitteeInfo) bool { - // XXX: Why is this top-level certificate even needed? if !bytes.Equal(c.Certificate, other.Certificate) { return false } + if !bytes.Equal(c.NextCertificate, other.NextCertificate) { + return false + } + if len(c.Addresses) != len(other.Addresses) { return false } @@ -202,6 +208,11 @@ func (c *CommitteeInfo) ParseCertificate() (*x509.Certificate, error) { return x509.ParseCertificate(c.Certificate) } +// ParseCertificate returns the parsed x509 next certificate. +func (c *CommitteeInfo) ParseNextCertificate() (*x509.Certificate, error) { + return x509.ParseCertificate(c.NextCertificate) +} + // P2PInfo contains information for connecting to this node via P2P transport. type P2PInfo struct { // ID is the unique identifier of the node on the P2P transport. diff --git a/go/genesis/genesis_test.go b/go/genesis/genesis_test.go index b0babf89e66..58d343b9621 100644 --- a/go/genesis/genesis_test.go +++ b/go/genesis/genesis_test.go @@ -1,7 +1,6 @@ package genesis import ( - "crypto/ed25519" "encoding/hex" "math" "testing" @@ -242,11 +241,9 @@ func TestGenesisSanityCheck(t *testing.T) { Addresses: []node.ConsensusAddress{testConsensusAddress}, }, } - nodeTLSSigner := memorySigner.NewFromRuntime(dummyCert.PrivateKey.(ed25519.PrivateKey)) nodeSigners := []signature.Signer{ nodeSigner, nodeP2PSigner, - nodeTLSSigner, nodeConsensusSigner, } signedTestNode := signNodeOrDie(nodeSigners, testNode) diff --git a/go/ias/proxy/client/client.go b/go/ias/proxy/client/client.go index a741b32d490..20f040f82ff 100644 --- a/go/ias/proxy/client/client.go +++ b/go/ias/proxy/client/client.go @@ -119,7 +119,7 @@ func New(identity *identity.Identity, proxyAddr, tlsCertFile string) (api.Endpoi certPool := x509.NewCertPool() certPool.AddCert(parsedCert) creds := credentials.NewTLS(&tls.Config{ - Certificates: []tls.Certificate{*identity.TLSCertificate}, + Certificates: []tls.Certificate{*identity.GetTLSCertificate()}, RootCAs: certPool, ServerName: proxy.CommonName, }) diff --git a/go/oasis-node/cmd/common/grpc/grpc.go b/go/oasis-node/cmd/common/grpc/grpc.go index 041e2ee7f61..045c598d7fd 100644 --- a/go/oasis-node/cmd/common/grpc/grpc.go +++ b/go/oasis-node/cmd/common/grpc/grpc.go @@ -14,6 +14,7 @@ import ( "google.golang.org/grpc" cmnGrpc "github.com/oasislabs/oasis-core/go/common/grpc" + "github.com/oasislabs/oasis-core/go/common/identity" "github.com/oasislabs/oasis-core/go/common/logging" "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common" ) @@ -50,9 +51,10 @@ func NewServerTCP(cert *tls.Certificate, installWrapper bool) (*cmnGrpc.Server, config := &cmnGrpc.ServerConfig{ Name: "internal", Port: uint16(viper.GetInt(CfgServerPort)), - Certificate: cert, + Identity: &identity.Identity{}, InstallWrapper: installWrapper, } + config.Identity.SetTLSCertificate(cert) return cmnGrpc.NewServer(config) } diff --git a/go/oasis-node/cmd/debug/byzantine/registry.go b/go/oasis-node/cmd/debug/byzantine/registry.go index ad433d8c76e..3bcf991118b 100644 --- a/go/oasis-node/cmd/debug/byzantine/registry.go +++ b/go/oasis-node/cmd/debug/byzantine/registry.go @@ -37,7 +37,7 @@ func registryRegisterNode(svc service.TendermintService, id *identity.Identity, var committeeAddresses []node.CommitteeAddress for _, addr := range addresses { committeeAddresses = append(committeeAddresses, node.CommitteeAddress{ - Certificate: id.TLSCertificate.Certificate[0], + Certificate: id.GetTLSCertificate().Certificate[0], Address: addr, }) } @@ -47,7 +47,7 @@ func registryRegisterNode(svc service.TendermintService, id *identity.Identity, EntityID: entityID, Expiration: 1000, Committee: node.CommitteeInfo{ - Certificate: id.TLSCertificate.Certificate[0], + Certificate: id.GetTLSCertificate().Certificate[0], Addresses: committeeAddresses, }, P2P: node.P2PInfo{ @@ -68,7 +68,6 @@ func registryRegisterNode(svc service.TendermintService, id *identity.Identity, registrationSigner, id.P2PSigner, id.ConsensusSigner, - id.TLSSigner, }, registry.RegisterGenesisNodeSignatureContext, nodeDesc, diff --git a/go/oasis-node/cmd/debug/byzantine/steps.go b/go/oasis-node/cmd/debug/byzantine/steps.go index e3edd27cad2..8680a378255 100644 --- a/go/oasis-node/cmd/debug/byzantine/steps.go +++ b/go/oasis-node/cmd/debug/byzantine/steps.go @@ -39,7 +39,7 @@ var ( func initDefaultIdentity(dataDir string) (*identity.Identity, error) { signerFactory := fileSigner.NewFactory(dataDir, signature.SignerNode, signature.SignerP2P, signature.SignerEntity, signature.SignerConsensus) - id, err := identity.LoadOrGenerate(dataDir, signerFactory) + id, err := identity.LoadOrGenerate(dataDir, signerFactory, false) if err != nil { return nil, errors.Wrap(err, "identity LoadOrGenerate") } diff --git a/go/oasis-node/cmd/debug/byzantine/storage.go b/go/oasis-node/cmd/debug/byzantine/storage.go index c5ec75c0677..1b0b6640c89 100644 --- a/go/oasis-node/cmd/debug/byzantine/storage.go +++ b/go/oasis-node/cmd/debug/byzantine/storage.go @@ -69,7 +69,7 @@ func dialNode(node *node.Node, opts grpc.DialOption) (*grpc.ClientConn, func(), } func newHonestNodeStorage(id *identity.Identity, node *node.Node) (*honestNodeStorage, error) { - opts, err := dialOptionForNode([]tls.Certificate{*id.TLSCertificate}, node) + opts, err := dialOptionForNode([]tls.Certificate{*id.GetTLSCertificate()}, node) if err != nil { return nil, errors.Wrap(err, "storage client DialOptionForNode") } diff --git a/go/oasis-node/cmd/identity/identity.go b/go/oasis-node/cmd/identity/identity.go index 1a40484b069..8f23789e7bb 100644 --- a/go/oasis-node/cmd/identity/identity.go +++ b/go/oasis-node/cmd/identity/identity.go @@ -44,7 +44,7 @@ func doNodeInit(cmd *cobra.Command, args []string) { // Provision the node identity. nodeSignerFactory := fileSigner.NewFactory(dataDir, signature.SignerNode, signature.SignerP2P, signature.SignerConsensus) - _, err := identity.LoadOrGenerate(dataDir, nodeSignerFactory) + _, err := identity.LoadOrGenerate(dataDir, nodeSignerFactory, true) if err != nil { logger.Error("failed to load or generate node identity", "err", err, diff --git a/go/oasis-node/cmd/node/node.go b/go/oasis-node/cmd/node/node.go index e0a2ff61804..b3e03da3295 100644 --- a/go/oasis-node/cmd/node/node.go +++ b/go/oasis-node/cmd/node/node.go @@ -521,7 +521,7 @@ func newNode(testNode bool) (*Node, error) { // Generate/Load the node identity. // TODO/hsm: Configure factory dynamically. signerFactory := fileSigner.NewFactory(dataDir, signature.SignerNode, signature.SignerP2P, signature.SignerConsensus) - node.Identity, err = identity.LoadOrGenerate(dataDir, signerFactory) + node.Identity, err = identity.LoadOrGenerate(dataDir, signerFactory, false) if err != nil { logger.Error("failed to load/generate identity", "err", err, diff --git a/go/oasis-node/cmd/registry/node/node.go b/go/oasis-node/cmd/registry/node/node.go index becc0f64807..6617842fd4b 100644 --- a/go/oasis-node/cmd/registry/node/node.go +++ b/go/oasis-node/cmd/registry/node/node.go @@ -91,7 +91,7 @@ func doConnect(cmd *cobra.Command) (*grpc.ClientConn, registry.Backend) { return conn, client } -func doInit(cmd *cobra.Command, args []string) { +func doInit(cmd *cobra.Command, args []string) { // nolint: gocyclo if err := cmdCommon.Init(); err != nil { cmdCommon.EarlyLogAndExit(err) } @@ -155,7 +155,7 @@ func doInit(cmd *cobra.Command, args []string) { // Provision the node identity. nodeSignerFactory := fileSigner.NewFactory(dataDir, signature.SignerNode, signature.SignerP2P, signature.SignerConsensus) - nodeIdentity, err := identity.LoadOrGenerate(dataDir, nodeSignerFactory) + nodeIdentity, err := identity.LoadOrGenerate(dataDir, nodeSignerFactory, false) if err != nil { logger.Error("failed to load or generate node identity", "err", err, @@ -163,12 +163,18 @@ func doInit(cmd *cobra.Command, args []string) { os.Exit(1) } + var nextCert []byte + if c := nodeIdentity.GetNextTLSCertificate(); c != nil { + nextCert = c.Certificate[0] + } + n := &node.Node{ ID: nodeIdentity.NodeSigner.Public(), EntityID: entityID, Expiration: viper.GetUint64(CfgExpiration), Committee: node.CommitteeInfo{ - Certificate: nodeIdentity.TLSCertificate.Certificate[0], + Certificate: nodeIdentity.GetTLSCertificate().Certificate[0], + NextCertificate: nextCert, }, P2P: node.P2PInfo{ ID: nodeIdentity.P2PSigner.Public(), @@ -262,7 +268,6 @@ func doInit(cmd *cobra.Command, args []string) { signers = append(signers, []signature.Signer{ nodeIdentity.P2PSigner, nodeIdentity.ConsensusSigner, - nodeIdentity.TLSSigner, }...) signed, err := node.MultiSignNode(signers, registry.RegisterGenesisNodeSignatureContext, n) diff --git a/go/oasis-node/cmd/storage/benchmark/benchmark.go b/go/oasis-node/cmd/storage/benchmark/benchmark.go index 64782117760..7e44d1fba9f 100644 --- a/go/oasis-node/cmd/storage/benchmark/benchmark.go +++ b/go/oasis-node/cmd/storage/benchmark/benchmark.go @@ -70,7 +70,7 @@ func doBenchmark(cmd *cobra.Command, args []string) { // nolint: gocyclo } // Create an identity. - ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory()) + ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory(), false) if err != nil { logger.Error("failed to generate a new identity", "err", err, diff --git a/go/oasis-test-runner/oasis/byzantine.go b/go/oasis-test-runner/oasis/byzantine.go index 920fe7e3d48..ebd2f5ed7f3 100644 --- a/go/oasis-test-runner/oasis/byzantine.go +++ b/go/oasis-test-runner/oasis/byzantine.go @@ -91,7 +91,7 @@ func (net *Network) NewByzantine(cfg *ByzantineCfg) (*Byzantine, error) { } // Pre-provision the node identity so that we can update the entity. - publicKey, err := net.provisionNodeIdentity(byzantineDir, cfg.IdentitySeed) + publicKey, err := net.provisionNodeIdentity(byzantineDir, cfg.IdentitySeed, false) if err != nil { return nil, errors.Wrap(err, "oasis/byzantine: failed to provision node identity") } diff --git a/go/oasis-test-runner/oasis/compute.go b/go/oasis-test-runner/oasis/compute.go index 01f9f0d3d38..a8bdb2dcb56 100644 --- a/go/oasis-test-runner/oasis/compute.go +++ b/go/oasis-test-runner/oasis/compute.go @@ -129,7 +129,7 @@ func (net *Network) NewCompute(cfg *ComputeCfg) (*Compute, error) { // Pre-provision the node identity so that we can update the entity. seed := fmt.Sprintf(computeIdentitySeedTemplate, len(net.computeWorkers)) - publicKey, err := net.provisionNodeIdentity(computeDir, seed) + publicKey, err := net.provisionNodeIdentity(computeDir, seed, false) if err != nil { return nil, errors.Wrap(err, "oasis/compute: failed to provision node identity") } diff --git a/go/oasis-test-runner/oasis/keymanager.go b/go/oasis-test-runner/oasis/keymanager.go index 45220e3b4f8..c956e5c0cf6 100644 --- a/go/oasis-test-runner/oasis/keymanager.go +++ b/go/oasis-test-runner/oasis/keymanager.go @@ -238,10 +238,16 @@ func (net *Network) NewKeymanager(cfg *KeymanagerCfg) (*Keymanager, error) { return nil, errors.Wrap(err, "oasis/keymanager: failed to create keymanager subdir") } + // If we're using sentry nodes, we need to have a static TLS certificate. + var persistTLS bool + if len(cfg.SentryIndices) > 0 { + persistTLS = true + } + // Pre-provision the node identity so that we can update the entity. // TODO: Use proper key manager index when multiple key managers are supported. seed := fmt.Sprintf(keymanagerIdentitySeedTemplate, 0) - publicKey, err := net.provisionNodeIdentity(kmDir, seed) + publicKey, err := net.provisionNodeIdentity(kmDir, seed, persistTLS) if err != nil { return nil, errors.Wrap(err, "oasis/keymanager: failed to provision node identity") } diff --git a/go/oasis-test-runner/oasis/oasis.go b/go/oasis-test-runner/oasis/oasis.go index 46e50451b45..69c513f42a5 100644 --- a/go/oasis-test-runner/oasis/oasis.go +++ b/go/oasis-test-runner/oasis/oasis.go @@ -27,6 +27,7 @@ import ( "github.com/oasislabs/oasis-core/go/oasis-node/cmd/genesis" "github.com/oasislabs/oasis-core/go/oasis-test-runner/env" "github.com/oasislabs/oasis-core/go/oasis-test-runner/log" + "github.com/oasislabs/oasis-core/go/worker/registration" ) const ( @@ -584,6 +585,10 @@ func (net *Network) startOasisNode( args = append(args, baseArgs...) args = append(args, extraArgs.vec...) + if len(net.sentries) == 0 && net.iasProxy == nil { + args = append(args, []string{"--" + registration.CfgRegistrationRotateCerts, "1"}...) + } + w, err := dir.NewLogWriter(logConsoleFile) if err != nil { return nil, nil, err @@ -701,7 +706,7 @@ func (net *Network) BasePath() string { return net.baseDir.String() } -func (net *Network) provisionNodeIdentity(dataDir *env.Dir, seed string) (signature.PublicKey, error) { +func (net *Network) provisionNodeIdentity(dataDir *env.Dir, seed string, persistTLS bool) (signature.PublicKey, error) { if net.cfg.DeterministicIdentities { if err := net.generateDeterministicNodeIdentity(dataDir, seed); err != nil { return signature.PublicKey{}, errors.Wrap(err, "oasis: failed to generate deterministic identity") @@ -709,7 +714,7 @@ func (net *Network) provisionNodeIdentity(dataDir *env.Dir, seed string) (signat } signerFactory := fileSigner.NewFactory(dataDir.String(), signature.SignerNode, signature.SignerP2P, signature.SignerConsensus) - nodeIdentity, err := identity.LoadOrGenerate(dataDir.String(), signerFactory) + nodeIdentity, err := identity.LoadOrGenerate(dataDir.String(), signerFactory, persistTLS) if err != nil { return signature.PublicKey{}, errors.Wrap(err, "oasis: failed to provision node identity") } diff --git a/go/oasis-test-runner/oasis/seed.go b/go/oasis-test-runner/oasis/seed.go index b26fed6cb54..8067fb064db 100644 --- a/go/oasis-test-runner/oasis/seed.go +++ b/go/oasis-test-runner/oasis/seed.go @@ -55,7 +55,7 @@ func (net *Network) newSeedNode() (*seedNode, error) { // to pass all the actual nodes in advance, instead of having to // start the node and fork out to `oasis-node debug tendermint show-node-id`. signerFactory := fileSigner.NewFactory(seedDir.String(), signature.SignerNode, signature.SignerP2P, signature.SignerConsensus) - seedIdentity, err := identity.LoadOrGenerate(seedDir.String(), signerFactory) + seedIdentity, err := identity.LoadOrGenerate(seedDir.String(), signerFactory, false) if err != nil { return nil, errors.Wrap(err, "oasis/seed: failed to provision seed identity") } diff --git a/go/oasis-test-runner/oasis/sentry.go b/go/oasis-test-runner/oasis/sentry.go index 315a90af9a9..6dd4ad32a39 100644 --- a/go/oasis-test-runner/oasis/sentry.go +++ b/go/oasis-test-runner/oasis/sentry.go @@ -105,7 +105,7 @@ func (net *Network) NewSentry(cfg *SentryCfg) (*Sentry, error) { // address to the validator so it can configure the sentry node's consensus // address as its consensus address. signerFactory := fileSigner.NewFactory(sentryDir.String(), signature.SignerNode, signature.SignerP2P, signature.SignerConsensus) - sentryIdentity, err := identity.LoadOrGenerate(sentryDir.String(), signerFactory) + sentryIdentity, err := identity.LoadOrGenerate(sentryDir.String(), signerFactory, true) if err != nil { net.logger.Error("failed to provision sentry identity", "err", err, diff --git a/go/oasis-test-runner/oasis/storage.go b/go/oasis-test-runner/oasis/storage.go index 6e03c9d4270..86a93bef409 100644 --- a/go/oasis-test-runner/oasis/storage.go +++ b/go/oasis-test-runner/oasis/storage.go @@ -148,9 +148,15 @@ func (net *Network) NewStorage(cfg *StorageCfg) (*Storage, error) { return nil, errors.Wrap(err, "oasis/storage: failed to create storage subdir") } + // If we're using sentry nodes, we need to have a static TLS certificate. + var persistTLS bool + if len(cfg.SentryIndices) > 0 { + persistTLS = true + } + // Pre-provision the node identity so that we can update the entity. seed := fmt.Sprintf(storageIdentitySeedTemplate, len(net.storageWorkers)) - publicKey, err := net.provisionNodeIdentity(storageDir, seed) + publicKey, err := net.provisionNodeIdentity(storageDir, seed, persistTLS) if err != nil { return nil, errors.Wrap(err, "oasis/storage: failed to provision node identity") } diff --git a/go/oasis-test-runner/oasis/validator.go b/go/oasis-test-runner/oasis/validator.go index 46e9c93b066..595a27007da 100644 --- a/go/oasis-test-runner/oasis/validator.go +++ b/go/oasis-test-runner/oasis/validator.go @@ -162,10 +162,16 @@ func (net *Network) NewValidator(cfg *ValidatorCfg) (*Validator, error) { consensusAddrs = append(consensusAddrs, &consensusAddr) } + // If we're using sentry nodes, we need to have a static TLS certificate. + var persistTLS bool + if len(val.sentries) > 0 { + persistTLS = true + } + // Load node's identity, so that we can pass the validator's Tendermint // address to sentry node(s) to configure it as a private peer. seed := fmt.Sprintf(validatorIdentitySeedTemplate, len(net.validators)) - valPublicKey, err := net.provisionNodeIdentity(valDir, seed) + valPublicKey, err := net.provisionNodeIdentity(valDir, seed, persistTLS) if err != nil { return nil, errors.Wrap(err, "oasis/validator: failed to provision node identity") } diff --git a/go/oasis-test-runner/scenario/e2e/registry_cli.go b/go/oasis-test-runner/scenario/e2e/registry_cli.go index 9b0bf6ab857..a2f0646d3ed 100644 --- a/go/oasis-test-runner/scenario/e2e/registry_cli.go +++ b/go/oasis-test-runner/scenario/e2e/registry_cli.go @@ -492,6 +492,7 @@ func (r *registryCLIImpl) initNode(childEnv *env.Env, ent *entity.Entity, entDir // Replace our testNode fields with the generated one, so we can just marshal both nodes and compare the output afterwards. testNode.ID = n.ID testNode.Committee.Certificate = n.Committee.Certificate + testNode.Committee.NextCertificate = n.Committee.NextCertificate testNode.P2P.ID = n.P2P.ID testNode.Consensus.ID = n.Consensus.ID for idx := range testNode.Committee.Addresses { @@ -513,6 +514,15 @@ func (r *registryCLIImpl) initNode(childEnv *env.Env, ent *entity.Entity, entDir if err != nil { return nil, err } + + // TLS certificates are regenerated each time, so replace them with new ones. + testNode.Committee.Certificate = n.Committee.Certificate + testNode.Committee.NextCertificate = n.Committee.NextCertificate + for idx := range testNode.Committee.Addresses { + testNode.Committee.Addresses[idx].Certificate = n.Committee.Certificate + } + testNodeStr, _ = json.Marshal(testNode) + nStr, _ = json.Marshal(n) if !bytes.Equal(nStr, testNodeStr) { return nil, fmt.Errorf("second run test node mismatch! Original node: %s, imported node: %s", testNodeStr, nStr) diff --git a/go/registry/api/api.go b/go/registry/api/api.go index 3aa6c88b4c7..86cb3138a78 100644 --- a/go/registry/api/api.go +++ b/go/registry/api/api.go @@ -572,18 +572,16 @@ func VerifyRegisterNodeArgs( // nolint: gocyclo ) return nil, nil, err } - certPub, err := verifyNodeCertificate(logger, &n) + _, err := verifyNodeCertificate(logger, &n) if err != nil { return nil, nil, err } - if !sigNode.MultiSigned.IsSignedBy(certPub) { - logger.Error("RegisterNode: not signed by TLS certificate key", - "signed_node", sigNode, - "node", n, - ) - return nil, nil, fmt.Errorf("%w: registration not signed by TLS certificate key", ErrInvalidArgument) + if n.Committee.NextCertificate != nil { + _, grr := verifyNodeNextCertificate(logger, &n) + if grr != nil { + return nil, nil, grr + } } - expectedSigners = append(expectedSigners, certPub) // Validate P2PInfo. if !n.P2P.ID.IsValid() { @@ -714,6 +712,38 @@ func verifyNodeCertificate(logger *logging.Logger, node *node.Node) (signature.P return certPub, nil } +func verifyNodeNextCertificate(logger *logging.Logger, node *node.Node) (signature.PublicKey, error) { + var certPub signature.PublicKey + + cert, err := node.Committee.ParseNextCertificate() + if err != nil { + logger.Error("RegisterNode: failed to parse committee certificate", + "err", err, + "node", node, + ) + return certPub, fmt.Errorf("%w: failed to parse committee certificate", ErrInvalidArgument) + } + + edPub, ok := cert.PublicKey.(goEd25519.PublicKey) + if !ok { + logger.Error("RegisterNode: incorrect committee certifiate signing algorithm", + "node", node, + ) + return certPub, fmt.Errorf("%w: incorrect committee certificate signing algorithm", ErrInvalidArgument) + } + + if err = certPub.UnmarshalBinary(edPub); err != nil { + // This should NEVER happen. + logger.Error("RegisterNode: malformed committee certificate signing key", + "err", err, + "node", node, + ) + return certPub, fmt.Errorf("%w: malformed committee certificate signing key", ErrInvalidArgument) + } + + return certPub, nil +} + // VerifyNodeRuntimeEnclaveIDs verifies TEE-specific attributes of the node's runtime. func VerifyNodeRuntimeEnclaveIDs(logger *logging.Logger, rt *node.Runtime, regRt *Runtime, ts time.Time) error { // If no TEE available, do nothing. diff --git a/go/registry/tests/tester.go b/go/registry/tests/tester.go index 852110993a3..1dd0af6e40c 100644 --- a/go/registry/tests/tester.go +++ b/go/registry/tests/tester.go @@ -597,12 +597,12 @@ func randomIdentity(rng *drbg.Drbg) *identity.Identity { ConsensusSigner: mustGenerateSigner(), } - var err error - ident.TLSCertificate, err = tls.Generate(identity.CommonName) + cert, err := tls.Generate(identity.CommonName) if err != nil { panic(err) } - ident.TLSSigner = memorySigner.NewFromRuntime(ident.TLSCertificate.PrivateKey.(ed25519.PrivateKey)) + ident.SetTLSCertificate(cert) + ident.TLSSigner = memorySigner.NewFromRuntime(cert.PrivateKey.(ed25519.PrivateKey)) return ident } @@ -628,7 +628,6 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, ent.Signer, nodeIdentity.P2PSigner, nodeIdentity.ConsensusSigner, - nodeIdentity.TLSSigner, } invalidIdentity := randomIdentity(rng) @@ -660,7 +659,7 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, nod.Node.P2P.Addresses = append(nod.Node.P2P.Addresses, addr) nod.Node.Consensus.ID = nodeIdentity.ConsensusSigner.Public() // Generate dummy TLS certificate. - nod.Node.Committee.Certificate = nodeIdentity.TLSCertificate.Certificate[0] + nod.Node.Committee.Certificate = nodeIdentity.GetTLSCertificate().Certificate[0] nod.Node.Committee.Addresses = []node.CommitteeAddress{ node.CommitteeAddress{ Certificate: nod.Node.Committee.Certificate, @@ -756,7 +755,6 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, nodeIdentity.NodeSigner, ent.Signer, nodeIdentity.ConsensusSigner, - nodeIdentity.TLSSigner, }, api.RegisterNodeSignatureContext, &invNode6, @@ -821,7 +819,6 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, nodeIdentity.NodeSigner, ent.Signer, nodeIdentity.P2PSigner, - nodeIdentity.TLSSigner, }, api.RegisterNodeSignatureContext, &invNode10, @@ -838,14 +835,13 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, invNode11 := *nod.Node invNode11.ID = invalidIdentity.NodeSigner.Public() invNode11.Consensus.ID = invalidIdentity.ConsensusSigner.Public() - invNode11.Committee.Certificate = invalidIdentity.TLSCertificate.Certificate[0] + invNode11.Committee.Certificate = invalidIdentity.GetTLSCertificate().Certificate[0] invalid11.signed, err = node.MultiSignNode( []signature.Signer{ invalidIdentity.NodeSigner, ent.Signer, invalidIdentity.ConsensusSigner, nodeIdentity.P2PSigner, - invalidIdentity.TLSSigner, }, api.RegisterNodeSignatureContext, &invNode11, @@ -862,14 +858,13 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, invNode12 := *nod.Node invNode12.ID = invalidIdentity.NodeSigner.Public() invNode12.P2P.ID = invalidIdentity.ConsensusSigner.Public() - invNode12.Committee.Certificate = invalidIdentity.TLSCertificate.Certificate[0] + invNode12.Committee.Certificate = invalidIdentity.GetTLSCertificate().Certificate[0] invalid12.signed, err = node.MultiSignNode( []signature.Signer{ invalidIdentity.NodeSigner, ent.Signer, nodeIdentity.ConsensusSigner, invalidIdentity.P2PSigner, - invalidIdentity.TLSSigner, }, api.RegisterNodeSignatureContext, &invNode12, @@ -893,7 +888,6 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, ent.Signer, invalidIdentity.ConsensusSigner, invalidIdentity.P2PSigner, - nodeIdentity.TLSSigner, }, api.RegisterNodeSignatureContext, &invNode13, @@ -945,14 +939,13 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, } newNode.P2P.ID = invalidIdentity.P2PSigner.Public() newNode.Consensus.ID = invalidIdentity.ConsensusSigner.Public() - newNode.Committee.Certificate = invalidIdentity.TLSCertificate.Certificate[0] + newNode.Committee.Certificate = invalidIdentity.GetTLSCertificate().Certificate[0] invalid14.signed, err = node.MultiSignNode( []signature.Signer{ nodeIdentity.NodeSigner, ent.Signer, invalidIdentity.ConsensusSigner, invalidIdentity.P2PSigner, - invalidIdentity.TLSSigner, }, api.RegisterNodeSignatureContext, newNode, diff --git a/go/runtime/committee/client.go b/go/runtime/committee/client.go index ffcce37b1f1..0df88421ab4 100644 --- a/go/runtime/committee/client.go +++ b/go/runtime/committee/client.go @@ -264,9 +264,11 @@ func (cc *committeeClient) updateConnectionLocked(n *node.Node) error { RootCAs: certPool, ServerName: identity.CommonName, } - if cc.clientIdentity != nil { + if cc.clientIdentity != nil && cc.clientIdentity.GetTLSCertificate() != nil { // Configure TLS client authentication if required. - tlsCfg.Certificates = []tls.Certificate{*cc.clientIdentity.TLSCertificate} + tlsCfg.GetClientCertificate = func(cri *tls.CertificateRequestInfo) (*tls.Certificate, error) { + return cc.clientIdentity.GetTLSCertificate(), nil + } } creds := credentials.NewTLS(&tlsCfg) diff --git a/go/sentry/client/client.go b/go/sentry/client/client.go index 3287b57ed01..9e153414052 100644 --- a/go/sentry/client/client.go +++ b/go/sentry/client/client.go @@ -45,9 +45,11 @@ func (c *Client) createConnection() error { certPool := x509.NewCertPool() certPool.AddCert(c.sentryCert) creds := credentials.NewTLS(&tls.Config{ - Certificates: []tls.Certificate{*c.nodeIdentity.TLSCertificate}, - RootCAs: certPool, - ServerName: identity.CommonName, + RootCAs: certPool, + ServerName: identity.CommonName, + GetClientCertificate: func(cri *tls.CertificateRequestInfo) (*tls.Certificate, error) { + return c.nodeIdentity.GetTLSCertificate(), nil + }, }) opts := grpc.WithTransportCredentials(creds) diff --git a/go/sentry/sentry.go b/go/sentry/sentry.go index 05041b2f01c..8ba7c101d92 100644 --- a/go/sentry/sentry.go +++ b/go/sentry/sentry.go @@ -40,7 +40,7 @@ func (b *backend) GetAddresses(ctx context.Context) (*api.SentryAddresses, error var committeeAddresses []node.CommitteeAddress for _, addr := range committeeAddrs { committeeAddresses = append(committeeAddresses, node.CommitteeAddress{ - Certificate: b.identity.TLSCertificate.Certificate[0], + Certificate: b.identity.GetTLSCertificate().Certificate[0], Address: addr, }) } diff --git a/go/storage/mkvs/urkel/interop/cmd/protocol_server.go b/go/storage/mkvs/urkel/interop/cmd/protocol_server.go index 515401f1f21..a34020540e2 100644 --- a/go/storage/mkvs/urkel/interop/cmd/protocol_server.go +++ b/go/storage/mkvs/urkel/interop/cmd/protocol_server.go @@ -45,7 +45,7 @@ func doProtoServer(cmd *cobra.Command, args []string) { genesisTestHelpers.SetTestChainContext() // Generate dummy identity. - ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory()) + ident, err := identity.LoadOrGenerate(dataDir, memorySigner.NewFactory(), false) if err != nil { logger.Error("failed to generate identity", "err", err, diff --git a/go/worker/common/committee/accessctl.go b/go/worker/common/committee/accessctl.go index 6d090b87b30..0022a7d1a10 100644 --- a/go/worker/common/committee/accessctl.go +++ b/go/worker/common/committee/accessctl.go @@ -29,10 +29,20 @@ func (ap AccessPolicy) AddRulesForCommittee(policy *accessctl.Policy, committee continue } + // Allow the node to perform actions from the given access policy. subject := accessctl.SubjectFromDER(node.Committee.Certificate) for _, action := range ap.Actions { policy.Allow(subject, action) } + + // Make sure to also allow the node to perform actions after it has + // rotated its TLS certificates. + if node.Committee.NextCertificate != nil { + subject := accessctl.SubjectFromDER(node.Committee.NextCertificate) + for _, action := range ap.Actions { + policy.Allow(subject, action) + } + } } } @@ -58,7 +68,15 @@ func (ap AccessPolicy) AddRulesForNodeRoles( for _, action := range ap.Actions { policy.Allow(subject, action) } - } + // Make sure to also allow the node to perform actions after is has + // rotated its TLS certificates. + if n.Committee.NextCertificate != nil { + subject := accessctl.SubjectFromDER(n.Committee.NextCertificate) + for _, action := range ap.Actions { + policy.Allow(subject, action) + } + } + } } } diff --git a/go/worker/common/worker.go b/go/worker/common/worker.go index 3b0ae738412..b370d5f4f2e 100644 --- a/go/worker/common/worker.go +++ b/go/worker/common/worker.go @@ -267,9 +267,9 @@ func New( // Create externally-accessible gRPC server. serverConfig := &grpc.ServerConfig{ - Name: "external", - Port: cfg.ClientPort, - Certificate: identity.TLSCertificate, + Name: "external", + Port: cfg.ClientPort, + Identity: identity, } grpc, err := grpc.NewServer(serverConfig) if err != nil { diff --git a/go/worker/registration/worker.go b/go/worker/registration/worker.go index 38b1e7b6c37..e551bcf8def 100644 --- a/go/worker/registration/worker.go +++ b/go/worker/registration/worker.go @@ -40,6 +40,9 @@ const ( // CfgRegistrationForceRegister overrides a previously saved deregistration // request. CfgRegistrationForceRegister = "worker.registration.force_register" + // CfgRegistrationRotateCerts sets the number of epochs that a node's TLS + // certificate should be valid for. + CfgRegistrationRotateCerts = "worker.registration.rotate_certs" ) var ( @@ -207,6 +210,37 @@ Loop: return case epoch = <-ch: // Epoch updated, check if we can submit a registration. + + // Check if we need to rotate the node's TLS certificate. + if !w.identity.DoNotRotate { + rotateTLSCertsPer := epochtime.EpochTime(viper.GetUint64(CfgRegistrationRotateCerts)) + if rotateTLSCertsPer != 0 && epoch%rotateTLSCertsPer == 0 { + baseEpoch, err := w.epochtime.GetBaseEpoch(w.ctx) + if err != nil { + w.logger.Error("failed to get base epoch, node TLS certificate rotation failed", + "new_epoch", epoch, + "err", err, + ) + } else { + // Rotate node TLS certificates once per epoch + // (but not on the first epoch). + // TODO: Make this time-based instead. + if epoch != baseEpoch { + err := w.identity.RotateCertificates() + if err != nil { + w.logger.Error("node TLS certificate rotation failed", + "new_epoch", epoch, + "err", err, + ) + } else { + w.logger.Info("node TLS certificates have been rotated", + "new_epoch", epoch, + ) + } + } + } + } + } case <-w.registerCh: // Notification that a role provider has been updated. } @@ -430,9 +464,18 @@ func (w *Worker) gatherCommitteeAddresses(sentryCommitteeAddrs []node.CommitteeA } for _, addr := range addrs { committeeAddresses = append(committeeAddresses, node.CommitteeAddress{ - Certificate: w.identity.TLSCertificate.Certificate[0], + Certificate: w.identity.GetTLSCertificate().Certificate[0], Address: addr, }) + // Make sure to also include the certificate that will be valid + // in the next epoch, so that the node remains reachable. + nextCert := w.identity.GetNextTLSCertificate() + if nextCert != nil { + committeeAddresses = append(committeeAddresses, node.CommitteeAddress{ + Certificate: nextCert.Certificate[0], + Address: addr, + }) + } } } @@ -469,13 +512,19 @@ func (w *Worker) registerNode(epoch epochtime.EpochTime, hook RegisterNodeHook) "epoch", epoch, ) + var nextCert []byte + if c := w.identity.GetNextTLSCertificate(); c != nil { + nextCert = c.Certificate[0] + } + identityPublic := w.identity.NodeSigner.Public() nodeDesc := node.Node{ ID: identityPublic, EntityID: w.entityID, Expiration: uint64(epoch) + 2, Committee: node.CommitteeInfo{ - Certificate: w.identity.TLSCertificate.Certificate[0], + Certificate: w.identity.GetTLSCertificate().Certificate[0], + NextCertificate: nextCert, }, P2P: node.P2PInfo{ ID: w.identity.P2PSigner.Public(), @@ -531,7 +580,6 @@ func (w *Worker) registerNode(epoch epochtime.EpochTime, hook RegisterNodeHook) w.registrationSigner, w.identity.P2PSigner, w.identity.ConsensusSigner, - w.identity.TLSSigner, } if !w.identity.NodeSigner.Public().Equal(w.registrationSigner.Public()) { // In the case where the registration signer is the entity signer @@ -745,6 +793,10 @@ func New( } } + if viper.GetUint64(CfgRegistrationRotateCerts) != 0 && identity.DoNotRotate { + return nil, fmt.Errorf("node TLS certificate rotation must not be enabled if using pre-generated TLS certificates") + } + w := &Worker{ workerCommonCfg: workerCommonCfg, store: serviceStore, @@ -828,6 +880,7 @@ func init() { Flags.String(CfgRegistrationEntity, "", "entity to use as the node owner in registrations") Flags.String(CfgDebugRegistrationPrivateKey, "", "private key to use to sign node registrations") Flags.Bool(CfgRegistrationForceRegister, false, "override a previously saved deregistration request") + Flags.Uint64(CfgRegistrationRotateCerts, 0, "rotate node TLS certificates every N epochs (0 to disable)") _ = Flags.MarkHidden(CfgDebugRegistrationPrivateKey) _ = viper.BindPFlags(Flags) diff --git a/go/worker/sentry/grpc/init.go b/go/worker/sentry/grpc/init.go index c127e1bd711..f640764482f 100644 --- a/go/worker/sentry/grpc/init.go +++ b/go/worker/sentry/grpc/init.go @@ -61,6 +61,7 @@ func initConnection(ident *identity.Identity) (*upstreamConn, error) { if err != nil { return nil, fmt.Errorf("failed to parse address: %s: %w", addr, err) } + upstreamCerts, err := configparser.ParseCertificateFiles([]string{certFile}) if err != nil { return nil, fmt.Errorf("failed to parse certificate file %s: %w", certFile, err) @@ -71,9 +72,11 @@ func initConnection(ident *identity.Identity) (*upstreamConn, error) { certPool.AddCert(cert) } creds := credentials.NewTLS(&tlsPkg.Config{ - Certificates: []tlsPkg.Certificate{*ident.TLSCertificate}, - RootCAs: certPool, - ServerName: identity.CommonName, + RootCAs: certPool, + ServerName: identity.CommonName, + GetClientCertificate: func(cri *tlsPkg.CertificateRequestInfo) (*tlsPkg.Certificate, error) { + return ident.GetTLSCertificate(), nil + }, }) // Dial node @@ -131,10 +134,10 @@ func New(identity *identity.Identity) (*Worker, error) { // Create externally-accessible proxy gRPC server. serverConfig := &cmnGrpc.ServerConfig{ - Name: "sentry-grpc", - Port: uint16(viper.GetInt(CfgClientPort)), - Certificate: identity.TLSCertificate, - AuthFunc: g.authFunction(), + Name: "sentry-grpc", + Port: uint16(viper.GetInt(CfgClientPort)), + Identity: identity, + AuthFunc: g.authFunction(), CustomOptions: []grpc.ServerOption{ // All unknown requests will be proxied to the upstream grpc server. grpc.UnknownServiceHandler(proxy.Handler(upstreamConn.conn)), diff --git a/go/worker/sentry/worker.go b/go/worker/sentry/worker.go index d6b57ac510f..0f40a4f1241 100644 --- a/go/worker/sentry/worker.go +++ b/go/worker/sentry/worker.go @@ -120,9 +120,9 @@ func New(backend api.Backend, identity *identity.Identity) (*Worker, error) { if w.enabled { grpcServer, err := grpc.NewServer(&grpc.ServerConfig{ - Name: "sentry", - Port: uint16(viper.GetInt(CfgControlPort)), - Certificate: identity.TLSCertificate, + Name: "sentry", + Port: uint16(viper.GetInt(CfgControlPort)), + Identity: identity, }) if err != nil { return nil, fmt.Errorf("worker/sentry: failed to create a new gRPC server: %w", err)