From b2ee5dcdd1427c7e92dd387c00254ec2d3df28a4 Mon Sep 17 00:00:00 2001 From: Jernej Kos Date: Thu, 14 May 2020 13:41:08 +0200 Subject: [PATCH] go/registry: Avoid storing full TLS certificates Previously the node registry descriptor contained full TLS certificates for talking with nodes via gRPC. This changes it so that only TLS public keys are used when verifying peer certificates for TLS authentication. This makes the registry descriptors smaller and also makes it easier to pass around TLS identities (as public keys are much shorter). Obviously, this change BREAKS the consensus protocol and all previously signed node descriptors. --- .changelog/2556.breaking.md | 11 ++ go/common/accessctl/accessctl.go | 24 ++- go/common/crypto/tls/verify.go | 121 +++++++++++++ go/common/crypto/tls/verify_test.go | 58 ++++++ go/common/grpc/credentials.go | 73 ++++++-- go/common/grpc/grpc.go | 16 ++ go/common/identity/identity.go | 47 +++-- go/common/identity/identity_test.go | 6 + go/common/node/address.go | 72 ++++---- go/common/node/address_test.go | 18 +- go/common/node/node.go | 59 +++--- .../tendermint/apps/registry/state/state.go | 50 +----- .../apps/registry/state/state_test.go | 52 +++--- .../tendermint/apps/scheduler/query.go | 2 +- go/genesis/genesis_test.go | 24 +-- go/oasis-node/cmd/debug/byzantine/registry.go | 14 +- go/oasis-node/cmd/debug/byzantine/storage.go | 25 ++- .../debug/txsource/workload/registration.go | 10 +- go/oasis-node/cmd/registry/node/node.go | 37 ++-- go/oasis-remote-signer/cmd/root.go | 18 +- go/oasis-test-runner/oasis/args.go | 16 +- go/oasis-test-runner/oasis/sentry.go | 3 + .../scenario/e2e/registry_cli.go | 52 +++--- go/oasis-test-runner/scenario/e2e/sentry.go | 4 +- go/registry/api/api.go | 170 +++++++----------- go/registry/api/legacy_v0.go | 64 +++++++ go/registry/api/sanity_check.go | 23 +-- go/registry/tests/tester.go | 36 ++-- go/runtime/committee/client.go | 51 ++---- go/sentry/api/api.go | 20 +-- go/sentry/api/grpc.go | 45 ++--- go/sentry/client/client.go | 33 ++-- go/sentry/sentry.go | 35 ++-- go/worker/common/committee/accessctl.go | 23 ++- go/worker/common/config.go | 35 ++-- go/worker/common/configparser/configparser.go | 28 --- go/worker/keymanager/watcher.go | 5 +- go/worker/keymanager/worker.go | 5 +- go/worker/registration/worker.go | 140 ++++++--------- go/worker/sentry/grpc/init.go | 43 ++--- go/worker/sentry/grpc/worker.go | 5 +- go/worker/sentry/worker.go | 20 +-- go/worker/storage/committee/node.go | 5 +- 43 files changed, 861 insertions(+), 737 deletions(-) create mode 100644 .changelog/2556.breaking.md create mode 100644 go/common/crypto/tls/verify.go create mode 100644 go/common/crypto/tls/verify_test.go create mode 100644 go/registry/api/legacy_v0.go diff --git a/.changelog/2556.breaking.md b/.changelog/2556.breaking.md new file mode 100644 index 00000000000..8d13e7449fb --- /dev/null +++ b/.changelog/2556.breaking.md @@ -0,0 +1,11 @@ +go/registry: Avoid storing full TLS certificates + +Previously the node registry descriptor contained full TLS certificates for +talking with nodes via gRPC. This changes it so that only TLS public keys are +used when verifying peer certificates for TLS authentication. + +This makes the registry descriptors smaller and also makes it easier to pass +around TLS identities (as public keys are much shorter). + +Obviously, this change BREAKS the consensus protocol and all previously +signed node descriptors. diff --git a/go/common/accessctl/accessctl.go b/go/common/accessctl/accessctl.go index ec32d5c9141..4f8ceac47d3 100644 --- a/go/common/accessctl/accessctl.go +++ b/go/common/accessctl/accessctl.go @@ -2,10 +2,11 @@ package accessctl import ( + "crypto/ed25519" "crypto/x509" "fmt" - "github.com/oasislabs/oasis-core/go/common/crypto/hash" + "github.com/oasislabs/oasis-core/go/common/crypto/signature" ) // Subject is an access control subject. @@ -14,14 +15,23 @@ type Subject string // SubjectFromX509Certificate returns a Subject from the given X.509 // certificate. func SubjectFromX509Certificate(cert *x509.Certificate) Subject { - return SubjectFromDER(cert.Raw) + pk, ok := cert.PublicKey.(ed25519.PublicKey) + if !ok { + // This should never happen if certificates are properly verified. + return "" + } + var spk signature.PublicKey + if err := spk.UnmarshalBinary(pk[:]); err != nil { + // This should NEVER happen. + return "" + } + + return SubjectFromPublicKey(spk) } -// SubjectFromDER returns a Subject from the given certificate's ASN.1 DER -// representation. To do so, it computes the hash of the DER representation. -func SubjectFromDER(der []byte) Subject { - h := hash.NewFromBytes(der) - return Subject(h.String()) +// SubjectFromPublicKey returns a Subject from the given public key. +func SubjectFromPublicKey(pubKey signature.PublicKey) Subject { + return Subject(pubKey.String()) } // Action is an access control action. diff --git a/go/common/crypto/tls/verify.go b/go/common/crypto/tls/verify.go new file mode 100644 index 00000000000..6197103704a --- /dev/null +++ b/go/common/crypto/tls/verify.go @@ -0,0 +1,121 @@ +package tls + +import ( + "crypto/ed25519" + "crypto/x509" + "fmt" + "time" + + "github.com/oasislabs/oasis-core/go/common/crypto/signature" +) + +// VerifyOptions are the certificate verification options. +type VerifyOptions struct { + // CommonName is the expected certificate common name. + CommonName string + + // Keys is the set of public keys that are allowed to sign the certificate. + Keys map[signature.PublicKey]bool + + // AllowUnknownKeys specifies whether any key will be allowed iff Keys is nil. + AllowUnknownKeys bool + + // AllowNoCertificate specifies whether connections presenting no certificates will be allowed. + AllowNoCertificate bool +} + +// VerifyCertificate verifies a TLS certificate as required by Oasis Core. Instead of using CAs, +// public key pinning is used and certificates must follow the template. +func VerifyCertificate(rawCerts [][]byte, opts VerifyOptions) error { + // Allowing no certificate is useful in case access control is performed by a higher layer. + if len(rawCerts) == 0 && opts.AllowNoCertificate { + return nil + } + + // Make sure there is only a single certificate. + if len(rawCerts) != 1 { + return fmt.Errorf("tls: expecting a single certificate (got: %d)", len(rawCerts)) + } + + cert, err := x509.ParseCertificate(rawCerts[0]) + if err != nil { + return fmt.Errorf("tls: bad X509 certificate: %w", err) + } + + // Public key should match the pinned key. + if cert.PublicKeyAlgorithm != x509.Ed25519 || cert.SignatureAlgorithm != x509.PureEd25519 { + return fmt.Errorf("tls: bad public key algorithm (expected: Ed25519)") + } + pk, ok := cert.PublicKey.(ed25519.PublicKey) + if !ok { + // This should never happen due to the above check. + return fmt.Errorf("tls: bad public key type (expected: Ed2551 got: %T)", cert.PublicKey) + } + if !opts.AllowUnknownKeys || opts.Keys != nil { + var spk signature.PublicKey + if err = spk.UnmarshalBinary(pk[:]); err != nil { + // This should NEVER happen. + return fmt.Errorf("tls: bad public key: %w", err) + } + if !opts.Keys[spk] { + return fmt.Errorf("tls: bad public key (%s)", spk) + } + } + + // Common name should match. + if cert.Subject.CommonName != opts.CommonName { + return fmt.Errorf("tls: bad common name (expected: %s got: %s)", + opts.CommonName, + cert.Subject.CommonName, + ) + } + + // Certificate serial number should match the template. + if cert.SerialNumber.Cmp(certTemplate.SerialNumber) != 0 { + return fmt.Errorf("tls: bad serial number (expected: %s got: %s)", + certTemplate.SerialNumber, + cert.SerialNumber, + ) + } + + // Certificate key usage should match the template. + if cert.KeyUsage != certTemplate.KeyUsage { + return fmt.Errorf("tls: bad key usage (expected: %d got: %d)", + certTemplate.KeyUsage, + cert.KeyUsage, + ) + } + + // Certificate extended key usage should match the template. + if len(cert.ExtKeyUsage) != len(certTemplate.ExtKeyUsage) || len(cert.UnknownExtKeyUsage) != 0 { + return fmt.Errorf("tls: bad extended key usage") + } + for i, eku := range certTemplate.ExtKeyUsage { + if eku != cert.ExtKeyUsage[i] { + return fmt.Errorf("tls: bad extended key usage (expected: %d got: %d)", + eku, + cert.ExtKeyUsage[i], + ) + } + } + + // There should be no extra extensions. + if len(cert.ExtraExtensions) != 0 || len(cert.UnhandledCriticalExtensions) != 0 { + return fmt.Errorf("tls: bad extensions") + } + + // Certificate should not be expired. + now := time.Now() + if now.Before(cert.NotBefore) { + return fmt.Errorf("tls: current time %s is before %s", now.Format(time.RFC3339), cert.NotBefore.Format(time.RFC3339)) + } else if now.After(cert.NotAfter) { + return fmt.Errorf("tls: current time %s is after %s", now.Format(time.RFC3339), cert.NotAfter.Format(time.RFC3339)) + } + + // Signature should be valid. + if err = cert.CheckSignature(cert.SignatureAlgorithm, cert.RawTBSCertificate, cert.Signature); err != nil { + return fmt.Errorf("tls: bad signature: %w", err) + } + + return nil +} diff --git a/go/common/crypto/tls/verify_test.go b/go/common/crypto/tls/verify_test.go new file mode 100644 index 00000000000..a21bef73c4b --- /dev/null +++ b/go/common/crypto/tls/verify_test.go @@ -0,0 +1,58 @@ +package tls + +import ( + "crypto/ed25519" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/oasislabs/oasis-core/go/common/crypto/signature" + "github.com/oasislabs/oasis-core/go/common/crypto/signature/signers/memory" +) + +func TestVerifyCertificate(t *testing.T) { + require := require.New(t) + + cert, err := Generate("my-common-name") + require.NoError(err, "Generate") + + signer := memory.NewFromRuntime(cert.PrivateKey.(ed25519.PrivateKey)) + signer2 := memory.NewTestSigner("common/crypto/tls: test signer") + + rawCerts := cert.Certificate + err = VerifyCertificate(rawCerts, VerifyOptions{ + CommonName: "my-common-name", + Keys: map[signature.PublicKey]bool{ + signer.Public(): true, + }, + }) + require.NoError(err, "VerifyCertificate") + + err = VerifyCertificate(rawCerts, VerifyOptions{ + CommonName: "my-common-name", + AllowUnknownKeys: true, + }) + require.NoError(err, "VerifyCertificate") + + err = VerifyCertificate(nil, VerifyOptions{ + CommonName: "my-common-name", + AllowNoCertificate: true, + }) + require.NoError(err, "VerifyCertificate") + + err = VerifyCertificate(rawCerts, VerifyOptions{ + CommonName: "other-common-name", + Keys: map[signature.PublicKey]bool{ + signer.Public(): true, + }, + }) + require.Error(err, "VerifyCertificate should fail with mismatched common name") + + err = VerifyCertificate(rawCerts, VerifyOptions{ + CommonName: "my-common-name", + Keys: map[signature.PublicKey]bool{ + signer2.Public(): true, + }, + }) + require.Error(err, "VerifyCertificate should fail with mismatched public key") +} diff --git a/go/common/grpc/credentials.go b/go/common/grpc/credentials.go index 66bc1095c66..c18244eebff 100644 --- a/go/common/grpc/credentials.go +++ b/go/common/grpc/credentials.go @@ -2,23 +2,62 @@ package grpc import ( "crypto/tls" - "crypto/x509" - "fmt" - "io/ioutil" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/security/advancedtls" + + "github.com/oasislabs/oasis-core/go/common/crypto/signature" + cmnTLS "github.com/oasislabs/oasis-core/go/common/crypto/tls" ) -// NewClientTLSConfigFromFile is a variant of the -// "google.golang.org/grpc/credentials".NewClientTLSFromFile function that -// returns a plain "crypto/tls".Config struct instead of wrapping it in the -// "google.golang.org/grpc/credentials".TransportCredentials object. -func NewClientTLSConfigFromFile(certFile, serverNameOverride string) (*tls.Config, error) { - b, err := ioutil.ReadFile(certFile) - if err != nil { - return nil, err - } - cp := x509.NewCertPool() - if !cp.AppendCertsFromPEM(b) { - return nil, fmt.Errorf("credentials: failed to append certificates") - } - return &tls.Config{ServerName: serverNameOverride, RootCAs: cp}, nil +// ClientOptions contains all the fields needed to configure a TLS client. +type ClientOptions struct { + // CommonName is the expected certificate common name. + CommonName string + + // ServerPubKeys is the set of public keys that are allowed to sign the server's certificate. If + // this field is set GetServerPubKeys will be ignored. + ServerPubKeys map[signature.PublicKey]bool + + // If GetServerPubKeys is set and ServerPubKeys is nil, GetServerPubKeys will be invoked every + // time when verifying the server certificates. + GetServerPubKeys func() (map[signature.PublicKey]bool, error) + + // If field Certificates is set, field GetClientCertificate will be ignored. The server will use + // Certificates every time when asked for a certificate, without performing certificate + // reloading. + Certificates []tls.Certificate + + // If GetClientCertificate is set and Certificates is nil, the server will invoke this function + // every time asked to present certificates to the client when a new connection is established. + // This is known as peer certificate reloading. + GetClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error) +} + +// NewClientCreds creates new client TLS transport credentials. +func NewClientCreds(opts *ClientOptions) (credentials.TransportCredentials, error) { + return advancedtls.NewClientCreds(&advancedtls.ClientOptions{ + Certificates: opts.Certificates, + GetClientCertificate: opts.GetClientCertificate, + VType: advancedtls.SkipVerification, + VerifyPeer: func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) { + var err error + keys := opts.ServerPubKeys + if keys == nil && opts.GetServerPubKeys != nil { + if keys, err = opts.GetServerPubKeys(); err != nil { + return nil, err + } + } + + err = cmnTLS.VerifyCertificate(params.RawCerts, cmnTLS.VerifyOptions{ + CommonName: opts.CommonName, + Keys: keys, + }) + if err != nil { + return nil, err + } + + return &advancedtls.VerificationResults{}, nil + }, + }) } diff --git a/go/common/grpc/grpc.go b/go/common/grpc/grpc.go index 1f68ff4e467..ee489faa0eb 100644 --- a/go/common/grpc/grpc.go +++ b/go/common/grpc/grpc.go @@ -4,6 +4,7 @@ package grpc import ( "context" "crypto/tls" + "crypto/x509" "fmt" "net" "os" @@ -22,6 +23,7 @@ import ( "google.golang.org/grpc/grpclog" "google.golang.org/grpc/keepalive" + cmnTLS "github.com/oasislabs/oasis-core/go/common/crypto/tls" "github.com/oasislabs/oasis-core/go/common/grpc/auth" "github.com/oasislabs/oasis-core/go/common/identity" "github.com/oasislabs/oasis-core/go/common/logging" @@ -306,6 +308,9 @@ type ServerConfig struct { // nolint: maligned InstallWrapper bool // AuthFunc is the authentication function for access control. AuthFunc auth.AuthenticationFunction + // ClientCommonName is the expected common name on client TLS certificates. If not specified, + // the default identity.CommonName will be used. + ClientCommonName string // CustomOptions is an array of extra options for the grpc server. CustomOptions []grpc.ServerOption } @@ -450,6 +455,10 @@ func NewServer(config *ServerConfig) (*Server, error) { // Default to NoAuth. config.AuthFunc = auth.NoAuth } + if config.ClientCommonName == "" { + // Default to identity.CommonName. + config.ClientCommonName = identity.CommonName + } var wrapper *grpcWrapper unaryInterceptors := []grpc.UnaryServerInterceptor{ logAdapter.unaryLogger, @@ -479,6 +488,13 @@ func NewServer(config *ServerConfig) (*Server, error) { if config.Identity != nil && config.Identity.GetTLSCertificate() != nil { tlsConfig := &tls.Config{ ClientAuth: clientAuthType, + VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + return cmnTLS.VerifyCertificate(rawCerts, cmnTLS.VerifyOptions{ + CommonName: config.ClientCommonName, + AllowUnknownKeys: true, + AllowNoCertificate: true, + }) + }, GetCertificate: func(ch *tls.ClientHelloInfo) (*tls.Certificate, error) { return config.Identity.GetTLSCertificate(), nil }, diff --git a/go/common/identity/identity.go b/go/common/identity/identity.go index e0795720461..b044fda3197 100644 --- a/go/common/identity/identity.go +++ b/go/common/identity/identity.go @@ -65,6 +65,8 @@ type Identity struct { tlsSigner signature.Signer // tlsCertificate is a certificate that can be used for TLS. tlsCertificate *tls.Certificate + // nextTLSSigner is a node TLS certificate signer that can be used in the next rotation. + nextTLSSigner signature.Signer // nextTLSCertificate is a certificate that can be used for TLS in the next rotation. nextTLSCertificate *tls.Certificate // tlsRotationNotifier is a notifier for certificate rotations. @@ -98,7 +100,7 @@ func (i *Identity) RotateCertificates() error { // Use the prepared certificate. if i.nextTLSCertificate != nil { i.tlsCertificate = i.nextTLSCertificate - i.tlsSigner = memory.NewFromRuntime(i.tlsCertificate.PrivateKey.(ed25519.PrivateKey)) + i.tlsSigner = i.nextTLSSigner } // Generate a new TLS certificate to be used in the next rotation. @@ -107,6 +109,7 @@ func (i *Identity) RotateCertificates() error { if err != nil { return err } + i.nextTLSSigner = memory.NewFromRuntime(i.nextTLSCertificate.PrivateKey.(ed25519.PrivateKey)) i.tlsRotationNotifier.Broadcast(struct{}{}) } @@ -122,14 +125,6 @@ func (i *Identity) GetTLSSigner() signature.Signer { return i.tlsSigner } -// SetTLSSigner sets the current TLS signer. -func (i *Identity) SetTLSSigner(s signature.Signer) { - i.Lock() - defer i.Unlock() - - i.tlsSigner = s -} - // GetTLSCertificate returns the current TLS certificate. func (i *Identity) GetTLSCertificate() *tls.Certificate { i.RLock() @@ -144,6 +139,15 @@ func (i *Identity) SetTLSCertificate(cert *tls.Certificate) { defer i.Unlock() i.tlsCertificate = cert + i.tlsSigner = memory.NewFromRuntime(cert.PrivateKey.(ed25519.PrivateKey)) +} + +// GetNextTLSSigner returns the next TLS signer. +func (i *Identity) GetNextTLSSigner() signature.Signer { + i.RLock() + defer i.RUnlock() + + return i.nextTLSSigner } // GetNextTLSCertificate returns the next TLS certificate. @@ -154,12 +158,19 @@ func (i *Identity) GetNextTLSCertificate() *tls.Certificate { return i.nextTLSCertificate } -// SetNextTLSCertificate sets the next TLS certificate. -func (i *Identity) SetNextTLSCertificate(nextCert *tls.Certificate) { - i.Lock() - defer i.Unlock() +// GetTLSPubKeys returns a list of currently valid TLS public keys. +func (i *Identity) GetTLSPubKeys() []signature.PublicKey { + i.RLock() + defer i.RUnlock() - i.nextTLSCertificate = nextCert + var pubKeys []signature.PublicKey + if i.tlsSigner != nil { + pubKeys = append(pubKeys, i.tlsSigner.Public()) + } + if i.nextTLSSigner != nil { + pubKeys = append(pubKeys, i.nextTLSSigner.Public()) + } + return pubKeys } // Load loads an identity. @@ -206,8 +217,9 @@ func doLoadOrGenerate(dataDir string, signerFactory signature.SignerFactory, sho } var ( - nextCert *tls.Certificate - dnr bool + nextCert *tls.Certificate + nextSigner signature.Signer + dnr bool ) // First, check if we can load the TLS certificate from disk. @@ -239,6 +251,8 @@ func doLoadOrGenerate(dataDir string, signerFactory signature.SignerFactory, sho if err != nil { return nil, err } + + nextSigner = memory.NewFromRuntime(nextCert.PrivateKey.(ed25519.PrivateKey)) } } @@ -264,6 +278,7 @@ func doLoadOrGenerate(dataDir string, signerFactory signature.SignerFactory, sho ConsensusSigner: signers[2], tlsSigner: memory.NewFromRuntime(cert.PrivateKey.(ed25519.PrivateKey)), tlsCertificate: cert, + nextTLSSigner: nextSigner, nextTLSCertificate: nextCert, DoNotRotateTLS: dnr, TLSSentryClientCertificate: sentryClientCert, diff --git a/go/common/identity/identity_test.go b/go/common/identity/identity_test.go index 9ffbf4f1152..834d6013e3d 100644 --- a/go/common/identity/identity_test.go +++ b/go/common/identity/identity_test.go @@ -22,6 +22,7 @@ func TestLoadOrGenerate(t *testing.T) { // Generate a new identity. identity, err := LoadOrGenerate(dataDir, factory, true) require.NoError(t, err, "LoadOrGenerate") + require.EqualValues(t, []signature.PublicKey{identity.GetTLSSigner().Public()}, identity.GetTLSPubKeys()) // Load an existing identity. identity2, err := LoadOrGenerate(dataDir, factory, false) @@ -31,6 +32,7 @@ func TestLoadOrGenerate(t *testing.T) { require.EqualValues(t, identity.ConsensusSigner, identity2.ConsensusSigner) require.EqualValues(t, identity.GetTLSSigner(), identity2.GetTLSSigner()) require.EqualValues(t, identity.GetTLSCertificate(), identity2.GetTLSCertificate()) + require.EqualValues(t, identity.GetTLSPubKeys(), identity2.GetTLSPubKeys()) dataDir2, err := ioutil.TempDir("", "oasis-identity-test2_") require.NoError(t, err, "create data dir (2)") @@ -39,6 +41,10 @@ func TestLoadOrGenerate(t *testing.T) { // Generate a new identity again, this time without persisting TLS certs. identity3, err := LoadOrGenerate(dataDir2, factory, false) require.NoError(t, err, "LoadOrGenerate (3)") + require.EqualValues(t, []signature.PublicKey{ + identity3.GetTLSSigner().Public(), + identity3.GetNextTLSSigner().Public(), + }, identity3.GetTLSPubKeys()) // Load it back. identity4, err := LoadOrGenerate(dataDir2, factory, false) diff --git a/go/common/node/address.go b/go/common/node/address.go index ed651b23c5f..7f295b0f67c 100644 --- a/go/common/node/address.go +++ b/go/common/node/address.go @@ -1,10 +1,7 @@ package node import ( - "bytes" - "crypto/x509" "encoding" - "encoding/base64" "errors" "fmt" "net" @@ -20,9 +17,8 @@ var ( // ErrConsensusAddressNoID is the error returned when a consensus address // doesn't have the ID@ part. ErrConsensusAddressNoID = errors.New("node: consensus address doesn't have ID@ part") - // ErrCommitteeAddressNoCertificate is the error returned when a committee address - // doesn't have the Certificate@ part. - ErrCommitteeAddressNoCertificate = errors.New("node: certificate address missing Certificate@ part") + // ErrTLSAddressNoPubKey is the error returned when a TLS address doesn't have the PubKey@ part. + ErrTLSAddressNoPubKey = errors.New("node: TLS address missing PubKey@ part") unroutableNetworks []net.IPNet @@ -140,63 +136,61 @@ func (ca *ConsensusAddress) String() string { return fmt.Sprintf("%s@%s", ca.ID, ca.Address) } -// CommitteeAddress represents an Oasis committee address that includes a -// server certificate and a TCP address. -// NOTE: The address certificate can be different from the actual node -// certificate to allow using a sentry node's addresses. -type CommitteeAddress struct { - // Certificate is the certificate for establishing TLS connections. - Certificate []byte `json:"certificate"` - // Address is the address at which the node can be reached +// TLSAddress represents an Oasis committee address that includes a TLS public key and a TCP +// address. +// +// NOTE: The address TLS public key can be different from the actual node TLS public key to allow +// using a sentry node's addresses. +type TLSAddress struct { + // PubKey is the public key used for establishing TLS connections. + PubKey signature.PublicKey `json:"pub_key"` + + // Address is the address at which the node can be reached. Address Address `json:"address"` } -// Equal compares vs another CommitteeInfo for equality. -func (ca *CommitteeAddress) Equal(other *CommitteeAddress) bool { - if !bytes.Equal(ca.Certificate, other.Certificate) { +// Equal compares vs another TLSAddress for equality. +func (ta *TLSAddress) Equal(other *TLSAddress) bool { + if !ta.PubKey.Equal(other.PubKey) { return false } - if !ca.Address.Equal(&other.Address) { + if !ta.Address.Equal(&other.Address) { return false } return true } -// ParseCertificate returns the parsed x509 certificate. -func (ca *CommitteeAddress) ParseCertificate() (*x509.Certificate, error) { - return x509.ParseCertificate(ca.Certificate) -} - // MarshalText implements the encoding.TextMarshaler interface. -func (ca *CommitteeAddress) MarshalText() ([]byte, error) { - certificateStr := base64.StdEncoding.EncodeToString(ca.Certificate[:]) - addrStr, err := ca.Address.MarshalText() +func (ta *TLSAddress) MarshalText() ([]byte, error) { + pubKeyStr, err := ta.PubKey.MarshalText() if err != nil { - return nil, fmt.Errorf("node: error marshalling committee address' TCP address: %w", err) + return nil, fmt.Errorf("node: error marshalling TLS address' public key: %w", err) } - return []byte(fmt.Sprintf("%s@%s", certificateStr, addrStr)), nil + addrStr, err := ta.Address.MarshalText() + if err != nil { + return nil, fmt.Errorf("node: error marshalling TLS address' TCP address: %w", err) + } + return []byte(fmt.Sprintf("%s@%s", pubKeyStr, addrStr)), nil } // UnmarshalText implements the encoding.TextUnmarshaler interface. -func (ca *CommitteeAddress) UnmarshalText(text []byte) error { +func (ta *TLSAddress) UnmarshalText(text []byte) error { spl := strings.Split(string(text), "@") if len(spl) != 2 { - return ErrCommitteeAddressNoCertificate + return ErrTLSAddressNoPubKey } - cert, err := base64.StdEncoding.DecodeString(spl[0]) - if err != nil { - return fmt.Errorf("node: unable to parse committee address' Certificate: %w", err) + if err := ta.PubKey.UnmarshalText([]byte(spl[0])); err != nil { + return fmt.Errorf("node: unable to parse TLS address' public key: %w", err) } - ca.Certificate = cert - if err := ca.Address.UnmarshalText([]byte(spl[1])); err != nil { - return fmt.Errorf("node: unable to parse committee address' TCP address: %w", err) + if err := ta.Address.UnmarshalText([]byte(spl[1])); err != nil { + return fmt.Errorf("node: unable to parse TLS address' TCP address: %w", err) } return nil } -// String returns a string representation of a committee address. -func (ca *CommitteeAddress) String() string { - return ca.Address.String() +// String returns a string representation of a TLS address. +func (ta *TLSAddress) String() string { + return ta.Address.String() } func init() { diff --git a/go/common/node/address_test.go b/go/common/node/address_test.go index a05f27214d2..ae1b49180a0 100644 --- a/go/common/node/address_test.go +++ b/go/common/node/address_test.go @@ -70,23 +70,23 @@ func TestConsensusAddress(t *testing.T) { } } -func TestCommitteeAddress(t *testing.T) { +func TestTLSAddress(t *testing.T) { type testCase struct { - committeeAddress string + tlsAddress string } testCases := []testCase{ { - committeeAddress: "MIIBSjCB8KADAgECAgEBMAoGCCqGSM49BAMCMBUxEzARBgNVBAMTCm9hc2lzLW5vZGUwHhcNMjAwMTE0MTQyMzU5WhcNMjEwMTE0MTUyMzU5WjAVMRMwEQYDVQQDEwpvYXNpcy1ub2RlMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAml6sfT1v+1fnwADmKLT0fsXkhJE1bQDol/utnC2SOApROfdK53koFdNjV+e7i6TMH7JijFls6GmkTd6nIC+jaMxMC8wDgYDVR0PAQH/BAQDAgKkMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAKBggqhkjOPQQDAgNJADBGAiEAhh8Y41Ahhe6YfX9gJlcKEfaZF1pSoErmnuu3//7twtUCIQC9fUavln6Inj/zEK2jYp9WNzaA6SqGkj01mhKJxlINPQ==@127.0.0.1:8000", + tlsAddress: "K1bw65BZsB8s2U6nGLE+eVCCBsJ3BTnK+k9s8hXN/pI=@127.0.0.1:8000", }, } for _, testCase := range testCases { - // Construct a CommitteeAddress object and check if text marshalling works. - var committeeAddress CommitteeAddress - require.NoError(t, committeeAddress.UnmarshalText([]byte(testCase.committeeAddress)), "error unmarshaling committee address") - committeeAddrBytes, err := committeeAddress.MarshalText() - require.NoError(t, err, "error marshalling commmittee address") - require.Equal(t, testCase.committeeAddress, string(committeeAddrBytes), "marshalled committee address does not match") + // Construct a TLSAddress object and check if text marshalling works. + var tlsAddress TLSAddress + require.NoError(t, tlsAddress.UnmarshalText([]byte(testCase.tlsAddress)), "error unmarshaling TLS address") + committeeAddrBytes, err := tlsAddress.MarshalText() + require.NoError(t, err, "error marshalling TLS address") + require.Equal(t, testCase.tlsAddress, string(committeeAddrBytes), "marshalled TLS address does not match") } } diff --git a/go/common/node/node.go b/go/common/node/node.go index ab85135cded..ab46e18c9b6 100644 --- a/go/common/node/node.go +++ b/go/common/node/node.go @@ -4,8 +4,6 @@ package node import ( - "bytes" - "crypto/x509" "errors" "fmt" "io" @@ -62,11 +60,10 @@ type Node struct { // nolint: maligned // Expiration is the epoch in which this node's commitment expires. Expiration uint64 `json:"expiration"` - // Committee contains information for connecting to this node as a committee - // member. - Committee CommitteeInfo `json:"committee"` + // TLS contains information for connecting to this node via TLS. + TLS TLSInfo `json:"tls"` - // P2P contains information for connecting to this node via P2P transport. + // P2P contains information for connecting to this node via P2P. P2P P2PInfo `json:"p2p"` // Consensus contains information for connecting to this node as a @@ -154,16 +151,21 @@ func (n *Node) ValidateBasic(strictVersion bool) error { return nil } -// AddRoles adds the Node roles +// AddRoles adds a new node role to the existing roles mask. func (n *Node) AddRoles(r RolesMask) { n.Roles |= r } -// HasRoles checks if Node has roles +// HasRoles checks if the node has the specified roles. func (n *Node) HasRoles(r RolesMask) bool { return n.Roles&r != 0 } +// OnlyHasRoles checks if the node only has the specified roles and no others. +func (n *Node) OnlyHasRoles(r RolesMask) bool { + return n.Roles == r +} + // IsExpired returns true if the node expiration epoch is strictly smaller // than the passed (current) epoch. func (n *Node) IsExpired(epoch uint64) bool { @@ -211,34 +213,33 @@ type Runtime struct { ExtraInfo []byte `json:"extra_info"` } -// CommitteeInfo contains information for connecting to this node as a -// committee member. -type CommitteeInfo struct { - // Certificate is the certificate for establishing TLS connections. - Certificate []byte `json:"certificate"` +// TLSInfo contains information for connecting to this node via TLS. +type TLSInfo struct { + // PubKey is the public key used for establishing TLS connections. + PubKey signature.PublicKey `json:"pub_key"` - // NextCertificate is the certificate that will be used for establishing - // TLS connections after certificate rotation (if enabled). - NextCertificate []byte `json:"next_certificate,omitempty"` + // NextPubKey is the public key that will be used for establishing TLS connections after + // certificate rotation (if enabled). + NextPubKey signature.PublicKey `json:"next_pub_key,omitempty"` - // Addresses is the list of committee addresses at which the node can be reached. - Addresses []CommitteeAddress `json:"addresses"` + // Addresses is the list of addresses at which the node can be reached. + Addresses []TLSAddress `json:"addresses"` } -// Equal compares vs another CommitteeInfo for equality. -func (c *CommitteeInfo) Equal(other *CommitteeInfo) bool { - if !bytes.Equal(c.Certificate, other.Certificate) { +// Equal compares vs another TLSInfo for equality. +func (t *TLSInfo) Equal(other *TLSInfo) bool { + if !t.PubKey.Equal(other.PubKey) { return false } - if !bytes.Equal(c.NextCertificate, other.NextCertificate) { + if !t.NextPubKey.Equal(other.NextPubKey) { return false } - if len(c.Addresses) != len(other.Addresses) { + if len(t.Addresses) != len(other.Addresses) { return false } - for i, ca := range c.Addresses { + for i, ca := range t.Addresses { if !ca.Equal(&other.Addresses[i]) { return false } @@ -247,16 +248,6 @@ func (c *CommitteeInfo) Equal(other *CommitteeInfo) bool { return true } -// ParseCertificate returns the parsed x509 certificate. -func (c *CommitteeInfo) ParseCertificate() (*x509.Certificate, error) { - return x509.ParseCertificate(c.Certificate) -} - -// ParseCertificate returns the parsed x509 next certificate. -func (c *CommitteeInfo) ParseNextCertificate() (*x509.Certificate, error) { - return x509.ParseCertificate(c.NextCertificate) -} - // P2PInfo contains information for connecting to this node via P2P transport. type P2PInfo struct { // ID is the unique identifier of the node on the P2P transport. diff --git a/go/consensus/tendermint/apps/registry/state/state.go b/go/consensus/tendermint/apps/registry/state/state.go index b230e1bf0b2..c72d4aaf3ab 100644 --- a/go/consensus/tendermint/apps/registry/state/state.go +++ b/go/consensus/tendermint/apps/registry/state/state.go @@ -1,13 +1,11 @@ package state import ( - "bytes" "context" "errors" "github.com/oasislabs/oasis-core/go/common" "github.com/oasislabs/oasis-core/go/common/cbor" - "github.com/oasislabs/oasis-core/go/common/crypto/hash" "github.com/oasislabs/oasis-core/go/common/crypto/signature" "github.com/oasislabs/oasis-core/go/common/entity" "github.com/oasislabs/oasis-core/go/common/keyformat" @@ -61,20 +59,15 @@ var ( // // Value is binary signature.PublicKey (node ID). keyMapKeyFmt = keyformat.New(0x17, &signature.PublicKey{}) - // certificateMapKeyFmt is the key format used for certificate-to-node-id map. - // This stores the hash-of-certificate to Node ID mappings. - // - // Value is binary signature.PublicKey (node ID). - certificateMapKeyFmt = keyformat.New(0x18, &hash.Hash{}) // suspendedRuntimeKeyFmt is the key format used for suspended runtimes. // // Value is CBOR-serialized signed runtime. - suspendedRuntimeKeyFmt = keyformat.New(0x19, &common.Namespace{}) + suspendedRuntimeKeyFmt = keyformat.New(0x18, &common.Namespace{}) // signedRuntimeByEntityKeyFmt is the key format used for signed runtime by entity // index. // // Value is empty. - signedRuntimeByEntityKeyFmt = keyformat.New(0x1a, &signature.PublicKey{}, &common.Namespace{}) + signedRuntimeByEntityKeyFmt = keyformat.New(0x19, &signature.PublicKey{}, &common.Namespace{}) ) // ImmutableState is the immutable registry state wrapper. @@ -520,8 +513,8 @@ func (s *ImmutableState) ConsensusParameters(ctx context.Context) (*registry.Con return ¶ms, nil } -// NodeByConsensusOrP2PKey looks up a specific node by its consensus or P2P key. -func (s *ImmutableState) NodeByConsensusOrP2PKey(ctx context.Context, key signature.PublicKey) (*node.Node, error) { +// NodeBySubKey looks up a specific node by its consensus, P2P or TLS key. +func (s *ImmutableState) NodeBySubKey(ctx context.Context, key signature.PublicKey) (*node.Node, error) { rawID, err := s.is.Get(ctx, keyMapKeyFmt.Encode(&key)) if err != nil { return nil, abciAPI.UnavailableStateError(err) @@ -537,29 +530,6 @@ func (s *ImmutableState) NodeByConsensusOrP2PKey(ctx context.Context, key signat return s.Node(ctx, id) } -// Hashes a node's committee certificate into a key for the certificate to node ID map. -func nodeCertificateToMapKey(cert []byte) hash.Hash { - return hash.NewFromBytes(cert) -} - -// NodeByCertificate looks up a specific node by its certificate. -func (s *ImmutableState) NodeByCertificate(ctx context.Context, cert []byte) (*node.Node, error) { - certHash := nodeCertificateToMapKey(cert) - rawID, err := s.is.Get(ctx, certificateMapKeyFmt.Encode(&certHash)) - if err != nil { - return nil, abciAPI.UnavailableStateError(err) - } - if rawID == nil { - return nil, registry.ErrNoSuchNode - } - - var id signature.PublicKey - if err := id.UnmarshalBinary(rawID); err != nil { - return nil, abciAPI.UnavailableStateError(err) - } - return s.Node(ctx, id) -} - func NewImmutableState(ctx context.Context, state abciAPI.ApplicationState, version int64) (*ImmutableState, error) { is, err := abciAPI.NewImmutableState(ctx, state, version) if err != nil { @@ -649,15 +619,13 @@ func (s *MutableState) SetNode(ctx context.Context, existingNode *node.Node, nod } // Committee TLS key. - if existingNode != nil && !bytes.Equal(existingNode.Committee.Certificate, node.Committee.Certificate) { + if existingNode != nil && !existingNode.TLS.PubKey.Equal(node.TLS.PubKey) { // Remove old TLS key mapping if it has changed. - certHash := nodeCertificateToMapKey(existingNode.Committee.Certificate) - if err = s.ms.Remove(ctx, certificateMapKeyFmt.Encode(&certHash)); err != nil { + if err = s.ms.Remove(ctx, keyMapKeyFmt.Encode(&existingNode.TLS.PubKey)); err != nil { return abciAPI.UnavailableStateError(err) } } - certHash := nodeCertificateToMapKey(node.Committee.Certificate) - if err = s.ms.Insert(ctx, certificateMapKeyFmt.Encode(&certHash), rawNodeID); err != nil { + if err = s.ms.Insert(ctx, keyMapKeyFmt.Encode(&node.TLS.PubKey), rawNodeID); err != nil { return abciAPI.UnavailableStateError(err) } @@ -687,9 +655,7 @@ func (s *MutableState) RemoveNode(ctx context.Context, node *node.Node) error { if err := s.ms.Remove(ctx, keyMapKeyFmt.Encode(&node.P2P.ID)); err != nil { return abciAPI.UnavailableStateError(err) } - - certHash := nodeCertificateToMapKey(node.Committee.Certificate) - if err := s.ms.Remove(ctx, certificateMapKeyFmt.Encode(&certHash)); err != nil { + if err := s.ms.Remove(ctx, keyMapKeyFmt.Encode(&node.TLS.PubKey)); err != nil { return abciAPI.UnavailableStateError(err) } diff --git a/go/consensus/tendermint/apps/registry/state/state_test.go b/go/consensus/tendermint/apps/registry/state/state_test.go index 43b074b1c57..f821c8e3582 100644 --- a/go/consensus/tendermint/apps/registry/state/state_test.go +++ b/go/consensus/tendermint/apps/registry/state/state_test.go @@ -20,6 +20,8 @@ var ( consensusSigner2 = memorySigner.NewTestSigner("consensus/tendermint/apps/registry/state: consensus signer 2") p2pSigner1 = memorySigner.NewTestSigner("consensus/tendermint/apps/registry/state: p2p signer 1") p2pSigner2 = memorySigner.NewTestSigner("consensus/tendermint/apps/registry/state: p2p signer 2") + tlsSigner1 = memorySigner.NewTestSigner("consensus/tendermint/apps/registry/state: tls signer 1") + tlsSigner2 = memorySigner.NewTestSigner("consensus/tendermint/apps/registry/state: tls signer 2") ) func mustMultiSignNode(t *testing.T, n *node.Node) *node.MultiSignedNode { @@ -48,8 +50,8 @@ func TestNodeUpdate(t *testing.T) { Consensus: node.ConsensusInfo{ ID: consensusSigner1.Public(), }, - Committee: node.CommitteeInfo{ - Certificate: []byte("this is a certificate"), + TLS: node.TLSInfo{ + PubKey: tlsSigner1.Public(), }, } err := s.SetNode(ctx, nil, &n, mustMultiSignNode(t, &n)) @@ -61,14 +63,14 @@ func TestNodeUpdate(t *testing.T) { resNode, err := s.NodeByConsensusAddress(ctx, consensusAddress) require.NoError(err, "consensus mapping should be there") require.EqualValues(n, *resNode, "returned node should be correct") - resNode, err = s.NodeByConsensusOrP2PKey(ctx, consensusSigner1.Public()) + resNode, err = s.NodeBySubKey(ctx, consensusSigner1.Public()) require.NoError(err, "consensus mapping should be there") require.EqualValues(n, *resNode, "returned node should be correct") - resNode, err = s.NodeByConsensusOrP2PKey(ctx, p2pSigner1.Public()) + resNode, err = s.NodeBySubKey(ctx, p2pSigner1.Public()) require.NoError(err, "P2P mapping should be there") require.EqualValues(n, *resNode, "returned node should be correct") - resNode, err = s.NodeByCertificate(ctx, n.Committee.Certificate) - require.NoError(err, "certificate mapping should be there") + resNode, err = s.NodeBySubKey(ctx, tlsSigner1.Public()) + require.NoError(err, "TLS mapping should be there") require.EqualValues(n, *resNode, "returned node should be correct") // Update the node with the same descriptor -- nothing should change. @@ -78,21 +80,21 @@ func TestNodeUpdate(t *testing.T) { resNode, err = s.NodeByConsensusAddress(ctx, consensusAddress) require.NoError(err, "consensus mapping should be there") require.EqualValues(n, *resNode, "returned node should be correct") - resNode, err = s.NodeByConsensusOrP2PKey(ctx, consensusSigner1.Public()) + resNode, err = s.NodeBySubKey(ctx, consensusSigner1.Public()) require.NoError(err, "consensus mapping should be there") require.EqualValues(n, *resNode, "returned node should be correct") - resNode, err = s.NodeByConsensusOrP2PKey(ctx, p2pSigner1.Public()) + resNode, err = s.NodeBySubKey(ctx, p2pSigner1.Public()) require.NoError(err, "P2P mapping should be there") require.EqualValues(n, *resNode, "returned node should be correct") - resNode, err = s.NodeByCertificate(ctx, n.Committee.Certificate) - require.NoError(err, "certificate mapping should be there") + resNode, err = s.NodeBySubKey(ctx, tlsSigner1.Public()) + require.NoError(err, "TLS mapping should be there") require.EqualValues(n, *resNode, "returned node should be correct") // Change the node's consensus/p2p/tls keys and check that indices have been updated. newNode := n newNode.P2P.ID = p2pSigner2.Public() newNode.Consensus.ID = consensusSigner2.Public() - newNode.Committee.Certificate = []byte("this is another certificate") + newNode.TLS.PubKey = tlsSigner2.Public() err = s.SetNode(ctx, &n, &newNode, mustMultiSignNode(t, &newNode)) require.NoError(err, "SetNode") @@ -101,27 +103,27 @@ func TestNodeUpdate(t *testing.T) { _, err = s.NodeByConsensusAddress(ctx, consensusAddress) require.Error(err, "old consensus mapping should be gone") require.Equal(registry.ErrNoSuchNode, err, "old consensus mapping should be gone") - _, err = s.NodeByConsensusOrP2PKey(ctx, consensusSigner1.Public()) + _, err = s.NodeBySubKey(ctx, consensusSigner1.Public()) require.Error(err, "old consensus mapping should be gone") require.Equal(registry.ErrNoSuchNode, err, "old consensus mapping should be gone") - _, err = s.NodeByConsensusOrP2PKey(ctx, p2pSigner1.Public()) + _, err = s.NodeBySubKey(ctx, p2pSigner1.Public()) require.Error(err, "old P2P mapping should be gone") require.Equal(registry.ErrNoSuchNode, err, "old P2P mapping should be gone") - _, err = s.NodeByCertificate(ctx, n.Committee.Certificate) - require.Error(err, "old certificate mapping should be gone") - require.Equal(registry.ErrNoSuchNode, err, "old certificate mapping should be gone") + _, err = s.NodeBySubKey(ctx, tlsSigner1.Public()) + require.Error(err, "old TLS mapping should be gone") + require.Equal(registry.ErrNoSuchNode, err, "old TLS mapping should be gone") resNode, err = s.NodeByConsensusAddress(ctx, newConsensusAddress) require.NoError(err, "new consensus mapping should be there") require.EqualValues(newNode, *resNode, "returned node should be correct") - resNode, err = s.NodeByConsensusOrP2PKey(ctx, consensusSigner2.Public()) + resNode, err = s.NodeBySubKey(ctx, consensusSigner2.Public()) require.NoError(err, "new consensus mapping should be there") require.EqualValues(newNode, *resNode, "returned node should be correct") - resNode, err = s.NodeByConsensusOrP2PKey(ctx, p2pSigner2.Public()) + resNode, err = s.NodeBySubKey(ctx, p2pSigner2.Public()) require.NoError(err, "new P2P mapping should be there") require.EqualValues(newNode, *resNode, "returned node should be correct") - resNode, err = s.NodeByCertificate(ctx, newNode.Committee.Certificate) - require.NoError(err, "new certificate mapping should be there") + resNode, err = s.NodeBySubKey(ctx, tlsSigner2.Public()) + require.NoError(err, "new TLS mapping should be there") require.EqualValues(newNode, *resNode, "returned node should be correct") // Remove a node and make sure all indices are gone. @@ -131,13 +133,13 @@ func TestNodeUpdate(t *testing.T) { _, err = s.NodeByConsensusAddress(ctx, newConsensusAddress) require.Error(err, "consensus mapping should be gone") require.Equal(registry.ErrNoSuchNode, err, "consensus mapping should be gone") - _, err = s.NodeByConsensusOrP2PKey(ctx, consensusSigner2.Public()) + _, err = s.NodeBySubKey(ctx, consensusSigner2.Public()) require.Error(err, "consensus mapping should be gone") require.Equal(registry.ErrNoSuchNode, err, "consensus mapping should be gone") - _, err = s.NodeByConsensusOrP2PKey(ctx, p2pSigner2.Public()) + _, err = s.NodeBySubKey(ctx, p2pSigner2.Public()) require.Error(err, "P2P mapping should be gone") require.Equal(registry.ErrNoSuchNode, err, "P2P mapping should be gone") - _, err = s.NodeByCertificate(ctx, newNode.Committee.Certificate) - require.Error(err, "certificate mapping should be gone") - require.Equal(registry.ErrNoSuchNode, err, "certificate mapping should be gone") + _, err = s.NodeBySubKey(ctx, tlsSigner2.Public()) + require.Error(err, "TLS mapping should be gone") + require.Equal(registry.ErrNoSuchNode, err, "TLS mapping should be gone") } diff --git a/go/consensus/tendermint/apps/scheduler/query.go b/go/consensus/tendermint/apps/scheduler/query.go index d649228c55a..a23d888d22b 100644 --- a/go/consensus/tendermint/apps/scheduler/query.go +++ b/go/consensus/tendermint/apps/scheduler/query.go @@ -71,7 +71,7 @@ func (sq *schedulerQuerier) Validators(ctx context.Context) ([]*scheduler.Valida // node identifiers for validators, because user queries are // likely more infrequent than all the business of actually // scheduling... - node, err := sq.regState.NodeByConsensusOrP2PKey(ctx, v) + node, err := sq.regState.NodeBySubKey(ctx, v) if err != nil { // Should NEVER happen. return nil, err diff --git a/go/genesis/genesis_test.go b/go/genesis/genesis_test.go index 13de540b926..6fa91095c69 100644 --- a/go/genesis/genesis_test.go +++ b/go/genesis/genesis_test.go @@ -1,7 +1,6 @@ package genesis import ( - "crypto/ed25519" "encoding/hex" "math" "testing" @@ -14,7 +13,6 @@ import ( "github.com/oasislabs/oasis-core/go/common/crypto/hash" "github.com/oasislabs/oasis-core/go/common/crypto/signature" memorySigner "github.com/oasislabs/oasis-core/go/common/crypto/signature/signers/memory" - "github.com/oasislabs/oasis-core/go/common/crypto/tls" "github.com/oasislabs/oasis-core/go/common/entity" "github.com/oasislabs/oasis-core/go/common/node" consensus "github.com/oasislabs/oasis-core/go/consensus/genesis" @@ -143,6 +141,7 @@ func TestGenesisSanityCheck(t *testing.T) { nodeSigner := memorySigner.NewTestSigner("node genesis sanity checks signer") nodeConsensusSigner := memorySigner.NewTestSigner("node consensus genesis sanity checks signer") nodeP2PSigner := memorySigner.NewTestSigner("node P2P genesis sanity checks signer") + nodeTLSSigner := memorySigner.NewTestSigner("node TLS genesis sanity checks signer") validPK := signer.Public() var validNS common.Namespace _ = validNS.UnmarshalBinary(validPK[:]) @@ -217,10 +216,6 @@ func TestGenesisSanityCheck(t *testing.T) { } signedTestRuntime := signRuntimeOrDie(signer, testRuntime) - dummyCert, err := tls.Generate("genesis sanity check dummy cert") - if err != nil { - panic(err) - } var testConsensusAddress node.ConsensusAddress _ = testConsensusAddress.UnmarshalText([]byte("AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBA=@127.0.0.1:1234")) var testAddress node.Address @@ -231,10 +226,10 @@ func TestGenesisSanityCheck(t *testing.T) { EntityID: testEntity.ID, Expiration: 10, Roles: node.RoleValidator, - Committee: node.CommitteeInfo{ - Certificate: dummyCert.Certificate[0], - Addresses: []node.CommitteeAddress{ - {Certificate: dummyCert.Certificate[0], Address: testAddress}, + TLS: node.TLSInfo{ + PubKey: nodeTLSSigner.Public(), + Addresses: []node.TLSAddress{ + {PubKey: nodeTLSSigner.Public(), Address: testAddress}, }, }, P2P: node.P2PInfo{ @@ -246,7 +241,6 @@ func TestGenesisSanityCheck(t *testing.T) { Addresses: []node.ConsensusAddress{testConsensusAddress}, }, } - nodeTLSSigner := memorySigner.NewFromRuntime(dummyCert.PrivateKey.(ed25519.PrivateKey)) nodeSigners := []signature.Signer{ nodeSigner, nodeP2PSigner, @@ -324,7 +318,7 @@ func TestGenesisSanityCheck(t *testing.T) { // First we define a helper function for calling the SanityCheck() on RuntimeStates. rtsSanityCheck := func(g roothashAPI.Genesis, isGenesis bool) error { for _, rts := range g.RuntimeStates { - if err = rts.SanityCheck(isGenesis); err != nil { + if err := rts.SanityCheck(isGenesis); err != nil { return err } } @@ -418,7 +412,7 @@ func TestGenesisSanityCheck(t *testing.T) { d = *testDoc te = *testEntity - signedBrokenEntity, err = entity.SignEntity(signer, signature.NewContext("genesis sanity check invalid ctx"), &te) + signedBrokenEntity, err := entity.SignEntity(signer, signature.NewContext("genesis sanity check invalid ctx"), &te) if err != nil { panic(err) } @@ -527,12 +521,12 @@ func TestGenesisSanityCheck(t *testing.T) { d = *testDoc tn = *testNode - tn.Committee.Certificate = []byte{1, 2, 3} + tn.TLS.PubKey = signature.PublicKey{} signedBrokenTestNode = signNodeOrDie(nodeSigners, &tn) d.Registry.Entities = []*entity.SignedEntity{signedEntityWithTestNode} d.Registry.Runtimes = []*registry.SignedRuntime{signedTestKMRuntime} d.Registry.Nodes = []*node.MultiSignedNode{signedBrokenTestNode} - require.Error(d.SanityCheck(), "node with invalid committee certificate should be rejected") + require.Error(d.SanityCheck(), "node with invalid TLS public key should be rejected") d = *testDoc tn = *testNode diff --git a/go/oasis-node/cmd/debug/byzantine/registry.go b/go/oasis-node/cmd/debug/byzantine/registry.go index 86f67445e54..1332f15838e 100644 --- a/go/oasis-node/cmd/debug/byzantine/registry.go +++ b/go/oasis-node/cmd/debug/byzantine/registry.go @@ -33,11 +33,11 @@ func registryRegisterNode(svc service.TendermintService, id *identity.Identity, } } - var committeeAddresses []node.CommitteeAddress + var tlsAddresses []node.TLSAddress for _, addr := range addresses { - committeeAddresses = append(committeeAddresses, node.CommitteeAddress{ - Certificate: id.GetTLSCertificate().Certificate[0], - Address: addr, + tlsAddresses = append(tlsAddresses, node.TLSAddress{ + PubKey: id.GetTLSSigner().Public(), + Address: addr, }) } @@ -46,9 +46,9 @@ func registryRegisterNode(svc service.TendermintService, id *identity.Identity, ID: id.NodeSigner.Public(), EntityID: entityID, Expiration: 1000, - Committee: node.CommitteeInfo{ - Certificate: id.GetTLSCertificate().Certificate[0], - Addresses: committeeAddresses, + TLS: node.TLSInfo{ + PubKey: id.GetTLSSigner().Public(), + Addresses: tlsAddresses, }, P2P: node.P2PInfo{ ID: id.P2PSigner.Public(), diff --git a/go/oasis-node/cmd/debug/byzantine/storage.go b/go/oasis-node/cmd/debug/byzantine/storage.go index 18761084989..525992b60e1 100644 --- a/go/oasis-node/cmd/debug/byzantine/storage.go +++ b/go/oasis-node/cmd/debug/byzantine/storage.go @@ -3,13 +3,11 @@ package byzantine import ( "context" "crypto/tls" - "crypto/x509" "fmt" "io" "google.golang.org/grpc" "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/credentials" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" @@ -33,20 +31,19 @@ type honestNodeStorage struct { } func dialOptionForNode(ourCerts []tls.Certificate, node *node.Node) (grpc.DialOption, error) { - certPool := x509.NewCertPool() - for _, addr := range node.Committee.Addresses { - nodeCert, err := addr.ParseCertificate() - if err != nil { - return nil, fmt.Errorf("failed to parse node's address certificate: %w", err) - } - certPool.AddCert(nodeCert) + tlsKeys := make(map[signature.PublicKey]bool) + for _, addr := range node.TLS.Addresses { + tlsKeys[addr.PubKey] = true } - creds := credentials.NewTLS(&tls.Config{ - Certificates: ourCerts, - RootCAs: certPool, - ServerName: identity.CommonName, + creds, err := cmnGrpc.NewClientCreds(&cmnGrpc.ClientOptions{ + CommonName: identity.CommonName, + ServerPubKeys: tlsKeys, + Certificates: ourCerts, }) + if err != nil { + return nil, err + } return grpc.WithTransportCredentials(creds), nil } @@ -61,7 +58,7 @@ func dialNode(node *node.Node, opts grpc.DialOption) (*grpc.ClientConn, error) { return nil, fmt.Errorf("failed dialing node: %w", err) } var resolverState resolver.State - for _, addr := range node.Committee.Addresses { + for _, addr := range node.TLS.Addresses { resolverState.Addresses = append(resolverState.Addresses, resolver.Address{Addr: addr.String()}) } manualResolver.UpdateState(resolverState) diff --git a/go/oasis-node/cmd/debug/txsource/workload/registration.go b/go/oasis-node/cmd/debug/txsource/workload/registration.go index 71023edc7fa..0be967cc8f4 100644 --- a/go/oasis-node/cmd/debug/txsource/workload/registration.go +++ b/go/oasis-node/cmd/debug/txsource/workload/registration.go @@ -100,12 +100,12 @@ func getNodeDesc(rng *rand.Rand, nodeIdentity *identity.Identity, entityID signa EntityID: entityID, Expiration: 0, Roles: availableRoles[rng.Intn(len(availableRoles))], - Committee: node.CommitteeInfo{ - Certificate: nodeIdentity.GetTLSCertificate().Certificate[0], - Addresses: []node.CommitteeAddress{ + TLS: node.TLSInfo{ + PubKey: nodeIdentity.GetTLSSigner().Public(), + Addresses: []node.TLSAddress{ { - Certificate: nodeIdentity.GetTLSCertificate().Certificate[0], - Address: nodeAddr, + PubKey: nodeIdentity.GetTLSSigner().Public(), + Address: nodeAddr, }, }, }, diff --git a/go/oasis-node/cmd/registry/node/node.go b/go/oasis-node/cmd/registry/node/node.go index 97e1dbf5b8b..542c4e70cc2 100644 --- a/go/oasis-node/cmd/registry/node/node.go +++ b/go/oasis-node/cmd/registry/node/node.go @@ -33,7 +33,7 @@ import ( const ( CfgEntityID = "node.entity_id" CfgExpiration = "node.expiration" - CfgCommitteeAddress = "node.committee_address" + CfgTLSAddress = "node.tls_address" CfgP2PAddress = "node.p2p_address" CfgConsensusAddress = "node.consensus_address" CfgRole = "node.role" @@ -170,9 +170,9 @@ func doInit(cmd *cobra.Command, args []string) { // nolint: gocyclo os.Exit(1) } - var nextCert []byte - if c := nodeIdentity.GetNextTLSCertificate(); c != nil { - nextCert = c.Certificate[0] + var nextPubKey signature.PublicKey + if s := nodeIdentity.GetNextTLSSigner(); s != nil { + nextPubKey = s.Public() } n := &node.Node{ @@ -180,9 +180,9 @@ func doInit(cmd *cobra.Command, args []string) { // nolint: gocyclo ID: nodeIdentity.NodeSigner.Public(), EntityID: entityID, Expiration: viper.GetUint64(CfgExpiration), - Committee: node.CommitteeInfo{ - Certificate: nodeIdentity.GetTLSCertificate().Certificate[0], - NextCertificate: nextCert, + TLS: node.TLSInfo{ + PubKey: nodeIdentity.GetTLSSigner().Public(), + NextPubKey: nextPubKey, }, P2P: node.P2PInfo{ ID: nodeIdentity.P2PSigner.Public(), @@ -211,21 +211,20 @@ func doInit(cmd *cobra.Command, args []string) { // nolint: gocyclo n.Runtimes = append(n.Runtimes, runtime) } - for _, v := range viper.GetStringSlice(CfgCommitteeAddress) { - - var committeeAddr node.CommitteeAddress - if committeeAddrErr := committeeAddr.UnmarshalText([]byte(v)); committeeAddrErr != nil { - if addrErr := committeeAddr.Address.UnmarshalText([]byte(v)); addrErr != nil { - logger.Error("failed to parse node's committee address", + for _, v := range viper.GetStringSlice(CfgTLSAddress) { + var tlsAddr node.TLSAddress + if tlsAddrErr := tlsAddr.UnmarshalText([]byte(v)); tlsAddrErr != nil { + if addrErr := tlsAddr.Address.UnmarshalText([]byte(v)); addrErr != nil { + logger.Error("failed to parse node's TLS address", "addrErr", addrErr, - "committeeAddrErr", committeeAddrErr, + "tlsAddrErr", tlsAddrErr, "addr", v, ) os.Exit(1) } - committeeAddr.Certificate = n.Committee.Certificate + tlsAddr.PubKey = n.TLS.PubKey } - n.Committee.Addresses = append(n.Committee.Addresses, committeeAddr) + n.TLS.Addresses = append(n.TLS.Addresses, tlsAddr) } for _, v := range viper.GetStringSlice(CfgP2PAddress) { @@ -239,8 +238,8 @@ func doInit(cmd *cobra.Command, args []string) { // nolint: gocyclo } n.P2P.Addresses = append(n.P2P.Addresses, addr) } - if n.HasRoles(maskCommitteeMember) && (len(n.Committee.Addresses) == 0 || len(n.P2P.Addresses) == 0) { - logger.Error("nodes that are committee members require at least 1 committee and 1 P2P address") + if n.HasRoles(maskCommitteeMember) && (len(n.TLS.Addresses) == 0 || len(n.P2P.Addresses) == 0) { + logger.Error("nodes that are committee members require at least 1 TLS and 1 P2P address") os.Exit(1) } @@ -420,7 +419,7 @@ func Register(parentCmd *cobra.Command) { func init() { flags.String(CfgEntityID, "", "Entity ID that controls this node") flags.Uint64(CfgExpiration, 0, "Epoch that the node registration should expire") - flags.StringSlice(CfgCommitteeAddress, nil, "Address(es) the node can be reached as a committee member of the form [Certificate@]ip:port (where Certificate@ part is optional and represents base64 encoded node certificate for TLS)") + flags.StringSlice(CfgTLSAddress, nil, "Address(es) the node can be reached over TLS of the form [PubKey@]ip:port (where PubKey@ part is optional and represents base64 encoded node TLS public key)") flags.StringSlice(CfgP2PAddress, nil, "Address(es) the node can be reached over the P2P transport") flags.StringSlice(CfgConsensusAddress, nil, "Address(es) the node can be reached as a consensus member of the form [ID@]ip:port (where the ID@ part is optional and ID represents the node's public key)") flags.StringSlice(CfgRole, nil, "Role(s) of the node. Supported values are \"compute-worker\", \"storage-worker\", \"transaction-scheduler\", \"key-manager\", \"merge-worker\", and \"validator\"") diff --git a/go/oasis-remote-signer/cmd/root.go b/go/oasis-remote-signer/cmd/root.go index 27c1e256236..b97d14e50ec 100644 --- a/go/oasis-remote-signer/cmd/root.go +++ b/go/oasis-remote-signer/cmd/root.go @@ -27,7 +27,12 @@ import ( cmdSigner "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/signer" ) -const cfgClientCertificate = "client.certificate" +const ( + cfgClientCertificate = "client.certificate" + + // clientCommonName is the common name on the client TLS certificates. + clientCommonName = "remote-signer-client" +) var ( rootCmd = &cobra.Command{ @@ -131,7 +136,7 @@ func doClientInit(cmd *cobra.Command, args []string) { _, err = tls.LoadOrGenerate( filepath.Join(dataDir, "remote_signer_client_cert.pem"), filepath.Join(dataDir, "remote_signer_client_key.pem"), - "remote-signer-client", + clientCommonName, ) return err }(); err != nil { @@ -172,10 +177,11 @@ func runRoot(cmd *cobra.Command, args []string) error { // Initialize the gRPC server. svrCfg := &grpc.ServerConfig{ - Name: "remote-signer", - Port: uint16(viper.GetInt(cmdGrpc.CfgServerPort)), - Identity: &identity.Identity{}, - AuthFunc: peerCertAuth.AuthFunc, + Name: "remote-signer", + Port: uint16(viper.GetInt(cmdGrpc.CfgServerPort)), + Identity: &identity.Identity{}, + AuthFunc: peerCertAuth.AuthFunc, + ClientCommonName: clientCommonName, } svrCfg.Identity.SetTLSCertificate(cert) svr, err := grpc.NewServer(svrCfg) diff --git a/go/oasis-test-runner/oasis/args.go b/go/oasis-test-runner/oasis/args.go index 8d8cb0b58bf..30c345b62b2 100644 --- a/go/oasis-test-runner/oasis/args.go +++ b/go/oasis-test-runner/oasis/args.go @@ -220,15 +220,6 @@ func (args *argBuilder) workerCommonSentryAddresses(addrs []string) *argBuilder return args } -func (args *argBuilder) workerCommonSentryCertFiles(certFiles []string) *argBuilder { - for _, certFile := range certFiles { - args.vec = append(args.vec, []string{ - "--" + workerCommon.CfgSentryCertFiles, certFile, - }...) - } - return args -} - func (args *argBuilder) workerSentryGrpcClientAddress(addrs []string) *argBuilder { for _, addr := range addrs { args.vec = append(args.vec, []string{ @@ -379,12 +370,11 @@ func (args *argBuilder) iasSPID(spid []byte) *argBuilder { } func (args *argBuilder) addSentries(sentries []*Sentry) *argBuilder { - var addrs, certFiles []string + var addrs []string for _, sentry := range sentries { - addrs = append(addrs, fmt.Sprintf("127.0.0.1:%d", sentry.controlPort)) - certFiles = append(certFiles, sentry.TLSCertPath()) + addrs = append(addrs, fmt.Sprintf("%s@127.0.0.1:%d", sentry.tlsPublicKey.String(), sentry.controlPort)) } - return args.workerCommonSentryAddresses(addrs).workerCommonSentryCertFiles(certFiles) + return args.workerCommonSentryAddresses(addrs) } func (args *argBuilder) addValidatorsAsSentryUpstreams(validators []*Validator) *argBuilder { diff --git a/go/oasis-test-runner/oasis/sentry.go b/go/oasis-test-runner/oasis/sentry.go index fbe1729d64f..4509c8ddc94 100644 --- a/go/oasis-test-runner/oasis/sentry.go +++ b/go/oasis-test-runner/oasis/sentry.go @@ -18,6 +18,7 @@ type Sentry struct { keymanagerIndices []int publicKey signature.PublicKey + tlsPublicKey signature.PublicKey tmAddress string consensusPort uint16 controlPort uint16 @@ -123,6 +124,7 @@ func (net *Network) NewSentry(cfg *SentryCfg) (*Sentry, error) { return nil, fmt.Errorf("oasis/sentry: failed to provision sentry identity: %w", err) } sentryPublicKey := sentryIdentity.NodeSigner.Public() + sentryTLSPublicKey := sentryIdentity.GetTLSSigner().Public() sentry := &Sentry{ Node: Node{ @@ -136,6 +138,7 @@ func (net *Network) NewSentry(cfg *SentryCfg) (*Sentry, error) { storageIndices: cfg.StorageIndices, keymanagerIndices: cfg.KeymanagerIndices, publicKey: sentryPublicKey, + tlsPublicKey: sentryTLSPublicKey, tmAddress: crypto.PublicKeyToTendermint(&sentryPublicKey).Address().String(), consensusPort: net.nextNodePort, controlPort: net.nextNodePort + 1, diff --git a/go/oasis-test-runner/scenario/e2e/registry_cli.go b/go/oasis-test-runner/scenario/e2e/registry_cli.go index 7ec8fd8e8b5..05d73ad13f4 100644 --- a/go/oasis-test-runner/scenario/e2e/registry_cli.go +++ b/go/oasis-test-runner/scenario/e2e/registry_cli.go @@ -385,20 +385,20 @@ func (r *registryCLIImpl) newTestNode(entityID signature.PublicKey) (*node.Node, testConsensusAddressesStr = append(testConsensusAddressesStr, a.String()) } - // Committee addresses. - testCommitteeAddresses := []node.CommitteeAddress{ + // TLS addresses. + testTLSAddresses := []node.TLSAddress{ { - Certificate: []byte{}, // Certificate is generated afterwards. - Address: testAddresses[0], + PubKey: signature.PublicKey{}, // Public key is generated afterwards. + Address: testAddresses[0], }, { - Certificate: []byte{}, // Certificate is generated afterwards. - Address: testAddresses[1], + PubKey: signature.PublicKey{}, // PublicKey is generated afterwards. + Address: testAddresses[1], }, } - testCommitteeAddressesStr := []string{} - for _, a := range testCommitteeAddresses { - testCommitteeAddressesStr = append(testCommitteeAddressesStr, a.String()) + testTLSAddressesStr := []string{} + for _, a := range testTLSAddresses { + testTLSAddressesStr = append(testTLSAddressesStr, a.String()) } testNode := node.Node{ @@ -406,9 +406,9 @@ func (r *registryCLIImpl) newTestNode(entityID signature.PublicKey) (*node.Node, ID: signature.PublicKey{}, // ID is generated afterwards. EntityID: entityID, Expiration: 42, - Committee: node.CommitteeInfo{ - Certificate: []byte{}, // Certificate is generated afterwards. - Addresses: testCommitteeAddresses, + TLS: node.TLSInfo{ + PubKey: signature.PublicKey{}, // Public key is generated afterwards. + Addresses: testTLSAddresses, }, P2P: node.P2PInfo{ ID: signature.PublicKey{}, // ID is generated afterwards. @@ -427,7 +427,7 @@ func (r *registryCLIImpl) newTestNode(entityID signature.PublicKey) (*node.Node, } _ = testNode.Runtimes[0].ID.UnmarshalHex("8000000000000000000000000000000000000000000000000000000000000000") - return &testNode, testAddressesStr, testConsensusAddressesStr, testCommitteeAddressesStr, nil + return &testNode, testAddressesStr, testConsensusAddressesStr, testTLSAddressesStr, nil } // initNode very "thoroughly" initializes new node and returns its instance. @@ -435,7 +435,7 @@ func (r *registryCLIImpl) initNode(childEnv *env.Env, ent *entity.Entity, entDir r.logger.Info("initializing new node") // testNode will be our fixture for testing the CLI. - testNode, testAddressesStr, testConsensusAddressesStr, testCommitteeAddressesStr, err := r.newTestNode(ent.ID) + testNode, testAddressesStr, testConsensusAddressesStr, testTLSAddressesStr, err := r.newTestNode(ent.ID) if err != nil { return nil, err } @@ -444,7 +444,7 @@ func (r *registryCLIImpl) initNode(childEnv *env.Env, ent *entity.Entity, entDir runInitNode := func() (*node.Node, error) { args := []string{ "registry", "node", "init", - "--" + cmdRegNode.CfgCommitteeAddress, strings.Join(testCommitteeAddressesStr, ","), + "--" + cmdRegNode.CfgTLSAddress, strings.Join(testTLSAddressesStr, ","), "--" + cmdRegNode.CfgConsensusAddress, strings.Join(testConsensusAddressesStr, ","), "--" + cmdRegNode.CfgEntityID, testNode.EntityID.String(), "--" + cmdRegNode.CfgExpiration, strconv.FormatUint(testNode.Expiration, 10), @@ -490,8 +490,8 @@ func (r *registryCLIImpl) initNode(childEnv *env.Env, ent *entity.Entity, entDir if !n.ID.IsValid() { return nil, errors.New("new node ID is not valid") } - if n.Committee.Certificate == nil || len(n.Committee.Certificate) == 0 { - return nil, errors.New("new node committee certificate is not set") + if !n.TLS.PubKey.IsValid() { + return nil, errors.New("new node TLS public key is not set") } if !n.P2P.ID.IsValid() { return nil, errors.New("new node P2P ID is not valid") @@ -502,12 +502,12 @@ func (r *registryCLIImpl) initNode(childEnv *env.Env, ent *entity.Entity, entDir // Replace our testNode fields with the generated one, so we can just marshal both nodes and compare the output afterwards. testNode.ID = n.ID - testNode.Committee.Certificate = n.Committee.Certificate - testNode.Committee.NextCertificate = n.Committee.NextCertificate + testNode.TLS.PubKey = n.TLS.PubKey + testNode.TLS.NextPubKey = n.TLS.NextPubKey testNode.P2P.ID = n.P2P.ID testNode.Consensus.ID = n.Consensus.ID - for idx := range testNode.Committee.Addresses { - testNode.Committee.Addresses[idx].Certificate = n.Committee.Certificate + for idx := range testNode.TLS.Addresses { + testNode.TLS.Addresses[idx].PubKey = n.TLS.PubKey } // Export both original and imported node to JSON and compare them. @@ -526,11 +526,11 @@ func (r *registryCLIImpl) initNode(childEnv *env.Env, ent *entity.Entity, entDir return nil, err } - // TLS certificates are regenerated each time, so replace them with new ones. - testNode.Committee.Certificate = n.Committee.Certificate - testNode.Committee.NextCertificate = n.Committee.NextCertificate - for idx := range testNode.Committee.Addresses { - testNode.Committee.Addresses[idx].Certificate = n.Committee.Certificate + // TLS keys are regenerated each time, so replace them with new ones. + testNode.TLS.PubKey = n.TLS.PubKey + testNode.TLS.NextPubKey = n.TLS.NextPubKey + for idx := range testNode.TLS.Addresses { + testNode.TLS.Addresses[idx].PubKey = n.TLS.PubKey } testNodeStr, _ = json.Marshal(testNode) diff --git a/go/oasis-test-runner/scenario/e2e/sentry.go b/go/oasis-test-runner/scenario/e2e/sentry.go index 1018bb68702..790b5bf3f28 100644 --- a/go/oasis-test-runner/scenario/e2e/sentry.go +++ b/go/oasis-test-runner/scenario/e2e/sentry.go @@ -57,11 +57,11 @@ func (s *sentryImpl) Fixture() (*oasis.NetworkFixture, error) { // +-------------+ +----------+ // // +-----------+ +----------+ - // | Storage 1 +<---------->+ Sentry 3 | + // | Storage 0 +<---------->+ Sentry 3 | // +-----------+ +----------+ // // +-----------+ +----------+ - // | Storage 2 +<---------->+ Sentry 4 | + // | Storage 1 +<---------->+ Sentry 4 | // +-----------+ +----------+ // // +------------+ +----------+ diff --git a/go/registry/api/api.go b/go/registry/api/api.go index af043b208ce..ccc0e9b1c9a 100644 --- a/go/registry/api/api.go +++ b/go/registry/api/api.go @@ -4,8 +4,6 @@ package api import ( "bytes" "context" - goEd25519 "crypto/ed25519" - "crypto/x509" "encoding/json" "fmt" "sort" @@ -167,8 +165,8 @@ var ( // ConsensusAddressRequiredRoles are the Node roles that require Consensus Address. ConsensusAddressRequiredRoles = node.RoleValidator - // CommitteeAddressRequiredRoles are the Node roles that require Committee Address. - CommitteeAddressRequiredRoles = (node.RoleComputeWorker | + // TLSAddressRequiredRoles are the Node roles that require TLS Address. + TLSAddressRequiredRoles = (node.RoleComputeWorker | node.RoleStorageWorker | node.RoleKeyManager) @@ -310,11 +308,8 @@ type NodeList struct { // NodeLookup interface implements various ways for the verification // functions to look-up nodes in the registry's state. type NodeLookup interface { - // Returns the node that corresponds to the given consensus or P2P ID. - NodeByConsensusOrP2PKey(ctx context.Context, key signature.PublicKey) (*node.Node, error) - - // Returns the node that corresponds to the given committee certificate. - NodeByCertificate(ctx context.Context, cert []byte) (*node.Node, error) + // NodeBySubKey looks up a specific node by its consensus, P2P or TLS key. + NodeBySubKey(ctx context.Context, key signature.PublicKey) (*node.Node, error) // Returns a list of all nodes. Nodes(ctx context.Context) ([]*node.Node, error) @@ -598,55 +593,53 @@ func VerifyRegisterNodeArgs( // nolint: gocyclo return nil, nil, err } - // Validate CommitteeInfo. - // Verify that certificate is well-formed. - if _, err := n.Committee.ParseCertificate(); err != nil { - logger.Error("RegisterNode: invalid committee TLS certificate", - "node", n, - "err", err, - ) - return nil, nil, fmt.Errorf("%w: invalid committee TLS certificate", ErrInvalidArgument) - } - committeeAddressRequired := n.HasRoles(CommitteeAddressRequiredRoles) - if err := verifyAddresses(params, committeeAddressRequired, n.Committee.Addresses); err != nil { - addrs, _ := json.Marshal(n.Committee.Addresses) - logger.Error("RegisterNode: missing/invalid committee addresses", - "node", n, - "committee_addrs", addrs, - ) - return nil, nil, err - } - - certPub, err := verifyNodeCertificate(logger, &n, false) - if err != nil { - return nil, nil, err - } + // Validate TLSInfo. + if n.DescriptorVersion == 0 { + // Old descriptor that used full TLS certificates instead of just public keys. We allow old + // descriptors iff this is the chain being initialized from genesis and the node only has + // the validator role (because validators do not expose any TLS services). + // TODO: Drop support for node descriptor version 0 (oasis-core#2918). + if !isGenesis && !isSanityCheck { + return nil, nil, fmt.Errorf("%w: v0 descriptor only allowed at genesis time", ErrInvalidArgument) + } + if !n.OnlyHasRoles(node.RoleValidator) { + logger.Error("RegisterNode: v0 descriptor for non-validator node", + "node", n, + ) + return nil, nil, fmt.Errorf("%w: v0 descriptor for non-validator node", ErrInvalidArgument) + } - if !sigNode.MultiSigned.IsSignedBy(certPub) { - if n.Committee.NextCertificate != nil { - nextCertPub, grr := verifyNodeCertificate(logger, &n, true) - if grr != nil { - return nil, nil, grr - } + legacyTLSKey, err := nodeV0parseTLSPubKey(logger, sigNode) + if err != nil { + return nil, nil, err + } - if !sigNode.MultiSigned.IsSignedBy(nextCertPub) { - logger.Error("RegisterNode: not signed by any TLS certificate key", - "signed_node", sigNode, - "node", n, - ) - return nil, nil, fmt.Errorf("%w: registration not signed by any TLS certificate key", ErrInvalidArgument) - } + expectedSigners = append(expectedSigners, legacyTLSKey) + } else { + if !n.TLS.PubKey.IsValid() { + logger.Error("RegisterNode: invalid TLS public key", + "node", n, + ) + return nil, nil, fmt.Errorf("%w: invalid TLS public key", ErrInvalidArgument) + } + tlsAddressRequired := n.HasRoles(TLSAddressRequiredRoles) + if err := verifyAddresses(params, tlsAddressRequired, n.TLS.Addresses); err != nil { + addrs, _ := json.Marshal(n.TLS.Addresses) + logger.Error("RegisterNode: missing/invalid committee addresses", + "node", n, + "committee_addrs", addrs, + ) + return nil, nil, err + } - expectedSigners = append(expectedSigners, nextCertPub) - } else { + if !sigNode.MultiSigned.IsSignedBy(n.TLS.PubKey) { logger.Error("RegisterNode: not signed by TLS certificate key", "signed_node", sigNode, "node", n, ) return nil, nil, fmt.Errorf("%w: registration not signed by TLS certificate key", ErrInvalidArgument) } - } else { - expectedSigners = append(expectedSigners, certPub) + expectedSigners = append(expectedSigners, n.TLS.PubKey) } // Validate P2PInfo. @@ -665,7 +658,7 @@ func VerifyRegisterNodeArgs( // nolint: gocyclo } expectedSigners = append(expectedSigners, n.P2P.ID) p2pAddressRequired := n.HasRoles(P2PAddressRequiredRoles) - if err = verifyAddresses(params, p2pAddressRequired, n.P2P.Addresses); err != nil { + if err := verifyAddresses(params, p2pAddressRequired, n.P2P.Addresses); err != nil { addrs, _ := json.Marshal(n.P2P.Addresses) logger.Error("RegisterNode: missing/invald P2P addresses", "node", n, @@ -674,20 +667,20 @@ func VerifyRegisterNodeArgs( // nolint: gocyclo return nil, nil, err } - // Make sure that the consensus and P2P keys, as well as the committee - // certificate are unique (between themselves and compared to other nodes). + // Make sure that the consensus, TLS and P2P keys are unique (between + // themselves and compared to other nodes). // // Note that if a key exists and belongs to the same node ID, this is not // counted as an error, since it is possible that the node descriptor is // just being updated (this check is called in both cases). - if n.Consensus.ID.Equal(n.P2P.ID) { - logger.Error("RegisterNode: node consensus and P2P IDs must differ", + if n.Consensus.ID.Equal(n.P2P.ID) || n.Consensus.ID.Equal(n.TLS.PubKey) || n.P2P.ID.Equal(n.TLS.PubKey) { + logger.Error("RegisterNode: node consensus, P2P and TLS keys must differ", "node", n, ) - return nil, nil, fmt.Errorf("%w: P2P and Consensus IDs not unique", ErrInvalidArgument) + return nil, nil, fmt.Errorf("%w: P2P, consensus and TLS keys not unique", ErrInvalidArgument) } - existingNode, err := nodeLookup.NodeByConsensusOrP2PKey(ctx, n.Consensus.ID) + existingNode, err := nodeLookup.NodeBySubKey(ctx, n.Consensus.ID) if err != nil && err != ErrNoSuchNode { logger.Error("RegisterNode: failed to get node by consensus ID", "err", err, @@ -703,7 +696,7 @@ func VerifyRegisterNodeArgs( // nolint: gocyclo return nil, nil, fmt.Errorf("%w: duplicate node consensus ID", ErrInvalidArgument) } - existingNode, err = nodeLookup.NodeByConsensusOrP2PKey(ctx, n.P2P.ID) + existingNode, err = nodeLookup.NodeBySubKey(ctx, n.P2P.ID) if err != nil && err != ErrNoSuchNode { logger.Error("RegisterNode: failed to get node by P2P ID", "err", err, @@ -719,19 +712,21 @@ func VerifyRegisterNodeArgs( // nolint: gocyclo return nil, nil, fmt.Errorf("%w: duplicate node P2P ID", ErrInvalidArgument) } - existingNode, err = nodeLookup.NodeByCertificate(ctx, n.Committee.Certificate) + existingNode, err = nodeLookup.NodeBySubKey(ctx, n.TLS.PubKey) if err != nil && err != ErrNoSuchNode { - logger.Error("RegisterNode: failed to get node by committee certificate", + logger.Error("RegisterNode: failed to get node by TLS public key", "err", err, + "tls_pub_key", n.TLS.PubKey.String(), ) return nil, nil, ErrInvalidArgument } - if existingNode != nil && existingNode.ID != n.ID { - logger.Error("RegisterNode: duplicate node committee certificate", + // TODO: Drop support for node descriptor version 0 (oasis-core#2918). + if existingNode != nil && existingNode.ID != n.ID && n.DescriptorVersion != 0 { + logger.Error("RegisterNode: duplicate node TLS public key", "node_id", n.ID, "existing_node_id", existingNode.ID, ) - return nil, nil, fmt.Errorf("%w: duplicate node committee certificate", ErrInvalidArgument) + return nil, nil, fmt.Errorf("%w: duplicate node TLS public key", ErrInvalidArgument) } // Ensure that only the expected signatures are present, and nothing more. @@ -746,49 +741,6 @@ func VerifyRegisterNodeArgs( // nolint: gocyclo return &n, runtimes, nil } -func verifyNodeCertificate(logger *logging.Logger, node *node.Node, useNextCert bool) (signature.PublicKey, error) { - var ( - cert *x509.Certificate - certPub signature.PublicKey - err error - ) - - if useNextCert { - cert, err = node.Committee.ParseNextCertificate() - } else { - cert, err = node.Committee.ParseCertificate() - } - if err != nil { - logger.Error("RegisterNode: failed to parse committee certificate", - "err", err, - "node", node, - "use_next_cert", useNextCert, - ) - return certPub, fmt.Errorf("%w: failed to parse committee certificate", ErrInvalidArgument) - } - - edPub, ok := cert.PublicKey.(goEd25519.PublicKey) - if !ok { - logger.Error("RegisterNode: incorrect committee certifiate signing algorithm", - "node", node, - "use_next_cert", useNextCert, - ) - return certPub, fmt.Errorf("%w: incorrect committee certificate signing algorithm", ErrInvalidArgument) - } - - if err = certPub.UnmarshalBinary(edPub); err != nil { - // This should NEVER happen. - logger.Error("RegisterNode: malformed committee certificate signing key", - "err", err, - "node", node, - "use_next_cert", useNextCert, - ) - return certPub, fmt.Errorf("%w: malformed committee certificate signing key", ErrInvalidArgument) - } - - return certPub, nil -} - // VerifyNodeRuntimeEnclaveIDs verifies TEE-specific attributes of the node's runtime. func VerifyNodeRuntimeEnclaveIDs(logger *logging.Logger, rt *node.Runtime, regRt *Runtime, ts time.Time) error { // If no TEE available, do nothing. @@ -896,13 +848,13 @@ func verifyAddresses(params *ConsensusParameters, addressRequired bool, addresse return err } } - case []node.CommitteeAddress: + case []node.TLSAddress: if len(addrs) == 0 && addressRequired { - return fmt.Errorf("%w: missing committee address", ErrInvalidArgument) + return fmt.Errorf("%w: missing TLS address", ErrInvalidArgument) } for _, v := range addrs { - if _, err := v.ParseCertificate(); err != nil { - return fmt.Errorf("%w: committee address certificate invalid", ErrInvalidArgument) + if !v.PubKey.IsValid() { + return fmt.Errorf("%w: TLS address public key invalid", ErrInvalidArgument) } if err := VerifyAddress(v.Address, params.DebugAllowUnroutableAddresses); err != nil { return err diff --git a/go/registry/api/legacy_v0.go b/go/registry/api/legacy_v0.go new file mode 100644 index 00000000000..6bd44f01fe0 --- /dev/null +++ b/go/registry/api/legacy_v0.go @@ -0,0 +1,64 @@ +package api + +// TODO: Remove this when dropping support for node descriptor version 0 (oasis-core#2918). + +import ( + goEd25519 "crypto/ed25519" + "crypto/x509" + "fmt" + + "github.com/oasislabs/oasis-core/go/common/cbor" + "github.com/oasislabs/oasis-core/go/common/crypto/signature" + "github.com/oasislabs/oasis-core/go/common/logging" + "github.com/oasislabs/oasis-core/go/common/node" +) + +func nodeV0parseTLSPubKey(logger *logging.Logger, sigNode *node.MultiSignedNode) (signature.PublicKey, error) { + var ( + cert *x509.Certificate + certPub signature.PublicKey + err error + ) + type v0Node struct { + // We are only interested in the old "committee certificate" field. + Committee struct { + Certificate []byte `json:"certificate"` + } `json:"committee"` + } + var node v0Node + if err = cbor.Unmarshal(sigNode.Blob, &node); err != nil { + logger.Error("RegisterNode: invalid v0 node descriptor", + "node", node, + "err", err, + ) + return certPub, ErrInvalidArgument + } + + cert, err = x509.ParseCertificate(node.Committee.Certificate) + if err != nil { + logger.Error("RegisterNode: failed to parse v0 committee certificate", + "err", err, + "node", node, + ) + return certPub, fmt.Errorf("%w: failed to parse v0 committee certificate", ErrInvalidArgument) + } + + edPub, ok := cert.PublicKey.(goEd25519.PublicKey) + if !ok { + logger.Error("RegisterNode: incorrect v0 committee certifiate signing algorithm", + "node", node, + ) + return certPub, fmt.Errorf("%w: incorrect v0 committee certificate signing algorithm", ErrInvalidArgument) + } + + if err = certPub.UnmarshalBinary(edPub); err != nil { + // This should NEVER happen. + logger.Error("RegisterNode: malformed v0 committee certificate signing key", + "err", err, + "node", node, + ) + return certPub, fmt.Errorf("%w: malformed v0 committee certificate signing key", ErrInvalidArgument) + } + + return certPub, nil +} diff --git a/go/registry/api/sanity_check.go b/go/registry/api/sanity_check.go index 29e9212686d..01fb187103b 100644 --- a/go/registry/api/sanity_check.go +++ b/go/registry/api/sanity_check.go @@ -6,7 +6,6 @@ import ( "time" "github.com/oasislabs/oasis-core/go/common" - "github.com/oasislabs/oasis-core/go/common/crypto/hash" "github.com/oasislabs/oasis-core/go/common/crypto/signature" "github.com/oasislabs/oasis-core/go/common/entity" "github.com/oasislabs/oasis-core/go/common/logging" @@ -147,8 +146,7 @@ func SanityCheckNodes( ) (NodeLookup, error) { // nolint: gocyclo nodeLookup := &sanityCheckNodeLookup{ - nodes: make(map[signature.PublicKey]*node.Node), - nodesCertHashes: make(map[hash.Hash]*node.Node), + nodes: make(map[signature.PublicKey]*node.Node), } for _, signedNode := range nodes { @@ -186,10 +184,8 @@ func SanityCheckNodes( // Add validated node to nodeLookup. nodeLookup.nodes[node.Consensus.ID] = node nodeLookup.nodes[node.P2P.ID] = node + nodeLookup.nodes[node.TLS.PubKey] = node nodeLookup.nodesList = append(nodeLookup.nodesList, node) - - h := hash.NewFromBytes(node.Committee.Certificate) - nodeLookup.nodesCertHashes[h] = node } return nodeLookup, nil @@ -382,13 +378,12 @@ func (r *sanityCheckRuntimeLookup) AllRuntimes(ctx context.Context) ([]*Runtime, // Node lookup used in sanity checks. type sanityCheckNodeLookup struct { - nodes map[signature.PublicKey]*node.Node - nodesCertHashes map[hash.Hash]*node.Node + nodes map[signature.PublicKey]*node.Node nodesList []*node.Node } -func (n *sanityCheckNodeLookup) NodeByConsensusOrP2PKey(ctx context.Context, key signature.PublicKey) (*node.Node, error) { +func (n *sanityCheckNodeLookup) NodeBySubKey(ctx context.Context, key signature.PublicKey) (*node.Node, error) { node, ok := n.nodes[key] if !ok { return nil, ErrNoSuchNode @@ -396,16 +391,6 @@ func (n *sanityCheckNodeLookup) NodeByConsensusOrP2PKey(ctx context.Context, key return node, nil } -func (n *sanityCheckNodeLookup) NodeByCertificate(ctx context.Context, cert []byte) (*node.Node, error) { - h := hash.NewFromBytes(cert) - - node, ok := n.nodesCertHashes[h] - if !ok { - return nil, ErrNoSuchNode - } - return node, nil -} - func (n *sanityCheckNodeLookup) Nodes(ctx context.Context) ([]*node.Node, error) { return n.nodesList, nil } diff --git a/go/registry/tests/tester.go b/go/registry/tests/tester.go index 1fed56544ea..f1e3ae2c4ea 100644 --- a/go/registry/tests/tester.go +++ b/go/registry/tests/tester.go @@ -4,7 +4,6 @@ package tests import ( "context" "crypto" - "crypto/ed25519" "errors" "net" "testing" @@ -685,7 +684,6 @@ func randomIdentity(rng *drbg.Drbg) *identity.Identity { panic(err) } ident.SetTLSCertificate(cert) - ident.SetTLSSigner(memorySigner.NewFromRuntime(cert.PrivateKey.(ed25519.PrivateKey))) return ident } @@ -744,11 +742,11 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, nod.Node.P2P.Addresses = append(nod.Node.P2P.Addresses, addr) nod.Node.Consensus.ID = nodeIdentity.ConsensusSigner.Public() // Generate dummy TLS certificate. - nod.Node.Committee.Certificate = nodeIdentity.GetTLSCertificate().Certificate[0] - nod.Node.Committee.Addresses = []node.CommitteeAddress{ - node.CommitteeAddress{ - Certificate: nod.Node.Committee.Certificate, - Address: addr, + nod.Node.TLS.PubKey = nodeIdentity.GetTLSSigner().Public() + nod.Node.TLS.Addresses = []node.TLSAddress{ + node.TLSAddress{ + PubKey: nod.Node.TLS.PubKey, + Address: addr, }, } @@ -771,24 +769,24 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, nod.invalidBefore = append(nod.invalidBefore, invalid1) } - // Add a registration with no committee addresses. + // Add a registration with no TLS addresses. invalid2 := &invalidNodeRegistration{ - descr: "register committee node without committee addresses", + descr: "register node without TLS addresses", } invNode2 := *nod.Node - invNode2.Committee.Addresses = nil + invNode2.TLS.Addresses = nil invalid2.signed, err = node.MultiSignNode(nodeSigners, api.RegisterNodeSignatureContext, &invNode2) if err != nil { return nil, err } nod.invalidBefore = append(nod.invalidBefore, invalid2) - // Add a registration with no committee certificate. + // Add a registration with no TLS public key. invalid3 := &invalidNodeRegistration{ - descr: "register committee node without committee certificate", + descr: "register node without TLS public key", } invNode3 := *nod.Node - invNode3.Committee.Certificate = nil + invNode3.TLS.PubKey = signature.PublicKey{} invalid3.signed, err = node.MultiSignNode( []signature.Signer{ nodeIdentity.NodeSigner, @@ -922,7 +920,7 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, invNode11 := *nod.Node invNode11.ID = invalidIdentity.NodeSigner.Public() invNode11.Consensus.ID = invalidIdentity.ConsensusSigner.Public() - invNode11.Committee.Certificate = invalidIdentity.GetTLSCertificate().Certificate[0] + invNode11.TLS.PubKey = invalidIdentity.GetTLSSigner().Public() invalid11.signed, err = node.MultiSignNode( []signature.Signer{ invalidIdentity.NodeSigner, @@ -946,7 +944,7 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, invNode12 := *nod.Node invNode12.ID = invalidIdentity.NodeSigner.Public() invNode12.P2P.ID = invalidIdentity.ConsensusSigner.Public() - invNode12.Committee.Certificate = invalidIdentity.GetTLSCertificate().Certificate[0] + invNode12.TLS.PubKey = invalidIdentity.GetTLSSigner().Public() invalid12.signed, err = node.MultiSignNode( []signature.Signer{ invalidIdentity.NodeSigner, @@ -1004,8 +1002,8 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, } nod.UpdatedNode.P2P.ID = nod.Node.P2P.ID nod.UpdatedNode.P2P.Addresses = append(nod.UpdatedNode.P2P.Addresses, addr) - nod.UpdatedNode.Committee.Certificate = nod.Node.Committee.Certificate - nod.UpdatedNode.Committee.Addresses = nod.Node.Committee.Addresses + nod.UpdatedNode.TLS.PubKey = nod.Node.TLS.PubKey + nod.UpdatedNode.TLS.Addresses = nod.Node.TLS.Addresses nod.UpdatedNode.Consensus.ID = nod.Node.Consensus.ID // This should remain the same or we'll get "node update not allowed". nod.SignedValidReRegistration, err = node.MultiSignNode(nodeSigners, api.RegisterNodeSignatureContext, nod.UpdatedNode) if err != nil { @@ -1027,11 +1025,11 @@ func (ent *TestEntity) NewTestNodes(nCompute int, nStorage int, idNonce []byte, Runtimes: newRuntimes, Roles: role, P2P: nod.Node.P2P, - Committee: nod.Node.Committee, + TLS: nod.Node.TLS, } newNode.P2P.ID = invalidIdentity.P2PSigner.Public() newNode.Consensus.ID = invalidIdentity.ConsensusSigner.Public() - newNode.Committee.Certificate = invalidIdentity.GetTLSCertificate().Certificate[0] + newNode.TLS.PubKey = invalidIdentity.GetTLSSigner().Public() invalid14.signed, err = node.MultiSignNode( []signature.Signer{ nodeIdentity.NodeSigner, diff --git a/go/runtime/committee/client.go b/go/runtime/committee/client.go index f4e59416943..b2280a45ed0 100644 --- a/go/runtime/committee/client.go +++ b/go/runtime/committee/client.go @@ -4,7 +4,6 @@ import ( "context" cryptorand "crypto/rand" "crypto/tls" - "crypto/x509" "fmt" "math/rand" "sync" @@ -13,7 +12,6 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" - "google.golang.org/grpc/security/advancedtls" "github.com/oasislabs/oasis-core/go/common/crypto/mathrand" "github.com/oasislabs/oasis-core/go/common/crypto/signature" @@ -152,9 +150,9 @@ type clientConnState struct { node *node.Node conn *grpc.ClientConn - resolver *manual.Resolver + tlsKeys map[signature.PublicKey]bool - certPool *x509.CertPool + resolver *manual.Resolver } // Refresh refreshes the node connection without closing the virtual connection. @@ -168,23 +166,12 @@ func (cs *clientConnState) Update(n *node.Node) error { // Update node descriptor. cs.node = n - // Update certificate pool. - certPool := x509.NewCertPool() - for _, addr := range n.Committee.Addresses { - nodeCert, err := addr.ParseCertificate() - if err != nil { - // This should never fail as the consensus layer should validate certificates. - return fmt.Errorf("failed to parse node certificate: %w", err) - } - - certPool.AddCert(nodeCert) - } - cs.certPool = certPool - - // Update addresses. The resolver will propagate addresses to the gRPC load balancer which will - // internally update subconns based on address changes. + // Update addresses and TLS keys. The resolver will propagate addresses to the gRPC load + // balancer which will internally update subconns based on address changes. var resolverState resolver.State - for _, addr := range n.Committee.Addresses { + cs.tlsKeys = make(map[signature.PublicKey]bool) + for _, addr := range n.TLS.Addresses { + cs.tlsKeys[addr.PubKey] = true resolverState.Addresses = append(resolverState.Addresses, resolver.Address{Addr: addr.String()}) } cs.resolver.UpdateState(resolverState) @@ -305,9 +292,9 @@ func (cc *committeeClient) updateConnectionLocked(n *node.Node) error { // If the connection to given node already exists, only update its addresses/certificates. var cs *clientConnState if cs = cc.conns[n.ID]; cs != nil { - // Only update connections if keys or addresses have changed. - if n.Committee.Equal(&cs.node.Committee) { - cc.logger.Debug("not updating connection as addresses have not changed", + // Only update connections if TLS keys or addresses have changed. + if n.TLS.Equal(&cs.node.TLS) { + cc.logger.Debug("not updating connection as TLS info has not changed", "node", n, ) return nil @@ -317,15 +304,13 @@ func (cc *committeeClient) updateConnectionLocked(n *node.Node) error { cs = new(clientConnState) // Create TLS credentials. - opts := advancedtls.ClientOptions{ - ServerNameOverride: identity.CommonName, // TODO: Consider using a custom VerifyPeer. - RootCertificateOptions: advancedtls.RootCertificateOptions{ - GetRootCAs: func(params *advancedtls.GetRootCAsParams) (*advancedtls.GetRootCAsResults, error) { - cc.RLock() - defer cc.RUnlock() - - return &advancedtls.GetRootCAsResults{TrustCerts: cs.certPool}, nil - }, + opts := cmnGrpc.ClientOptions{ + CommonName: identity.CommonName, + GetServerPubKeys: func() (map[signature.PublicKey]bool, error) { + cc.RLock() + keys := cs.tlsKeys + cc.RUnlock() + return keys, nil }, } if cc.clientIdentity != nil { @@ -339,7 +324,7 @@ func (cc *committeeClient) updateConnectionLocked(n *node.Node) error { } } - creds, err := advancedtls.NewClientCreds(&opts) + creds, err := cmnGrpc.NewClientCreds(&opts) if err != nil { return fmt.Errorf("failed to create TLS client credentials: %w", err) } diff --git a/go/sentry/api/api.go b/go/sentry/api/api.go index adcb778ab14..810fc312c79 100644 --- a/go/sentry/api/api.go +++ b/go/sentry/api/api.go @@ -4,25 +4,25 @@ package api import ( "context" + "github.com/oasislabs/oasis-core/go/common/crypto/signature" "github.com/oasislabs/oasis-core/go/common/node" ) -// SentryAddresses contains sentry node consensus and committee addresses. +// SentryAddresses contains sentry node consensus and TLS addresses. type SentryAddresses struct { - Consensus []node.ConsensusAddress - Committee []node.CommitteeAddress + Consensus []node.ConsensusAddress `json:"consensus"` + TLS []node.TLSAddress `json:"tls"` } // Backend is a sentry backend implementation. type Backend interface { - // Get addresses returns the list of consensus and committee addresses of - // the sentry node. + // Get addresses returns the list of consensus and TLS addresses of the sentry node. GetAddresses(context.Context) (*SentryAddresses, error) - // SetUpstreamTLSCertificates notifies the sentry node of the new - // TLS certificates used by its upstream node. - SetUpstreamTLSCertificates(context.Context, [][]byte) error + // SetUpstreamTLSPubKeys notifies the sentry node of the new TLS public keys used by its + // upstream node. + SetUpstreamTLSPubKeys(context.Context, []signature.PublicKey) error - // GetUpstreamTLSCertificates returns the TLS certificates of the sentry node's upstream node. - GetUpstreamTLSCertificates(context.Context) ([][]byte, error) + // GetUpstreamTLSPubKeys returns the TLS public keys of the sentry node's upstream node. + GetUpstreamTLSPubKeys(context.Context) ([]signature.PublicKey, error) } diff --git a/go/sentry/api/grpc.go b/go/sentry/api/grpc.go index 3ea2607703b..fc665abc1ef 100644 --- a/go/sentry/api/grpc.go +++ b/go/sentry/api/grpc.go @@ -5,6 +5,7 @@ import ( "google.golang.org/grpc" + "github.com/oasislabs/oasis-core/go/common/crypto/signature" cmnGrpc "github.com/oasislabs/oasis-core/go/common/grpc" ) @@ -15,11 +16,11 @@ var ( // methodGetAddresses is the GetAddresses method. methodGetAddresses = serviceName.NewMethod("GetAddresses", nil) - // methodSetUpstreamTLSCertificates is the SetUpstreamTLSCertificates method. - methodSetUpstreamTLSCertificates = serviceName.NewMethod("SetUpstreamTLSCertificates", [][]byte{}) + // methodSetUpstreamTLSPubKeys is the SetUpstreamTLSPubKeys method. + methodSetUpstreamTLSPubKeys = serviceName.NewMethod("SetUpstreamTLSPubKeys", []signature.PublicKey{}) - // methodGetUpstreamTLSCertificates is the GetUpstreamTLSCertificates method. - methodGetUpstreamTLSCertificates = serviceName.NewMethod("GetUpstreamTLSCertificates", nil) + // methodGetUpstreamTLSPubKeys is the GetUpstreamTLSPubKeys method. + methodGetUpstreamTLSPubKeys = serviceName.NewMethod("GetUpstreamTLSPubKeys", nil) // serviceDesc is the gRPC service descriptor. serviceDesc = grpc.ServiceDesc{ @@ -31,12 +32,12 @@ var ( Handler: handlerGetAddresses, }, { - MethodName: methodSetUpstreamTLSCertificates.ShortName(), - Handler: handlerSetUpstreamTLSCertificates, + MethodName: methodSetUpstreamTLSPubKeys.ShortName(), + Handler: handlerSetUpstreamTLSPubKeys, }, { - MethodName: methodGetUpstreamTLSCertificates.ShortName(), - Handler: handlerGetUpstreamTLSCertificates, + MethodName: methodGetUpstreamTLSPubKeys.ShortName(), + Handler: handlerGetUpstreamTLSPubKeys, }, }, Streams: []grpc.StreamDesc{}, @@ -62,44 +63,44 @@ func handlerGetAddresses( // nolint: golint return interceptor(ctx, nil, info, handler) } -func handlerSetUpstreamTLSCertificates( // nolint: golint +func handlerSetUpstreamTLSPubKeys( // nolint: golint srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor, ) (interface{}, error) { - var req [][]byte + var req []signature.PublicKey if err := dec(&req); err != nil { return nil, err } if interceptor == nil { - return nil, srv.(Backend).SetUpstreamTLSCertificates(ctx, req) + return nil, srv.(Backend).SetUpstreamTLSPubKeys(ctx, req) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: methodSetUpstreamTLSCertificates.FullName(), + FullMethod: methodSetUpstreamTLSPubKeys.FullName(), } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return nil, srv.(Backend).SetUpstreamTLSCertificates(ctx, *req.(*[][]byte)) + return nil, srv.(Backend).SetUpstreamTLSPubKeys(ctx, *req.(*[]signature.PublicKey)) } return interceptor(ctx, &req, info, handler) } -func handlerGetUpstreamTLSCertificates( // nolint: golint +func handlerGetUpstreamTLSPubKeys( // nolint: golint srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor, ) (interface{}, error) { if interceptor == nil { - return srv.(Backend).GetUpstreamTLSCertificates(ctx) + return srv.(Backend).GetUpstreamTLSPubKeys(ctx) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: methodGetUpstreamTLSCertificates.FullName(), + FullMethod: methodGetUpstreamTLSPubKeys.FullName(), } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(Backend).GetUpstreamTLSCertificates(ctx) + return srv.(Backend).GetUpstreamTLSPubKeys(ctx) } return interceptor(ctx, nil, info, handler) } @@ -121,16 +122,16 @@ func (c *sentryClient) GetAddresses(ctx context.Context) (*SentryAddresses, erro return &rsp, nil } -func (c *sentryClient) SetUpstreamTLSCertificates(ctx context.Context, certs [][]byte) error { - if err := c.conn.Invoke(ctx, methodSetUpstreamTLSCertificates.FullName(), certs, nil); err != nil { +func (c *sentryClient) SetUpstreamTLSPubKeys(ctx context.Context, pubKeys []signature.PublicKey) error { + if err := c.conn.Invoke(ctx, methodSetUpstreamTLSPubKeys.FullName(), pubKeys, nil); err != nil { return err } return nil } -func (c *sentryClient) GetUpstreamTLSCertificates(ctx context.Context) ([][]byte, error) { - var rsp [][]byte - if err := c.conn.Invoke(ctx, methodGetUpstreamTLSCertificates.FullName(), nil, &rsp); err != nil { +func (c *sentryClient) GetUpstreamTLSPubKeys(ctx context.Context) ([]signature.PublicKey, error) { + var rsp []signature.PublicKey + if err := c.conn.Invoke(ctx, methodGetUpstreamTLSPubKeys.FullName(), nil, &rsp); err != nil { return nil, err } return rsp, nil diff --git a/go/sentry/client/client.go b/go/sentry/client/client.go index 2a84baa53dd..31474730303 100644 --- a/go/sentry/client/client.go +++ b/go/sentry/client/client.go @@ -3,12 +3,11 @@ package client import ( "crypto/tls" - "crypto/x509" "fmt" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" + "github.com/oasislabs/oasis-core/go/common/crypto/signature" cmnGrpc "github.com/oasislabs/oasis-core/go/common/grpc" "github.com/oasislabs/oasis-core/go/common/identity" "github.com/oasislabs/oasis-core/go/common/logging" @@ -24,10 +23,8 @@ type Client struct { logger *logging.Logger - sentryAddress *node.Address - sentryCert *x509.Certificate - - nodeIdentity *identity.Identity + sentryAddress node.TLSAddress + identity *identity.Identity conn *grpc.ClientConn } @@ -42,13 +39,16 @@ func (c *Client) Close() { func (c *Client) createConnection() error { // Setup a secure gRPC connection. - certPool := x509.NewCertPool() - certPool.AddCert(c.sentryCert) - creds := credentials.NewTLS(&tls.Config{ - RootCAs: certPool, - ServerName: identity.CommonName, - Certificates: []tls.Certificate{*c.nodeIdentity.TLSSentryClientCertificate}, + creds, err := cmnGrpc.NewClientCreds(&cmnGrpc.ClientOptions{ + CommonName: identity.CommonName, + ServerPubKeys: map[signature.PublicKey]bool{ + c.sentryAddress.PubKey: true, + }, + Certificates: []tls.Certificate{*c.identity.TLSSentryClientCertificate}, }) + if err != nil { + return err + } opts := grpc.WithTransportCredentials(creds) conn, err := cmnGrpc.Dial(c.sentryAddress.String(), opts) // nolint: staticcheck @@ -65,16 +65,11 @@ func (c *Client) createConnection() error { } // New creates a new sentry client. -func New( - sentryAddress *node.Address, - sentryCert *x509.Certificate, - nodeIdentity *identity.Identity, -) (*Client, error) { +func New(sentryAddress node.TLSAddress, identity *identity.Identity) (*Client, error) { c := &Client{ logger: logging.GetLogger("sentry/client"), sentryAddress: sentryAddress, - sentryCert: sentryCert, - nodeIdentity: nodeIdentity, + identity: identity, } if err := c.createConnection(); err != nil { diff --git a/go/sentry/sentry.go b/go/sentry/sentry.go index 912ac59b09e..a4a9c54e832 100644 --- a/go/sentry/sentry.go +++ b/go/sentry/sentry.go @@ -6,6 +6,7 @@ import ( "fmt" "sync" + "github.com/oasislabs/oasis-core/go/common/crypto/signature" "github.com/oasislabs/oasis-core/go/common/identity" "github.com/oasislabs/oasis-core/go/common/logging" "github.com/oasislabs/oasis-core/go/common/node" @@ -24,7 +25,7 @@ type backend struct { consensus consensus.Backend identity *identity.Identity - upstreamTLSCertificates [][]byte + upstreamTLSPubKeys []signature.PublicKey } func (b *backend) GetAddresses(ctx context.Context) (*api.SentryAddresses, error) { @@ -37,48 +38,48 @@ func (b *backend) GetAddresses(ctx context.Context) (*api.SentryAddresses, error "addresses", consensusAddrs, ) - // Committee addresses - only available if gRPC sentry is enabled. - committeeAddrs, err := grpcSentry.GetNodeAddresses() + // TLS addresses -- only available if gRPC sentry is enabled. + tlsAddrs, err := grpcSentry.GetNodeAddresses() if err != nil { return nil, fmt.Errorf("sentry: error obtaining sentry worker addresses: %w", err) } - var committeeAddresses []node.CommitteeAddress + var tlsAddresses []node.TLSAddress - for _, addr := range committeeAddrs { - committeeAddresses = append(committeeAddresses, node.CommitteeAddress{ - Certificate: b.identity.GetTLSCertificate().Certificate[0], - Address: addr, + for _, addr := range tlsAddrs { + tlsAddresses = append(tlsAddresses, node.TLSAddress{ + PubKey: b.identity.GetTLSSigner().Public(), + Address: addr, }) // Make sure to also include the certificate that will be valid // in the next epoch, so that the node remains reachable. - if nextCert := b.identity.GetNextTLSCertificate(); nextCert != nil { - committeeAddresses = append(committeeAddresses, node.CommitteeAddress{ - Certificate: nextCert.Certificate[0], - Address: addr, + if nextSigner := b.identity.GetNextTLSSigner(); nextSigner != nil { + tlsAddresses = append(tlsAddresses, node.TLSAddress{ + PubKey: nextSigner.Public(), + Address: addr, }) } } return &api.SentryAddresses{ - Committee: committeeAddresses, Consensus: consensusAddrs, + TLS: tlsAddresses, }, nil } -func (b *backend) SetUpstreamTLSCertificates(ctx context.Context, certs [][]byte) error { +func (b *backend) SetUpstreamTLSPubKeys(ctx context.Context, pubKeys []signature.PublicKey) error { b.Lock() defer b.Unlock() - b.upstreamTLSCertificates = certs + b.upstreamTLSPubKeys = pubKeys return nil } -func (b *backend) GetUpstreamTLSCertificates(ctx context.Context) ([][]byte, error) { +func (b *backend) GetUpstreamTLSPubKeys(ctx context.Context) ([]signature.PublicKey, error) { b.RLock() defer b.RUnlock() - return b.upstreamTLSCertificates, nil + return b.upstreamTLSPubKeys, nil } // New constructs a new sentry Backend instance. diff --git a/go/worker/common/committee/accessctl.go b/go/worker/common/committee/accessctl.go index 0022a7d1a10..b9dc73e8457 100644 --- a/go/worker/common/committee/accessctl.go +++ b/go/worker/common/committee/accessctl.go @@ -1,9 +1,8 @@ package committee import ( - "crypto/x509" - "github.com/oasislabs/oasis-core/go/common/accessctl" + "github.com/oasislabs/oasis-core/go/common/crypto/signature" "github.com/oasislabs/oasis-core/go/common/logging" "github.com/oasislabs/oasis-core/go/common/node" "github.com/oasislabs/oasis-core/go/runtime/committee" @@ -30,15 +29,15 @@ func (ap AccessPolicy) AddRulesForCommittee(policy *accessctl.Policy, committee } // Allow the node to perform actions from the given access policy. - subject := accessctl.SubjectFromDER(node.Committee.Certificate) + subject := accessctl.SubjectFromPublicKey(node.TLS.PubKey) for _, action := range ap.Actions { policy.Allow(subject, action) } // Make sure to also allow the node to perform actions after it has // rotated its TLS certificates. - if node.Committee.NextCertificate != nil { - subject := accessctl.SubjectFromDER(node.Committee.NextCertificate) + if node.TLS.NextPubKey.IsValid() { + subject := accessctl.SubjectFromPublicKey(node.TLS.NextPubKey) for _, action := range ap.Actions { policy.Allow(subject, action) } @@ -46,10 +45,10 @@ func (ap AccessPolicy) AddRulesForCommittee(policy *accessctl.Policy, committee } } -// AddCertPolicy augments the given policy by allowing actions in the current AccessPolicy -// to given certificate. -func (ap AccessPolicy) AddCertPolicy(policy *accessctl.Policy, cert *x509.Certificate) { - subject := accessctl.SubjectFromX509Certificate(cert) +// AddPublicKeyPolicy augments the given policy by allowing actions in the current AccessPolicy +// to given TLS public key. +func (ap AccessPolicy) AddPublicKeyPolicy(policy *accessctl.Policy, pubKey signature.PublicKey) { + subject := accessctl.SubjectFromPublicKey(pubKey) for _, action := range ap.Actions { policy.Allow(subject, action) } @@ -64,15 +63,15 @@ func (ap AccessPolicy) AddRulesForNodeRoles( ) { for _, n := range nodes { if n.HasRoles(roles) { - subject := accessctl.SubjectFromDER(n.Committee.Certificate) + subject := accessctl.SubjectFromPublicKey(n.TLS.PubKey) for _, action := range ap.Actions { policy.Allow(subject, action) } // Make sure to also allow the node to perform actions after is has // rotated its TLS certificates. - if n.Committee.NextCertificate != nil { - subject := accessctl.SubjectFromDER(n.Committee.NextCertificate) + if n.TLS.NextPubKey.IsValid() { + subject := accessctl.SubjectFromPublicKey(n.TLS.NextPubKey) for _, action := range ap.Actions { policy.Allow(subject, action) } diff --git a/go/worker/common/config.go b/go/worker/common/config.go index cdcba0b0b8e..82c81dc7370 100644 --- a/go/worker/common/config.go +++ b/go/worker/common/config.go @@ -1,7 +1,6 @@ package common import ( - "crypto/x509" "fmt" "time" @@ -26,12 +25,9 @@ var ( cfgClientAddresses = "worker.client.addresses" - // CfgSentryAddresses configures addresses of sentry nodes the worker - // should connect to. + // CfgSentryAddresses configures addresses and public keys of sentry nodes the worker should + // connect to. CfgSentryAddresses = "worker.sentry.address" - // CfgSentryCertFiles configures paths to certificates of the sentry nodes - // the worker should connect to. - CfgSentryCertFiles = "worker.sentry.cert_file" // CfgRuntimeProvisioner configures the runtime provisioner. CfgRuntimeProvisioner = "worker.runtime.provisioner" @@ -70,10 +66,9 @@ const ( // Config contains common worker config. type Config struct { // nolint: maligned - ClientPort uint16 - ClientAddresses []node.Address - SentryAddresses []node.Address - SentryCertificates []*x509.Certificate + ClientPort uint16 + ClientAddresses []node.Address + SentryAddresses []node.TLSAddress // RuntimeHost contains configuration for a worker that hosts runtimes. It may be nil if the // worker is not configured to host runtimes. @@ -127,21 +122,20 @@ func NewConfig(ias ias.Endpoint) (*Config, error) { return nil, err } - sentryAddresses, err := configparser.ParseAddressList(viper.GetStringSlice(CfgSentryAddresses)) - if err != nil { - return nil, err - } - - sentryCerts, err := configparser.ParseCertificateFiles(viper.GetStringSlice(CfgSentryCertFiles)) - if err != nil { - return nil, err + // Parse sentry configuration. + var sentryAddresses []node.TLSAddress + for _, v := range viper.GetStringSlice(CfgSentryAddresses) { + var tlsAddr node.TLSAddress + if err = tlsAddr.UnmarshalText([]byte(v)); err != nil { + return nil, fmt.Errorf("worker: bad sentry address (%s): %w", v, err) + } + sentryAddresses = append(sentryAddresses, tlsAddr) } cfg := Config{ ClientPort: uint16(viper.GetInt(CfgClientPort)), ClientAddresses: clientAddresses, SentryAddresses: sentryAddresses, - SentryCertificates: sentryCerts, StorageCommitTimeout: viper.GetDuration(cfgStorageCommitTimeout), logger: logging.GetLogger("worker/config"), } @@ -234,8 +228,7 @@ func NewConfig(ias ias.Endpoint) (*Config, error) { func init() { Flags.Uint16(CfgClientPort, 9100, "Port to use for incoming gRPC client connections") Flags.StringSlice(cfgClientAddresses, []string{}, "Address/port(s) to use for client connections when registering this node (if not set, all non-loopback local interfaces will be used)") - Flags.StringSlice(CfgSentryAddresses, []string{}, fmt.Sprintf("Address(es) of sentry node(s) to connect to (each address should have a corresponding certificate file set in %s)", CfgSentryCertFiles)) - Flags.StringSlice(CfgSentryCertFiles, []string{}, fmt.Sprintf("Certificate file(s) of sentry node(s) to connect to (each certificate file should have a corresponding address set in %s)", CfgSentryAddresses)) + Flags.StringSlice(CfgSentryAddresses, []string{}, fmt.Sprintf("Address(es) of sentry node(s) to connect to of the form [PubKey@]ip:port (where PubKey@ part represents base64 encoded node TLS public key)")) Flags.String(CfgRuntimeProvisioner, RuntimeProvisionerSandboxed, "Runtime provisioner to use") Flags.String(CfgRuntimeSGXLoader, "", "(for SGX runtimes) Path to SGXS runtime loader binary") diff --git a/go/worker/common/configparser/configparser.go b/go/worker/common/configparser/configparser.go index 046ffd0cf67..032e1eea3ab 100644 --- a/go/worker/common/configparser/configparser.go +++ b/go/worker/common/configparser/configparser.go @@ -1,14 +1,11 @@ package configparser import ( - tlsPkg "crypto/tls" - "crypto/x509" "fmt" "net" "strconv" "github.com/oasislabs/oasis-core/go/common" - "github.com/oasislabs/oasis-core/go/common/crypto/tls" "github.com/oasislabs/oasis-core/go/common/node" ) @@ -42,31 +39,6 @@ func ParseAddressList(addresses []string) ([]node.Address, error) { return output, nil } -// ParseCertificateFiles parses certificate files. -func ParseCertificateFiles(certFiles []string) ([]*x509.Certificate, error) { - certs := make([]*x509.Certificate, 0, len(certFiles)) - var err error - - for _, certFile := range certFiles { - var tlsCert *tlsPkg.Certificate - tlsCert, err = tls.LoadCertificate(certFile) - if err != nil { - return nil, fmt.Errorf("failed to load certificate file %v: %w", certFile, err) - } - if len(tlsCert.Certificate) != 1 { - return nil, fmt.Errorf("certificate file %v should contain exactly 1 certificate in the chain", certFile) - } - var x509Cert *x509.Certificate - x509Cert, err = x509.ParseCertificate(tlsCert.Certificate[0]) - if err != nil { - return nil, fmt.Errorf("failed to parse certificate file %v: %w", certFile, err) - } - certs = append(certs, x509Cert) - } - - return certs, nil -} - // GetRuntimes parses hex strings to PublicKeys func GetRuntimes(runtimeIDsHex []string) ([]common.Namespace, error) { var runtimes []common.Namespace diff --git a/go/worker/keymanager/watcher.go b/go/worker/keymanager/watcher.go index 822dd1391d7..e5ebe1147e4 100644 --- a/go/worker/keymanager/watcher.go +++ b/go/worker/keymanager/watcher.go @@ -74,9 +74,8 @@ func (knw *kmNodeWatcher) watchNodes() { // Rebuild the access policy, something has changed. policy := accessctl.NewPolicy() - sentryCerts := knw.w.commonWorker.GetConfig().SentryCertificates - for _, cert := range sentryCerts { - sentryNodesPolicy.AddCertPolicy(&policy, cert) + for _, addr := range knw.w.commonWorker.GetConfig().SentryAddresses { + sentryNodesPolicy.AddPublicKeyPolicy(&policy, addr.PubKey) } var nodes []*node.Node diff --git a/go/worker/keymanager/worker.go b/go/worker/keymanager/worker.go index fbb409ee933..275e238ef29 100644 --- a/go/worker/keymanager/worker.go +++ b/go/worker/keymanager/worker.go @@ -553,9 +553,8 @@ func (crw *clientRuntimeWatcher) updateExternalServicePolicyLocked(snapshot *com } // Apply rules for configured sentry nodes. - sentryCerts := crw.w.commonWorker.GetConfig().SentryCertificates - for _, cert := range sentryCerts { - sentryNodesPolicy.AddCertPolicy(&policy, cert) + for _, addr := range crw.w.commonWorker.GetConfig().SentryAddresses { + sentryNodesPolicy.AddPublicKeyPolicy(&policy, addr.PubKey) } crw.w.grpcPolicy.SetAccessPolicy(policy, crw.node.Runtime.ID()) diff --git a/go/worker/registration/worker.go b/go/worker/registration/worker.go index 988e5b08032..2bc99f4d31c 100644 --- a/go/worker/registration/worker.go +++ b/go/worker/registration/worker.go @@ -2,7 +2,6 @@ package registration import ( "context" - "crypto/x509" "fmt" "sync" "sync/atomic" @@ -117,8 +116,7 @@ type Worker struct { // nolint: maligned entityID signature.PublicKey registrationSigner signature.Signer - sentryAddresses []node.Address - sentryCerts []*x509.Certificate + sentryAddresses []node.TLSAddress runtimeRegistry runtimeRegistry.Registry epochtime epochtime.Backend @@ -148,43 +146,27 @@ func DebugForceAllowUnroutableAddresses() { func (w *Worker) registrationLoop() { // nolint: gocyclo // If we have any sentry nodes, let them know about our TLS certs. - sentryAddrs := w.sentryAddresses - sentryCerts := w.sentryCerts - if len(sentryAddrs) > 0 { - for i, sentryAddr := range sentryAddrs { - var numRetries uint - + if len(w.sentryAddresses) > 0 { + pubKeys := w.identity.GetTLSPubKeys() + for _, sentryAddr := range w.sentryAddresses { pushCerts := func() error { - client, err := sentryClient.New(&sentryAddr, sentryCerts[i], w.identity) + client, err := sentryClient.New(sentryAddr, w.identity) if err != nil { - if numRetries < 60 { - numRetries++ - return err - } - return backoff.Permanent(err) + return err } defer client.Close() - certs := [][]byte{} - if c := w.identity.GetTLSCertificate(); c != nil { - certs = append(certs, c.Certificate[0]) - } - if c := w.identity.GetNextTLSCertificate(); c != nil { - certs = append(certs, c.Certificate[0]) - } - - err = client.SetUpstreamTLSCertificates(w.ctx, certs) + err = client.SetUpstreamTLSPubKeys(w.ctx, pubKeys) if err != nil { return err } - return nil } - sched := backoff.NewConstantBackOff(1 * time.Second) + sched := backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 60) err := backoff.Retry(pushCerts, backoff.WithContext(sched, w.ctx)) if err != nil { - w.logger.Error("unable to push upstream TLS certificates to sentry node!", + w.logger.Error("unable to push upstream TLS certificates to sentry node", "err", err, "sentry_address", sentryAddr, ) @@ -273,14 +255,14 @@ Loop: "err", err, ) } else { - cert1 := w.identity.GetTLSCertificate().Certificate[0] - cert2 := w.identity.GetNextTLSCertificate().Certificate[0] + pub1 := w.identity.GetTLSSigner().Public() + pub2 := w.identity.GetNextTLSSigner().Public() tlsRotationPending = true w.logger.Info("node TLS certificates have been rotated", "new_epoch", epoch, - "new_cert1", accessctl.SubjectFromDER(cert1), - "new_cert2", accessctl.SubjectFromDER(cert2), + "new_pub1", accessctl.SubjectFromPublicKey(pub1), + "new_pub2", accessctl.SubjectFromPublicKey(pub2), ) } } @@ -499,42 +481,41 @@ func (w *Worker) gatherConsensusAddresses(sentryConsensusAddrs []node.ConsensusA return validatedAddrs, nil } -func (w *Worker) gatherCommitteeAddresses(sentryCommitteeAddrs []node.CommitteeAddress) ([]node.CommitteeAddress, error) { - var committeeAddresses []node.CommitteeAddress +func (w *Worker) gatherTLSAddresses(sentryTLSAddrs []node.TLSAddress) ([]node.TLSAddress, error) { + var tlsAddresses []node.TLSAddress switch len(w.sentryAddresses) > 0 { // If sentry nodes are used, use sentry addresses. case true: - committeeAddresses = sentryCommitteeAddrs - // Otherwise gather committee addresses. + tlsAddresses = sentryTLSAddrs + // Otherwise gather TLS addresses. case false: addrs, err := w.workerCommonCfg.GetNodeAddresses() if err != nil { return nil, fmt.Errorf("worker/registration: failed to register node: unable to get node addresses: %w", err) } for _, addr := range addrs { - committeeAddresses = append(committeeAddresses, node.CommitteeAddress{ - Certificate: w.identity.GetTLSCertificate().Certificate[0], - Address: addr, + tlsAddresses = append(tlsAddresses, node.TLSAddress{ + PubKey: w.identity.GetTLSSigner().Public(), + Address: addr, }) // Make sure to also include the certificate that will be valid // in the next epoch, so that the node remains reachable. - if nextCert := w.identity.GetNextTLSCertificate(); nextCert != nil { - committeeAddresses = append(committeeAddresses, node.CommitteeAddress{ - Certificate: nextCert.Certificate[0], - Address: addr, + if nextSigner := w.identity.GetNextTLSSigner(); nextSigner != nil { + tlsAddresses = append(tlsAddresses, node.TLSAddress{ + PubKey: nextSigner.Public(), + Address: addr, }) } } } // Filter out any potentially invalid addresses. - var validatedAddrs []node.CommitteeAddress - for _, addr := range committeeAddresses { - if _, err := addr.ParseCertificate(); err != nil { - w.logger.Error("worker/registration: skipping node address due to invalid certificate", + var validatedAddrs []node.TLSAddress + for _, addr := range tlsAddresses { + if !addr.PubKey.IsValid() { + w.logger.Error("worker/registration: skipping node address due to invalid public key", "addr", addr, - "err", err, ) continue } @@ -550,7 +531,7 @@ func (w *Worker) gatherCommitteeAddresses(sentryCommitteeAddrs []node.CommitteeA } if len(validatedAddrs) == 0 { - return nil, fmt.Errorf("worker/registration: node has no valid committee addresses") + return nil, fmt.Errorf("worker/registration: node has no valid TLS addresses") } return validatedAddrs, nil @@ -563,9 +544,9 @@ func (w *Worker) registerNode(epoch epochtime.EpochTime, hook RegisterNodeHook) "node_id", identityPublic.String(), ) - var nextCert []byte - if c := w.identity.GetNextTLSCertificate(); c != nil { - nextCert = c.Certificate[0] + var nextPubKey signature.PublicKey + if s := w.identity.GetNextTLSSigner(); s != nil { + nextPubKey = s.Public() } nodeDesc := node.Node{ @@ -573,9 +554,9 @@ func (w *Worker) registerNode(epoch epochtime.EpochTime, hook RegisterNodeHook) ID: identityPublic, EntityID: w.entityID, Expiration: uint64(epoch) + 2, - Committee: node.CommitteeInfo{ - Certificate: w.identity.GetTLSCertificate().Certificate[0], - NextCertificate: nextCert, + TLS: node.TLSInfo{ + PubKey: w.identity.GetTLSSigner().Public(), + NextPubKey: nextPubKey, }, P2P: node.P2PInfo{ ID: w.identity.P2PSigner.Public(), @@ -599,9 +580,9 @@ func (w *Worker) registerNode(epoch epochtime.EpochTime, hook RegisterNodeHook) } var sentryConsensusAddrs []node.ConsensusAddress - var sentryCommitteeAddrs []node.CommitteeAddress + var sentryTLSAddrs []node.TLSAddress if len(w.sentryAddresses) > 0 { - sentryConsensusAddrs, sentryCommitteeAddrs = w.querySentries() + sentryConsensusAddrs, sentryTLSAddrs = w.querySentries() } // Add Consensus Addresses if required. @@ -613,13 +594,13 @@ func (w *Worker) registerNode(epoch epochtime.EpochTime, hook RegisterNodeHook) nodeDesc.Consensus.Addresses = addrs } - // Add Committee Addresses if required. - if nodeDesc.HasRoles(registry.CommitteeAddressRequiredRoles) { - addrs, err := w.gatherCommitteeAddresses(sentryCommitteeAddrs) + // Add TLS Addresses if required. + if nodeDesc.HasRoles(registry.TLSAddressRequiredRoles) { + addrs, err := w.gatherTLSAddresses(sentryTLSAddrs) if err != nil { - return fmt.Errorf("error gathering committee addresses: %w", err) + return fmt.Errorf("error gathering TLS addresses: %w", err) } - nodeDesc.Committee.Addresses = addrs + nodeDesc.TLS.Addresses = addrs } // Add P2P Addresses if required. @@ -660,16 +641,15 @@ func (w *Worker) registerNode(epoch epochtime.EpochTime, hook RegisterNodeHook) return nil } -func (w *Worker) querySentries() ([]node.ConsensusAddress, []node.CommitteeAddress) { +func (w *Worker) querySentries() ([]node.ConsensusAddress, []node.TLSAddress) { var consensusAddrs []node.ConsensusAddress - var committeeAddrs []node.CommitteeAddress + var tlsAddrs []node.TLSAddress var err error - sentryAddrs := w.sentryAddresses - sentryCerts := w.sentryCerts - for i, sentryAddr := range sentryAddrs { + pubKeys := w.identity.GetTLSPubKeys() + for _, sentryAddr := range w.sentryAddresses { var client *sentryClient.Client - client, err = sentryClient.New(&sentryAddr, sentryCerts[i], w.identity) + client, err = sentryClient.New(sentryAddr, w.identity) if err != nil { w.logger.Warn("failed to create client to a sentry node", "err", err, @@ -690,14 +670,7 @@ func (w *Worker) querySentries() ([]node.ConsensusAddress, []node.CommitteeAddre } // Keep sentries updated with our latest TLS certificates. - certs := [][]byte{} - if c := w.identity.GetTLSCertificate(); c != nil { - certs = append(certs, c.Certificate[0]) - } - if c := w.identity.GetNextTLSCertificate(); c != nil { - certs = append(certs, c.Certificate[0]) - } - err = client.SetUpstreamTLSCertificates(w.ctx, certs) + err = client.SetUpstreamTLSPubKeys(w.ctx, pubKeys) if err != nil { w.logger.Warn("failed to provide upstream TLS certificates to sentry node", "err", err, @@ -706,23 +679,21 @@ func (w *Worker) querySentries() ([]node.ConsensusAddress, []node.CommitteeAddre } consensusAddrs = append(consensusAddrs, sentryAddresses.Consensus...) - committeeAddrs = append(committeeAddrs, sentryAddresses.Committee...) + tlsAddrs = append(tlsAddrs, sentryAddresses.TLS...) } if len(consensusAddrs) == 0 { - errMsg := "failed to obtain any consensus address from the configured sentry nodes" - w.logger.Error(errMsg, - "sentry_addresses", sentryAddrs, + w.logger.Error("failed to obtain any consensus address from the configured sentry nodes", + "sentry_addresses", w.sentryAddresses, ) } - if len(committeeAddrs) == 0 { - errMsg := "failed to obtain any committee address from the configured sentry nodes" - w.logger.Error(errMsg, - "sentry_addresses", sentryAddrs, + if len(tlsAddrs) == 0 { + w.logger.Error("failed to obtain any TLS address from the configured sentry nodes", + "sentry_addresses", w.sentryAddresses, ) } - return consensusAddrs, committeeAddrs + return consensusAddrs, tlsAddrs } // RequestDeregistration requests that the node not register itself in the next epoch. @@ -876,7 +847,6 @@ func New( delegate: delegate, entityID: entityID, sentryAddresses: workerCommonCfg.SentryAddresses, - sentryCerts: workerCommonCfg.SentryCertificates, registrationSigner: registrationSigner, runtimeRegistry: runtimeRegistry, epochtime: epochtime, diff --git a/go/worker/sentry/grpc/init.go b/go/worker/sentry/grpc/init.go index ab7d5525ab3..1f4818a9fa2 100644 --- a/go/worker/sentry/grpc/init.go +++ b/go/worker/sentry/grpc/init.go @@ -4,13 +4,11 @@ package grpc import ( "context" tlsPkg "crypto/tls" - "crypto/x509" "fmt" flag "github.com/spf13/pflag" "github.com/spf13/viper" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" @@ -74,35 +72,32 @@ func initConnection(ctx context.Context, logger *logging.Logger, ident *identity "upstream_node_id", upstreamNodeIDRaw, ) - // Get upstream node's certificates. - upstreamCerts, err := backend.GetUpstreamTLSCertificates(ctx) + // Get upstream node's TLS public keys. + upstreamPubKeys, err := backend.GetUpstreamTLSPubKeys(ctx) if err != nil { - return nil, fmt.Errorf("failed to get upstream node's TLS certificates: %w", err) + return nil, fmt.Errorf("failed to get upstream node's TLS public keys: %w", err) } - if len(upstreamCerts) == 0 { - return nil, fmt.Errorf("upstream node has no defined TLS certificates") + if len(upstreamPubKeys) == 0 { + return nil, fmt.Errorf("upstream node has no defined TLS public keys") } - logger.Info("found certificates for upstream node", - "num_certs", len(upstreamCerts), + logger.Info("found public keys for upstream node", + "num_keys", len(upstreamPubKeys), ) - certPool := x509.NewCertPool() - for _, cert := range upstreamCerts { - // Parse cert and add it to the pool. - parsedCert, grr := x509.ParseCertificate(cert) - if grr != nil { - // This should never happen. - return nil, fmt.Errorf("unable to parse certificate: %w", grr) - } - certPool.AddCert(parsedCert) + pubKeys := make(map[signature.PublicKey]bool) + for _, pk := range upstreamPubKeys { + pubKeys[pk] = true } - creds := credentials.NewTLS(&tlsPkg.Config{ - RootCAs: certPool, - ServerName: identity.CommonName, + creds, err := cmnGrpc.NewClientCreds(&cmnGrpc.ClientOptions{ + CommonName: identity.CommonName, + ServerPubKeys: pubKeys, GetClientCertificate: func(cri *tlsPkg.CertificateRequestInfo) (*tlsPkg.Certificate, error) { return ident.GetTLSCertificate(), nil }, }) + if err != nil { + return nil, fmt.Errorf("failed to create TLS credentials: %w", err) + } // Dial node manualResolver := manual.NewBuilderWithScheme("oasis-core-resolver") @@ -123,9 +118,9 @@ func initConnection(ctx context.Context, logger *logging.Logger, ident *identity manualResolver.UpdateState(resolverState) return &upstreamConn{ - nodeID: upstreamNodeID, - certs: upstreamCerts, - conn: conn, + nodeID: upstreamNodeID, + pubKeys: upstreamPubKeys, + conn: conn, }, nil } diff --git a/go/worker/sentry/grpc/worker.go b/go/worker/sentry/grpc/worker.go index 882928165e8..5e93ae70d40 100644 --- a/go/worker/sentry/grpc/worker.go +++ b/go/worker/sentry/grpc/worker.go @@ -57,8 +57,8 @@ type Worker struct { // nolint: maligned type upstreamConn struct { // ID of the upstream node. nodeID signature.PublicKey - // TLS certificates for the upstream node. - certs [][]byte + // TLS public keys for the upstream node. + pubKeys []signature.PublicKey // Client connection to the upstream node. conn *grpc.ClientConn } @@ -258,6 +258,7 @@ func (g *Worker) Name() string { // Stop halts the worker. func (g *Worker) Stop() { if !g.enabled { + close(g.quitCh) close(g.stopCh) return } diff --git a/go/worker/sentry/worker.go b/go/worker/sentry/worker.go index 998b11805a5..b4655e0c89b 100644 --- a/go/worker/sentry/worker.go +++ b/go/worker/sentry/worker.go @@ -40,7 +40,6 @@ type Worker struct { grpcServer *grpc.Server quitCh chan struct{} - initCh chan struct{} logger *logging.Logger } @@ -71,7 +70,14 @@ func (w *Worker) Start() error { return err } - close(w.initCh) + // Stop the gRPC server when the worker quits. + go func() { + defer close(w.quitCh) + + <-w.grpcWorker.Quit() + w.logger.Debug("sentry gRPC worker quit, stopping sentry gRPC server") + w.grpcServer.Stop() + }() return nil } @@ -84,8 +90,7 @@ func (w *Worker) Stop() { } w.grpcWorker.Stop() - w.grpcServer.Stop() - close(w.quitCh) + // The gRPC server will terminate once the worker quits. } // Enabled returns true if worker is enabled. @@ -114,7 +119,6 @@ func New(backend api.Backend, identity *identity.Identity) (*Worker, error) { enabled: Enabled(), backend: backend, quitCh: make(chan struct{}), - initCh: make(chan struct{}), logger: logging.GetLogger("worker/sentry"), } @@ -139,12 +143,6 @@ func New(backend api.Backend, identity *identity.Identity) (*Worker, error) { } w.grpcWorker = sentryGrpcWorker - // Stop in case of grpc/worker quitting. - go func() { - <-w.grpcWorker.Quit() - w.Stop() - }() - return w, nil } diff --git a/go/worker/storage/committee/node.go b/go/worker/storage/committee/node.go index 0579f6b912f..494777943e6 100644 --- a/go/worker/storage/committee/node.go +++ b/go/worker/storage/committee/node.go @@ -317,9 +317,8 @@ func (n *Node) updateExternalServicePolicyLocked(snapshot *committee.EpochSnapsh policy := accessctl.NewPolicy() // Add policy for configured sentry nodes. - sentryCerts := n.workerCommonCfg.SentryCertificates - for _, cert := range sentryCerts { - sentryNodesPolicy.AddCertPolicy(&policy, cert) + for _, addr := range n.workerCommonCfg.SentryAddresses { + sentryNodesPolicy.AddPublicKeyPolicy(&policy, addr.PubKey) } for _, xc := range snapshot.GetExecutorCommittees() {