diff --git a/pkg/security/certificate_loader.go b/pkg/security/certificate_loader.go index 9eb7928c62c1..b23d6b98eae9 100644 --- a/pkg/security/certificate_loader.go +++ b/pkg/security/certificate_loader.go @@ -72,6 +72,9 @@ const ( _ PemUsage = iota // CAPem describes the main CA certificate. CAPem + // TenantCAPem describes the CA certificate used to broker authN/Z for SQL + // tenants wishing to access the KV layer. + TenantCAPem // ClientCAPem describes the CA certificate used to verify client certificates. ClientCAPem // UICAPem describes the CA certificate used to verify the Admin UI server certificate. @@ -83,6 +86,8 @@ const ( UIPem // ClientPem describes a client certificate. ClientPem + // TenantClientPem describes a SQL tenant client certificate. + TenantClientPem // Maximum allowable permissions. maxKeyPermissions os.FileMode = 0700 @@ -94,7 +99,7 @@ const ( ) func isCA(usage PemUsage) bool { - return usage == CAPem || usage == ClientCAPem || usage == UICAPem + return usage == CAPem || usage == ClientCAPem || usage == TenantCAPem || usage == UICAPem } func (p PemUsage) String() string { @@ -103,6 +108,8 @@ func (p PemUsage) String() string { return "CA" case ClientCAPem: return "Client CA" + case TenantCAPem: + return "Tenant CA" case UICAPem: return "UI CA" case NodePem: @@ -185,6 +192,11 @@ func CertInfoFromFilename(filename string) (*CertInfo, error) { if numParts != 2 { return nil, errors.Errorf("client CA certificate filename should match ca-client%s", certExtension) } + case `ca-tenant`: + fileUsage = TenantCAPem + if numParts != 2 { + return nil, errors.Errorf("tenant CA certificate filename should match ca%s", certExtension) + } case `ca-ui`: fileUsage = UICAPem if numParts != 2 { @@ -202,11 +214,18 @@ func CertInfoFromFilename(filename string) (*CertInfo, error) { } case `client`: fileUsage = ClientPem - // strip prefix and suffix and re-join middle parts. + // Strip prefix and suffix and re-join middle parts. name = strings.Join(parts[1:numParts-1], `.`) if len(name) == 0 { return nil, errors.Errorf("client certificate filename should match client.%s", certExtension) } + case `tenant`: + fileUsage = TenantClientPem + // Strip prefix and suffix and re-join middle parts. + name = strings.Join(parts[1:numParts-1], `.`) + if len(name) == 0 { + return nil, errors.Errorf("tenant certificate filename should match tenant.%s", certExtension) + } default: return nil, errors.Errorf("unknown prefix %q", prefix) } diff --git a/pkg/security/certificate_manager.go b/pkg/security/certificate_manager.go index c1eb4d8aef17..c289ef92d427 100644 --- a/pkg/security/certificate_manager.go +++ b/pkg/security/certificate_manager.go @@ -195,6 +195,18 @@ func (cm *CertificateManager) CACertPath() string { // CACertFilename returns the expected file name for the CA certificate. func CACertFilename() string { return "ca" + certExtension } +// TenantCACertPath returns the expected file path for the Tenant CA +// certificate. +func (cm *CertificateManager) TenantCACertPath() string { + return filepath.Join(cm.certsDir, TenantCACertFilename()) +} + +// TenantCACertFilename returns the expected file name for the Tenant CA +// certificate. +func TenantCACertFilename() string { + return "ca-tenant" + certExtension +} + // ClientCACertPath returns the expected file path for the CA certificate // used to verify client certificates. func (cm *CertificateManager) ClientCACertPath() string { @@ -209,12 +221,22 @@ func (cm *CertificateManager) UICACertPath() string { // NodeCertPath returns the expected file path for the node certificate. func (cm *CertificateManager) NodeCertPath() string { - return filepath.Join(cm.certsDir, "node"+certExtension) + return filepath.Join(cm.certsDir, NodeCertFilename()) +} + +// NodeCertFilename returns the expected file name for the node certificate. +func NodeCertFilename() string { + return "node" + certExtension } // NodeKeyPath returns the expected file path for the node key. func (cm *CertificateManager) NodeKeyPath() string { - return filepath.Join(cm.certsDir, "node"+keyExtension) + return filepath.Join(cm.certsDir, NodeKeyFilename()) +} + +// NodeKeyFilename returns the expected file name for the node key. +func NodeKeyFilename() string { + return "node" + keyExtension } // UICertPath returns the expected file path for the UI certificate. @@ -227,6 +249,26 @@ func (cm *CertificateManager) UIKeyPath() string { return filepath.Join(cm.certsDir, "ui"+keyExtension) } +// TenantClientCertPath returns the expected file path for the user's certificate. +func (cm *CertificateManager) TenantClientCertPath(tenantIdentifier string) string { + return filepath.Join(cm.certsDir, TenantClientCertFilename(tenantIdentifier)) +} + +// TenantClientCertFilename returns the expected file name for the user's certificate. +func TenantClientCertFilename(tenantIdentifier string) string { + return "tenant." + tenantIdentifier + certExtension +} + +// TenantClientKeyPath returns the expected file path for the tenant's key. +func (cm *CertificateManager) TenantClientKeyPath(tenantIdentifier string) string { + return filepath.Join(cm.certsDir, TenantClientKeyFilename(tenantIdentifier)) +} + +// TenantClientKeyFilename returns the expected file name for the user's key. +func TenantClientKeyFilename(tenantIdentifier string) string { + return "tenant." + tenantIdentifier + keyExtension +} + // ClientCertPath returns the expected file path for the user's certificate. func (cm *CertificateManager) ClientCertPath(user string) string { return filepath.Join(cm.certsDir, ClientCertFilename(user)) diff --git a/pkg/security/certs.go b/pkg/security/certs.go index 8f38a812df8c..73e3aad523c5 100644 --- a/pkg/security/certs.go +++ b/pkg/security/certs.go @@ -86,6 +86,18 @@ func CreateCAPair( return createCACertAndKey(certsDir, caKeyPath, CAPem, keySize, lifetime, allowKeyReuse, overwrite) } +// CreateTenantCAPair creates a tenant CA pair. The private key is written to +// caKeyPath and the public key is created in certsDir. +func CreateTenantCAPair( + certsDir, caKeyPath string, + keySize int, + lifetime time.Duration, + allowKeyReuse bool, + overwrite bool, +) error { + return createCACertAndKey(certsDir, caKeyPath, TenantCAPem, keySize, lifetime, allowKeyReuse, overwrite) +} + // CreateClientCAPair creates a client CA certificate and associated key. func CreateClientCAPair( certsDir, caKeyPath string, @@ -132,7 +144,7 @@ func createCACertAndKey( if len(certsDir) == 0 { return errors.New("the path to the certs directory is required") } - if caType != CAPem && caType != ClientCAPem && caType != UICAPem { + if caType != CAPem && caType != TenantCAPem && caType != ClientCAPem && caType != UICAPem { return fmt.Errorf("caType argument to createCACertAndKey must be one of CAPem (%d), ClientCAPem (%d), or UICAPem (%d), got: %d", CAPem, ClientCAPem, UICAPem, caType) } @@ -194,10 +206,14 @@ func createCACertAndKey( switch caType { case CAPem: certPath = cm.CACertPath() + case TenantCAPem: + certPath = cm.TenantCACertPath() case ClientCAPem: certPath = cm.ClientCACertPath() case UICAPem: certPath = cm.UICACertPath() + default: + return errors.Newf("unknown CA type %v", caType) } var existingCertificates []*pem.Block @@ -424,6 +440,93 @@ func CreateClientPair( return nil } +// TenantClientPair are client certs for use with multi-tenancy. +type TenantClientPair struct { + PrivateKey *rsa.PrivateKey + Cert []byte +} + +// CreateTenantClientPair creates a key and certificate for use as client certs +// when communicating with the KV layer. The tenant CA cert and key must load +// properly. If multiple certificates exist in the CA cert, the first one is +// used. +// +// To write the returned TenantClientPair to disk, use WriteTenantClientPair. +func CreateTenantClientPair( + certsDir, caKeyPath string, + keySize int, + lifetime time.Duration, + tenantIdentifier string, + hosts []string, +) (*TenantClientPair, error) { + if len(caKeyPath) == 0 { + return nil, errors.New("the path to the CA key is required") + } + if len(certsDir) == 0 { + return nil, errors.New("the path to the certs directory is required") + } + + // The certificate manager expands the env for the certs directory. + // For consistency, we need to do this for the key as well. + caKeyPath = os.ExpandEnv(caKeyPath) + + // Create a certificate manager with "create dir if not exist". + cm, err := NewCertificateManagerFirstRun(certsDir) + if err != nil { + return nil, err + } + + caCertPath := cm.TenantCACertPath() + + // Load the CA pair. + caCert, caPrivateKey, err := loadCACertAndKey(caCertPath, caKeyPath) + if err != nil { + return nil, err + } + + // Generate certificates and keys. + clientKey, err := rsa.GenerateKey(rand.Reader, keySize) + if err != nil { + return nil, errors.Errorf("could not generate new tenant key: %v", err) + } + + clientCert, err := GenerateClientCert( + caCert, caPrivateKey, clientKey.Public(), lifetime, tenantIdentifier, hosts..., + ) + if err != nil { + return nil, errors.Errorf("error creating tenant certificate and key: %s", err) + } + return &TenantClientPair{ + PrivateKey: clientKey, + Cert: clientCert, + }, nil +} + +// WriteTenantClientPair writes a TenantClientPair into certsDir. +func WriteTenantClientPair(certsDir string, cp *TenantClientPair, overwrite bool) error { + cm, err := NewCertificateManagerFirstRun(certsDir) + if err != nil { + return err + } + cert, err := x509.ParseCertificate(cp.Cert) + if err != nil { + return err + } + tenantIdentifier := cert.Subject.CommonName + certPath := cm.TenantClientCertPath(tenantIdentifier) + if err := writeCertificateToFile(certPath, cp.Cert, overwrite); err != nil { + return errors.Errorf("error writing tenant certificate to %s: %v", certPath, err) + } + log.Infof(context.Background(), "Wrote SQL tenant client certificate: %s", certPath) + + keyPath := cm.TenantClientKeyPath(tenantIdentifier) + if err := writeKeyToFile(keyPath, cp.PrivateKey, overwrite); err != nil { + return errors.Errorf("error writing tenant key to %s: %v", keyPath, err) + } + log.Infof(context.Background(), "Generated tenant key: %s", keyPath) + return nil +} + // PEMContentsToX509 takes raw pem-encoded contents and attempts to parse into // x509.Certificate objects. func PEMContentsToX509(contents []byte) ([]*x509.Certificate, error) { diff --git a/pkg/security/certs_tenant_test.go b/pkg/security/certs_tenant_test.go new file mode 100644 index 000000000000..dbf4480f2b2e --- /dev/null +++ b/pkg/security/certs_tenant_test.go @@ -0,0 +1,129 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package security_test + +import ( + "fmt" + "io/ioutil" + "net" + "net/http" + "path/filepath" + "testing" + "time" + + "github.com/cockroachdb/cockroach/pkg/security" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/stretchr/testify/require" +) + +// TestTenantCertificates creates a tenant CA and from it client certificates +// for a tenant. It then sets up a smoke test that verifies that the tenant +// can use its client certificates to connect to a https server that trusts +// the tenant CA. +// +// This foreshadows upcoming work on multi-tenancy, see: +// https://github.com/cockroachdb/cockroach/issues/49105 +// https://github.com/cockroachdb/cockroach/issues/47898 +func TestTenantCertificates(t *testing.T) { + defer leaktest.AfterTest(t)() + + // Don't mock assets in this test, we're creating our own one-off certs. + security.ResetAssetLoader() + defer ResetTest() + + certsDir, cleanup := tempDir(t) + defer cleanup() + + // Make certs for the tenant CA (= auth broker). In production, these would be + // given to a dedicated service. + tenantCAKey := filepath.Join(certsDir, "tenant-ca-name-irrelevant.key") + require.NoError(t, security.CreateTenantCAPair( + certsDir, + tenantCAKey, + 2048, + 100000*time.Hour, // basically long-lived + false, // allowKeyReuse + false, // overwrite + )) + + // That dedicated service can make client certs for a tenant as follows: + const tenant = "gromphadorhina-portentosa" + tenantCerts, err := security.CreateTenantClientPair( + certsDir, tenantCAKey, testKeySize, 48*time.Hour, tenant, []string{"127.0.0.1"}, + ) + require.NoError(t, err) + // We write the certs to disk, though in production this would not necessarily + // happen (it may be enough to just have them in-mem, we will see). + require.NoError(t, security.WriteTenantClientPair(certsDir, tenantCerts, false /* overwrite */)) + + // To make this example work we also need certs that the server can use. We + // need something here that the tenant will trust. We just make node certs + // out of convenience. In production, these would be auxiliary certs. + dummyCAKeyPath := filepath.Join(certsDir, "name-does-not-matter-too.key") + require.NoError(t, security.CreateCAPair( + certsDir, dummyCAKeyPath, testKeySize, 1000*time.Hour, false, false, + )) + require.NoError(t, security.CreateNodePair( + certsDir, dummyCAKeyPath, testKeySize, 500*time.Hour, false, []string{"127.0.0.1"})) + + dummyCACertPath := filepath.Join(certsDir, security.CACertFilename()) + + // Now set up the config a server would use. The client will trust it based on + // the dummy CA and dummy node certs, and it will validate incoming + // connections based on the tenant CA. + serverTLSConfig, err := security.LoadServerTLSConfig( + dummyCACertPath, + filepath.Join(certsDir, security.TenantCACertFilename()), + filepath.Join(certsDir, security.NodeCertFilename()), + filepath.Join(certsDir, security.NodeKeyFilename()), + ) + require.NoError(t, err) + + // The client in turn trusts the dummy CA and presents its tenant certs to the + // server (which will validate them using the tenant CA). + clientTLSConfig, err := security.LoadClientTLSConfig( + dummyCACertPath, + filepath.Join(certsDir, security.TenantClientCertFilename(tenant)), + filepath.Join(certsDir, security.TenantClientKeyFilename(tenant)), + ) + require.NoError(t, err) + + // Set up a HTTPS server using server TLS config, set up a http client using the + // client TLS config, make a request. + + ln, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + defer ln.Close() + + httpServer := http.Server{ + Addr: ln.Addr().String(), + TLSConfig: serverTLSConfig, + Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + fmt.Fprint(w, "hello, tenant ", req.TLS.PeerCertificates[0].Subject.CommonName) + }), + } + defer httpServer.Close() + go func() { + _ = httpServer.ServeTLS(ln, "", "") + }() + + httpClient := http.Client{Transport: &http.Transport{ + TLSClientConfig: clientTLSConfig, + }} + defer httpClient.CloseIdleConnections() + + resp, err := httpClient.Get("https://" + ln.Addr().String()) + require.NoError(t, err) + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "hello, tenant "+tenant, string(b)) +} diff --git a/pkg/security/certs_test.go b/pkg/security/certs_test.go index 5c50c609d04a..c5219d8eae7c 100644 --- a/pkg/security/certs_test.go +++ b/pkg/security/certs_test.go @@ -27,25 +27,32 @@ import ( "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" ) const testKeySize = 1024 -func TestGenerateCACert(t *testing.T) { - defer leaktest.AfterTest(t)() - // Do not mock cert access for this test. - security.ResetAssetLoader() - defer ResetTest() - +// tempDir is like testutils.TempDir but avoids a circular import. +func tempDir(t *testing.T) (string, func()) { certsDir, err := ioutil.TempDir("", "certs_test") if err != nil { t.Fatal(err) } - defer func() { + return certsDir, func() { if err := os.RemoveAll(certsDir); err != nil { t.Fatal(err) } - }() + } +} + +func TestGenerateCACert(t *testing.T) { + defer leaktest.AfterTest(t)() + // Do not mock cert access for this test. + security.ResetAssetLoader() + defer ResetTest() + + certsDir, cleanup := tempDir(t) + defer cleanup() cm, err := security.NewCertificateManager(certsDir) if err != nil { @@ -108,6 +115,54 @@ func TestGenerateCACert(t *testing.T) { } } +func TestGenerateTenantCerts(t *testing.T) { + defer leaktest.AfterTest(t)() + // Do not mock cert access for this test. + security.ResetAssetLoader() + defer ResetTest() + + certsDir, cleanup := tempDir(t) + defer cleanup() + + caKeyFile := filepath.Join(certsDir, "name-must-not-matter.key") + require.NoError(t, security.CreateTenantCAPair( + certsDir, + caKeyFile, + testKeySize, + 48*time.Hour, + false, // allowKeyReuse + false, // overwrite + )) + + cp, err := security.CreateTenantClientPair( + certsDir, + caKeyFile, + testKeySize, + time.Hour, + "tenant999", + []string{"1.2.3.4", "apples.and.banan.as"}, + ) + require.NoError(t, err) + require.NoError(t, security.WriteTenantClientPair(certsDir, cp, false)) + + cl := security.NewCertificateLoader(certsDir) + require.NoError(t, cl.Load()) + infos := cl.Certificates() + for _, info := range infos { + require.NoError(t, info.Error) + } + require.Len(t, infos, 2) + require.Equal(t, security.TenantCAPem, infos[0].FileUsage) + require.Equal(t, security.TenantClientPem, infos[1].FileUsage) + require.Equal(t, "tenant999", infos[1].Name) + require.Equal(t, []string{"apples.and.banan.as"}, infos[1].ParsedCertificates[0].DNSNames) + require.Len(t, infos[1].ParsedCertificates[0].IPAddresses, 1) + require.Equal(t, + "1.2.3.4", + infos[1].ParsedCertificates[0].IPAddresses[0].String(), + ) +} + func TestGenerateNodeCerts(t *testing.T) { defer leaktest.AfterTest(t)() // Do not mock cert access for this test. diff --git a/pkg/security/x509.go b/pkg/security/x509.go index c305842d5e24..4d695c8656e9 100644 --- a/pkg/security/x509.go +++ b/pkg/security/x509.go @@ -130,13 +130,7 @@ func GenerateServerCert( // Both server and client authentication are allowed (for inter-node RPC). template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth} - for _, h := range hosts { - if ip := net.ParseIP(h); ip != nil { - template.IPAddresses = append(template.IPAddresses, ip) - } else { - template.DNSNames = append(template.DNSNames, h) - } - } + addHostsToTemplate(template, hosts) certBytes, err := x509.CreateCertificate(rand.Reader, template, caCert, nodePublicKey, caPrivateKey) if err != nil { @@ -146,6 +140,16 @@ func GenerateServerCert( return certBytes, nil } +func addHostsToTemplate(template *x509.Certificate, hosts []string) { + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, h) + } + } +} + // GenerateUIServerCert generates a server certificate for the Admin UI and returns the cert bytes. // Takes in the CA cert and private key, the UI cert public key, the certificate lifetime, // and the list of hosts/ip addresses this certificate applies to. @@ -169,13 +173,7 @@ func GenerateUIServerCert( // Only server authentication is allowed. template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} - for _, h := range hosts { - if ip := net.ParseIP(h); ip != nil { - template.IPAddresses = append(template.IPAddresses, ip) - } else { - template.DNSNames = append(template.DNSNames, h) - } - } + addHostsToTemplate(template, hosts) certBytes, err := x509.CreateCertificate(rand.Reader, template, caCert, certPublicKey, caPrivateKey) if err != nil { @@ -188,12 +186,16 @@ func GenerateUIServerCert( // GenerateClientCert generates a client certificate and returns the cert bytes. // Takes in the CA cert and private key, the client public key, the certificate lifetime, // and the username. +// +// This is used both for vanilla CockroachDB user client certs as well as for the +// multi-tenancy KV auth broker (in which case the user is a SQL tenant). func GenerateClientCert( caCert *x509.Certificate, caPrivateKey crypto.PrivateKey, clientPublicKey crypto.PublicKey, lifetime time.Duration, user string, + hosts ...string, ) ([]byte, error) { // TODO(marc): should we add extra checks? @@ -201,7 +203,7 @@ func GenerateClientCert( return nil, errors.Errorf("user cannot be empty") } - // Create template for "user". + // Create template for user. template, err := newTemplate(user, lifetime) if err != nil { return nil, err @@ -215,6 +217,7 @@ func GenerateClientCert( // Set client-specific fields. // Client authentication only. template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + addHostsToTemplate(template, hosts) certBytes, err := x509.CreateCertificate(rand.Reader, template, caCert, clientPublicKey, caPrivateKey) if err != nil {