security: add tenant server certificates

This is an addendum to cockroachdb#49864. I had added a tenant CA and tenant client
certs but did not add tenant server CA and certs. However, this is
necessary:a KV server run in a multi-tenancy environment will use

- tenant server certs (issued by tenant server CA) to host TLS-enabled
  endpoint (for use by SQL tenant processes)
- tenant client CA (to validate incoming connections)
- node server certs (for intra-KV traffic).

The expectation is that the tenant server certs/CA will be using
external PKI (i.e. publicly trusted certs), but I learned quickly
while prototyping cockroachdb#47898 that it will well be worth it to generate
these certs internally as well.

Release note: None

pkg/security/certificate_loader.go

@@ -72,16 +72,22 @@ const (
 	_ PemUsage = iota
 	// CAPem describes the main CA certificate.
 	CAPem
-	// TenantCAPem describes the CA certificate used to broker authN/Z for SQL
+	// TenantServerCAPem describes the CA certificate used to host endpoints that
+	// tenants will access.
+	TenantServerCAPem
+	// TenantClientCAPem describes the CA certificate used to broker authN/Z for SQL
 	// tenants wishing to access the KV layer.
-	TenantCAPem
+	TenantClientCAPem
 	// ClientCAPem describes the CA certificate used to verify client certificates.
 	ClientCAPem
 	// UICAPem describes the CA certificate used to verify the Admin UI server certificate.
 	UICAPem
 	// NodePem describes the server certificate for the node, possibly a combined server/client
 	// certificate for user Node if a separate 'client.node.crt' is not present.
 	NodePem
+	// TenantServerPem describes the server certificate for hosting endpoints accessible
+	// to SQL tenants.
+	TenantServerPem
 	// UIPem describes the server certificate for the admin UI.
 	UIPem
 	// ClientPem describes a client certificate.
@@ -99,7 +105,7 @@ const (
 )
 
 func isCA(usage PemUsage) bool {
-	return usage == CAPem || usage == ClientCAPem || usage == TenantCAPem || usage == UICAPem
+	return usage == CAPem || usage == TenantServerCAPem || usage == ClientCAPem || usage == TenantClientCAPem || usage == UICAPem
 }
 
 func (p PemUsage) String() string {
@@ -108,8 +114,10 @@ func (p PemUsage) String() string {
 		return "CA"
 	case ClientCAPem:
 		return "Client CA"
-	case TenantCAPem:
-		return "Tenant CA"
+	case TenantServerCAPem:
+		return "Tenant Server CA"
+	case TenantClientCAPem:
+		return "Tenant Client CA"
 	case UICAPem:
 		return "UI CA"
 	case NodePem:
@@ -192,8 +200,13 @@ func CertInfoFromFilename(filename string) (*CertInfo, error) {
 		if numParts != 2 {
 			return nil, errors.Errorf("client CA certificate filename should match ca-client%s", certExtension)
 		}
-	case `ca-tenant`:
-		fileUsage = TenantCAPem
+	case `ca-server-tenant`:
+		fileUsage = TenantServerCAPem
+		if numParts != 2 {
+			return nil, errors.Errorf("tenant CA certificate filename should match ca%s", certExtension)
+		}
+	case `ca-client-tenant`:
+		fileUsage = TenantClientCAPem
 		if numParts != 2 {
 			return nil, errors.Errorf("tenant CA certificate filename should match ca%s", certExtension)
 		}
@@ -207,6 +220,11 @@ func CertInfoFromFilename(filename string) (*CertInfo, error) {
 		if numParts != 2 {
 			return nil, errors.Errorf("node certificate filename should match node%s", certExtension)
 		}
+	case `server-tenant`:
+		fileUsage = TenantServerPem
+		if numParts != 2 {
+			return nil, errors.Errorf("tenant server certificate filename should match server-tenant%s", certExtension)
+		}
 	case `ui`:
 		fileUsage = UIPem
 		if numParts != 2 {
@@ -219,12 +237,12 @@ func CertInfoFromFilename(filename string) (*CertInfo, error) {
 		if len(name) == 0 {
 			return nil, errors.Errorf("client certificate filename should match client.<user>%s", certExtension)
 		}
-	case `tenant`:
+	case `client-tenant`:
 		fileUsage = TenantClientPem
 		// Strip prefix and suffix and re-join middle parts.
 		name = strings.Join(parts[1:numParts-1], `.`)
 		if len(name) == 0 {
-			return nil, errors.Errorf("tenant certificate filename should match tenant.<tenantid>%s", certExtension)
+			return nil, errors.Errorf("tenant certificate filename should match client-tenant.<tenantid>%s", certExtension)
 		}
 	default:
 		return nil, errors.Errorf("unknown prefix %q", prefix)

pkg/security/certificate_manager.go

@@ -106,6 +106,9 @@ type CertificateManager struct {
 	uiCert         *CertInfo // optional: server certificate for the admin UI.
 	clientCerts    map[string]*CertInfo
 
+	// Certs only used with multi-tenancy.
+	tenantServerCACert, tenantServerCert, tenantClientCACert, tenantClientCert *CertInfo
+
 	// TLS configs. Initialized lazily. Wiped on every successful Load().
 	// Server-side config.
 	serverConfig *tls.Config
@@ -195,16 +198,28 @@ func (cm *CertificateManager) CACertPath() string {
 // CACertFilename returns the expected file name for the CA certificate.
 func CACertFilename() string { return "ca" + certExtension }
 
-// TenantCACertPath returns the expected file path for the Tenant CA
+// TenantServerCACertPath returns the expected file path for the Tenant server
+// CA certificate.
+func (cm *CertificateManager) TenantServerCACertPath() string {
+	return filepath.Join(cm.certsDir, TenantServerCACertFilename())
+}
+
+// TenantServerCACertFilename returns the expected file name for the Tenant server CA
+// certificate.
+func TenantServerCACertFilename() string {
+	return "ca-server-tenant" + certExtension
+}
+
+// TenantClientCACertPath returns the expected file path for the Tenant client CA
 // certificate.
-func (cm *CertificateManager) TenantCACertPath() string {
-	return filepath.Join(cm.certsDir, TenantCACertFilename())
+func (cm *CertificateManager) TenantClientCACertPath() string {
+	return filepath.Join(cm.certsDir, TenantClientCACertFilename())
 }
 
-// TenantCACertFilename returns the expected file name for the Tenant CA
+// TenantClientCACertFilename returns the expected file name for the Tenant CA
 // certificate.
-func TenantCACertFilename() string {
-	return "ca-tenant" + certExtension
+func TenantClientCACertFilename() string {
+	return "ca-client-tenant" + certExtension
 }
 
 // ClientCACertPath returns the expected file path for the CA certificate
@@ -239,6 +254,29 @@ func NodeKeyFilename() string {
 	return "node" + keyExtension
 }
 
+// TenantServerCertPath returns the expected file path for the tenant server
+// certificate.
+func (cm *CertificateManager) TenantServerCertPath() string {
+	return filepath.Join(cm.certsDir, TenantServerCertFilename())
+}
+
+// TenantServerCertFilename returns the expected file name for the tenant server
+// certificate.
+func TenantServerCertFilename() string {
+	return "server-tenant" + certExtension
+}
+
+// TenantServerKeyPath returns the expected file path for the tenant server key.
+func (cm *CertificateManager) TenantServerKeyPath() string {
+	return filepath.Join(cm.certsDir, TenantServerKeyFilename())
+}
+
+// TenantServerKeyFilename returns the expected file name for the tenant server
+// key.
+func TenantServerKeyFilename() string {
+	return "server-tenant" + keyExtension
+}
+
 // UICertPath returns the expected file path for the UI certificate.
 func (cm *CertificateManager) UICertPath() string {
 	return filepath.Join(cm.certsDir, "ui"+certExtension)
@@ -256,7 +294,7 @@ func (cm *CertificateManager) TenantClientCertPath(tenantIdentifier string) stri
 
 // TenantClientCertFilename returns the expected file name for the user's certificate.
 func TenantClientCertFilename(tenantIdentifier string) string {
-	return "tenant." + tenantIdentifier + certExtension
+	return "client-tenant." + tenantIdentifier + certExtension
 }
 
 // TenantClientKeyPath returns the expected file path for the tenant's key.
@@ -266,7 +304,7 @@ func (cm *CertificateManager) TenantClientKeyPath(tenantIdentifier string) strin
 
 // TenantClientKeyFilename returns the expected file name for the user's key.
 func TenantClientKeyFilename(tenantIdentifier string) string {
-	return "tenant." + tenantIdentifier + keyExtension
+	return "client-tenant." + tenantIdentifier + keyExtension
 }
 
 // ClientCertPath returns the expected file path for the user's certificate.
@@ -373,6 +411,7 @@ func (cm *CertificateManager) LoadCertificates() error {
 	}
 
 	var caCert, clientCACert, uiCACert, nodeCert, uiCert, nodeClientCert *CertInfo
+	var tenantServerCACert, tenantServerCert, tenantClientCACert, tenantClientCert *CertInfo
 	clientCerts := make(map[string]*CertInfo)
 	for _, ci := range cl.Certificates() {
 		switch ci.FileUsage {
@@ -384,13 +423,23 @@ func (cm *CertificateManager) LoadCertificates() error {
 			uiCACert = ci
 		case NodePem:
 			nodeCert = ci
+		case TenantServerCAPem:
+			tenantServerCACert = ci
+		case TenantServerPem:
+			tenantServerCert = ci
+		case TenantClientPem:
+			tenantClientCert = ci
+		case TenantClientCAPem:
+			tenantClientCACert = ci
 		case UIPem:
 			uiCert = ci
 		case ClientPem:
 			clientCerts[ci.Name] = ci
 			if ci.Name == NodeUser {
 				nodeClientCert = ci
 			}
+		default:
+			return errors.Errorf("unsupported certificate %v", ci.Filename)
 		}
 	}
 
@@ -416,6 +465,19 @@ func (cm *CertificateManager) LoadCertificates() error {
 		if err := checkCertIsValid(uiCert); checkCertIsValid(cm.uiCert) == nil && err != nil {
 			return makeError(err, "reload would lose valid UI certificate")
 		}
+
+		if err := checkCertIsValid(tenantServerCACert); checkCertIsValid(cm.tenantServerCACert) == nil && err != nil {
+			return makeError(err, "reload would lose valid tenant server CA certificate")
+		}
+		if err := checkCertIsValid(tenantServerCert); checkCertIsValid(cm.tenantServerCert) == nil && err != nil {
+			return makeError(err, "reload would lose valid tenant server certificate")
+		}
+		if err := checkCertIsValid(tenantClientCACert); checkCertIsValid(cm.tenantClientCACert) == nil && err != nil {
+			return makeError(err, "reload would lose valid tenant client CA certificate")
+		}
+		if err := checkCertIsValid(tenantClientCert); checkCertIsValid(cm.tenantClientCert) == nil && err != nil {
+			return makeError(err, "reload would lose valid tenant client certificate")
+		}
 	}
 
 	if nodeClientCert == nil && nodeCert != nil {
@@ -442,6 +504,11 @@ func (cm *CertificateManager) LoadCertificates() error {
 	cm.uiServerConfig = nil
 	cm.clientConfig = nil
 
+	cm.tenantServerCACert = tenantServerCACert
+	cm.tenantServerCert = tenantServerCert
+	cm.tenantClientCACert = tenantClientCACert
+	cm.tenantClientCert = tenantClientCert
+
 	cm.updateMetricsLocked()
 	return nil
 }