[v8] Client Certificate Authentication for GCP Cloud SQL (#10059)

* Vendor go-mysql update

* Client Certificate Authentication for GCP Cloud SQL (#9991)

Allow users to secure GCP Cloud SQL instances by setting "Allow only SSL
connections", which enforces client certificate authentication.

This implementation does not require any configuration changes for Teleport
users. Teleport will detect whether client certificate authentication is
required and handle either case automatically.

Client certificates are ephemeral. They are created for every connection by
calling the GCP Cloud SQL API's GenerateEphemeralCert function. Certificates
are only created when the destination Cloud SQL instance is configured to
require client certificate authentication. The configuration is detected by
requesting instance settings from the GCP Cloud SQL API on every connection
attempt.

A special case was implemented for MySQL. MySQL servers in GCP Cloud SQL do not
trust the ephemeral certificate's CA but GCP Cloud Proxy does. To work around
this issue, the implementation will connect to the MySQL Cloud Proxy port using
a TLS dialer instead of the default MySQL port when client certificate
authentication is required.

The common.CloudClients interface and implementation now return an interface
(GCPSQLAdminClient) from the GetGCPSQLAdminClient function instead of the GCP
client's sqladmin.Service. Returning an interface simplified calling code and
allowed for the client to be mocked for testing.

Existing GCP Cloud SQL tests are configured to not require client certificate
authentication by default. A new test named TestGCPRequireSSL was created to
simulate client certificate authentication for both Postgres and MySQL. This
required some minor changes to the test server code.

A new ConnectWithDialer function was added to the
github.com/gravitational/go-mysql fork. This function is available upstream in
v1.4.0 but other changes upstream resulted in a number of errors and a panic
processing network packets. So instead of upgrading, the dialer function was
copied to the Teleport fork and a custom version was created instead:
v1.1.1-teleport.1.

go.mod

@@ -213,6 +213,6 @@ replace (
 	github.com/coreos/go-oidc => github.com/gravitational/go-oidc v0.0.5
 	github.com/gogo/protobuf => github.com/gravitational/protobuf v1.3.2-0.20201123192827-2b9fcfaffcbf
 	github.com/gravitational/teleport/api => ./api
-	github.com/siddontang/go-mysql v1.1.0 => github.com/gravitational/go-mysql v1.1.1-0.20210212011549-886316308a77
+	github.com/siddontang/go-mysql v1.1.0 => github.com/gravitational/go-mysql v1.1.1-teleport.1
 	github.com/sirupsen/logrus => github.com/gravitational/logrus v1.4.4-0.20210817004754-047e20245621
 )

go.sum

@@ -394,8 +394,8 @@ github.com/gravitational/configure v0.0.0-20180808141939-c3428bd84c23 h1:havbccu
 github.com/gravitational/configure v0.0.0-20180808141939-c3428bd84c23/go.mod h1:XL9nebvlfNVvRzRPWdDcWootcyA0l7THiH/A+W1233g=
 github.com/gravitational/form v0.0.0-20151109031454-c4048f792f70 h1:To76nCJtM3DI0mdq3nGLzXqTV1wNOJByxv01+u9/BxM=
 github.com/gravitational/form v0.0.0-20151109031454-c4048f792f70/go.mod h1:88hFR45MpUd23d2vNWE/dYtesU50jKsbz0I9kH7UaBY=
-github.com/gravitational/go-mysql v1.1.1-0.20210212011549-886316308a77 h1:ivambM2XeST8qfxeSm+0Y8CP/DlNbS3o/9tSF2KtGFk=
-github.com/gravitational/go-mysql v1.1.1-0.20210212011549-886316308a77/go.mod h1:re0JQZ1Cy5dVlIDGq0YksfDIla/GRZlxqOoC0XPSSGE=
+github.com/gravitational/go-mysql v1.1.1-teleport.1 h1:062V8u0juCyUvpYMdkYch8JDDw7wf5rdhKaIfhnojDg=
+github.com/gravitational/go-mysql v1.1.1-teleport.1/go.mod h1:re0JQZ1Cy5dVlIDGq0YksfDIla/GRZlxqOoC0XPSSGE=
 github.com/gravitational/go-oidc v0.0.5 h1:kxsCknoOZ+KqIAoYLLdHuQcvcc+SrQlnT7xxIM8oo6o=
 github.com/gravitational/go-oidc v0.0.5/go.mod h1:SevmOUNdOB0aD9BAIgjptZ6oHkKxMZZgA70nwPfgU/w=
 github.com/gravitational/kingpin v2.1.11-0.20190130013101-742f2714c145+incompatible h1:CfyZl3nyo9K5lLqOmqvl9/IElY1UCnOWKZiQxJ8HKdA=

lib/srv/db/access_test.go

@@ -55,6 +55,7 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 	"go.mongodb.org/mongo-driver/x/mongo/driver/wiremessage"
+	sqladmin "google.golang.org/api/sqladmin/v1beta4"
 )
 
 func TestMain(m *testing.M) {
@@ -357,6 +358,70 @@ func TestAccessMySQLServerPacket(t *testing.T) {
 	require.NoError(t, err)
 }
 
+// TestGCPRequireSSL tests connecting to GCP Cloud SQL Postgres and MySQL
+// databases with an ephemeral client certificate.
+func TestGCPRequireSSL(t *testing.T) {
+	ctx := context.Background()
+	user := "alice"
+	testCtx := setupTestContext(ctx, t)
+	testCtx.createUserAndRole(ctx, t, user, "admin", []string{types.Wildcard}, []string{types.Wildcard})
+
+	// Generate ephemeral cert returned from mock GCP API.
+	ephemeralCert, err := common.MakeTestClientTLSCert(common.TestClientConfig{
+		AuthClient: testCtx.authClient,
+		AuthServer: testCtx.authServer,
+		Cluster:    testCtx.clusterName,
+		Username:   user,
+	})
+	require.NoError(t, err)
+
+	// Setup database servers for Postgres and MySQL with a mock GCP API that
+	// will require SSL and return the ephemeral certificate created above.
+	testCtx.server = testCtx.setupDatabaseServer(ctx, t, agentParams{
+		Databases: []types.Database{
+			withCloudSQLPostgres("postgres", cloudSQLAuthToken)(t, ctx, testCtx),
+			withCloudSQLMySQLTLS("mysql", user, cloudSQLPassword)(t, ctx, testCtx),
+		},
+		GCPSQL: &cloud.GCPSQLAdminClientMock{
+			EphemeralCert: ephemeralCert,
+			DatabaseInstance: &sqladmin.DatabaseInstance{
+				Settings: &sqladmin.Settings{
+					IpConfiguration: &sqladmin.IpConfiguration{
+						RequireSsl: true,
+					},
+				},
+			},
+		},
+	})
+	go testCtx.startHandlingConnections()
+
+	// Try to connect to postgres.
+	pgConn, err := testCtx.postgresClient(ctx, user, "postgres", "postgres", "postgres")
+	require.NoError(t, err)
+
+	// Execute a query.
+	pgResult, err := pgConn.Exec(ctx, "select 1").ReadAll()
+	require.NoError(t, err)
+	require.Equal(t, []*pgconn.Result{postgres.TestQueryResponse}, pgResult)
+
+	// Disconnect.
+	err = pgConn.Close(ctx)
+	require.NoError(t, err)
+
+	// Try to connect to MySQL.
+	mysqlConn, err := testCtx.mysqlClient(user, "mysql", user)
+	require.NoError(t, err)
+
+	// Execute a query.
+	mysqlResult, err := mysqlConn.Execute("select 1")
+	require.NoError(t, err)
+	require.Equal(t, mysql.TestQueryResponse, mysqlResult)
+
+	// Disconnect.
+	err = mysqlConn.Close()
+	require.NoError(t, err)
+}
+
 // TestAccessMongoDB verifies access scenarios to a MongoDB database based
 // on the configured RBAC rules.
 func TestAccessMongoDB(t *testing.T) {
@@ -985,12 +1050,25 @@ type agentParams struct {
 	OnReconcile func(types.Databases)
 	// NoStart indicates server should not be started.
 	NoStart bool
+	// GCPSQL defines the GCP Cloud SQL mock to use for GCP API calls.
+	GCPSQL *cloud.GCPSQLAdminClientMock
 }
 
 func (p *agentParams) setDefaults(c *testContext) {
 	if p.HostID == "" {
 		p.HostID = c.hostID
 	}
+	if p.GCPSQL == nil {
+		p.GCPSQL = &cloud.GCPSQLAdminClientMock{
+			DatabaseInstance: &sqladmin.DatabaseInstance{
+				Settings: &sqladmin.Settings{
+					IpConfiguration: &sqladmin.IpConfiguration{
+						RequireSsl: false,
+					},
+				},
+			},
+		}
+	}
 }
 
 func (c *testContext) setupDatabaseServer(ctx context.Context, t *testing.T, p agentParams) *Server {
@@ -1056,6 +1134,7 @@ func (c *testContext) setupDatabaseServer(ctx context.Context, t *testing.T, p a
 			RDS:      &cloud.RDSMock{},
 			Redshift: &cloud.RedshiftMock{},
 			IAM:      &cloud.IAMMock{},
+			GCPSQL:   p.GCPSQL,
 		},
 	})
 	require.NoError(t, err)
@@ -1317,6 +1396,46 @@ func withCloudSQLMySQL(name, authUser, authToken string) withDatabaseOption {
 	}
 }
 
+// withCloudSQLMySQLTLS creates a test MySQL server that simulates GCP Cloud SQL
+// and requires client authentication using an ephemeral client certificate.
+func withCloudSQLMySQLTLS(name, authUser, authToken string) withDatabaseOption {
+	return func(t *testing.T, ctx context.Context, testCtx *testContext) types.Database {
+		mysqlServer, err := mysql.NewTestServer(common.TestServerConfig{
+			Name:       name,
+			AuthClient: testCtx.authClient,
+			AuthUser:   authUser,
+			AuthToken:  authToken,
+			// Cloud SQL presented certificate must have <project-id>:<instance-id>
+			// in its CN.
+			CN: "project-1:instance-1",
+			// Enable TLS listener.
+			ListenTLS: true,
+		})
+		require.NoError(t, err)
+		go mysqlServer.Serve()
+		t.Cleanup(func() { mysqlServer.Close() })
+		database, err := types.NewDatabaseV3(types.Metadata{
+			Name: name,
+		}, types.DatabaseSpecV3{
+			Protocol:      defaults.ProtocolMySQL,
+			URI:           net.JoinHostPort("localhost", mysqlServer.Port()),
+			DynamicLabels: dynamicLabels,
+			GCP: types.GCPCloudSQL{
+				ProjectID:  "project-1",
+				InstanceID: "instance-1",
+			},
+			// Set CA cert to pass cert validation.
+			CACert: string(testCtx.hostCA.GetActiveKeys().TLS[0].Cert),
+		})
+		require.NoError(t, err)
+		testCtx.mysql[name] = testMySQL{
+			db:       mysqlServer,
+			resource: database,
+		}
+		return database
+	}
+}
+
 func withAzureMySQL(name, authUser, authToken string) withDatabaseOption {
 	return func(t *testing.T, ctx context.Context, testCtx *testContext) types.Database {
 		mysqlServer, err := mysql.NewTestServer(common.TestServerConfig{

lib/srv/db/cloud/gcp.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2022 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloud
+
+import (
+	"context"
+	"crypto/tls"
+
+	"github.com/gravitational/teleport/lib/srv/db/common"
+	"github.com/gravitational/trace"
+)
+
+// GetGCPRequireSSL requests settings for the project/instance in session from GCP
+// and returns true when the instance requires SSL. An access denied error is
+// returned when an unauthorized error is returned from GCP.
+func GetGCPRequireSSL(ctx context.Context, sessionCtx *common.Session, gcpClient common.GCPSQLAdminClient) (requireSSL bool, err error) {
+	dbi, err := gcpClient.GetDatabaseInstance(ctx, sessionCtx)
+	if err != nil {
+		err = common.ConvertError(err)
+		if trace.IsAccessDenied(err) {
+			return false, trace.Wrap(err, `Could not get GCP database instance settings:
+
+  %v
+
+Make sure Teleport db service has "Cloud SQL Admin" GCP IAM role,
+or "cloudsql.instances.get" IAM permission.`, err)
+		}
+		return false, trace.Wrap(err, "Failed to get Cloud SQL instance information for %q.", common.GCPServerName(sessionCtx))
+	} else if dbi.Settings == nil || dbi.Settings.IpConfiguration == nil {
+		return false, trace.BadParameter("Failed to find Cloud SQL settings for %q. GCP returned %+v.", common.GCPServerName(sessionCtx), dbi)
+	}
+	return dbi.Settings.IpConfiguration.RequireSsl, nil
+}
+
+// AppendGCPClientCert calls the GCP API to generate an ephemeral certificate
+// and adds it to the TLS config. An access denied error is returned when the
+// generate call fails.
+func AppendGCPClientCert(ctx context.Context, sessionCtx *common.Session, gcpClient common.GCPSQLAdminClient, tlsConfig *tls.Config) error {
+	cert, err := gcpClient.GenerateEphemeralCert(ctx, sessionCtx)
+	if err != nil {
+		err = common.ConvertError(err)
+		if trace.IsAccessDenied(err) {
+			return trace.Wrap(err, `Cloud not generate GCP ephemeral client certificate:
+
+  %v
+
+Make sure Teleport db service has "Cloud SQL Admin" GCP IAM role,
+or "cloudsql.sslCerts.createEphemeral" IAM permission.`, err)
+		}
+		return trace.Wrap(err, "Failed to generate GCP ephemeral client certificate for %q.", common.GCPServerName(sessionCtx))
+	}
+	tlsConfig.Certificates = []tls.Certificate{*cert}
+	return nil
+}

lib/srv/db/cloud/mocks.go

@@ -17,6 +17,9 @@ limitations under the License.
 package cloud
 
 import (
+	"context"
+	"crypto/tls"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/service/iam"
@@ -27,7 +30,9 @@ import (
 	"github.com/aws/aws-sdk-go/service/redshift/redshiftiface"
 	"github.com/aws/aws-sdk-go/service/sts"
 	"github.com/aws/aws-sdk-go/service/sts/stsiface"
+	"github.com/gravitational/teleport/lib/srv/db/common"
 	"github.com/gravitational/trace"
+	sqladmin "google.golang.org/api/sqladmin/v1beta4"
 )
 
 // STSMock mocks AWS STS API.
@@ -307,3 +312,23 @@ func (m *IAMMockUnauth) GetUserPolicyWithContext(ctx aws.Context, input *iam.Get
 func (m *IAMMockUnauth) PutUserPolicyWithContext(ctx aws.Context, input *iam.PutUserPolicyInput, options ...request.Option) (*iam.PutUserPolicyOutput, error) {
 	return nil, trace.AccessDenied("unauthorized")
 }
+
+// GCPSQLAdminClientMock implements the common.GCPSQLAdminClient interface for tests.
+type GCPSQLAdminClientMock struct {
+	// DatabaseInstance is returned from GetDatabaseInstance.
+	DatabaseInstance *sqladmin.DatabaseInstance
+	// EphemeralCert is returned from GenerateEphemeralCert.
+	EphemeralCert *tls.Certificate
+}
+
+func (g *GCPSQLAdminClientMock) UpdateUser(ctx context.Context, sessionCtx *common.Session, user *sqladmin.User) error {
+	return nil
+}
+
+func (g *GCPSQLAdminClientMock) GetDatabaseInstance(ctx context.Context, sessionCtx *common.Session) (*sqladmin.DatabaseInstance, error) {
+	return g.DatabaseInstance, nil
+}
+
+func (g *GCPSQLAdminClientMock) GenerateEphemeralCert(ctx context.Context, sessionCtx *common.Session) (*tls.Certificate, error) {
+	return g.EphemeralCert, nil
+}

lib/srv/db/common/auth.go

@@ -251,11 +251,8 @@ func (a *dbAuth) GetCloudSQLPassword(ctx context.Context, sessionCtx *Session) (
 }
 
 // updateCloudSQLUser makes a request to Cloud SQL API to update the provided user.
-func (a *dbAuth) updateCloudSQLUser(ctx context.Context, sessionCtx *Session, gcpCloudSQL *sqladmin.Service, user *sqladmin.User) error {
-	_, err := gcpCloudSQL.Users.Update(
-		sessionCtx.Database.GetGCP().ProjectID,
-		sessionCtx.Database.GetGCP().InstanceID,
-		user).Name(sessionCtx.DatabaseUser).Host("%").Context(ctx).Do()
+func (a *dbAuth) updateCloudSQLUser(ctx context.Context, sessionCtx *Session, gcpCloudSQL GCPSQLAdminClient, user *sqladmin.User) error {
+	err := gcpCloudSQL.UpdateUser(ctx, sessionCtx, user)
 	if err != nil {
 		return trace.AccessDenied(`Could not update Cloud SQL user %q password:
 
@@ -354,7 +351,7 @@ func (a *dbAuth) getTLSConfigVerifyFull(ctx context.Context, sessionCtx *Session
 		// Cloud SQL server presented certificates encode instance names as
 		// "<project-id>:<instance-id>" in CommonName. This is verified against
 		// the ServerName in a custom connection verification step (see below).
-		tlsConfig.ServerName = fmt.Sprintf("%v:%v", sessionCtx.Database.GetGCP().ProjectID, sessionCtx.Database.GetGCP().InstanceID)
+		tlsConfig.ServerName = GCPServerName(sessionCtx)
 		// This just disables default verification.
 		tlsConfig.InsecureSkipVerify = true
 		// This will verify CN and cert chain on each connection.

lib/srv/db/common/cloud.go

@@ -39,7 +39,6 @@ import (
 	"github.com/gravitational/trace"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/api/option"
-	sqladmin "google.golang.org/api/sqladmin/v1beta4"
 	"google.golang.org/grpc"
 )
 
@@ -58,7 +57,7 @@ type CloudClients interface {
 	// GetGCPIAMClient returns GCP IAM client.
 	GetGCPIAMClient(context.Context) (*gcpcredentials.IamCredentialsClient, error)
 	// GetGCPSQLAdminClient returns GCP Cloud SQL Admin client.
-	GetGCPSQLAdminClient(context.Context) (*sqladmin.Service, error)
+	GetGCPSQLAdminClient(context.Context) (GCPSQLAdminClient, error)
 	// GetAzureCredential returns Azure default token credential chain.
 	GetAzureCredential() (azcore.TokenCredential, error)
 	// Closer closes all initialized clients.
@@ -78,7 +77,7 @@ type cloudClients struct {
 	// gcpIAM is the cached GCP IAM client.
 	gcpIAM *gcpcredentials.IamCredentialsClient
 	// gcpSQLAdmin is the cached GCP Cloud SQL Admin client.
-	gcpSQLAdmin *sqladmin.Service
+	gcpSQLAdmin GCPSQLAdminClient
 	// azureCredential is the cached Azure credential.
 	azureCredential azcore.TokenCredential
 	// mtx is used for locking.
@@ -144,7 +143,7 @@ func (c *cloudClients) GetGCPIAMClient(ctx context.Context) (*gcpcredentials.Iam
 }
 
 // GetGCPSQLAdminClient returns GCP Cloud SQL Admin client.
-func (c *cloudClients) GetGCPSQLAdminClient(ctx context.Context) (*sqladmin.Service, error) {
+func (c *cloudClients) GetGCPSQLAdminClient(ctx context.Context) (GCPSQLAdminClient, error) {
 	c.mtx.RLock()
 	if c.gcpSQLAdmin != nil {
 		defer c.mtx.RUnlock()
@@ -211,14 +210,14 @@ func (c *cloudClients) initGCPIAMClient(ctx context.Context) (*gcpcredentials.Ia
 	return gcpIAM, nil
 }
 
-func (c *cloudClients) initGCPSQLAdminClient(ctx context.Context) (*sqladmin.Service, error) {
+func (c *cloudClients) initGCPSQLAdminClient(ctx context.Context) (GCPSQLAdminClient, error) {
 	c.mtx.Lock()
 	defer c.mtx.Unlock()
 	if c.gcpSQLAdmin != nil { // If some other thread already got here first.
 		return c.gcpSQLAdmin, nil
 	}
 	logrus.Debug("Initializing GCP Cloud SQL Admin client.")
-	gcpSQLAdmin, err := sqladmin.NewService(ctx)
+	gcpSQLAdmin, err := NewGCPSQLAdminClient(ctx)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -248,6 +247,7 @@ type TestCloudClients struct {
 	Redshift     redshiftiface.RedshiftAPI
 	IAM          iamiface.IAMAPI
 	STS          stsiface.STSAPI
+	GCPSQL       GCPSQLAdminClient
 }
 
 // GetAWSSession returns AWS session for the specified region.
@@ -286,10 +286,8 @@ func (c *TestCloudClients) GetGCPIAMClient(ctx context.Context) (*gcpcredentials
 }
 
 // GetGCPSQLAdminClient returns GCP Cloud SQL Admin client.
-func (c *TestCloudClients) GetGCPSQLAdminClient(ctx context.Context) (*sqladmin.Service, error) {
-	return sqladmin.NewService(ctx,
-		option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), // Insecure must be set for unauth client.
-		option.WithoutAuthentication())
+func (c *TestCloudClients) GetGCPSQLAdminClient(ctx context.Context) (GCPSQLAdminClient, error) {
+	return c.GCPSQL, nil
 }
 
 // GetAzureCredential returns default Azure token credential chain.