From d59e54320c3c4afcf4890f68e5986890fdca4a22 Mon Sep 17 00:00:00 2001 From: Tiago Silva Date: Wed, 17 Jul 2024 12:50:51 +0100 Subject: [PATCH] [sec_scan][20] add `ReportSecrets` forwarder to proxy's gRPC insecure server This PR implements a `ReportSecrets` forwarder from Proxy server to Auth server. The goal is to allow clients to hit the proxy insecure gRPC server (credentialless) and proxy will forward requests to the AuthServer on behalf of the client. This is required because the client doesn't have valid credentials and it wasn't possible for it to reach auth server via reversetunnel when the cluster uses `separate` mode. This PR is part of https://github.com/gravitational/access-graph/issues/637. Signed-off-by: Tiago Silva --- lib/secretsscanner/client/client.go | 178 ++++++++++++++++++++ lib/secretsscanner/proxy/proxy.go | 133 +++++++++++++++ lib/secretsscanner/proxy/proxy_test.go | 221 +++++++++++++++++++++++++ lib/service/service.go | 26 ++- lib/service/service_test.go | 3 +- 5 files changed, 555 insertions(+), 6 deletions(-) create mode 100644 lib/secretsscanner/client/client.go create mode 100644 lib/secretsscanner/proxy/proxy.go create mode 100644 lib/secretsscanner/proxy/proxy_test.go diff --git a/lib/secretsscanner/client/client.go b/lib/secretsscanner/client/client.go new file mode 100644 index 0000000000000..aee8340ef9100 --- /dev/null +++ b/lib/secretsscanner/client/client.go @@ -0,0 +1,178 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package client + +import ( + "context" + "crypto/tls" + "log/slog" + "slices" + + "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" + "golang.org/x/net/http2" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + "github.com/gravitational/teleport/api/client" + "github.com/gravitational/teleport/api/constants" + apidefaults "github.com/gravitational/teleport/api/defaults" + accessgraphsecretsv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/accessgraph/v1" + "github.com/gravitational/teleport/api/metadata" + "github.com/gravitational/teleport/lib/srv/alpnproxy/common" + "github.com/gravitational/teleport/lib/utils" +) + +// Client is a client for the SecretsScannerService. +type Client interface { + // ReportSecrets is used by trusted devices to report secrets found on the host that could be used to bypass Teleport. + // The client (device) should first authenticate using the [ReportSecretsRequest.device_assertion] flow. Please refer to + // the [teleport.devicetrust.v1.AssertDeviceRequest] and [teleport.devicetrust.v1.AssertDeviceResponse] messages for more details. + // + // Once the device is asserted, the client can send the secrets using the [ReportSecretsRequest.private_keys] field + // and then close the client side of the stream. + // + // -> ReportSecrets (client) [1 or more] + // -> CloseStream (client) + // <- TerminateStream (server) + // + // Any failure in the assertion ceremony will result in the stream being terminated by the server. All secrets + // reported by the client before the assertion terminates will be ignored and result in the stream being terminated. + ReportSecrets(ctx context.Context, opts ...grpc.CallOption) (accessgraphsecretsv1pb.SecretsScannerService_ReportSecretsClient, error) + // Close closes the client connection. + Close() error +} + +// ClientConfig specifies parameters for the client to dial credentialless via the proxy. +type ClientConfig struct { + // ProxyServer is the address of the proxy server + ProxyServer string + // CipherSuites is a list of cipher suites to use for TLS client connection + CipherSuites []uint16 + // Clock specifies the time provider. Will be used to override the time anchor + // for TLS certificate verification. + // Defaults to real clock if unspecified + Clock clockwork.Clock + // Insecure trusts the certificates from the Auth Server or Proxy during registration without verification. + Insecure bool + // Log is the logger. + Log *slog.Logger +} + +// NewSecretsScannerServiceClient creates a new SecretsScannerServiceClient that connects to the proxy +// gRPC server that does not require authentication (credentialless) to report secrets found during scanning. +func NewSecretsScannerServiceClient(ctx context.Context, cfg ClientConfig) (Client, error) { + if cfg.ProxyServer == "" { + return nil, trace.BadParameter("missing ProxyServer") + } + if cfg.Clock == nil { + cfg.Clock = clockwork.NewRealClock() + } + if cfg.Log == nil { + cfg.Log = slog.Default() + } + + grpcConn, err := proxyConn(ctx, cfg) + if err != nil { + return nil, trace.Wrap(err, "failed to connect to the proxy") + } + + return &secretsSvcClient{ + SecretsScannerServiceClient: accessgraphsecretsv1pb.NewSecretsScannerServiceClient(grpcConn), + conn: grpcConn, + }, nil +} + +type secretsSvcClient struct { + accessgraphsecretsv1pb.SecretsScannerServiceClient + conn *grpc.ClientConn +} + +func (c *secretsSvcClient) Close() error { + return c.conn.Close() +} + +// proxyConn attempts to connect to the proxy insecure grpc server. +// The Proxy's TLS cert will be verified using the host's root CA pool +// (PKI) unless the --insecure flag was passed. +func proxyConn( + ctx context.Context, params ClientConfig, +) (*grpc.ClientConn, error) { + tlsConfig := utils.TLSConfig(params.CipherSuites) + tlsConfig.Time = params.Clock.Now + // set NextProtos for TLS routing, the actual protocol will be h2 + tlsConfig.NextProtos = []string{string(common.ProtocolProxyGRPCInsecure), http2.NextProtoTLS} + + if params.Insecure { + tlsConfig.InsecureSkipVerify = true + params.Log.WarnContext(ctx, "Connecting to the cluster without validating the identity of the Proxy Server.") + } + + // Check if proxy is behind a load balancer. If so, the connection upgrade + // will verify the load balancer's cert using system cert pool. This + // provides the same level of security as the client only verifies Proxy's + // web cert against system cert pool when connection upgrade is not + // required. + // + // With the ALPN connection upgrade, the tunneled TLS Routing request will + // skip verify as the Proxy server will present its host cert which is not + // fully verifiable at this point since the client does not have the host + // CAs yet before completing registration. + alpnConnUpgrade := client.IsALPNConnUpgradeRequired(ctx, params.ProxyServer, params.Insecure) + if alpnConnUpgrade && !params.Insecure { + tlsConfig.InsecureSkipVerify = true + tlsConfig.VerifyConnection = verifyALPNUpgradedConn(params.Clock) + } + + dialer := client.NewDialer( + ctx, + apidefaults.DefaultIdleTimeout, + apidefaults.DefaultIOTimeout, + client.WithInsecureSkipVerify(params.Insecure), + client.WithALPNConnUpgrade(alpnConnUpgrade), + ) + + conn, err := grpc.NewClient( + params.ProxyServer, + grpc.WithContextDialer(client.GRPCContextDialer(dialer)), + grpc.WithUnaryInterceptor(metadata.UnaryClientInterceptor), + grpc.WithStreamInterceptor(metadata.StreamClientInterceptor), + grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), + ) + return conn, trace.Wrap(err) +} + +// verifyALPNUpgradedConn is a tls.Config.VerifyConnection callback function +// used by the tunneled TLS Routing request to verify the host cert of a Proxy +// behind a L7 load balancer. +// +// Since the client has not obtained the cluster CAs at this point, the +// presented cert cannot be fully verified yet. For now, this function only +// checks if "teleport.cluster.local" is present as one of the DNS names and +// verifies the cert is not expired. +func verifyALPNUpgradedConn(clock clockwork.Clock) func(tls.ConnectionState) error { + return func(server tls.ConnectionState) error { + for _, cert := range server.PeerCertificates { + if slices.Contains(cert.DNSNames, constants.APIDomain) && clock.Now().Before(cert.NotAfter) { + return nil + } + } + return trace.AccessDenied("server is not a Teleport proxy or server certificate is expired") + } +} diff --git a/lib/secretsscanner/proxy/proxy.go b/lib/secretsscanner/proxy/proxy.go new file mode 100644 index 0000000000000..1820725c65b88 --- /dev/null +++ b/lib/secretsscanner/proxy/proxy.go @@ -0,0 +1,133 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Package proxy implements a proxy service that proxies requests from the proxy unauthenticated +// gRPC service to the Auth's secret service. +package proxy + +import ( + "context" + "errors" + "io" + "log/slog" + + "github.com/gravitational/trace" + + accessgraphsecretsv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/accessgraph/v1" +) + +// AuthClient is a subset of the full Auth API that must be connected +type AuthClient interface { + AccessGraphSecretsScannerClient() accessgraphsecretsv1pb.SecretsScannerServiceClient +} + +// ServiceConfig is the configuration for the Service. +type ServiceConfig struct { + // AuthClient is the client to the Auth service. + AuthClient AuthClient + // Log is the logger. + Log *slog.Logger +} + +// New creates a new Service. +func New(cfg ServiceConfig) (*Service, error) { + if cfg.AuthClient == nil { + return nil, trace.BadParameter("missing AuthClient") + } + if cfg.Log == nil { + cfg.Log = slog.Default() + } + return &Service{ + authClient: cfg.AuthClient, + log: cfg.Log, + }, nil +} + +// Service is a service that proxies requests from the proxy to the Auth's secret service. +// It only implements the ReportSecrets method of the SecretsScannerService because it is the only method that needs to be proxied +// from the proxy to the Auth's secret service. +type Service struct { + accessgraphsecretsv1pb.UnimplementedSecretsScannerServiceServer + // authClient is the client to the Auth service. + authClient AuthClient + + log *slog.Logger +} + +func (s *Service) ReportSecrets(client accessgraphsecretsv1pb.SecretsScannerService_ReportSecretsServer) error { + ctx, cancel := context.WithCancel(client.Context()) + defer cancel() + upstream, err := s.authClient.AccessGraphSecretsScannerClient().ReportSecrets(ctx) + if err != nil { + return trace.Wrap(err) + } + + errCh := make(chan error, 1) + go func() { + err := trace.Wrap(s.forwardClientToServer(ctx, client, upstream)) + if err != nil { + cancel() + } + errCh <- err + }() + + err = s.forwardServerToClient(ctx, client, upstream) + return trace.NewAggregate(err, <-errCh) +} + +func (s *Service) forwardClientToServer(ctx context.Context, + client accessgraphsecretsv1pb.SecretsScannerService_ReportSecretsServer, + server accessgraphsecretsv1pb.SecretsScannerService_ReportSecretsClient) (err error) { + for { + req, err := client.Recv() + if errors.Is(err, io.EOF) { + if err := server.CloseSend(); err != nil { + s.log.WarnContext(ctx, "Failed to close upstream stream", "error", err) + } + break + } + if err != nil { + s.log.WarnContext(ctx, "Failed to receive from client stream", "error", err) + return trace.Wrap(err) + } + if err := server.Send(req); err != nil { + s.log.WarnContext(ctx, "Failed to send to upstream stream", "error", err) + return trace.Wrap(err) + } + } + return nil +} + +func (s *Service) forwardServerToClient(ctx context.Context, + client accessgraphsecretsv1pb.SecretsScannerService_ReportSecretsServer, + server accessgraphsecretsv1pb.SecretsScannerService_ReportSecretsClient) (err error) { + for { + out, err := server.Recv() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + s.log.WarnContext(ctx, "Failed to receive from upstream stream", "error", err) + return trace.Wrap(err) + } + if err := client.Send(out); err != nil { + s.log.WarnContext(ctx, "Failed to send to client stream", "error", err) + return trace.Wrap(err) + } + } +} diff --git a/lib/secretsscanner/proxy/proxy_test.go b/lib/secretsscanner/proxy/proxy_test.go new file mode 100644 index 0000000000000..d5271d268aab0 --- /dev/null +++ b/lib/secretsscanner/proxy/proxy_test.go @@ -0,0 +1,221 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package proxy + +import ( + "context" + "crypto/tls" + "errors" + "io" + "net" + "testing" + + "github.com/gravitational/trace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + "github.com/gravitational/teleport/api/defaults" + accessgraphsecretsv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/accessgraph/v1" + devicepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/devicetrust/v1" + "github.com/gravitational/teleport/lib/fixtures" + secretscannerclient "github.com/gravitational/teleport/lib/secretsscanner/client" +) + +func TestProxy(t *testing.T) { + // Disable the TLS routing connection upgrade + t.Setenv(defaults.TLSRoutingConnUpgradeEnvVar, "false") + + authClient := newFakefakeSecretsScannerSvc(t) + + lis, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + + newProxyService(t, lis, authClient) + ctx := context.Background() + + client, err := secretscannerclient.NewSecretsScannerServiceClient(ctx, secretscannerclient.ClientConfig{ + ProxyServer: lis.Addr().String(), + Insecure: true, + }) + require.NoError(t, err) + + stream, err := client.ReportSecrets(ctx) + require.NoError(t, err) + + // Send the device assertion init message + err = stream.Send(&accessgraphsecretsv1pb.ReportSecretsRequest{ + Payload: &accessgraphsecretsv1pb.ReportSecretsRequest_DeviceAssertion{ + DeviceAssertion: &devicepb.AssertDeviceRequest{ + Payload: &devicepb.AssertDeviceRequest_Init{ + Init: &devicepb.AssertDeviceInit{}, + }, + }, + }, + }) + require.NoError(t, err) + + // Receive the device assertion challenge message + msg, err := stream.Recv() + require.NoError(t, err) + assert.NotNil(t, msg.GetDeviceAssertion().GetChallenge()) + + // Send the device assertion challenge response message + err = stream.Send(&accessgraphsecretsv1pb.ReportSecretsRequest{ + Payload: &accessgraphsecretsv1pb.ReportSecretsRequest_DeviceAssertion{ + DeviceAssertion: &devicepb.AssertDeviceRequest{ + Payload: &devicepb.AssertDeviceRequest_ChallengeResponse{ + ChallengeResponse: &devicepb.AuthenticateDeviceChallengeResponse{Signature: []byte("response")}, + }, + }, + }, + }) + require.NoError(t, err) + + // Receive the device assertion response message + msg, err = stream.Recv() + require.NoError(t, err) + assert.NotNil(t, msg.GetDeviceAssertion().GetDeviceAsserted()) + + // Send close message + err = stream.CloseSend() + require.NoError(t, err) + + // Receive the termination message + _, err = stream.Recv() + require.ErrorIs(t, err, io.EOF) + +} + +func newFakefakeSecretsScannerSvc(t *testing.T) *fakeSecretsClient { + lis, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + + server := grpc.NewServer() + accessgraphsecretsv1pb.RegisterSecretsScannerServiceServer(server, &fakeSecretsScannerSvc{}) + go func() { + err := server.Serve(lis) + assert.NoError(t, err) + }() + t.Cleanup(server.GracefulStop) + + client, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + + return &fakeSecretsClient{ + SecretsScannerServiceClient: accessgraphsecretsv1pb.NewSecretsScannerServiceClient(client), + } + +} + +type fakeSecretsClient struct { + accessgraphsecretsv1pb.SecretsScannerServiceClient +} + +func (s *fakeSecretsClient) AccessGraphSecretsScannerClient() accessgraphsecretsv1pb.SecretsScannerServiceClient { + return s +} + +type fakeSecretsScannerSvc struct { + accessgraphsecretsv1pb.UnimplementedSecretsScannerServiceServer +} + +func (f *fakeSecretsScannerSvc) ReportSecrets(in accessgraphsecretsv1pb.SecretsScannerService_ReportSecretsServer) error { + msg, err := in.Recv() + if err != nil { + return trace.Wrap(err) + } + + if msg.GetDeviceAssertion().GetInit() == nil { + return trace.BadParameter("missing device init") + } + + err = in.Send(&accessgraphsecretsv1pb.ReportSecretsResponse{ + Payload: &accessgraphsecretsv1pb.ReportSecretsResponse_DeviceAssertion{ + DeviceAssertion: &devicepb.AssertDeviceResponse{ + Payload: &devicepb.AssertDeviceResponse_Challenge{ + Challenge: &devicepb.AuthenticateDeviceChallenge{Challenge: []byte("challenge")}, + }, + }, + }, + }) + if err != nil { + return trace.Wrap(err) + } + msg, err = in.Recv() + if err != nil { + return trace.Wrap(err) + } + + if msg.GetDeviceAssertion().GetChallengeResponse() == nil { + return trace.BadParameter("missing device challenge") + } + + err = in.Send(&accessgraphsecretsv1pb.ReportSecretsResponse{ + Payload: &accessgraphsecretsv1pb.ReportSecretsResponse_DeviceAssertion{ + DeviceAssertion: &devicepb.AssertDeviceResponse{ + Payload: &devicepb.AssertDeviceResponse_DeviceAsserted{ + DeviceAsserted: &devicepb.DeviceAsserted{}, + }, + }, + }, + }) + if err != nil { + return trace.Wrap(err) + } + + msg, err = in.Recv() + if errors.Is(err, io.EOF) { + return nil + } + return trace.BadParameter("unexpected message") +} + +func newProxyService(t *testing.T, lis net.Listener, authClient AuthClient) { + localTLSConfig, err := fixtures.LocalTLSConfig() + require.NoError(t, err) + + tlsConfig := localTLSConfig.TLS.Clone() + tlsConfig.InsecureSkipVerify = true + tlsConfig.ClientAuth = tls.RequestClientCert + tlsConfig.RootCAs = nil + + s := grpc.NewServer( + grpc.Creds( + credentials.NewTLS(tlsConfig), + ), + ) + t.Cleanup(s.GracefulStop) + + proxy, err := New(ServiceConfig{ + AuthClient: authClient, + }, + ) + require.NoError(t, err) + + accessgraphsecretsv1pb.RegisterSecretsScannerServiceServer(s, proxy) + + go func() { + err := s.Serve(lis) + assert.NoError(t, err) + }() + +} diff --git a/lib/service/service.go b/lib/service/service.go index e5803e2738ff4..a57274ec7cfeb 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -73,6 +73,7 @@ import ( "github.com/gravitational/teleport/api/client/webclient" "github.com/gravitational/teleport/api/constants" apidefaults "github.com/gravitational/teleport/api/defaults" + accessgraphsecretsv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/accessgraph/v1" integrationpb "github.com/gravitational/teleport/api/gen/proto/go/teleport/integration/v1" kubeproto "github.com/gravitational/teleport/api/gen/proto/go/teleport/kube/v1" transportpb "github.com/gravitational/teleport/api/gen/proto/go/teleport/transport/v1" @@ -140,6 +141,7 @@ import ( "github.com/gravitational/teleport/lib/resumption" "github.com/gravitational/teleport/lib/reversetunnel" "github.com/gravitational/teleport/lib/reversetunnelclient" + secretsscannerproxy "github.com/gravitational/teleport/lib/secretsscanner/proxy" "github.com/gravitational/teleport/lib/service/servicecfg" "github.com/gravitational/teleport/lib/services" "github.com/gravitational/teleport/lib/services/local" @@ -4138,7 +4140,6 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error { return trace.Wrap(err) } alpnRouter, reverseTunnelALPNRouter := setupALPNRouter(listeners, serverTLSConfig, cfg) - alpnAddr := "" if listeners.alpn != nil { alpnAddr = listeners.alpn.Addr().String() @@ -4987,8 +4988,10 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error { grpcServerMTLS *grpc.Server ) if alpnRouter != nil { - grpcServerPublic = process.initPublicGRPCServer(proxyLimiter, conn, listeners.grpcPublic) - + grpcServerPublic, err = process.initPublicGRPCServer(proxyLimiter, conn, listeners.grpcPublic) + if err != nil { + return trace.Wrap(err) + } grpcServerMTLS, err = process.initSecureGRPCServer( initSecureGRPCServerCfg{ limiter: proxyLimiter, @@ -6317,7 +6320,7 @@ func (process *TeleportProcess) initPublicGRPCServer( limiter *limiter.Limiter, conn *Connector, listener net.Listener, -) *grpc.Server { +) (*grpc.Server, error) { server := grpc.NewServer( grpc.ChainUnaryInterceptor( interceptors.GRPCServerUnaryErrorInterceptor, @@ -6348,11 +6351,24 @@ func (process *TeleportProcess) initPublicGRPCServer( ) joinServiceServer := joinserver.NewJoinServiceGRPCServer(conn.Client) proto.RegisterJoinServiceServer(server, joinServiceServer) + + accessGraphProxySvc, err := secretsscannerproxy.New( + secretsscannerproxy.ServiceConfig{ + AuthClient: conn.Client, + Log: process.logger, + }) + if err != nil { + return nil, trace.Wrap(err) + + } + + accessgraphsecretsv1pb.RegisterSecretsScannerServiceServer(server, accessGraphProxySvc) + process.RegisterCriticalFunc("proxy.grpc.public", func() error { process.logger.InfoContext(process.ExitContext(), "Starting proxy gRPC server.", "listen_address", listener.Addr()) return trace.Wrap(server.Serve(listener)) }) - return server + return server, nil } // initSecureGRPCServer creates and registers a gRPC server that uses mTLS for diff --git a/lib/service/service_test.go b/lib/service/service_test.go index 6bf7bd98e77e5..c65199403ac9f 100644 --- a/lib/service/service_test.go +++ b/lib/service/service_test.go @@ -1299,7 +1299,8 @@ func TestProxyGRPCServers(t *testing.T) { }) // Insecure gRPC server. - insecureGRPC := process.initPublicGRPCServer(limiter, testConnector, insecureListener) + insecureGRPC, err := process.initPublicGRPCServer(limiter, testConnector, insecureListener) + require.NoError(t, err) t.Cleanup(insecureGRPC.GracefulStop) proxyLockWatcher, err := services.NewLockWatcher(context.Background(), services.LockWatcherConfig{