From a00337397f1ab861847dbcf37e47fc7166242838 Mon Sep 17 00:00:00 2001 From: Renat Tuktarov Date: Tue, 15 Jun 2021 17:56:17 +0300 Subject: [PATCH] feat: using certloader.ClientConfigBuilder to prepare tls.Config Signed-off-by: Renat Tuktarov --- backend/client/client.go | 2 +- backend/go.mod | 1 + backend/go.sum | 1 + backend/internal/config/config.go | 56 +++++++++++++++ backend/internal/config/init.go | 108 +++++++++++++++++++++++++++++ backend/internal/grpc/tls.go | 22 ++++++ backend/internal/msg/logs.go | 23 +++--- backend/main.go | 46 +++++++----- backend/{ => pkg}/logger/logger.go | 0 backend/server/get-events.go | 2 +- backend/server/get-flows.go | 12 +++- backend/server/get-status.go | 26 ++++++- backend/server/helpers/helpers.go | 2 +- backend/server/server.go | 56 ++++++++------- 14 files changed, 299 insertions(+), 58 deletions(-) create mode 100644 backend/internal/config/config.go create mode 100644 backend/internal/config/init.go create mode 100644 backend/internal/grpc/tls.go rename backend/{ => pkg}/logger/logger.go (100%) diff --git a/backend/client/client.go b/backend/client/client.go index c24a06b33..4a88fb613 100644 --- a/backend/client/client.go +++ b/backend/client/client.go @@ -9,7 +9,7 @@ import ( "google.golang.org/grpc" "github.com/cilium/cilium/api/v1/flow" - "github.com/cilium/hubble-ui/backend/logger" + "github.com/cilium/hubble-ui/backend/pkg/logger" "github.com/cilium/hubble-ui/backend/proto/ui" ) diff --git a/backend/go.mod b/backend/go.mod index 71814dec4..1b47d0747 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -7,6 +7,7 @@ require ( github.com/cilium/hubble v0.6.1 github.com/golang/protobuf v1.5.0 github.com/google/gops v0.3.17 + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.7.0 google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.26.0 diff --git a/backend/go.sum b/backend/go.sum index 227df6001..6f1006578 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -652,6 +652,7 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go new file mode 100644 index 000000000..fdf329f64 --- /dev/null +++ b/backend/internal/config/config.go @@ -0,0 +1,56 @@ +package config + +import ( + "crypto/tls" + "fmt" + + "github.com/pkg/errors" + + "github.com/cilium/cilium/pkg/crypto/certloader" + "github.com/cilium/hubble-ui/backend/pkg/logger" +) + +var ( + log = logger.New("config") + + CACertLoadError = errors.New("failed to read CA certificate") + CACertInvalidError = errors.New("CA certificate is not properly PEM-encoded") +) + +const ( + TLSAddrPrefix = "tls://" +) + +type Config struct { + // The address of hubble-relay instance + RelayAddr string + + // The port which will be used to listen to on grpc server setup + UIServerPort string + + TLSToRelayEnabled bool + // The meaning of this flags is the same as in + // https://github.com/cilium/hubble/blob/master/cmd/common/config/flags.go + TLSToRelayAllowInsecure bool + TLSRelayServerName string + TLSRelayCACertFiles []string + TLSRelayClientCertFile string + TLSRelayClientKeyFile string + + relayClientConfig certloader.ClientConfigBuilder +} + +func (cfg *Config) UIServerListenAddr() string { + return fmt.Sprintf("0.0.0.0:%s", cfg.UIServerPort) +} + +func (cfg *Config) AsRelayClientTLSConfig() (*tls.Config, error) { + if cfg.relayClientConfig == nil { + return nil, errors.New("hubble-ui backend is running with TLS disabled") + } + + return cfg.relayClientConfig.ClientConfig(&tls.Config{ + InsecureSkipVerify: cfg.TLSToRelayAllowInsecure, + ServerName: cfg.TLSRelayServerName, + }), nil +} diff --git a/backend/internal/config/init.go b/backend/internal/config/init.go new file mode 100644 index 000000000..c4095465d --- /dev/null +++ b/backend/internal/config/init.go @@ -0,0 +1,108 @@ +package config + +import ( + "os" + "strconv" + "strings" + + "github.com/cilium/cilium/pkg/crypto/certloader" + "github.com/cilium/hubble-ui/backend/internal/msg" + "github.com/cilium/hubble-ui/backend/pkg/logger" +) + +const ( + UIServerDefaultPort = "8090" +) + +func Init() (*Config, error) { + cfg := &Config{} + + setupRelayAddr(cfg) + setupUIServerPort(cfg) + + if err := setupTLSOptions(cfg); err != nil { + return nil, err + } + + return cfg, nil +} + +func setupRelayAddr(cfg *Config) { + relayAddr, ok := os.LookupEnv("FLOWS_API_ADDR") + if !ok { + relayAddr = "localhost:50051" + log.Warnf(msg.ServerSetupUsingDefRelayAddr, relayAddr) + } + + cfg.RelayAddr = relayAddr +} + +func setupUIServerPort(cfg *Config) { + port, ok := os.LookupEnv("EVENTS_SERVER_PORT") + if !ok { + port = UIServerDefaultPort + log.Warnf(msg.ServerSetupUsingDefPort, port) + } + + cfg.UIServerPort = port +} + +func setupTLSOptions(cfg *Config) error { + tlsEnabledStr, _ := os.LookupEnv("TLS_TO_RELAY_ENABLED") + tlsServerName, _ := os.LookupEnv("TLS_RELAY_SERVER_NAME") + tlsAllowInsecureStr, _ := os.LookupEnv("TLS_TO_RELAY_ALLOW_INSECURE") + tlsCACertFiles, _ := os.LookupEnv("TLS_RELAY_CA_CERT_FILES") + tlsClientCertFile, _ := os.LookupEnv("TLS_RELAY_CLIENT_CERT_FILE") + tlsClientKeyFile, _ := os.LookupEnv("TLS_RELAY_CLIENT_KEY_FILE") + + cfg.TLSRelayServerName = tlsServerName + + tlsAllowInsecure, _ := strconv.ParseBool(tlsAllowInsecureStr) + if tlsAllowInsecure { + log.Warnf(msg.ServerSetupTLSAllowInsecureDef, tlsAllowInsecure) + } + + tlsToRelayEnabled, _ := strconv.ParseBool(tlsEnabledStr) + + cfg.TLSToRelayEnabled = tlsToRelayEnabled + cfg.TLSToRelayAllowInsecure = tlsAllowInsecure + cfg.TLSRelayCACertFiles = make([]string, 0) + for _, caCertPath := range strings.Split(tlsCACertFiles, ",") { + trimmed := strings.TrimSpace(caCertPath) + if len(trimmed) == 0 { + continue + } + + cfg.TLSRelayCACertFiles = append(cfg.TLSRelayCACertFiles, trimmed) + } + + cfg.TLSRelayClientCertFile = tlsClientCertFile + cfg.TLSRelayClientKeyFile = tlsClientKeyFile + + tlsState := "disabled" + if tlsToRelayEnabled { + tlsState = "enabled" + } + + log.Infof(msg.ServerSetupTLSInitState, tlsState) + + if !tlsToRelayEnabled { + return nil + } + + relayClientConfig, err := certloader.NewWatchedClientConfig( + logger.New("tls-config-watcher"), + cfg.TLSRelayCACertFiles, + cfg.TLSRelayClientCertFile, + cfg.TLSRelayClientKeyFile, + ) + + if err != nil { + return err + } + + cfg.relayClientConfig = relayClientConfig + log.Infof(msg.ServerSetupTLSInitWithNCACerts, len(cfg.TLSRelayCACertFiles)) + + return nil +} diff --git a/backend/internal/grpc/tls.go b/backend/internal/grpc/tls.go new file mode 100644 index 000000000..4a881ca9c --- /dev/null +++ b/backend/internal/grpc/tls.go @@ -0,0 +1,22 @@ +package grpc + +import ( + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + "github.com/cilium/hubble-ui/backend/internal/config" +) + +func TransportSecurityToRelay(cfg *config.Config) (grpc.DialOption, error) { + if !cfg.TLSToRelayEnabled { + return grpc.WithInsecure(), nil + } + + tlsConfig, err := cfg.AsRelayClientTLSConfig() + if err != nil { + return nil, err + } + + creds := credentials.NewTLS(tlsConfig) + return grpc.WithTransportCredentials(creds), nil +} diff --git a/backend/internal/msg/logs.go b/backend/internal/msg/logs.go index a57d5fae2..7b7ba6827 100644 --- a/backend/internal/msg/logs.go +++ b/backend/internal/msg/logs.go @@ -10,15 +10,20 @@ const ( ZeroIdentityInSourceOrDest = "invalid (zero) identity in source / dest services\n" PrintZeroIdentityFlowJson = "zero identity flow json: %v\n" - ServerSetupGRPCClientError = "failed to setup grpc client: %v\n" - ServerSetupK8sClientsetError = "failed to create k8s clientset: %v\n" - ServerSetupNoRelayAddrError = "hubble-relay addr is empty, flows fetching is impossible\n" - ServerSetupRelayClientReady = "hubble-relay grpc client created (hubble-relay addr: %s)\n" - ServerSetupRunError = "failed to run server: %v\n" - ServerSetupUsingDefRelayAddr = "hubble-relay addr is set to default (%s)\n" - ServerSetupUsingDefPort = "server port is set to default (%s)\n" - ServerSetupListenError = "failed to listen: %v\n" - ServerSetupListeningAt = "listening at: %s\n" + ServerSetupGRPCClientError = "failed to setup grpc client: %v\n" + ServerSetupK8sClientsetError = "failed to create k8s clientset: %v\n" + ServerSetupNoRelayAddrError = "hubble-relay addr is empty, flows fetching is impossible\n" + ServerSetupRelayClientReady = "hubble-relay grpc client created (hubble-relay addr: %s)\n" + ServerSetupRunError = "failed to run server: %v\n" + ServerSetupUsingDefRelayAddr = "hubble-relay addr is set to default (%s)\n" + ServerSetupUsingDefPort = "server port is set to default (%s)\n" + ServerSetupListenError = "failed to listen: %v\n" + ServerSetupListeningAt = "listening at: %s\n" + ServerSetupConfigInitError = "failed to initialize hubble-ui backend: %s\n" + ServerSetupTLSAllowInsecureDef = "TLSAllowInsecure option set to: %v\n" + ServerSetupTLSInitWithNCACerts = "initialized with %d CA certificates\n" + ServerSetupTLSInitState = "initialized with TLS %s\n" + ServerSetupRelayClientError = "failed to create a hubble-relay client from user context %v\n" SendConnStateError = "failed to send connection state (%s) to the client: %v\n" SendK8sUnavailableNotifError = "failed to send k8s unavailable notification: %v\n" diff --git a/backend/main.go b/backend/main.go index e611e8743..9ff5ad58e 100644 --- a/backend/main.go +++ b/backend/main.go @@ -4,6 +4,7 @@ import ( "fmt" "net" "os" + "strconv" // "github.com/cilium/cilium/pkg/logging" // "github.com/cilium/cilium/pkg/logging/logfields" @@ -13,8 +14,9 @@ import ( "google.golang.org/grpc" "github.com/cilium/hubble-ui/backend/client" + "github.com/cilium/hubble-ui/backend/internal/config" "github.com/cilium/hubble-ui/backend/internal/msg" - "github.com/cilium/hubble-ui/backend/logger" + "github.com/cilium/hubble-ui/backend/pkg/logger" "github.com/cilium/hubble-ui/backend/server" ) @@ -48,19 +50,9 @@ func setupListener() net.Listener { return listener } -func getObserverAddr() string { - observerAddr, ok := os.LookupEnv("FLOWS_API_ADDR") - if !ok { - observerAddr = "0.0.0.0:50051" - log.Warnf(msg.ServerSetupUsingDefRelayAddr, observerAddr) - } - - return observerAddr -} - -func runServer() { - observerAddr := getObserverAddr() - srv := server.New(observerAddr) +func runServer(cfg *config.Config) { + // observerAddr := getObserverAddr() + srv := server.New(cfg) if err := srv.Run(); err != nil { log.Errorf(msg.ServerSetupRunError, err) @@ -74,8 +66,8 @@ func runServer() { grpcServer.Serve(listener) } -func runClient() { - addr := getServerAddr() +func runClient(cfg *config.Config) { + addr := cfg.UIServerListenAddr() log.Infof("connecting to server: %s\n", addr) cl := client.New(addr) @@ -91,7 +83,11 @@ func getMode() string { return "server" } -func main() { +func runGops() { + if enabled, _ := strconv.ParseBool(os.Getenv("GOPS_ENABLED")); !enabled { + return + } + gopsPort := "0" if gopsPortEnv := os.Getenv("GOPS_PORT"); gopsPortEnv != "" { gopsPort = gopsPortEnv @@ -99,17 +95,29 @@ func main() { // Open socket for using gops to get stacktraces of the agent. addr := fmt.Sprintf("127.0.0.1:%s", gopsPort) addrField := logrus.Fields{"address": addr} + if err := gops.Listen(gops.Options{ Addr: addr, ReuseSocketAddrAndPort: true, }); err != nil { log.WithError(err).WithFields(addrField).Fatal("Cannot start gops server") } + log.WithFields(addrField).Info("Started gops server") +} + +func main() { + runGops() + + cfg, err := config.Init() + if err != nil { + log.Errorf(msg.ServerSetupConfigInitError, err.Error()) + os.Exit(1) + } if mode := getMode(); mode == "server" { - runServer() + runServer(cfg) } else { - runClient() + runClient(cfg) } } diff --git a/backend/logger/logger.go b/backend/pkg/logger/logger.go similarity index 100% rename from backend/logger/logger.go rename to backend/pkg/logger/logger.go diff --git a/backend/server/get-events.go b/backend/server/get-events.go index 11913874e..b2069424d 100644 --- a/backend/server/get-events.go +++ b/backend/server/get-events.go @@ -34,7 +34,7 @@ func (srv *UIServer) GetEvents( var flowCancel func() if eventsRequested.FlowsRequired() { - flowCancel, flowResponses, flowErrors = srv.GetFlows(req) + flowCancel, flowResponses, flowErrors = srv.GetFlows(stream.Context(), req) defer flowCancel() } diff --git a/backend/server/get-flows.go b/backend/server/get-flows.go index 9c298c871..f7bc33f5e 100644 --- a/backend/server/get-flows.go +++ b/backend/server/get-flows.go @@ -19,7 +19,9 @@ import ( "github.com/cilium/hubble-ui/backend/server/helpers" ) -func (srv *UIServer) GetFlows(req *ui.GetEventsRequest) ( +func (srv *UIServer) GetFlows( + streamContext context.Context, req *ui.GetEventsRequest, +) ( context.CancelFunc, chan *ui.GetEventsResponse, chan error, ) { // TODO: handle context cancellation @@ -33,7 +35,13 @@ func (srv *UIServer) GetFlows(req *ui.GetEventsRequest) ( retry := func(attempt int) error { log.Infof(msg.GetFlowsConnectingToRelay, attempt) - fs, err := srv.hubbleClient.GetFlows(ctx, flowsRequest) + client, err := srv.GetHubbleClientFromContext(streamContext) + if err != nil { + log.Errorf(msg.ServerSetupRelayClientError, err) + return err + } + + fs, err := client.hubble.GetFlows(ctx, flowsRequest) if err != nil { log.Errorf(msg.GetFlowsConnectionAttemptError, attempt, err) return err diff --git a/backend/server/get-status.go b/backend/server/get-status.go index d28cf8ba0..99deed17a 100644 --- a/backend/server/get-status.go +++ b/backend/server/get-status.go @@ -14,7 +14,22 @@ import ( func (srv *UIServer) GetStatus(ctx context.Context, _ *ui.GetStatusRequest) ( *ui.GetStatusResponse, error, ) { - ss, err := srv.hubbleClient.ServerStatus( + var client *HubbleClient = nil + parentClient := ctx.Value("hubbleClient") + + if parentClient != nil { + client = parentClient.(*HubbleClient) + } else { + cl, err := srv.GetHubbleClientFromContext(ctx) + if err != nil { + log.Errorf(msg.ServerSetupRelayClientError, err) + return nil, err + } + + client = cl + } + + ss, err := client.hubble.ServerStatus( ctx, &observer.ServerStatusRequest{}, ) @@ -63,6 +78,15 @@ func (srv *UIServer) RunStatusChecker(req *ui.GetStatusRequest) ( return } } + + client, err := srv.GetHubbleClientFromContext(ctx) + if err != nil { + log.Errorf(msg.ServerSetupRelayClientError, err) + sendError(err) + return + } + + ctx = context.WithValue(ctx, "hubbleClient", client) F: for { var lastError error = nil diff --git a/backend/server/helpers/helpers.go b/backend/server/helpers/helpers.go index f26a28c92..112f087c6 100644 --- a/backend/server/helpers/helpers.go +++ b/backend/server/helpers/helpers.go @@ -7,7 +7,7 @@ import ( "github.com/cilium/hubble-ui/backend/domain/flow" "github.com/cilium/hubble-ui/backend/domain/link" "github.com/cilium/hubble-ui/backend/domain/service" - "github.com/cilium/hubble-ui/backend/logger" + "github.com/cilium/hubble-ui/backend/pkg/logger" "github.com/cilium/hubble-ui/backend/proto/ui" "github.com/golang/protobuf/ptypes" ) diff --git a/backend/server/server.go b/backend/server/server.go index 64c09f37c..2be8e95a8 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -18,11 +18,13 @@ import ( "github.com/cilium/cilium/api/v1/observer" cilium_backoff "github.com/cilium/cilium/pkg/backoff" + grpc_helpers "github.com/cilium/hubble-ui/backend/internal/grpc" "github.com/cilium/hubble-ui/backend/proto/ui" "github.com/cilium/hubble-ui/backend/domain/cache" + "github.com/cilium/hubble-ui/backend/internal/config" "github.com/cilium/hubble-ui/backend/internal/msg" - "github.com/cilium/hubble-ui/backend/logger" + "github.com/cilium/hubble-ui/backend/pkg/logger" ) var ( @@ -32,18 +34,21 @@ var ( type UIServer struct { ui.UnimplementedUIServer - relayAddr string + cfg *config.Config relayConnParams *grpc.ConnectParams - hubbleClient observer.ObserverClient - grpcConnection *grpc.ClientConn k8s kubernetes.Interface dataCache *cache.DataCache } -func New(relayAddr string) *UIServer { +type HubbleClient struct { + hubble observer.ObserverClient + grpcConnection *grpc.ClientConn +} + +func New(cfg *config.Config) *UIServer { return &UIServer{ - relayAddr: relayAddr, + cfg: cfg, relayConnParams: &grpc.ConnectParams{ Backoff: backoff.Config{ BaseDelay: 1.0 * time.Second, @@ -67,12 +72,6 @@ func (srv *UIServer) newRetries() *cilium_backoff.Exponential { } func (srv *UIServer) Run() error { - err := srv.SetupGrpcClient() - if err != nil { - log.Errorf(msg.ServerSetupGRPCClientError, err) - os.Exit(1) - } - k8s, err := createClientset() if err != nil { log.Errorf(msg.ServerSetupK8sClientsetError, err) @@ -84,26 +83,35 @@ func (srv *UIServer) Run() error { return nil } -func (srv *UIServer) SetupGrpcClient() error { - if len(srv.relayAddr) == 0 { - return fmt.Errorf(msg.ServerSetupNoRelayAddrError) +func (srv *UIServer) GetHubbleClientFromContext(ctx context.Context) ( + *HubbleClient, error, +) { + relayAddr := srv.cfg.RelayAddr + if len(relayAddr) == 0 { + return nil, fmt.Errorf(msg.ServerSetupNoRelayAddrError) } - conn, err := grpc.Dial( - srv.relayAddr, - grpc.WithInsecure(), + transportDialOpt, err := grpc_helpers.TransportSecurityToRelay(srv.cfg) + if err != nil { + return nil, err + } + + dialOpts := []grpc.DialOption{ + transportDialOpt, grpc.WithConnectParams(*srv.relayConnParams), - ) + } + conn, err := grpc.Dial(relayAddr, dialOpts...) if err != nil { - return err + return nil, err } - log.Infof(msg.ServerSetupRelayClientReady, srv.relayAddr) - srv.hubbleClient = observer.NewObserverClient(conn) - srv.grpcConnection = conn + log.Infof(msg.ServerSetupRelayClientReady, relayAddr) - return nil + return &HubbleClient{ + hubble: observer.NewObserverClient(conn), + grpcConnection: conn, + }, nil } func (srv *UIServer) RetryIfGrpcUnavailable(