From 711746339ccfc466dfd56d7d740abbbcb9eabe1c Mon Sep 17 00:00:00 2001 From: danehans Date: Wed, 1 Jun 2022 15:16:34 -0700 Subject: [PATCH 1/5] WIP: Manager Design Doc Signed-off-by: danehans --- docs/design/COMPONENT_MANAGER.md | 310 +++++++++++++++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100644 docs/design/COMPONENT_MANAGER.md diff --git a/docs/design/COMPONENT_MANAGER.md b/docs/design/COMPONENT_MANAGER.md new file mode 100644 index 00000000000..9eedf50e76f --- /dev/null +++ b/docs/design/COMPONENT_MANAGER.md @@ -0,0 +1,310 @@ +Component Manager Design +=================== + +## Motivation + +[Issue 43][issue_43] specifies the need for lifecycle management of Envoy Gateway system components that run as Go +routines. + +??? +The Component Manager can decide which services, e.g. xDS server to instantiate based on a provided +configuration. For example, if the xDs server configuration parameter is not included in the provided configuration, the +xDS server component will not be instantiated). This approach adds optionality of running all components as one process +if needed (mainly useful for standalone VM controller cases or app dev local dev workflow testing) or as separate +processes/containers in production. +??? + +## Goals + +* Gracefully manage the software services that comprise Envoy Gateway. + +## Non-Goals + +* TODO + +## Proposal + +It was [decided][issue_60] to use the Runnable interface for providing goroutine management. TODO + +A `Manager` is created.... + +A `serve` argument is introduced to the `envoy-gateway` command. The `serve` argument encapsulates the management of +supported Envoy Gateway services, e.g. xDS Server, Provisioner, etc. +```go +// gateway/internal/cmd/root.go + +package cmd + +import ( + "github.com/spf13/cobra" + + "github.com/envoyproxy/gateway/internal/cmd/serve" +) + +// GetRootCommand returns the root cobra command to be executed +// by main. +func GetRootCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "envoy-gateway", + Short: "Manages Envoy Proxy as a standalone or Kubernetes-based application gateway", + } + + cmd.AddCommand(serve.NewCommand()) + + return cmd +} +``` + +The serve package: +- Exposes EG config. +- Processes, validates, etc. command line flags and config file. +- Starts EG services. + +```go +// gateway/internal/cmd/server/server.go + +package serve + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" + restclient "k8s.io/client-go/rest" + + "github.com/envoyproxy/gateway/apis/config/v1alpha1" + "github.com/envoyproxy/gateway/apis/config/v1alpha1/validation" +) + +// NewCommand allows the serve package to be run as a Cobra Command. +func NewCommand() *cobra.Command { + flags := NewGatewayFlags() + + gatewayCfg, err := NewGatewayConfig() + if err != nil { + os.Exit(1) + } + + cmd := &cobra.Command{ + Use: "server", + Short: "Server serves Envoy Gateway services, e.g. xDS Server, Provisioner, etc.", + RunE: func(cmd *cobra.Command, args []string) error { + // validate the initial GatewayFlags + if err := ValidateGatewayFlags(flags); err != nil { + return fmt.Errorf("failed to validate envoy gateway flags: %w", err) + } + + // load envoy gateway config file, if provided. + if cfgFile := flags.ConfigFile; len(cfgFile) > 0 { + gatewayCfg, err = loadConfigFile(cfgFile) + if err != nil { + return fmt.Errorf("failed to load envoy gateway config file, error: %w, path: %s", err, cfgFile) + } + } + + // Validate the local configuration (command line flags + config file). + if err := validation.ValidateGatewayConfig(gatewayCfg); err != nil { + return fmt.Errorf("failed to validate envoy gateway configuration, error: %w, path: %s", + err, gatewayCfg) + } + + // construct a GatewayServer from flags + config. + gatewayServer := &GatewayServer{ + GatewayFlags: *flags, + GatewayConfig: *gatewayCfg, + } + + // run envoy gateway + return run(context.TODO, gatewayServer) + }, + } + + addFlags(flags) + + return cmd +} + +// Load the config file specified by name. +func loadConfigFile(name string) (*v1alpha1.GatewayConfig, error) { + // Read config file from disk. + // Load and return the GatewayConfig or an error if a configuration can't be loaded. +} + +// Run the specified GatewayServer with the provided context. +func run(ctx context.Context, s *GatewayServer, gatewayDeps *gateway.Dependencies) error { + // validate the initial GatewayServer. + if err := options.ValidateGatewayServer(s); err != nil { + return err + } + + // If KubeConfig is nil then consider Envoy Gateway in "standalone" mode, e.g. virtual machine + // and don't create k8s client. + + clientConfig, err := buildGatewayClientConfig(ctx, s) + if err != nil { + return err + } + + return nil +} + +// buildGatewayClientConfig constructs the appropriate client config for Envoy Gateway. +func buildGatewayClientConfig(ctx context.Context, s *GatewayServer) (*restclient.Config, func(), error) { + if s.KubeConfig != "" { + return clientcmd.BuildConfigFromFlags("", s.KubeConfig) + } + return rest.InClusterConfig() +} +``` + +A subset of the Envoy Gateway's configuration parameters may be set by a configuration file, as a substitute for +command-line flags. Providing parameters via a config file is the recommended approach as it simplifies deployment +and configuration management. +```go +// gateway/internal/cmd/serve/options.go + +package serve + +import ( + "github.com/spf13/pflag" + + "github.com/envoyproxy/gateway/apis/config/v1alpha1" + "github.com/envoyproxy/gateway/apis/config/validation" + "github.com/envoyproxy/gateway/internal/cmd/serve" +) + +// GatewayFlags contains Envoy Gateway configuration flags. +type GatewayFlags struct { + // KubeConfig is the path to the kubeconfig file. + KubeConfig string + // ConfigFile is the name of the Envoy Gateway configuration file. + ConfigFile string + // XdsService should be set to true to enable the xDS Server. + XdsService bool + // ProvisionerService should be set to true to enable the Provisioner. + ProvisionerService bool + // TODO: Expose other serve start flags. +} + +// NewGatewayFlags will create a new GatewayFlags with default values. +func NewGatewayFlags() *GatewayFlags { + return &GatewayFlags{ + XdsService: true, + ProvisionerService: true, + // TODO: Set other default serve flags. + } +} + +// ValidateGatewayFlags validates Envoy Gateway's configuration flags. +func ValidateGatewayFlags(f *GatewayFlags) error { +} + +// NewGatewayConfig will create a new GatewayConfig with default values. +func NewGatewayConfig() (*v1alpha1.GatewayConfig, error) { + // Create a Scheme that understands the types in the gateway.config API group. + // Create an instance of the GatewayConfig object, set defaults, and return. +} + +// GatewayServer encapsulates all the necessary parameters for starting Envoy Gateway. +// These can either be set via command line or in the config file. +type GatewayServer struct { + GatewayFlags + v1alpha1.GatewayConfig +} + +// ValidateGatewayServer validates configuration of GatewayServer and returns +// an error if the input configuration is invalid. +func ValidateGatewayServer(s *GatewayServer) error { + // Validate flags and config file. +} + +// AddFlags adds flags for a specific GatewayFlags to the specified FlagSet. +func (f *GatewayFlags) AddFlags(fs *pflag.FlagSet) { + fs := pflag.NewFlagSet("", pflag.ExitOnError) + // Providing --kubeconfig enables Kubernetes API server mode. Omitting --kubeconfig enables standalone mode. + fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file for connecting to Kube API server.") + // Omit the --config flag to use the built-in default configuration values. + // Command-line flags override configuration from this file. + fs.StringVar(&f.ConfigFile, "config", f.ConfigFile, "The Envoy Gateway configuration file.") + fs.BoolVar(&f.XdsService, "xds-service", f.XdsService, "Set to false to disable the xds server.") + fs.BoolVar(&f.ProvisionerService, "provisioner-service", f.ProvisionerService, + "Set to false to disable the provisioner server.") +} +``` + +The --config flag specifies the path of the Envoy Gateway configuration file. Envoy Gateway will load its config from +this file. Command line flags which target the same value as a config file will override that value. If --config is +provided and values are not specified via the command line, the defaults for the config file apply. + +In `gateway/internal/componentmanager/manager.go`: +```go +package componentmanager + +import ( + "fmt" + + "github.com/envoyproxy/gateway/internal/provisioner" + "github.com/envoyproxy/gateway/internal/xds" + + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const ( + // DefaultEnvoyImage is the default Envoy image to use when unspecified. + DefaultEnvoyImage = "docker.io/envoyproxy/envoy:v1.23.0" +) + +// ComponentManager manages Envoy Gateway components. It sets up dependencies and defines the topology +// of Envoy Gateway and its managed components, wiring them together, properly shutting them down, etc. +type ComponentManager struct { + manager manager.Manager + config *Config +} + +// Config is configuration of a ComponentManager. +type Config struct { + // LeaderElection determines whether to use leader election. + LeaderElection bool + // Provisioner + Provisioner provisioner.Config + // XdsServer + XdsServer xds.ServerConfig +} + +// DefaultConfig returns a ComponentManager config using default values. +func DefaultConfig() *Config { + return &Config{ + EnvoyImage: DefaultEnvoyImage, + } +} + +// New creates a new ComponentManager from cliCfg and operatorConfig. +func New(cliCfg *rest.Config, mgrCfg *Config) (*ComponentManager, error) { + mgrOpts := manager.Options{ + LeaderElection: mgrCfg.LeaderElection, + } + mgr, err := ctrl.NewManager(cliCfg, mgrOpts) + if err != nil { + return nil, fmt.Errorf("failed to create manager: %w", err) + } + + // Create and register components. + if _, err := provisioner.New(mgr, provisioner.Config{ + EnvoyImage: mgrCfg.EnvoyImage, + }); err != nil { + return nil, fmt.Errorf("failed to create provisioner: %w", err) + } + if _, err := xds.NewServer(mgr, xds.ServerConfig{ + BindAddress: mgrCfg.EnvoyImage, + }); err != nil { + return nil, fmt.Errorf("failed to create contour controller: %w", err) + } +} +``` + +[issue_43]: https://github.com/envoyproxy/gateway/issues/43 +[issue_60]: https://github.com/envoyproxy/gateway/issues/60 + From ae5b5aa4b740171fe54d53aa6b60d278041deb1a Mon Sep 17 00:00:00 2001 From: danehans Date: Thu, 2 Jun 2022 13:04:23 -0700 Subject: [PATCH 2/5] Config Management Design Signed-off-by: danehans --- docs/design/COMPONENT_MANAGER.md | 310 ------------------------------- docs/design/CONFIG_MANAGEMENT.md | 238 ++++++++++++++++++++++++ 2 files changed, 238 insertions(+), 310 deletions(-) delete mode 100644 docs/design/COMPONENT_MANAGER.md create mode 100644 docs/design/CONFIG_MANAGEMENT.md diff --git a/docs/design/COMPONENT_MANAGER.md b/docs/design/COMPONENT_MANAGER.md deleted file mode 100644 index 9eedf50e76f..00000000000 --- a/docs/design/COMPONENT_MANAGER.md +++ /dev/null @@ -1,310 +0,0 @@ -Component Manager Design -=================== - -## Motivation - -[Issue 43][issue_43] specifies the need for lifecycle management of Envoy Gateway system components that run as Go -routines. - -??? -The Component Manager can decide which services, e.g. xDS server to instantiate based on a provided -configuration. For example, if the xDs server configuration parameter is not included in the provided configuration, the -xDS server component will not be instantiated). This approach adds optionality of running all components as one process -if needed (mainly useful for standalone VM controller cases or app dev local dev workflow testing) or as separate -processes/containers in production. -??? - -## Goals - -* Gracefully manage the software services that comprise Envoy Gateway. - -## Non-Goals - -* TODO - -## Proposal - -It was [decided][issue_60] to use the Runnable interface for providing goroutine management. TODO - -A `Manager` is created.... - -A `serve` argument is introduced to the `envoy-gateway` command. The `serve` argument encapsulates the management of -supported Envoy Gateway services, e.g. xDS Server, Provisioner, etc. -```go -// gateway/internal/cmd/root.go - -package cmd - -import ( - "github.com/spf13/cobra" - - "github.com/envoyproxy/gateway/internal/cmd/serve" -) - -// GetRootCommand returns the root cobra command to be executed -// by main. -func GetRootCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "envoy-gateway", - Short: "Manages Envoy Proxy as a standalone or Kubernetes-based application gateway", - } - - cmd.AddCommand(serve.NewCommand()) - - return cmd -} -``` - -The serve package: -- Exposes EG config. -- Processes, validates, etc. command line flags and config file. -- Starts EG services. - -```go -// gateway/internal/cmd/server/server.go - -package serve - -import ( - "context" - "fmt" - "os" - - "github.com/spf13/cobra" - restclient "k8s.io/client-go/rest" - - "github.com/envoyproxy/gateway/apis/config/v1alpha1" - "github.com/envoyproxy/gateway/apis/config/v1alpha1/validation" -) - -// NewCommand allows the serve package to be run as a Cobra Command. -func NewCommand() *cobra.Command { - flags := NewGatewayFlags() - - gatewayCfg, err := NewGatewayConfig() - if err != nil { - os.Exit(1) - } - - cmd := &cobra.Command{ - Use: "server", - Short: "Server serves Envoy Gateway services, e.g. xDS Server, Provisioner, etc.", - RunE: func(cmd *cobra.Command, args []string) error { - // validate the initial GatewayFlags - if err := ValidateGatewayFlags(flags); err != nil { - return fmt.Errorf("failed to validate envoy gateway flags: %w", err) - } - - // load envoy gateway config file, if provided. - if cfgFile := flags.ConfigFile; len(cfgFile) > 0 { - gatewayCfg, err = loadConfigFile(cfgFile) - if err != nil { - return fmt.Errorf("failed to load envoy gateway config file, error: %w, path: %s", err, cfgFile) - } - } - - // Validate the local configuration (command line flags + config file). - if err := validation.ValidateGatewayConfig(gatewayCfg); err != nil { - return fmt.Errorf("failed to validate envoy gateway configuration, error: %w, path: %s", - err, gatewayCfg) - } - - // construct a GatewayServer from flags + config. - gatewayServer := &GatewayServer{ - GatewayFlags: *flags, - GatewayConfig: *gatewayCfg, - } - - // run envoy gateway - return run(context.TODO, gatewayServer) - }, - } - - addFlags(flags) - - return cmd -} - -// Load the config file specified by name. -func loadConfigFile(name string) (*v1alpha1.GatewayConfig, error) { - // Read config file from disk. - // Load and return the GatewayConfig or an error if a configuration can't be loaded. -} - -// Run the specified GatewayServer with the provided context. -func run(ctx context.Context, s *GatewayServer, gatewayDeps *gateway.Dependencies) error { - // validate the initial GatewayServer. - if err := options.ValidateGatewayServer(s); err != nil { - return err - } - - // If KubeConfig is nil then consider Envoy Gateway in "standalone" mode, e.g. virtual machine - // and don't create k8s client. - - clientConfig, err := buildGatewayClientConfig(ctx, s) - if err != nil { - return err - } - - return nil -} - -// buildGatewayClientConfig constructs the appropriate client config for Envoy Gateway. -func buildGatewayClientConfig(ctx context.Context, s *GatewayServer) (*restclient.Config, func(), error) { - if s.KubeConfig != "" { - return clientcmd.BuildConfigFromFlags("", s.KubeConfig) - } - return rest.InClusterConfig() -} -``` - -A subset of the Envoy Gateway's configuration parameters may be set by a configuration file, as a substitute for -command-line flags. Providing parameters via a config file is the recommended approach as it simplifies deployment -and configuration management. -```go -// gateway/internal/cmd/serve/options.go - -package serve - -import ( - "github.com/spf13/pflag" - - "github.com/envoyproxy/gateway/apis/config/v1alpha1" - "github.com/envoyproxy/gateway/apis/config/validation" - "github.com/envoyproxy/gateway/internal/cmd/serve" -) - -// GatewayFlags contains Envoy Gateway configuration flags. -type GatewayFlags struct { - // KubeConfig is the path to the kubeconfig file. - KubeConfig string - // ConfigFile is the name of the Envoy Gateway configuration file. - ConfigFile string - // XdsService should be set to true to enable the xDS Server. - XdsService bool - // ProvisionerService should be set to true to enable the Provisioner. - ProvisionerService bool - // TODO: Expose other serve start flags. -} - -// NewGatewayFlags will create a new GatewayFlags with default values. -func NewGatewayFlags() *GatewayFlags { - return &GatewayFlags{ - XdsService: true, - ProvisionerService: true, - // TODO: Set other default serve flags. - } -} - -// ValidateGatewayFlags validates Envoy Gateway's configuration flags. -func ValidateGatewayFlags(f *GatewayFlags) error { -} - -// NewGatewayConfig will create a new GatewayConfig with default values. -func NewGatewayConfig() (*v1alpha1.GatewayConfig, error) { - // Create a Scheme that understands the types in the gateway.config API group. - // Create an instance of the GatewayConfig object, set defaults, and return. -} - -// GatewayServer encapsulates all the necessary parameters for starting Envoy Gateway. -// These can either be set via command line or in the config file. -type GatewayServer struct { - GatewayFlags - v1alpha1.GatewayConfig -} - -// ValidateGatewayServer validates configuration of GatewayServer and returns -// an error if the input configuration is invalid. -func ValidateGatewayServer(s *GatewayServer) error { - // Validate flags and config file. -} - -// AddFlags adds flags for a specific GatewayFlags to the specified FlagSet. -func (f *GatewayFlags) AddFlags(fs *pflag.FlagSet) { - fs := pflag.NewFlagSet("", pflag.ExitOnError) - // Providing --kubeconfig enables Kubernetes API server mode. Omitting --kubeconfig enables standalone mode. - fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file for connecting to Kube API server.") - // Omit the --config flag to use the built-in default configuration values. - // Command-line flags override configuration from this file. - fs.StringVar(&f.ConfigFile, "config", f.ConfigFile, "The Envoy Gateway configuration file.") - fs.BoolVar(&f.XdsService, "xds-service", f.XdsService, "Set to false to disable the xds server.") - fs.BoolVar(&f.ProvisionerService, "provisioner-service", f.ProvisionerService, - "Set to false to disable the provisioner server.") -} -``` - -The --config flag specifies the path of the Envoy Gateway configuration file. Envoy Gateway will load its config from -this file. Command line flags which target the same value as a config file will override that value. If --config is -provided and values are not specified via the command line, the defaults for the config file apply. - -In `gateway/internal/componentmanager/manager.go`: -```go -package componentmanager - -import ( - "fmt" - - "github.com/envoyproxy/gateway/internal/provisioner" - "github.com/envoyproxy/gateway/internal/xds" - - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -const ( - // DefaultEnvoyImage is the default Envoy image to use when unspecified. - DefaultEnvoyImage = "docker.io/envoyproxy/envoy:v1.23.0" -) - -// ComponentManager manages Envoy Gateway components. It sets up dependencies and defines the topology -// of Envoy Gateway and its managed components, wiring them together, properly shutting them down, etc. -type ComponentManager struct { - manager manager.Manager - config *Config -} - -// Config is configuration of a ComponentManager. -type Config struct { - // LeaderElection determines whether to use leader election. - LeaderElection bool - // Provisioner - Provisioner provisioner.Config - // XdsServer - XdsServer xds.ServerConfig -} - -// DefaultConfig returns a ComponentManager config using default values. -func DefaultConfig() *Config { - return &Config{ - EnvoyImage: DefaultEnvoyImage, - } -} - -// New creates a new ComponentManager from cliCfg and operatorConfig. -func New(cliCfg *rest.Config, mgrCfg *Config) (*ComponentManager, error) { - mgrOpts := manager.Options{ - LeaderElection: mgrCfg.LeaderElection, - } - mgr, err := ctrl.NewManager(cliCfg, mgrOpts) - if err != nil { - return nil, fmt.Errorf("failed to create manager: %w", err) - } - - // Create and register components. - if _, err := provisioner.New(mgr, provisioner.Config{ - EnvoyImage: mgrCfg.EnvoyImage, - }); err != nil { - return nil, fmt.Errorf("failed to create provisioner: %w", err) - } - if _, err := xds.NewServer(mgr, xds.ServerConfig{ - BindAddress: mgrCfg.EnvoyImage, - }); err != nil { - return nil, fmt.Errorf("failed to create contour controller: %w", err) - } -} -``` - -[issue_43]: https://github.com/envoyproxy/gateway/issues/43 -[issue_60]: https://github.com/envoyproxy/gateway/issues/60 - diff --git a/docs/design/CONFIG_MANAGEMENT.md b/docs/design/CONFIG_MANAGEMENT.md new file mode 100644 index 00000000000..7d8404e0552 --- /dev/null +++ b/docs/design/CONFIG_MANAGEMENT.md @@ -0,0 +1,238 @@ +Config Management Design +=================== + +## Motivation + +[Issue 43][issue_43] specifies the need for lifecycle management of Envoy Gateway system components that run as Go +routines. + +## Goals + +* Gracefully manage the services that comprise Envoy Gateway. + +## Non-Goals + +* Provide a detailed Envoy Gateway configuration specification. + +## Proposal + +Introduce a `serve` argument to the `envoy-gateway` command. The `serve` argument encapsulates the management of Envoy +Gateway. +```go +// gateway/internal/cmd/root.go + +package cmd + +import ( + "github.com/spf13/cobra" + + "github.com/envoyproxy/gateway/internal/cmd/serve" +) + +// GetRootCommand returns the root cobra command to be executed +// by main. +func GetRootCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "envoy-gateway", + Short: "Manages Envoy Proxy as a standalone or Kubernetes-based application gateway", + } + + cmd.AddCommand(serve.AddCommand()) + + return cmd +} +``` + +The serve package provides the following functionality: +- Allows the serve package to be run as a Cobra Command. +- Validates, processes, etc. command line flags and the Envoy Gateway config file. +- Constructs a object, e.g. `GatewayServer`, based on the validated flags/config. This object encapsulates all the +necessary parameters for running Envoy Gateway. +- Runs Envoy Gateway based on the provided `GatewayServer`. +```go +// gateway/internal/cmd/serve/serve.go + +package serve + +import ( + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/envoyproxy/gateway/apis/config/v1alpha1" +) + +// AddCommand allows the serve package to be run as a Cobra Command. +func AddCommand() *cobra.Command { + flags := NewGatewayFlags() + + gatewayCfg, _ := NewGatewayConfig() + + cmd := &cobra.Command{ + Use: "serve", + Short: "Serve serves Envoy Gateway", + RunE: func(cmd *cobra.Command, args []string) error { + // If provided, validate flags and config file. + // If provided, load the envoy gateway config file. + // Construct a GatewayServer object from the GatewayConfig type. + // Run Envoy Gateway using the provided GatewayServer. + }, + } + + return cmd +} + +// GatewayFlags contains Envoy Gateway configuration flags. +type GatewayFlags struct { + // Add fields for command line flags, e.g. config file, enabled services, etc. +} + +// NewGatewayFlags will create a new GatewayFlags with default values. +func NewGatewayFlags() *GatewayFlags { + return &GatewayFlags{ + // Set flag defaults, e.g. enabled services, config file path, etc. + } +} + +// ValidateGatewayFlags validates Envoy Gateway's configuration flags. +func ValidateGatewayFlags(f *GatewayFlags) error { +} + +// NewGatewayConfig will create a new GatewayConfig with default values. +func NewGatewayConfig() (*v1alpha1.GatewayConfig, error) { + // Create a Scheme that understands the types in the gateway.config API group. + // Create an instance of the GatewayConfig object, set defaults, and return. +} + +// GatewayServer encapsulates all the necessary parameters for starting Envoy Gateway. +// These can either be set via command line or in the config file. +type GatewayServer struct { + GatewayFlags + // TBD Envoy Gateway config CRD. + Config v1alpha1.GatewayConfig +} + +// ValidateGatewayServer validates configuration of GatewayServer and returns +// an error if the input configuration is invalid. +func (s *GatewayServer) ValidateGatewayServer() error { + // Validate flags and config file. +} + +// AddFlags adds flags for a specific GatewayFlags to the specified FlagSet. +func (f *GatewayFlags) AddFlags(fs *pflag.FlagSet) { + // Add command line flags, e.g. config file, enabled services, etc. +} +``` +A subset of the Envoy Gateway's configuration parameters may be set by a configuration file, as a substitute for +command-line flags. Providing parameters via a config file is the recommended approach as it simplifies deployment +and configuration management. The config file is defined by the `GatewayConfig` struct. The config file must be a YAML +representation of the parameters in this struct. For example: +```yaml +apiVersion: gateway.envoy.io/v1alpha1 +kind: GatewayConfig +foo: + bar: baz +... +``` +The `--config` flag specifies the path of the Envoy Gateway configuration file. Envoy Gateway will load its config from +this file. Command line flags which target the same value as a config file will override that value. If `--config` is +provided and values are not specified via the command line, the defaults for the config file are applied. + +The `serve` command calls `New()` and `Start()` to create and run Envoy Gateway. [Manager][mgr] creates controllers, +e.g. GatewayClass controller, and provides shared dependencies such as clients, caches, etc. Non-controller +processes, e.g. xDS server, implement the [Runnable][runnable] interface. This allows Manager to manage these processes +in the same manner as controller-based processes. +```go +package manager + +import ( + "context" + + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/envoyproxy/gateway/internal/kubernetes/gateway" + "github.com/envoyproxy/gateway/internal/kubernetes/gatewayclass" + "github.com/envoyproxy/gateway/internal/kubernetes/httproute" +) + +// GatewayManager sets-up dependencies and defines the topology of Envoy Gateway +// managed components, wiring them together. +type GatewayManager struct { + client client.Client + manager manager.Manager + config *v1alpha1.GatewayConfig +} + +// New creates a new Envoy Gateway using the provided configs. +// TODO: A Manager requires a rest config to construct a clutser. This means +// using Manager to manage services requires access to a k8s cluster. +// xref: https://github.com/kubernetes-sigs/controller-runtime/blob/v0.12.1/pkg/cluster/cluster.go#L145-L205 +func New(cliCfg *rest.Config, s *GatewayServer) (*GatewayManager, error) { + mgrOpts := manager.Options{ + // Add the scheme with types supported by Envoy Gateway. + // Add other manager configuration options, e.g. metrics bind address, + // leader election, etc. from GatewayServer. + } + mgr, _ := ctrl.NewManager(cliCfg, mgrOpts) + + // Create and register components with the manager. + if !s.Config.StandAlone { + // Create and register the kube controllers. + gcCfg := &gatewayclass.ControllerConfig{ + // GatewayClass controller config. + } + if _, err := gatewayclass.NewController(mgr, gcCfg); err != nil { + return nil, err + } + gwCfg := &gateway.ControllerConfig{ + // Gateway controller config. + } + if _, err := gateway.NewController(mgr, gwCfg); err != nil { + return nil, err + } + httpCfg := &controller.HttpRouteConfig{ + // HTTPRoute controller config. + } + if _, err := httproute.NewController(mgr, httpCfg); err != nil { + return nil, err + } + } + if s.Config.XdsService { + // The xds service will implement the Runnable interface, so it can be added to manager. + x, _ := xds.NewServer() + if err := mgr.Add(x); err != nil { + return nil, err + } + } + if s.Config.ProvisionerService { + // The provisioner service will implement the Runnable interface, so it can be added to manager. + p, _ := provisioner.NewServer() + if err := mgr.Add(p); err != nil { + return nil, err + } + } +} + +// Start starts Envoy Gateway manager, waiting for it to exit or an explicit stop. +func (g *GatewayManager) Start(ctx context.Context) error { + errChan := make(chan error) + go func() { + errChan <- g.manager.Start(ctx) + }() + + // Wait for the manager to exit or an explicit stop. + select { + case <-ctx.Done(): + return nil + case err := <-errChan: + return err + } +} +``` + +[issue_43]: https://github.com/envoyproxy/gateway/issues/43 +[issue_60]: https://github.com/envoyproxy/gateway/issues/60 +[mgr]: https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/manager#Manager +[runnable]: https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/manager#Runnable From aee49391d8b46e28716b715e10fbc86709622369 Mon Sep 17 00:00:00 2001 From: danehans Date: Mon, 6 Jun 2022 11:38:40 -0700 Subject: [PATCH 3/5] Updates Design to use run.Group and BootstrapConfig Signed-off-by: danehans --- docs/design/CONFIG_MANAGEMENT.md | 213 +++++++++++++++++-------------- 1 file changed, 118 insertions(+), 95 deletions(-) diff --git a/docs/design/CONFIG_MANAGEMENT.md b/docs/design/CONFIG_MANAGEMENT.md index 7d8404e0552..25f1658901e 100644 --- a/docs/design/CONFIG_MANAGEMENT.md +++ b/docs/design/CONFIG_MANAGEMENT.md @@ -37,7 +37,7 @@ func GetRootCommand() *cobra.Command { Short: "Manages Envoy Proxy as a standalone or Kubernetes-based application gateway", } - cmd.AddCommand(serve.AddCommand()) + cmd.AddCommand(serve.NewCommand()) return cmd } @@ -46,26 +46,47 @@ func GetRootCommand() *cobra.Command { The serve package provides the following functionality: - Allows the serve package to be run as a Cobra Command. - Validates, processes, etc. command line flags and the Envoy Gateway config file. -- Constructs a object, e.g. `GatewayServer`, based on the validated flags/config. This object encapsulates all the +- Constructs an object, e.g. `GatewayServer`, based on the validated flags/config. This object encapsulates all the necessary parameters for running Envoy Gateway. - Runs Envoy Gateway based on the provided `GatewayServer`. + +A subset of the Envoy Gateway's configuration parameters may be set by a configuration file, as a substitute for +command-line flags. Providing parameters via a config file is the recommended approach as it simplifies deployment +and configuration management. The config file is defined by the `BootstrapConfig` struct. The config file must be a YAML +representation of the parameters in this struct. For example: +```yaml +apiVersion: gateway.envoy.io/v1alpha1 +kind: BootstrapConfig +foo: + bar: baz +... +``` +The `--config` flag specifies the path of the Envoy Gateway configuration file. Envoy Gateway will load its config from +this file. Command line flags which target the same value as a config file will override that value. If `--config` is +provided and values are not specified via the command line, the defaults for the config file are applied. ```go // gateway/internal/cmd/serve/serve.go package serve import ( + "context" + + "github.com/oklog/run" "github.com/spf13/cobra" "github.com/spf13/pflag" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" "github.com/envoyproxy/gateway/apis/config/v1alpha1" + kube "github.com/envoyproxy/gateway/internal/kubernetes" ) -// AddCommand allows the serve package to be run as a Cobra Command. -func AddCommand() *cobra.Command { - flags := NewGatewayFlags() +// NewCommand creates the "serve" *cobra.Command object with default parameters. +func NewCommand() *cobra.Command { + flags := NewBootstrapFlags() - gatewayCfg, _ := NewGatewayConfig() + cfg, _ := NewBootstrapConfig() cmd := &cobra.Command{ Use: "serve", @@ -73,42 +94,48 @@ func AddCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { // If provided, validate flags and config file. // If provided, load the envoy gateway config file. - // Construct a GatewayServer object from the GatewayConfig type. + // Construct a GatewayServer object from the BootstrapConfig type. // Run Envoy Gateway using the provided GatewayServer. + return run(GatewayServer) }, } return cmd } -// GatewayFlags contains Envoy Gateway configuration flags. -type GatewayFlags struct { - // Add fields for command line flags, e.g. config file, enabled services, etc. +// BootstrapFlags contains Envoy Gateway bootstrap configuration flags. +type BootstrapFlags struct { + // Add fields for command line flags, e.g. bootstrap config file, enabled services, etc. } -// NewGatewayFlags will create a new GatewayFlags with default values. -func NewGatewayFlags() *GatewayFlags { - return &GatewayFlags{ +// NewBootstrapFlags will create a new BootstrapFlags with default values. +func NewBootstrapFlags() *BootstrapFlags { + return &BootstrapFlags{ // Set flag defaults, e.g. enabled services, config file path, etc. } } -// ValidateGatewayFlags validates Envoy Gateway's configuration flags. -func ValidateGatewayFlags(f *GatewayFlags) error { +// AddFlags adds flags for a specific BootstrapFlags to the specified FlagSet. +func (f *BootstrapFlags) AddFlags(fs *pflag.FlagSet) { + // Add command line flags, e.g. config file, enabled services, etc. } -// NewGatewayConfig will create a new GatewayConfig with default values. -func NewGatewayConfig() (*v1alpha1.GatewayConfig, error) { +// ValidateBootstrapFlags validates Envoy Gateway's bootstrap configuration flags. +func (f *BootstrapFlags) ValidateBootstrapFlags() error { +} + +// NewBootstrapConfig will create a new BootstrapConfig with default values. +func NewBootstrapConfig() (*v1alpha1.BootstrapConfig, error) { // Create a Scheme that understands the types in the gateway.config API group. - // Create an instance of the GatewayConfig object, set defaults, and return. + // Create an instance of the BootstrapConfig object, set defaults, and return. } // GatewayServer encapsulates all the necessary parameters for starting Envoy Gateway. // These can either be set via command line or in the config file. type GatewayServer struct { - GatewayFlags - // TBD Envoy Gateway config CRD. - Config v1alpha1.GatewayConfig + BootstrapFlags + // TBD Envoy Gateway bootstrap config CRD. + Config v1alpha1.BootstrapConfig } // ValidateGatewayServer validates configuration of GatewayServer and returns @@ -117,32 +144,45 @@ func (s *GatewayServer) ValidateGatewayServer() error { // Validate flags and config file. } -// AddFlags adds flags for a specific GatewayFlags to the specified FlagSet. -func (f *GatewayFlags) AddFlags(fs *pflag.FlagSet) { - // Add command line flags, e.g. config file, enabled services, etc. +// Run runs the provided GatewayServer. +func run(s *GatewayServer) error { + var g run.Group + + // Set up the channels for the watcher, operator, and metrics using + // the context provided from the controller runtime. + signal, cancel := context.WithCancel(signals.SetupSignalHandler()) + defer cancel() + + if s.Config.Kubernetes != nil { + // GetConfig creates a *rest.Config for talking to a Kubernetes API server. + // If --kubeconfig is set, it will use the kubeconfig file at that location. + // Otherwise, it will assume running in cluster and use the cluster provided kubeconfig. + cfg, _ := config.GetConfig() + + // Set up and start the manager for managing k8s controllers. + mgr, _ := kube.NewManager(cfg, s) + + g.Add(func() error { + return mgr.Start(signal) + }, func(error) { + cancel() + }) + } + // Repeat for other components, e.g. Message Server, Provisioner, etc. + + return nil } ``` -A subset of the Envoy Gateway's configuration parameters may be set by a configuration file, as a substitute for -command-line flags. Providing parameters via a config file is the recommended approach as it simplifies deployment -and configuration management. The config file is defined by the `GatewayConfig` struct. The config file must be a YAML -representation of the parameters in this struct. For example: -```yaml -apiVersion: gateway.envoy.io/v1alpha1 -kind: GatewayConfig -foo: - bar: baz -... -``` -The `--config` flag specifies the path of the Envoy Gateway configuration file. Envoy Gateway will load its config from -this file. Command line flags which target the same value as a config file will override that value. If `--config` is -provided and values are not specified via the command line, the defaults for the config file are applied. +The `run()` function uses `run.Group` to manage goroutine lifecycles. A zero-value `run.Group` is created and actors, +e.g. the Kubernetes Manager, are added to it. -The `serve` command calls `New()` and `Start()` to create and run Envoy Gateway. [Manager][mgr] creates controllers, -e.g. GatewayClass controller, and provides shared dependencies such as clients, caches, etc. Non-controller -processes, e.g. xDS server, implement the [Runnable][runnable] interface. This allows Manager to manage these processes -in the same manner as controller-based processes. +If Kubernetes is set in BootstrapConfig, `NewManager()` and +`Manager.Start()` are called to create and run Kubernetes controllers. [Manager][mgr] creates controllers, e.g. +GatewayClass controller, and provides shared dependencies such as clients, caches, etc. ```go -package manager +// gateway/internal/kubernetes/manager.go + +package kubernetes import ( "context" @@ -152,74 +192,57 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - "github.com/envoyproxy/gateway/internal/kubernetes/gateway" - "github.com/envoyproxy/gateway/internal/kubernetes/gatewayclass" - "github.com/envoyproxy/gateway/internal/kubernetes/httproute" + "github.com/envoyproxy/internal/cmd/serve" + "github.com/envoyproxy/internal/kubernetes/gateway" + "github.com/envoyproxy/internal/kubernetes/gatewayclass" + "github.com/envoyproxy/internal/kubernetes/httproute" ) -// GatewayManager sets-up dependencies and defines the topology of Envoy Gateway +// Manager sets-up dependencies and defines the topology of Envoy Gateway // managed components, wiring them together. -type GatewayManager struct { - client client.Client - manager manager.Manager - config *v1alpha1.GatewayConfig +type Manager struct { + client client.Client + + manager.Manager } -// New creates a new Envoy Gateway using the provided configs. -// TODO: A Manager requires a rest config to construct a clutser. This means -// using Manager to manage services requires access to a k8s cluster. -// xref: https://github.com/kubernetes-sigs/controller-runtime/blob/v0.12.1/pkg/cluster/cluster.go#L145-L205 -func New(cliCfg *rest.Config, s *GatewayServer) (*GatewayManager, error) { +// NewManager creates a new Manager for managing k8s controllers. +func NewManager(cfg *rest.Config, s *serve.GatewayServer) (*Manager, error) { mgrOpts := manager.Options{ // Add the scheme with types supported by Envoy Gateway. // Add other manager configuration options, e.g. metrics bind address, // leader election, etc. from GatewayServer. } - mgr, _ := ctrl.NewManager(cliCfg, mgrOpts) - - // Create and register components with the manager. - if !s.Config.StandAlone { - // Create and register the kube controllers. - gcCfg := &gatewayclass.ControllerConfig{ - // GatewayClass controller config. - } - if _, err := gatewayclass.NewController(mgr, gcCfg); err != nil { - return nil, err - } - gwCfg := &gateway.ControllerConfig{ - // Gateway controller config. - } - if _, err := gateway.NewController(mgr, gwCfg); err != nil { - return nil, err - } - httpCfg := &controller.HttpRouteConfig{ - // HTTPRoute controller config. - } - if _, err := httproute.NewController(mgr, httpCfg); err != nil { - return nil, err - } + mgr, _ := ctrl.NewManager(cfg, mgrOpts) + + // Create and register the kube controllers. + gcCfg := &gatewayclass.Config{ + // GatewayClass controller config. } - if s.Config.XdsService { - // The xds service will implement the Runnable interface, so it can be added to manager. - x, _ := xds.NewServer() - if err := mgr.Add(x); err != nil { - return nil, err - } - } - if s.Config.ProvisionerService { - // The provisioner service will implement the Runnable interface, so it can be added to manager. - p, _ := provisioner.NewServer() - if err := mgr.Add(p); err != nil { - return nil, err - } - } + if _, err := gatewayclass.NewController(mgr, gcCfg); err != nil { + return nil, err + } + gwCfg := &gateway.Config{ + // Gateway controller config. + } + if _, err := gateway.NewController(mgr, gwCfg); err != nil { + return nil, err + } + httpCfg := &httproute.Config{ + // HTTPRoute controller config. + } + if _, err := httproute.NewController(mgr, httpCfg); err != nil { + return nil, err + } + + return &Manager{mgr.GetClient(), mgr}, nil } // Start starts Envoy Gateway manager, waiting for it to exit or an explicit stop. -func (g *GatewayManager) Start(ctx context.Context) error { +func (m *Manager) Start(ctx context.Context) error { errChan := make(chan error) go func() { - errChan <- g.manager.Start(ctx) + errChan <- m.Start(ctx) }() // Wait for the manager to exit or an explicit stop. From ea6dcd316773e8c7e5cbc794480f9cc40c27dbca Mon Sep 17 00:00:00 2001 From: danehans Date: Mon, 6 Jun 2022 14:55:36 -0700 Subject: [PATCH 4/5] Updates cli flags for specifying config file Signed-off-by: danehans --- docs/design/CONFIG_MANAGEMENT.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/design/CONFIG_MANAGEMENT.md b/docs/design/CONFIG_MANAGEMENT.md index 25f1658901e..eeaf5a412ea 100644 --- a/docs/design/CONFIG_MANAGEMENT.md +++ b/docs/design/CONFIG_MANAGEMENT.md @@ -61,9 +61,11 @@ foo: bar: baz ... ``` -The `--config` flag specifies the path of the Envoy Gateway configuration file. Envoy Gateway will load its config from -this file. Command line flags which target the same value as a config file will override that value. If `--config` is -provided and values are not specified via the command line, the defaults for the config file are applied. + +Envoy Gateway will follow Envoy's approach to command line flags for specifying configuration by supporting +[--config-path][cfg_path] and [--config-yaml][cfg_file]. Command line flags which target the same value as a config file +will override that value. If either config flag is provided and individual values are not specified via the command +line, the defaults for the config file are applied. ```go // gateway/internal/cmd/serve/serve.go @@ -259,3 +261,5 @@ func (m *Manager) Start(ctx context.Context) error { [issue_60]: https://github.com/envoyproxy/gateway/issues/60 [mgr]: https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/manager#Manager [runnable]: https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/manager#Runnable +[cfg_path]: https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-c +[cfg_file]: https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-config-yaml \ No newline at end of file From 176b5071a39da29d9378825151c4871f90b10a85 Mon Sep 17 00:00:00 2001 From: danehans Date: Tue, 7 Jun 2022 10:10:01 -0700 Subject: [PATCH 5/5] Fixes API group and removes cfg-file flag Signed-off-by: danehans --- docs/design/CONFIG_MANAGEMENT.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/design/CONFIG_MANAGEMENT.md b/docs/design/CONFIG_MANAGEMENT.md index eeaf5a412ea..f39f74a3154 100644 --- a/docs/design/CONFIG_MANAGEMENT.md +++ b/docs/design/CONFIG_MANAGEMENT.md @@ -55,7 +55,7 @@ command-line flags. Providing parameters via a config file is the recommended ap and configuration management. The config file is defined by the `BootstrapConfig` struct. The config file must be a YAML representation of the parameters in this struct. For example: ```yaml -apiVersion: gateway.envoy.io/v1alpha1 +apiVersion: gateway.envoyproxy.io/v1alpha1 kind: BootstrapConfig foo: bar: baz @@ -63,9 +63,9 @@ foo: ``` Envoy Gateway will follow Envoy's approach to command line flags for specifying configuration by supporting -[--config-path][cfg_path] and [--config-yaml][cfg_file]. Command line flags which target the same value as a config file -will override that value. If either config flag is provided and individual values are not specified via the command -line, the defaults for the config file are applied. +[--config-path][cfg_path]. Command line flags which target the same value as a config file will override that value. If +either config flag is provided and individual values are not specified via the command line, the defaults for the config +file are applied. ```go // gateway/internal/cmd/serve/serve.go @@ -262,4 +262,3 @@ func (m *Manager) Start(ctx context.Context) error { [mgr]: https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/manager#Manager [runnable]: https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/manager#Runnable [cfg_path]: https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-c -[cfg_file]: https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-config-yaml \ No newline at end of file