[Elastic Agent] Add options to bootstrap Fleet Server with TLS (#24142)

* Add support for SSL with bootstraping fleet-server.

* Run mage fmt.

* Fix issues with enrollment w/ fleet-server.

* Add changelog.

x-pack/elastic-agent/CHANGELOG.next.asciidoc

@@ -71,3 +71,4 @@
 - Add metrics collection for Agent {pull}22793[22793]
 - Add support for Fleet Server {pull}23736[23736]
 - Add support for enrollment with local bootstrap of Fleet Server {pull}23865[23865]
+- Add TLS support for Fleet Server {pull}24142[24142]

x-pack/elastic-agent/pkg/agent/application/config.go

@@ -5,6 +5,7 @@
 package application
 
 import (
+	"github.com/elastic/beats/v7/libbeat/common/transport/tlscommon"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
@@ -27,7 +28,7 @@ func createFleetConfigFromEnroll(accessAPIKey string, kbn *kibana.Config) (*conf
 	return cfg, nil
 }
 
-func createFleetServerBootstrapConfig(connStr string, policyID string) (*configuration.FleetAgentConfig, error) {
+func createFleetServerBootstrapConfig(connStr string, policyID string, host string, port uint16, cert string, key string) (*configuration.FleetAgentConfig, error) {
 	es, err := configuration.ElasticsearchFromConnStr(connStr)
 	if err != nil {
 		return nil, err
@@ -39,10 +40,20 @@ func createFleetServerBootstrapConfig(connStr string, policyID string) (*configu
 		Output: configuration.FleetServerOutputConfig{
 			Elasticsearch: es,
 		},
+		Host: host,
+		Port: port,
 	}
 	if policyID != "" {
 		cfg.Server.Policy = &configuration.FleetServerPolicyConfig{ID: policyID}
 	}
+	if cert != "" || key != "" {
+		cfg.Server.TLS = &tlscommon.Config{
+			Certificate: tlscommon.CertificateConfig{
+				Certificate: cert,
+				Key:         key,
+			},
+		}
+	}
 
 	if err := cfg.Valid(); err != nil {
 		return nil, errors.New(err, "invalid enrollment options", errors.TypeConfig)

x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go

@@ -15,25 +15,27 @@ import (
 	"os"
 	"time"
 
-	"github.com/elastic/beats/v7/libbeat/common/backoff"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/proto"
-
 	"gopkg.in/yaml.v2"
 
+	"github.com/elastic/beats/v7/libbeat/common/backoff"
+
 	"github.com/elastic/beats/v7/libbeat/common/transport/tlscommon"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/proto"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/authority"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
 )
 
 const (
-	waitingForAgent       = "waiting for Elastic Agent to start"
-	waitingForFleetServer = "waiting for Elastic Agent to start Fleet Server"
+	waitingForAgent        = "waiting for Elastic Agent to start"
+	waitingForFleetServer  = "waiting for Elastic Agent to start Fleet Server"
+	defaultFleetServerPort = 8220
 )
 
 var (
@@ -84,6 +86,11 @@ type EnrollCmdOption struct {
 	Staging              string
 	FleetServerConnStr   string
 	FleetServerPolicyID  string
+	FleetServerHost      string
+	FleetServerPort      uint16
+	FleetServerCert      string
+	FleetServerCertKey   string
+	FleetServerInsecure  bool
 }
 
 func (e *EnrollCmdOption) kibanaConfig() (*kibana.Config, error) {
@@ -140,46 +147,40 @@ func NewEnrollCmdWithStore(
 	configPath string,
 	store store,
 ) (*EnrollCmd, error) {
-
-	cfg, err := options.kibanaConfig()
-	if err != nil {
-		return nil, errors.New(
-			err, "Error",
-			errors.TypeConfig,
-			errors.M(errors.MetaKeyURI, options.URL))
-	}
-
-	client, err := fleetapi.NewWithConfig(log, cfg)
-	if err != nil {
-		return nil, errors.New(
-			err, "Error",
-			errors.TypeNetwork,
-			errors.M(errors.MetaKeyURI, options.URL))
-	}
-
 	return &EnrollCmd{
-		log:          log,
-		client:       client,
-		options:      options,
-		kibanaConfig: cfg,
-		configStore:  store,
+		log:         log,
+		options:     options,
+		configStore: store,
 	}, nil
 }
 
 // Execute tries to enroll the agent into Fleet.
 func (c *EnrollCmd) Execute(ctx context.Context) error {
+	var err error
 	if c.options.FleetServerConnStr != "" {
-		err := c.fleetServerBootstrap(ctx)
+		err = c.fleetServerBootstrap(ctx)
 		if err != nil {
 			return err
 		}
+	}
+
+	c.kibanaConfig, err = c.options.kibanaConfig()
+	if err != nil {
+		return errors.New(
+			err, "Error",
+			errors.TypeConfig,
+			errors.M(errors.MetaKeyURI, c.options.URL))
+	}
 
-		// enroll should use localhost as fleet-server is now running
-		// it must also restart
-		c.options.URL = "http://localhost:8000"
+	c.client, err = fleetapi.NewWithConfig(c.log, c.kibanaConfig)
+	if err != nil {
+		return errors.New(
+			err, "Error",
+			errors.TypeNetwork,
+			errors.M(errors.MetaKeyURI, c.options.URL))
 	}
 
-	err := c.enrollWithBackoff(ctx)
+	err = c.enrollWithBackoff(ctx)
 	if err != nil {
 		return errors.New(err, "fail to enroll")
 	}
@@ -198,7 +199,15 @@ func (c *EnrollCmd) fleetServerBootstrap(ctx context.Context) error {
 		return errors.New("failed to communicate with elastic-agent daemon; is elastic-agent running?")
 	}
 
-	fleetConfig, err := createFleetServerBootstrapConfig(c.options.FleetServerConnStr, c.options.FleetServerPolicyID)
+	err = c.prepareFleetTLS()
+	if err != nil {
+		return err
+	}
+
+	fleetConfig, err := createFleetServerBootstrapConfig(
+		c.options.FleetServerConnStr, c.options.FleetServerPolicyID,
+		c.options.FleetServerHost, c.options.FleetServerPort,
+		c.options.FleetServerCert, c.options.FleetServerCertKey)
 	configToStore := map[string]interface{}{
 		"fleet": fleetConfig,
 	}
@@ -222,6 +231,53 @@ func (c *EnrollCmd) fleetServerBootstrap(ctx context.Context) error {
 	return nil
 }
 
+func (c *EnrollCmd) prepareFleetTLS() error {
+	host := c.options.FleetServerHost
+	if host == "" {
+		host = "localhost"
+	}
+	port := c.options.FleetServerPort
+	if port == 0 {
+		port = defaultFleetServerPort
+	}
+	if c.options.FleetServerCert != "" && c.options.FleetServerCertKey == "" {
+		return errors.New("certificate private key is required when certificate provided")
+	}
+	if c.options.FleetServerCertKey != "" && c.options.FleetServerCert == "" {
+		return errors.New("certificate is required when certificate private key is provided")
+	}
+	if c.options.FleetServerCert == "" && c.options.FleetServerCertKey == "" {
+		if c.options.FleetServerInsecure {
+			// running insecure, force the binding to localhost (unless specified)
+			if c.options.FleetServerHost == "" {
+				c.options.FleetServerHost = "localhost"
+			}
+			c.options.URL = fmt.Sprintf("http://%s:%d", host, port)
+			c.options.Insecure = true
+			return nil
+		}
+
+		c.log.Info("Generating self-signed certificate for Fleet Server")
+		hostname, err := os.Hostname()
+		if err != nil {
+			return err
+		}
+		ca, err := authority.NewCA()
+		if err != nil {
+			return err
+		}
+		pair, err := ca.GeneratePairWithName(hostname)
+		if err != nil {
+			return err
+		}
+		c.options.FleetServerCert = string(pair.Crt)
+		c.options.FleetServerCertKey = string(pair.Key)
+		c.options.URL = fmt.Sprintf("https://%s:%d", hostname, port)
+		c.options.CAs = []string{string(ca.Crt())}
+	}
+	return nil
+}
+
 func (c *EnrollCmd) daemonReload(ctx context.Context) error {
 	daemon := client.New()
 	err := daemon.Connect(ctx)
@@ -276,6 +332,9 @@ func (c *EnrollCmd) enroll(ctx context.Context) error {
 	}
 
 	fleetConfig, err := createFleetConfigFromEnroll(resp.Item.AccessAPIKey, c.kibanaConfig)
+	if err != nil {
+		return err
+	}
 	agentConfig := map[string]interface{}{
 		"id": resp.Item.ID,
 	}
@@ -286,7 +345,10 @@ func (c *EnrollCmd) enroll(ctx context.Context) error {
 		}
 	}
 	if c.options.FleetServerConnStr != "" {
-		serverConfig, err := createFleetServerBootstrapConfig(c.options.FleetServerConnStr, c.options.FleetServerPolicyID)
+		serverConfig, err := createFleetServerBootstrapConfig(
+			c.options.FleetServerConnStr, c.options.FleetServerPolicyID,
+			c.options.FleetServerHost, c.options.FleetServerPort,
+			c.options.FleetServerCert, c.options.FleetServerCertKey)
 		if err != nil {
 			return err
 		}
@@ -400,10 +462,12 @@ func waitForFleetServer(ctx context.Context, log *logger.Logger) error {
 				resChan <- waitResult{}
 				break
 			}
-			appMsg := fmt.Sprintf("Fleet Server - %s", app.Message)
-			if msg != appMsg {
-				msg = appMsg
-				log.Info(appMsg)
+			if app.Message != "" {
+				appMsg := fmt.Sprintf("Fleet Server - %s", app.Message)
+				if msg != appMsg {
+					msg = appMsg
+					log.Info(appMsg)
+				}
 			}
 		}
 	}()

x-pack/elastic-agent/pkg/agent/cmd/enroll.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"strconv"
 	"syscall"
 
 	"github.com/spf13/cobra"
@@ -52,6 +53,11 @@ func addEnrollFlags(cmd *cobra.Command) {
 	cmd.Flags().StringP("enrollment-token", "t", "", "Enrollment token to use to enroll Agent into Fleet")
 	cmd.Flags().StringP("fleet-server", "", "", "Start and run a Fleet Server along side this Elastic Agent")
 	cmd.Flags().StringP("fleet-server-policy", "", "", "Start and run a Fleet Server on this specific policy")
+	cmd.Flags().StringP("fleet-server-host", "", "", "Fleet Server HTTP binding host (overrides the policy)")
+	cmd.Flags().Uint16P("fleet-server-port", "", 0, "Fleet Server HTTP binding port (overrides the policy)")
+	cmd.Flags().StringP("fleet-server-cert", "", "", "Certificate to use for exposed Fleet Server HTTPS endpoint")
+	cmd.Flags().StringP("fleet-server-cert-key", "", "", "Private key to use for exposed Fleet Server HTTPS endpoint")
+	cmd.Flags().BoolP("fleet-server-insecure-http", "", false, "Expose Fleet Server over HTTP (not recommended; insecure)")
 	cmd.Flags().StringP("certificate-authorities", "a", "", "Comma separated list of root certificate for server verifications")
 	cmd.Flags().StringP("ca-sha256", "p", "", "Comma separated list of certificate authorities hash pins used for certificate verifications")
 	cmd.Flags().BoolP("insecure", "i", false, "Allow insecure connection to Kibana")
@@ -70,6 +76,11 @@ func buildEnrollmentFlags(cmd *cobra.Command, url string, token string) []string
 	}
 	fServer, _ := cmd.Flags().GetString("fleet-server")
 	fPolicy, _ := cmd.Flags().GetString("fleet-server-policy")
+	fHost, _ := cmd.Flags().GetString("fleet-server-host")
+	fPort, _ := cmd.Flags().GetUint16("fleet-server-port")
+	fCert, _ := cmd.Flags().GetString("fleet-server-cert")
+	fCertKey, _ := cmd.Flags().GetString("fleet-server-cert-key")
+	fInsecure, _ := cmd.Flags().GetBool("fleet-server-insecure-http")
 	ca, _ := cmd.Flags().GetString("certificate-authorities")
 	sha256, _ := cmd.Flags().GetString("ca-sha256")
 	insecure, _ := cmd.Flags().GetBool("insecure")
@@ -92,6 +103,25 @@ func buildEnrollmentFlags(cmd *cobra.Command, url string, token string) []string
 		args = append(args, "--fleet-server-policy")
 		args = append(args, fPolicy)
 	}
+	if fHost != "" {
+		args = append(args, "--fleet-server-host")
+		args = append(args, fHost)
+	}
+	if fPort > 0 {
+		args = append(args, "--fleet-server-port")
+		args = append(args, strconv.Itoa(int(fPort)))
+	}
+	if fCert != "" {
+		args = append(args, "--fleet-server-cert")
+		args = append(args, fCert)
+	}
+	if fCertKey != "" {
+		args = append(args, "--fleet-server-cert-key")
+		args = append(args, fCertKey)
+	}
+	if fInsecure {
+		args = append(args, "--fleet-server-insecure-http")
+	}
 	if ca != "" {
 		args = append(args, "--certificate-authorities")
 		args = append(args, ca)
@@ -170,6 +200,11 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args
 	enrollmentToken, _ := cmd.Flags().GetString("enrollment-token")
 	fServer, _ := cmd.Flags().GetString("fleet-server")
 	fPolicy, _ := cmd.Flags().GetString("fleet-server-policy")
+	fHost, _ := cmd.Flags().GetString("fleet-server-host")
+	fPort, _ := cmd.Flags().GetUint16("fleet-server-port")
+	fCert, _ := cmd.Flags().GetString("fleet-server-cert")
+	fCertKey, _ := cmd.Flags().GetString("fleet-server-cert-key")
+	fInsecure, _ := cmd.Flags().GetBool("fleet-server-insecure-http")
 
 	caStr, _ := cmd.Flags().GetString("certificate-authorities")
 	CAs := cli.StringToSlice(caStr)
@@ -189,6 +224,11 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args
 		Staging:              staging,
 		FleetServerConnStr:   fServer,
 		FleetServerPolicyID:  fPolicy,
+		FleetServerHost:      fHost,
+		FleetServerPort:      fPort,
+		FleetServerCert:      fCert,
+		FleetServerCertKey:   fCertKey,
+		FleetServerInsecure:  fInsecure,
 	}
 
 	c, err := application.NewEnrollCmd(

x-pack/elastic-agent/pkg/agent/configuration/fleet_server.go

@@ -16,6 +16,9 @@ type FleetServerConfig struct {
 	Bootstrap bool                     `config:"bootstrap" yaml:"bootstrap,omitempty"`
 	Policy    *FleetServerPolicyConfig `config:"policy" yaml:"policy,omitempty"`
 	Output    FleetServerOutputConfig  `config:"output" yaml:"output,omitempty"`
+	Host      string                   `config:"host" yaml:"host,omitempty"`
+	Port      uint16                   `config:"port" yaml:"port,omitempty"`
+	TLS       *tlscommon.Config        `config:"ssl" yaml:"ssl,omitempty"`
 }
 
 // FleetServerPolicyConfig is the configuration for the policy Fleet Server should run on.