Set User-Agent in HTTP requests (#654)

* Set User-Agent in HTTP requests

* Adjust integration test

.gitignore

@@ -4,7 +4,7 @@
 .vscode/
 
 bin/
-build/
+/build/
 
 fleet-server
 fleet_server

Makefile

@@ -24,7 +24,8 @@ endif
 
 PLATFORM_TARGETS=$(addprefix release-, $(PLATFORMS))
 COMMIT=$(shell git rev-parse --short HEAD)
-LDFLAGS=-w -s -X main.Version=${VERSION} -X main.Commit=${COMMIT}
+NOW=$(shell date -u '+%Y-%m-%dT%H:%M:%SZ')
+LDFLAGS=-w -s -X main.Version=${VERSION} -X main.Commit=${COMMIT} -X main.BuildTime=$(NOW) 
 CMD_COLOR_ON=\033[32m\xE2\x9c\x93
 CMD_COLOR_OFF=\033[0m
 

NOTICE.txt

@@ -483,11 +483,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-elasticsearch/v7
-Version: v7.13.1
+Version: v7.5.1-0.20210823155509-845c8efe54a7
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v7@v7.13.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v7@v7.5.1-0.20210823155509-845c8efe54a7/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

cmd/fleet/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/elastic/go-ucfg/yaml"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/action"
+	"github.com/elastic/fleet-server/v7/internal/pkg/build"
 	"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
 	"github.com/elastic/fleet-server/v7/internal/pkg/cache"
 	"github.com/elastic/fleet-server/v7/internal/pkg/checkin"
@@ -49,6 +50,8 @@ const (
 	kServiceName               = "fleet-server"
 	kAgentMode                 = "agent-mode"
 	kAgentModeRestartLoopDelay = 2 * time.Second
+
+	kUAFleetServer = "Fleet-Server"
 )
 
 func installSignalHandler() context.Context {
@@ -95,7 +98,7 @@ func initLogger(cfg *config.Config, version, commit string) (*logger.Logger, err
 	return l, err
 }
 
-func getRunCommand(version, commit string) func(cmd *cobra.Command, args []string) error {
+func getRunCommand(bi build.Info) func(cmd *cobra.Command, args []string) error {
 	return func(cmd *cobra.Command, args []string) error {
 		cfgObject := cmd.Flags().Lookup("E").Value.(*config.Flag)
 		cliCfg := cfgObject.Config()
@@ -112,12 +115,12 @@ func getRunCommand(version, commit string) func(cmd *cobra.Command, args []strin
 			if err != nil {
 				return err
 			}
-			l, err = initLogger(cfg, version, commit)
+			l, err = initLogger(cfg, bi.Version, bi.Commit)
 			if err != nil {
 				return err
 			}
 
-			agent, err := NewAgentMode(cliCfg, os.Stdin, version, l)
+			agent, err := NewAgentMode(cliCfg, os.Stdin, bi, l)
 			if err != nil {
 				return err
 			}
@@ -141,12 +144,12 @@ func getRunCommand(version, commit string) func(cmd *cobra.Command, args []strin
 				return err
 			}
 
-			l, err = initLogger(cfg, version, commit)
+			l, err = initLogger(cfg, bi.Version, bi.Commit)
 			if err != nil {
 				return err
 			}
 
-			srv, err := NewFleetServer(cfg, version, status.NewLog())
+			srv, err := NewFleetServer(cfg, bi, status.NewLog())
 			if err != nil {
 				return err
 			}
@@ -164,11 +167,11 @@ func getRunCommand(version, commit string) func(cmd *cobra.Command, args []strin
 	}
 }
 
-func NewCommand(version, commit string) *cobra.Command {
+func NewCommand(bi build.Info) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   kServiceName,
 		Short: "Fleet Server controls a fleet of Elastic Agents",
-		RunE:  getRunCommand(version, commit),
+		RunE:  getRunCommand(bi),
 	}
 	cmd.Flags().StringP("config", "c", "fleet-server.yml", "Configuration for Fleet Server")
 	cmd.Flags().Bool(kAgentMode, false, "Running under execution of the Elastic Agent")
@@ -182,9 +185,8 @@ type firstCfg struct {
 }
 
 type AgentMode struct {
-	cliCfg  *ucfg.Config
-	version string
-
+	cliCfg      *ucfg.Config
+	bi          build.Info
 	reloadables []reload.Reloadable
 
 	agent client.Client
@@ -197,12 +199,12 @@ type AgentMode struct {
 	startChan    chan struct{}
 }
 
-func NewAgentMode(cliCfg *ucfg.Config, reader io.Reader, version string, reloadables ...reload.Reloadable) (*AgentMode, error) {
+func NewAgentMode(cliCfg *ucfg.Config, reader io.Reader, bi build.Info, reloadables ...reload.Reloadable) (*AgentMode, error) {
 	var err error
 
 	a := &AgentMode{
 		cliCfg:      cliCfg,
-		version:     version,
+		bi:          bi,
 		reloadables: reloadables,
 	}
 	a.agent, err = client.NewFromReader(reader, a)
@@ -247,7 +249,7 @@ func (a *AgentMode) Run(ctx context.Context) error {
 	srvCtx, srvCancel := context.WithCancel(ctx)
 	defer srvCancel()
 	log.Info().Msg("received initial configuration starting Fleet Server")
-	srv, err := NewFleetServer(cfg.cfg, a.version, status.NewChained(status.NewLog(), a.agent))
+	srv, err := NewFleetServer(cfg.cfg, a.bi, status.NewChained(status.NewLog(), a.agent))
 	if err != nil {
 		// unblock startChan even though there was an error
 		a.startChan <- struct{}{}
@@ -384,7 +386,7 @@ func (a *AgentMode) OnError(err error) {
 }
 
 type FleetServer struct {
-	ver      string
+	bi       build.Info
 	verCon   version.Constraints
 	policyId string
 
@@ -395,8 +397,8 @@ type FleetServer struct {
 }
 
 // NewFleetServer creates the actual fleet server service.
-func NewFleetServer(cfg *config.Config, verStr string, reporter status.Reporter) (*FleetServer, error) {
-	verCon, err := buildVersionConstraint(verStr)
+func NewFleetServer(cfg *config.Config, bi build.Info, reporter status.Reporter) (*FleetServer, error) {
+	verCon, err := buildVersionConstraint(bi.Version)
 	if err != nil {
 		return nil, err
 	}
@@ -407,7 +409,7 @@ func NewFleetServer(cfg *config.Config, verStr string, reporter status.Reporter)
 	}
 
 	return &FleetServer{
-		ver:      verStr,
+		bi:       bi,
 		verCon:   verCon,
 		cfg:      cfg,
 		cfgCh:    make(chan *config.Config, 1),
@@ -646,8 +648,8 @@ func initRuntime(cfg *config.Config) {
 	}
 }
 
-func initBulker(ctx context.Context, cfg *config.Config) (*bulk.Bulker, error) {
-	es, err := es.NewClient(ctx, cfg, false)
+func (f *FleetServer) initBulker(ctx context.Context, cfg *config.Config) (*bulk.Bulker, error) {
+	es, err := es.NewClient(ctx, cfg, false, es.WithUserAgent(kUAFleetServer, f.bi))
 	if err != nil {
 		return nil, err
 	}
@@ -675,7 +677,7 @@ func (f *FleetServer) runServer(ctx context.Context, cfg *config.Config) (err er
 	defer bulkCancel()
 
 	// Create the bulker subsystem
-	bulker, err := initBulker(bulkCtx, cfg)
+	bulker, err := f.initBulker(bulkCtx, cfg)
 	if err != nil {
 		return err
 	}
@@ -723,13 +725,13 @@ func (f *FleetServer) runSubsystems(ctx context.Context, cfg *config.Config, g *
 	esCli := bulker.Client()
 
 	// Check version compatibility with Elasticsearch
-	err = ver.CheckCompatibility(ctx, esCli, f.ver)
+	err = ver.CheckCompatibility(ctx, esCli, f.bi.Version)
 	if err != nil {
 		return fmt.Errorf("failed version compatibility check with elasticsearch: %w", err)
 	}
 
 	// Monitoring es client, longer timeout, no retries
-	monCli, err := es.NewClient(ctx, cfg, true)
+	monCli, err := es.NewClient(ctx, cfg, true, es.WithUserAgent(kUAFleetServer, f.bi))
 	if err != nil {
 		return err
 	}
@@ -744,7 +746,7 @@ func (f *FleetServer) runSubsystems(ctx context.Context, cfg *config.Config, g *
 	}
 
 	g.Go(loggedRunFunc(ctx, "Policy index monitor", pim.Run))
-	cord := coordinator.NewMonitor(cfg.Fleet, f.ver, bulker, pim, coordinator.NewCoordinatorZero)
+	cord := coordinator.NewMonitor(cfg.Fleet, f.bi.Version, bulker, pim, coordinator.NewCoordinatorZero)
 	g.Go(loggedRunFunc(ctx, "Coordinator policy monitor", cord.Run))
 
 	// Policy monitor

cmd/fleet/metrics.go

@@ -35,7 +35,7 @@ var (
 func (f *FleetServer) initMetrics(ctx context.Context, cfg *config.Config) (*api.Server, error) {
 	registry := monitoring.GetNamespace("info").GetRegistry()
 	if registry.Get("version") == nil {
-		monitoring.NewString(registry, "version").Set(f.ver)
+		monitoring.NewString(registry, "version").Set(f.bi.Version)
 	}
 	if registry.Get("name") == nil {
 		monitoring.NewString(registry, "name").Set(kServiceName)

cmd/fleet/server_integration_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/elastic/fleet-server/v7/internal/pkg/build"
 	"github.com/elastic/fleet-server/v7/internal/pkg/config"
 	"github.com/elastic/fleet-server/v7/internal/pkg/logger"
 	"github.com/elastic/fleet-server/v7/internal/pkg/sleep"
@@ -76,7 +77,7 @@ func startTestServer(ctx context.Context) (*tserver, error) {
 	cfg.Inputs[0].Server = *srvcfg
 	log.Info().Uint16("port", port).Msg("Test fleet server")
 
-	srv, err := NewFleetServer(cfg, serverVersion, status.NewLog())
+	srv, err := NewFleetServer(cfg, build.Info{Version: serverVersion}, status.NewLog())
 	if err != nil {
 		return nil, err
 	}

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/dgraph-io/ristretto v0.1.0
 	github.com/elastic/beats/v7 v7.11.1
 	github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6
-	github.com/elastic/go-elasticsearch/v7 v7.13.1
+	github.com/elastic/go-elasticsearch/v7 v7.5.1-0.20210823155509-845c8efe54a7
 	github.com/elastic/go-ucfg v0.8.3
 	github.com/gofrs/uuid v3.3.0+incompatible
 	github.com/google/go-cmp v0.4.0

go.sum

@@ -253,8 +253,8 @@ github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQ
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdboCbArMF/nSCDUXgQuWTeoMmE/z8607X+k7ng=
 github.com/elastic/go-concert v0.0.4 h1:pzgYCmJ/xMJsW8PSk33inAWZ065hrwSeP79TpwAbsLE=
 github.com/elastic/go-concert v0.0.4/go.mod h1:9MtFarjXroUgmm0m6HY3NSe1XiKhdktiNRRj9hWvIaM=
-github.com/elastic/go-elasticsearch/v7 v7.13.1 h1:PaM3V69wPlnwR+ne50rSKKn0RNDYnnOFQcuGEI0ce80=
-github.com/elastic/go-elasticsearch/v7 v7.13.1/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
+github.com/elastic/go-elasticsearch/v7 v7.5.1-0.20210823155509-845c8efe54a7 h1:Nq382VeELkUSC7y8JIXBNj0YfOqmq/d8mX+crl4xdrM=
+github.com/elastic/go-elasticsearch/v7 v7.5.1-0.20210823155509-845c8efe54a7/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-libaudit/v2 v2.1.0 h1:yWSKoGaoWLGFPjqWrQ4gwtuM77pTk7K4CsPxXss8he4=
 github.com/elastic/go-libaudit/v2 v2.1.0/go.mod h1:MM/l/4xV7ilcl+cIblL8Zn448J7RZaDwgNLE4gNKYPg=
 github.com/elastic/go-licenser v0.3.1 h1:RmRukU/JUmts+rpexAw0Fvt2ly7VVu6mw8z4HrEzObU=

internal/pkg/build/build.go

@@ -0,0 +1,20 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package build
+
+import "time"
+
+type Info struct {
+	Version, Commit string
+	BuildTime       time.Time
+}
+
+func Time(stime string) time.Time {
+	t, err := time.Parse(time.RFC3339, stime)
+	if err != nil {
+		return time.Time{}
+	}
+	return t
+}

internal/pkg/es/client.go

@@ -8,14 +8,19 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"runtime"
 
+	"github.com/elastic/fleet-server/v7/internal/pkg/build"
 	"github.com/elastic/fleet-server/v7/internal/pkg/config"
 
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/rs/zerolog/log"
 )
 
-func NewClient(ctx context.Context, cfg *config.Config, longPoll bool) (*elasticsearch.Client, error) {
+type ConfigOption func(config elasticsearch.Config)
+
+func NewClient(ctx context.Context, cfg *config.Config, longPoll bool, opts ...ConfigOption) (*elasticsearch.Client, error) {
 	escfg, err := cfg.Output.Elasticsearch.ToESConfig(longPoll)
 	if err != nil {
 		return nil, err
@@ -24,6 +29,11 @@ func NewClient(ctx context.Context, cfg *config.Config, longPoll bool) (*elastic
 	user := cfg.Output.Elasticsearch.Username
 	mcph := cfg.Output.Elasticsearch.MaxConnPerHost
 
+	// Apply configuration options
+	for _, opt := range opts {
+		opt(escfg)
+	}
+
 	log.Debug().
 		Strs("addr", addr).
 		Str("user", user).
@@ -50,6 +60,24 @@ func NewClient(ctx context.Context, cfg *config.Config, longPoll bool) (*elastic
 	return es, nil
 }
 
+func WithUserAgent(name string, bi build.Info) func(config elasticsearch.Config) {
+	return func(config elasticsearch.Config) {
+		ua := userAgent(name, bi)
+		// Set User-Agent header
+		if config.Header == nil {
+			config.Header = http.Header{}
+		}
+		config.Header.Set("User-Agent", ua)
+	}
+}
+
+func userAgent(name string, bi build.Info) string {
+	return fmt.Sprintf("Elastic-%s/%s (%s; %s; %s; %s)",
+		name,
+		bi.Version, runtime.GOOS, runtime.GOARCH,
+		bi.Commit, bi.BuildTime)
+}
+
 type InfoResponse struct {
 	ClusterName string `json:"cluster_name"`
 	ClusterUUID string `json:"cluster_uuid"`

main.go

@@ -14,17 +14,23 @@ import (
 	"os"
 
 	"github.com/elastic/fleet-server/v7/cmd/fleet"
+	"github.com/elastic/fleet-server/v7/internal/pkg/build"
 )
 
 const defaultVersion = "8.0.0"
 
 var (
-	Version string = defaultVersion
-	Commit  string
+	Version   string = defaultVersion
+	Commit    string
+	BuildTime string
 )
 
 func main() {
-	cmd := fleet.NewCommand(Version, Commit)
+	cmd := fleet.NewCommand(build.Info{
+		Version:   Version,
+		Commit:    Commit,
+		BuildTime: build.Time(BuildTime),
+	})
 	if err := cmd.Execute(); err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)