beater: telemetry for tail-based sampling config (elastic#4360)

* beater: telemetry for tail-based sampling config

beater/beater.go

@@ -145,7 +145,9 @@ func (bt *beater) Run(b *beat.Beat) error {
 	}
 
 	// send configs to telemetry
-	recordConfigs(b.Info, bt.config, bt.rawConfig, bt.logger)
+	if err := recordConfigs(b.Info, bt.config, bt.rawConfig); err != nil {
+		bt.logger.Errorf("Error recording telemetry data", err)
+	}
 
 	tracer, tracerServer, err := bt.initTracing(b)
 	if err != nil {

beater/telemetry.go

@@ -23,7 +23,6 @@ import (
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/idxmgmt/ilm"
-	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/monitoring"
 )
 
@@ -45,6 +44,8 @@ type configTelemetry struct {
 	jaegerGRPCEnabled         *monitoring.Bool
 	jaegerHTTPEnabled         *monitoring.Bool
 	sslEnabled                *monitoring.Bool
+	tailSamplingEnabled       *monitoring.Bool
+	tailSamplingPolicies      *monitoring.Int
 }
 
 var configMonitors = &configTelemetry{
@@ -63,13 +64,14 @@ var configMonitors = &configTelemetry{
 	jaegerGRPCEnabled:         monitoring.NewBool(apmRegistry, "jaeger.grpc.enabled"),
 	jaegerHTTPEnabled:         monitoring.NewBool(apmRegistry, "jaeger.http.enabled"),
 	sslEnabled:                monitoring.NewBool(apmRegistry, "ssl.enabled"),
+	tailSamplingEnabled:       monitoring.NewBool(apmRegistry, "sampling.tail.enabled"),
+	tailSamplingPolicies:      monitoring.NewInt(apmRegistry, "sampling.tail.policies"),
 }
 
-func recordConfigs(info beat.Info, apmCfg *config.Config, rootCfg *common.Config, logger *logp.Logger) {
+func recordConfigs(info beat.Info, apmCfg *config.Config, rootCfg *common.Config) error {
 	indexManagementCfg, err := idxmgmt.NewIndexManagementConfig(info, rootCfg)
 	if err != nil {
-		logger.Errorf("Error recording telemetry data", err)
-		return
+		return err
 	}
 	configMonitors.rumEnabled.Set(apmCfg.RumConfig.IsEnabled())
 	configMonitors.apiKeysEnabled.Set(apmCfg.APIKeyConfig.IsEnabled())
@@ -87,4 +89,11 @@ func recordConfigs(info beat.Info, apmCfg *config.Config, rootCfg *common.Config
 	configMonitors.ilmSetupRequirePolicy.Set(indexManagementCfg.ILM.Setup.RequirePolicy)
 	mode := indexManagementCfg.ILM.Mode
 	configMonitors.ilmEnabled.Set(mode == ilm.ModeAuto || mode == ilm.ModeEnabled)
+
+	tailSamplingEnabled := apmCfg.Sampling.Tail != nil && apmCfg.Sampling.Tail.Enabled
+	configMonitors.tailSamplingEnabled.Set(tailSamplingEnabled)
+	if tailSamplingEnabled {
+		configMonitors.tailSamplingPolicies.Set(int64(len(apmCfg.Sampling.Tail.Policies)))
+	}
+	return nil
 }

beater/telemetry_test.go

@@ -20,9 +20,8 @@ package beater
 import (
 	"testing"
 
-	"github.com/elastic/beats/v7/libbeat/logp"
-
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/apm-server/beater/config"
 	"github.com/elastic/beats/v7/libbeat/beat"
@@ -55,7 +54,7 @@ func TestRecordConfigs(t *testing.T) {
 			},
 		},
 	})
-	recordConfigs(info, apmCfg, rootCfg, logp.NewLogger("beater"))
+	require.NoError(t, recordConfigs(info, apmCfg, rootCfg))
 
 	assert.Equal(t, configMonitors.ilmSetupEnabled.Get(), true)
 	assert.Equal(t, configMonitors.rumEnabled.Get(), false)

changelogs/head.asciidoc

@@ -20,3 +20,4 @@ https://github.com/elastic/apm-server/compare/7.10\...master[View commits]
 * Monitoring for aggregation of transaction metrics {pull}4287[4287]
 * Log warnings in aggregation of transaction metrics when grouping limit is reached {pull}4313[4313]
 * Configurable tail-based sampling policies {pull}4320[4320]
+* Monitoring and telemetry for tail-based sampling {pull}4346[4346] {pull}4360[4360]

systemtest/sampling_test.go

@@ -147,6 +147,21 @@ func TestTailSampling(t *testing.T) {
 	// Make sure apm-server.sampling.tail metrics are published. Metric values are unit tested.
 	doc := getBeatsMonitoringStats(t, srv1, nil)
 	assert.True(t, gjson.GetBytes(doc.RawSource, "beats_stats.metrics.apm-server.sampling.tail").Exists())
+
+	// Check tail-sampling config is reported in telemetry.
+	var state struct {
+		APMServer struct {
+			Sampling struct {
+				Tail struct {
+					Enabled  bool
+					Policies int
+				}
+			}
+		} `mapstructure:"apm-server"`
+	}
+	getBeatsMonitoringState(t, srv1, &state)
+	assert.True(t, state.APMServer.Sampling.Tail.Enabled)
+	assert.Equal(t, 1, state.APMServer.Sampling.Tail.Policies)
 }
 
 func TestTailSamplingUnlicensed(t *testing.T) {