x-pack/filebeat/input/cel: make redact configuration recommended (#36008

)

Ideally this would be a hard requirement, but that would be a breaking
change, so just log at WARN if the configuration is missing.

CHANGELOG.next.asciidoc

@@ -147,6 +147,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix duplicate ID panic in filestream metrics. {issue}35964[35964] {pull}35972[35972]
 - Improve error reporting and fix IPv6 handling of TCP and UDP metric collection. {pull}35996[35996]
 - Fix handling of NUL-terminated log lines in Fortinet Firewall module. {issue}36026[36026] {pull}36027[36027]
+- Make redact field configuration recommended in CEL input and log warning if missing. {pull}36008[36008]
 
 *Heartbeat*
 

x-pack/filebeat/docs/inputs/input-cel.asciidoc

@@ -631,6 +631,37 @@ Whether to use the host's local time rather that UTC for timestamping rotated lo
 
 This determines whether rotated logs should be gzip compressed.
 
+[float]
+==== `redact`
+
+During debug level logging, the `state` object and the resulting evaluation result are included in logs. This may result in leaking of secrets. In order to prevent this, fields may be redacted or deleted from the logged `state`. The `redact` configuration allows users to configure this field redaction behaviour. For safety reasons if the `redact` configuration is missing a warning is logged.
+
+In the case of no-required redaction an empty `redact.fields` configuration should be used to silence the logged warning.
+
+["source","yaml",subs="attributes"]
+----
+- type: cel
+  redact:
+    fields: ~
+----
+
+As an example, if a user-constructed Basic Authentication request is used in a CEL program the password can be redacted like so
+
+["source","yaml",subs="attributes"]
+----
+filebeat.inputs:
+- type: cel
+  resource.url: http://localhost:9200/_search
+  state:
+    user: [email protected]
+    password: P@$$W0₹D
+  redact:
+    fields: password
+    delete: true
+----
+
+Note that fields under the `auth` configuration hierarchy are not exposed to the `state` and so do not need to be redacted. For this reason it is preferable to use these for authentication over the request construction shown above where possible.
+
 [float]
 ==== `redact.fields`
 

x-pack/filebeat/input/cel/config.go

@@ -15,6 +15,7 @@ import (
 
 	"gopkg.in/natefinch/lumberjack.v2"
 
+	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/transport/httpcommon"
 )
 
@@ -44,7 +45,7 @@ type config struct {
 	// available if no stored cursor exists.
 	State map[string]interface{} `config:"state"`
 	// Redact is the debug log state redaction configuration.
-	Redact redact `config:"redact"`
+	Redact *redact `config:"redact"`
 
 	// Auth is the authentication config for connection to an HTTP
 	// API endpoint.
@@ -65,6 +66,10 @@ type redact struct {
 }
 
 func (c config) Validate() error {
+	if c.Redact == nil {
+		logp.L().Named("input.cel").Warn("missing recommended 'redact' configuration: " +
+			"see documentation for details: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_redact")
+	}
 	if c.Interval <= 0 {
 		return errors.New("interval must be greater than 0")
 	}

x-pack/filebeat/input/cel/config_test.go

@@ -112,7 +112,8 @@ func TestConfigMustFailWithInvalidResource(t *testing.T) {
 		}
 		cfg := conf.MustNewConfigFrom(m)
 		conf := defaultConfig()
-		conf.Program = "{}" // Provide an empty program to avoid validation error from that.
+		conf.Program = "{}"     // Provide an empty program to avoid validation error from that.
+		conf.Redact = &redact{} // Make sure we pass the redact requirement.
 		err := cfg.Unpack(&conf)
 		if fmt.Sprint(err) != fmt.Sprint(test.want) {
 			t.Errorf("unexpected error return from Unpack: got:%v want:%v", err, test.want)
@@ -458,7 +459,8 @@ func TestConfigOauth2Validation(t *testing.T) {
 			test.input["resource.url"] = "localhost"
 			cfg := conf.MustNewConfigFrom(test.input)
 			conf := defaultConfig()
-			conf.Program = "{}" // Provide an empty program to avoid validation error from that.
+			conf.Program = "{}"     // Provide an empty program to avoid validation error from that.
+			conf.Redact = &redact{} // Make sure we pass the redact requirement.
 			err := cfg.Unpack(&conf)
 
 			if fmt.Sprint(err) != fmt.Sprint(test.wantErr) {
@@ -509,7 +511,8 @@ func TestKeepAliveSetting(t *testing.T) {
 			test.input["resource.url"] = "localhost"
 			cfg := conf.MustNewConfigFrom(test.input)
 			conf := defaultConfig()
-			conf.Program = "{}" // Provide an empty program to avoid validation error from that.
+			conf.Program = "{}"     // Provide an empty program to avoid validation error from that.
+			conf.Redact = &redact{} // Make sure we pass the redact requirement.
 			err := cfg.Unpack(&conf)
 			if fmt.Sprint(err) != fmt.Sprint(test.wantErr) {
 				t.Errorf("unexpected error return from Unpack: got: %v want: %v", err, test.wantErr)

x-pack/filebeat/input/cel/input_test.go

@@ -1202,6 +1202,7 @@ func TestInput(t *testing.T) {
 			cfg := conf.MustNewConfigFrom(test.config)
 
 			conf := defaultConfig()
+			conf.Redact = &redact{} // Make sure we pass the redact requirement.
 			err := cfg.Unpack(&conf)
 			if err != nil {
 				t.Fatalf("unexpected error unpacking config: %v", err)