From 25ed53423f94ccf0ab1b03f63376b324c427e986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 14 Apr 2021 15:53:16 +0200 Subject: [PATCH 1/2] Strip Azure EventHub connection string in debug logs (#25066) * Strip Azure EventHub connection string in debug logs (cherry picked from commit 96fa570d98632d59e8b0e7ad5f1c140ccbd019fe) --- CHANGELOG.next.asciidoc | 28 +++++++++++++++++++ x-pack/filebeat/input/azureeventhub/input.go | 17 ++++++++++- .../input/azureeventhub/input_test.go | 24 ++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 394996489e38..0e1af4fee8cd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -156,6 +156,34 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - system/package: Fix an error that can occur while trying to persist package metadata. {issue}18536[18536] {pull}18887[18887] - system/socket: Fix dataset using 100% CPU and becoming unresponsive in some scenarios. {pull}19033[19033] {pull}19764[19764] - system/socket: Fixed tracking of long-running connections. {pull}19033[19033] +- system/package: Fix librpm loading on Fedora 31/32. {pull}NNNN[NNNN] +- file_integrity: Create fsnotify watcher only when starting file_integrity module {pull}19505[19505] +- auditd: Fix spelling of anomaly in `event.category`. +- auditd: Fix typo in `event.action` of `removed-user-role-from`. {pull}19300[19300] +- auditd: Fix typo in `event.action` of `used-suspicious-link`. {pull}19300[19300] +- system/socket: Fix kprobe grouping to allow running more than one instance. {pull}20325[20325] +- system/socket: Fixed a crash due to concurrent map read and write. {issue}21192[21192] {pull}21690[21690] +- file_integrity: stop monitoring excluded paths {issue}21278[21278] {pull}21282[21282] +- auditd: Fix an error condition causing a lot of `audit_send_reply` kernel threads being created. {pull}22673[22673] +- system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693] +- system/socket: Having some CPUs unavailable to Auditbeat could cause startup errors or event loss. {pull}22827[22827] +- Note incompatibility of system/socket on ARM. {pull}23381[23381] + +*Filebeat* + +- Fix mapping of fortinet.firewall.mem as integer. {pull}19335[19335] +- Ensure all zeek timestamps include millisecond precision. {issue}14599[14599] {pull}16766[16766] +- Fix s3 input hanging with GetObjectRequest API call by adding context_timeout config. {issue}15502[15502] {pull}15590[15590] +- Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656] +- Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765] +- Fix mapping error when zeek weird logs do not contain IP addresses. {pull}15906[15906] +- Improve `elasticsearch/audit` fileset to handle timestamps correctly. {pull}15942[15942] +- Prevent Elasticsearch from spewing log warnings about redundant wildcards when setting up ingest pipelines for the `elasticsearch` module. {issue}15840[15840] {pull}15900[15900] +- Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088] +- Fix a connection error in httpjson input. {pull}16123[16123] +- Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523] +- Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277] +- Strip Azure Eventhub connection string in debug logs. {pulll}[] *Filebeat* diff --git a/x-pack/filebeat/input/azureeventhub/input.go b/x-pack/filebeat/input/azureeventhub/input.go index 010e2807b560..5f0d1b3df5ee 100644 --- a/x-pack/filebeat/input/azureeventhub/input.go +++ b/x-pack/filebeat/input/azureeventhub/input.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "sync" "time" @@ -80,7 +81,7 @@ func NewInput( in := &azureInput{ config: config, - log: logp.NewLogger(fmt.Sprintf("%s input", inputName)).With("connection string", config.ConnectionString), + log: logp.NewLogger(fmt.Sprintf("%s input", inputName)).With("connection string", stripConnectionString(config.ConnectionString)), context: inputContext, workerCtx: workerCtx, workerCancel: workerCancel, @@ -235,3 +236,17 @@ func (a *azureInput) parseMultipleMessages(bMessage []byte) []string { } return messages } + +// Strip connection string to remove sensitive information +// A connection string should look like this: +// Endpoint=sb://dummynamespace.servicebus.windows.net/;SharedAccessKeyName=DummyAccessKeyName;SharedAccessKey=5dOntTRytoC24opYThisAsit3is2B+OGY1US/fuL3ly= +// This code will remove everything after ';' so key information is stripped +func stripConnectionString(c string) string { + if parts := strings.SplitN(c, ";", 2); len(parts) == 2 { + return parts[0] + } + + // We actually expect the string to have the documented format + // if we reach here something is wrong, so let's stay on the safe side + return "(redacted)" +} diff --git a/x-pack/filebeat/input/azureeventhub/input_test.go b/x-pack/filebeat/input/azureeventhub/input_test.go index 8537e7529f54..d4268d9d96a4 100644 --- a/x-pack/filebeat/input/azureeventhub/input_test.go +++ b/x-pack/filebeat/input/azureeventhub/input_test.go @@ -120,6 +120,30 @@ func TestNewInputDone(t *testing.T) { inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) } +func TestStripConnectionString(t *testing.T) { + tests := []struct { + connectionString, expected string + }{ + { + "Endpoint=sb://something", + "(redacted)", + }, + { + "Endpoint=sb://dummynamespace.servicebus.windows.net/;SharedAccessKeyName=DummyAccessKeyName;SharedAccessKey=5dOntTRytoC24opYThisAsit3is2B+OGY1US/fuL3ly=", + "Endpoint=sb://dummynamespace.servicebus.windows.net/", + }, + { + "Endpoint=sb://dummynamespace.servicebus.windows.net/;SharedAccessKey=5dOntTRytoC24opYThisAsit3is2B+OGY1US/fuL3ly=", + "Endpoint=sb://dummynamespace.servicebus.windows.net/", + }, + } + + for _, tt := range tests { + res := stripConnectionString(tt.connectionString) + assert.Equal(t, res, tt.expected) + } +} + type stubOutleter struct { sync.Mutex cond *sync.Cond From a15e65dd98c1d283aa7ff0783909f60830c090aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 14 Apr 2021 16:10:24 +0200 Subject: [PATCH 2/2] Fix changelog --- CHANGELOG.next.asciidoc | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0e1af4fee8cd..019fdd6765d8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -156,34 +156,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - system/package: Fix an error that can occur while trying to persist package metadata. {issue}18536[18536] {pull}18887[18887] - system/socket: Fix dataset using 100% CPU and becoming unresponsive in some scenarios. {pull}19033[19033] {pull}19764[19764] - system/socket: Fixed tracking of long-running connections. {pull}19033[19033] -- system/package: Fix librpm loading on Fedora 31/32. {pull}NNNN[NNNN] -- file_integrity: Create fsnotify watcher only when starting file_integrity module {pull}19505[19505] -- auditd: Fix spelling of anomaly in `event.category`. -- auditd: Fix typo in `event.action` of `removed-user-role-from`. {pull}19300[19300] -- auditd: Fix typo in `event.action` of `used-suspicious-link`. {pull}19300[19300] -- system/socket: Fix kprobe grouping to allow running more than one instance. {pull}20325[20325] -- system/socket: Fixed a crash due to concurrent map read and write. {issue}21192[21192] {pull}21690[21690] -- file_integrity: stop monitoring excluded paths {issue}21278[21278] {pull}21282[21282] -- auditd: Fix an error condition causing a lot of `audit_send_reply` kernel threads being created. {pull}22673[22673] -- system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693] -- system/socket: Having some CPUs unavailable to Auditbeat could cause startup errors or event loss. {pull}22827[22827] -- Note incompatibility of system/socket on ARM. {pull}23381[23381] - -*Filebeat* - -- Fix mapping of fortinet.firewall.mem as integer. {pull}19335[19335] -- Ensure all zeek timestamps include millisecond precision. {issue}14599[14599] {pull}16766[16766] -- Fix s3 input hanging with GetObjectRequest API call by adding context_timeout config. {issue}15502[15502] {pull}15590[15590] -- Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656] -- Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765] -- Fix mapping error when zeek weird logs do not contain IP addresses. {pull}15906[15906] -- Improve `elasticsearch/audit` fileset to handle timestamps correctly. {pull}15942[15942] -- Prevent Elasticsearch from spewing log warnings about redundant wildcards when setting up ingest pipelines for the `elasticsearch` module. {issue}15840[15840] {pull}15900[15900] -- Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088] -- Fix a connection error in httpjson input. {pull}16123[16123] -- Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523] -- Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277] -- Strip Azure Eventhub connection string in debug logs. {pulll}[] *Filebeat* @@ -269,6 +241,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861] - Fix google_workspace and okta modules pagination when next page template is empty. {pull}24967[24967] - Fix IPtables Pipeline and Ubiquiti dashboard. {issue}24878[24878] {pull}24928[24928] +- Strip Azure Eventhub connection string in debug logs. {pulll}25066[25066] *Heartbeat*