From e82332d47b955a1afcb16298502b601a63001ff6 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Wed, 8 May 2024 16:43:41 +0930 Subject: [PATCH 1/2] x-pack/filebeat/input/entityanalytics/provider/azuread: fix query handling (#39420) (cherry picked from commit f5bb6423fab42759e638b296ea39266a0a14a1b7) --- CHANGELOG.next.asciidoc | 38 +++++++++++++++++++ x-pack/filebeat/input/awss3/input_test.go | 2 + .../provider/azuread/fetcher/graph/graph.go | 22 ++++++----- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 66feb1b4c58..3c8f65875e3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -75,6 +75,44 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix handling of Juniper SRX structured data when there is no leading junos element. {issue}36270[36270] {pull}36308[36308] - Fix Filebeat Cisco module with missing escape character {issue}36325[36325] {pull}36326[36326] - Added a fix for Crowdstrike pipeline handling process arrays {pull}36496[36496] +- Fix m365_defender cursor value and query building. {pull}37116[37116] +- Fix TCP/UDP metric queue length parsing base. {pull}37714[37714] +- Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799] +- [threatintel] MISP pagination fixes {pull}37898[37898] +- Fix file handle leak when handling errors in filestream {pull}37973[37973] +- Fix a race condition that could crash Filebeat with a "negative WaitGroup counter" error {pull}38094[38094] +- Prevent HTTPJSON holding response bodies between executions. {issue}35219[35219] {pull}38116[38116] +- Fix "failed processing S3 event for object key" error on aws-s3 input when key contains the "+" character {issue}38012[38012] {pull}38125[38125] +- Fix duplicated addition of regexp extension in CEL input. {pull}38181[38181] +- Fix the incorrect values generated by the uri_parts processor. {pull}38216[38216] +- Fix HTTPJSON handling of empty object bodies in POST requests. {issue}33961[33961] {pull}38290[38290] +- Fix PEM key validation for CEL and HTTPJSON inputs. {pull}38405[38405] +- Fix filebeat gcs input panic {pull}38407[38407] +- Rename `activity_guid` to `activity_id` in ETW input events to suit other Windows inputs. {pull}38530[38530] +- Add missing provider registration and fix published entity for Active Directory entityanalytics provider. {pull}38645[38645] +- Fix handling of un-parsed JSON in O365 module. {issue}37800[37800] {pull}38709[38709] +- Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488] +- Fix indexing failures by re-enabling event normalisation in netflow input. {issue}38703[38703] {pull}38780[38780] +- Fix handling of truncated files in Filestream {issue}38070[38070] {pull}38416[38416] +- Fix panic when more than 32767 pipeline clients are active. {issue}38197[38197] {pull}38556[38556] +- Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488] +- [threatintel] MISP splitting fix for empty responses {issue}38739[38739] {pull}38917[38917] +- Fix a bug in cloudwatch task allocation that could skip some logs {issue}38918[38918] {pull}38953[38953] +- Prevent GCP Pub/Sub input blockage by increasing default value of `max_outstanding_messages` {issue}35029[35029] {pull}38985[38985] +- entity-analytics input: Improve structured logging. {pull}38990[38990] +- Fix config validation for CEL and HTTPJSON inputs when using password grant authentication and `client.id` or `client.secret` are not present. {pull}38962[38962] +- Updated Websocket input title to align with existing inputs {pull}39006[39006] +- Restore netflow input on Windows {pull}39024[39024] +- Upgrade azure-event-hubs-go and azure-storage-blob-go dependencies. {pull}38861[38861] +- Fix concurrency/error handling bugs in the AWS S3 input that could drop data and prevent ingestion of large buckets. {pull}39131[39131] +- Fix EntraID query handling. {issue}39419[39419] {pull}39420[39420] + +*Heartbeat* + +- Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] +- Fix setuid root when running under cgroups v2. {pull}37794[37794] +- Adjust State loader to only retry when response code status is 5xx {pull}37981[37981] +- Reset prctl dumpable flag after cap drop. {pull}38269[38269] *Heartbeat* diff --git a/x-pack/filebeat/input/awss3/input_test.go b/x-pack/filebeat/input/awss3/input_test.go index 8a195eb3084..82c72cfbb95 100644 --- a/x-pack/filebeat/input/awss3/input_test.go +++ b/x-pack/filebeat/input/awss3/input_test.go @@ -9,6 +9,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + aws "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" ) func TestGetProviderFromDomain(t *testing.T) { diff --git a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go index 6cabdf887e8..8eb1db153a3 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go +++ b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go @@ -31,9 +31,10 @@ import ( const ( defaultAPIEndpoint = "https://graph.microsoft.com/v1.0" - defaultGroupsQuery = "$select=displayName,members" - defaultUsersQuery = "$select=accountEnabled,userPrincipalName,mail,displayName,givenName,surname,jobTitle,officeLocation,mobilePhone,businessPhones" - defaultDevicesQuery = "$select=accountEnabled,deviceId,displayName,operatingSystem,operatingSystemVersion,physicalIds,extensionAttributes,alternativeSecurityIds" + queryName = "$select" + defaultGroupsQuery = "displayName,members" + defaultUsersQuery = "accountEnabled,userPrincipalName,mail,displayName,givenName,surname,jobTitle,officeLocation,mobilePhone,businessPhones" + defaultDevicesQuery = "accountEnabled,deviceId,displayName,operatingSystem,operatingSystemVersion,physicalIds,extensionAttributes,alternativeSecurityIds" apiGroupType = "#microsoft.graph.group" apiUserType = "#microsoft.graph.user" @@ -353,21 +354,21 @@ func New(cfg *config.C, logger *logp.Logger, auth authenticator.Authenticator) ( if err != nil { return nil, fmt.Errorf("invalid groups URL endpoint: %w", err) } - groupsURL.RawQuery = url.QueryEscape(formatQuery(c.Select.GroupQuery, defaultGroupsQuery)) + groupsURL.RawQuery = formatQuery(queryName, c.Select.GroupQuery, defaultGroupsQuery) f.groupsURL = groupsURL.String() usersURL, err := url.Parse(f.conf.APIEndpoint + "/users/delta") if err != nil { return nil, fmt.Errorf("invalid users URL endpoint: %w", err) } - usersURL.RawQuery = url.QueryEscape(formatQuery(c.Select.UserQuery, defaultUsersQuery)) + usersURL.RawQuery = formatQuery(queryName, c.Select.UserQuery, defaultUsersQuery) f.usersURL = usersURL.String() devicesURL, err := url.Parse(f.conf.APIEndpoint + "/devices/delta") if err != nil { return nil, fmt.Errorf("invalid devices URL endpoint: %w", err) } - devicesURL.RawQuery = url.QueryEscape(formatQuery(c.Select.DeviceQuery, defaultDevicesQuery)) + devicesURL.RawQuery = formatQuery(queryName, c.Select.DeviceQuery, defaultDevicesQuery) f.devicesURL = devicesURL.String() // The API takes a departure from the query approach here, so we @@ -382,11 +383,12 @@ func New(cfg *config.C, logger *logp.Logger, auth authenticator.Authenticator) ( return &f, nil } -func formatQuery(query []string, dflt string) string { - if len(query) == 0 { - return dflt +func formatQuery(name string, query []string, dflt string) string { + q := dflt + if len(query) != 0 { + q = strings.Join(query, ",") } - return "$select=" + strings.Join(query, ",") + return url.Values{name: []string{q}}.Encode() } // newUserFromAPI translates an API-representation of a user to a fetcher.User. From c68341729e61d713c21ec0b1f7e8a0ce8c813dc5 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 9 May 2024 10:04:11 +0930 Subject: [PATCH 2/2] remove irrelevant backport changes * changelog entries * aws import --- CHANGELOG.next.asciidoc | 37 ----------------------- x-pack/filebeat/input/awss3/input_test.go | 2 -- 2 files changed, 39 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3c8f65875e3..ba93b1d5905 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -75,47 +75,10 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix handling of Juniper SRX structured data when there is no leading junos element. {issue}36270[36270] {pull}36308[36308] - Fix Filebeat Cisco module with missing escape character {issue}36325[36325] {pull}36326[36326] - Added a fix for Crowdstrike pipeline handling process arrays {pull}36496[36496] -- Fix m365_defender cursor value and query building. {pull}37116[37116] -- Fix TCP/UDP metric queue length parsing base. {pull}37714[37714] -- Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799] -- [threatintel] MISP pagination fixes {pull}37898[37898] -- Fix file handle leak when handling errors in filestream {pull}37973[37973] -- Fix a race condition that could crash Filebeat with a "negative WaitGroup counter" error {pull}38094[38094] -- Prevent HTTPJSON holding response bodies between executions. {issue}35219[35219] {pull}38116[38116] -- Fix "failed processing S3 event for object key" error on aws-s3 input when key contains the "+" character {issue}38012[38012] {pull}38125[38125] -- Fix duplicated addition of regexp extension in CEL input. {pull}38181[38181] -- Fix the incorrect values generated by the uri_parts processor. {pull}38216[38216] -- Fix HTTPJSON handling of empty object bodies in POST requests. {issue}33961[33961] {pull}38290[38290] -- Fix PEM key validation for CEL and HTTPJSON inputs. {pull}38405[38405] -- Fix filebeat gcs input panic {pull}38407[38407] -- Rename `activity_guid` to `activity_id` in ETW input events to suit other Windows inputs. {pull}38530[38530] -- Add missing provider registration and fix published entity for Active Directory entityanalytics provider. {pull}38645[38645] -- Fix handling of un-parsed JSON in O365 module. {issue}37800[37800] {pull}38709[38709] -- Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488] -- Fix indexing failures by re-enabling event normalisation in netflow input. {issue}38703[38703] {pull}38780[38780] -- Fix handling of truncated files in Filestream {issue}38070[38070] {pull}38416[38416] -- Fix panic when more than 32767 pipeline clients are active. {issue}38197[38197] {pull}38556[38556] -- Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488] -- [threatintel] MISP splitting fix for empty responses {issue}38739[38739] {pull}38917[38917] -- Fix a bug in cloudwatch task allocation that could skip some logs {issue}38918[38918] {pull}38953[38953] -- Prevent GCP Pub/Sub input blockage by increasing default value of `max_outstanding_messages` {issue}35029[35029] {pull}38985[38985] -- entity-analytics input: Improve structured logging. {pull}38990[38990] -- Fix config validation for CEL and HTTPJSON inputs when using password grant authentication and `client.id` or `client.secret` are not present. {pull}38962[38962] -- Updated Websocket input title to align with existing inputs {pull}39006[39006] -- Restore netflow input on Windows {pull}39024[39024] -- Upgrade azure-event-hubs-go and azure-storage-blob-go dependencies. {pull}38861[38861] -- Fix concurrency/error handling bugs in the AWS S3 input that could drop data and prevent ingestion of large buckets. {pull}39131[39131] - Fix EntraID query handling. {issue}39419[39419] {pull}39420[39420] *Heartbeat* -- Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] -- Fix setuid root when running under cgroups v2. {pull}37794[37794] -- Adjust State loader to only retry when response code status is 5xx {pull}37981[37981] -- Reset prctl dumpable flag after cap drop. {pull}38269[38269] - -*Heartbeat* - *Metricbeat* diff --git a/x-pack/filebeat/input/awss3/input_test.go b/x-pack/filebeat/input/awss3/input_test.go index 82c72cfbb95..8a195eb3084 100644 --- a/x-pack/filebeat/input/awss3/input_test.go +++ b/x-pack/filebeat/input/awss3/input_test.go @@ -9,8 +9,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - - aws "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" ) func TestGetProviderFromDomain(t *testing.T) {