From 2d3aa03e4ac54f8bbbd88a2365666e95b8c2bbb3 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 10 Apr 2020 12:08:07 +0200 Subject: [PATCH 1/5] add option to select a different azure env --- filebeat/docs/modules/azure.asciidoc | 7 ++++++ .../docs/inputs/input-azure-eventhub.asciidoc | 11 +++++--- x-pack/filebeat/filebeat.reference.yml | 12 ++++++--- x-pack/filebeat/input/azureeventhub/config.go | 2 ++ x-pack/filebeat/input/azureeventhub/eph.go | 25 ++++++++++++++++++- x-pack/filebeat/module/azure/_meta/config.yml | 12 ++++++--- .../filebeat/module/azure/_meta/docs.asciidoc | 7 ++++++ .../activitylogs/config/azure-eventhub.yml | 1 + .../module/azure/activitylogs/manifest.yml | 1 + .../azure/auditlogs/config/azure-eventhub.yml | 1 + .../module/azure/auditlogs/manifest.yml | 1 + .../signinlogs/config/azure-eventhub.yml | 1 + .../module/azure/signinlogs/manifest.yml | 1 + x-pack/filebeat/modules.d/azure.yml.disabled | 12 ++++++--- 14 files changed, 78 insertions(+), 16 deletions(-) diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 5d52e33beac..8fa93a584b2 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -43,6 +43,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -52,6 +53,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" signinlogs: enabled: false @@ -61,6 +63,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" ``` @@ -90,6 +93,10 @@ The name of the storage account the state/offsets will be stored and updated. _string_ The storage account key, this key will be used to authorize access to data in your storage account. +`override_resource_manager_endpoint` :: +_string_ +Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. + include::../include/what-happens.asciidoc[] include::../include/gs-link.asciidoc[] diff --git a/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc b/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc index 15b628169ce..f5954de0c52 100644 --- a/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc @@ -28,6 +28,8 @@ Example configuration: storage_account: "azureeph" storage_account_key: "....." storage_account_container: "" + override_resource_manager_endpoint: "" + ---- ==== Configuration options @@ -36,7 +38,7 @@ The `azure-eventhub` input supports the following configuration: ==== `eventhub` -The name of the eventhub users would like to read from. +The name of the eventhub users would like to read from, field required. ==== `consumer_group` @@ -50,14 +52,17 @@ A Blob Storage account is required in order to store/retrieve/update the offset ==== `storage_account` -The name of the storage account. +The name of the storage account. Required. ==== `storage_account_key` -The storage account key, this key will be used to authorize access to data in your storage account. +The storage account key, this key will be used to authorize access to data in your storage account, option is required. ==== `storage_account_container` Optional, the name of the storage account container you would like to store the offset information in. +==== `override_resource_manager_endpoint` + +Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index d42397123fb..1674d6d5e60 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -288,16 +288,18 @@ filebeat.modules: activitylogs: enabled: true var: - # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: "insights-operational-logs" - # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" - # the name of the storage account the state/offsets will be stored and updated. + # the name of the storage account the state/offsets will be stored and updated storage_account: "" - #The storage account key, this key will be used to authorize access to data in your storage account. + # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint + override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -307,6 +309,7 @@ filebeat.modules: # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -315,6 +318,7 @@ filebeat.modules: # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" #--------------------------------- CEF Module --------------------------------- - module: cef diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index b567b25e35b..bfd60b99a3f 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -17,6 +17,8 @@ type azureInputConfig struct { SAName string `config:"storage_account"` SAKey string `config:"storage_account_key"` SAContainer string `config:"storage_account_container"` + // by default the azure public environment is used, to override, users can provide a specific resource manager endpoint + OverrideEnvironment string `config:"override_resource_manager_endpoint"` } const ephContainerName = "filebeat" diff --git a/x-pack/filebeat/input/azureeventhub/eph.go b/x-pack/filebeat/input/azureeventhub/eph.go index 8848483c8be..fa63b8f46f3 100644 --- a/x-pack/filebeat/input/azureeventhub/eph.go +++ b/x-pack/filebeat/input/azureeventhub/eph.go @@ -23,7 +23,11 @@ func (a *azureInput) runWithEPH() error { if err != nil { return err } - leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(cred, a.config.SAName, a.config.SAContainer, azure.PublicCloud) + env, err := getAzureEnvironment(a.config.OverrideEnvironment) + if err != nil { + return err + } + leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(cred, a.config.SAName, a.config.SAContainer, env) if err != nil { return err } @@ -74,3 +78,22 @@ func (a *azureInput) runWithEPH() error { } return nil } + +func getAzureEnvironment(overrideResManager string) (azure.Environment, error) { + // if no overrride is set then the azure public cloud is used + if overrideResManager == "" { + return azure.PublicCloud, nil + } + // users can select from one of the already defined azure cloud envs + var environments = map[string]azure.Environment{ + azure.ChinaCloud.ResourceManagerEndpoint: azure.ChinaCloud, + azure.GermanCloud.ResourceManagerEndpoint: azure.GermanCloud, + azure.PublicCloud.ResourceManagerEndpoint: azure.PublicCloud, + azure.USGovernmentCloud.ResourceManagerEndpoint: azure.USGovernmentCloud, + } + if env, ok := environments[overrideResManager]; ok { + return env, nil + } + // can retrieve hybrid env from the resource manager endpoint + return azure.EnvironmentFromURL(overrideResManager) +} diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index 7509037c28e..557c96664b8 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -3,16 +3,18 @@ activitylogs: enabled: true var: - # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: "insights-operational-logs" - # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" - # the name of the storage account the state/offsets will be stored and updated. + # the name of the storage account the state/offsets will be stored and updated storage_account: "" - #The storage account key, this key will be used to authorize access to data in your storage account. + # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint + override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -22,6 +24,7 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -30,3 +33,4 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index 5bf7bb576d0..fe5b3235ab3 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -38,6 +38,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -47,6 +48,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" signinlogs: enabled: false @@ -56,6 +58,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + override_resource_manager_endpoint: "" ``` @@ -85,6 +88,10 @@ The name of the storage account the state/offsets will be stored and updated. _string_ The storage account key, this key will be used to authorize access to data in your storage account. +`override_resource_manager_endpoint` :: +_string_ +Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. + include::../include/what-happens.asciidoc[] include::../include/gs-link.asciidoc[] diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index b89bebb30f9..e4f8da96b8a 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -4,3 +4,4 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} +override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml index 7375b6e42a4..e7078b075f3 100644 --- a/x-pack/filebeat/module/azure/activitylogs/manifest.yml +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -10,6 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key + - name: override_resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index 01796611504..b01a4d7f8ed 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -4,4 +4,5 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} +override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/auditlogs/manifest.yml b/x-pack/filebeat/module/azure/auditlogs/manifest.yml index d6cd469718b..0f193cf6b0d 100644 --- a/x-pack/filebeat/module/azure/auditlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/auditlogs/manifest.yml @@ -10,6 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key + - name: override_resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index b89bebb30f9..e4f8da96b8a 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -4,3 +4,4 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} +override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/signinlogs/manifest.yml b/x-pack/filebeat/module/azure/signinlogs/manifest.yml index f68109af4a1..7f82837828b 100644 --- a/x-pack/filebeat/module/azure/signinlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/signinlogs/manifest.yml @@ -10,6 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key + - name: override_resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index c8003fbcf96..25beff24479 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -6,16 +6,18 @@ activitylogs: enabled: true var: - # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: "insights-operational-logs" - # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" - # the name of the storage account the state/offsets will be stored and updated. + # the name of the storage account the state/offsets will be stored and updated storage_account: "" - #The storage account key, this key will be used to authorize access to data in your storage account. + # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint + override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -25,6 +27,7 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -33,3 +36,4 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" + # override_resource_manager_endpoint: "" From f1dc7da81c7e20b84fb9dcb9231158aed146a125 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 10 Apr 2020 12:19:39 +0200 Subject: [PATCH 2/5] changelog --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/azureeventhub/eph.go | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 55de0e08f3b..ac4d631e1c4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -250,6 +250,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add dashboard for Google Cloud Audit and AWS CloudTrail. {pull}17379[17379] - Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}17491[17491] - Release Google Cloud module as GA. {pull}17511[17511] +- Add config option to select a different azure cloud env in the azure-eventhub input and azure module. {issue}17649[17649] {pull}17659[17659] *Heartbeat* diff --git a/x-pack/filebeat/input/azureeventhub/eph.go b/x-pack/filebeat/input/azureeventhub/eph.go index fa63b8f46f3..bab54a45223 100644 --- a/x-pack/filebeat/input/azureeventhub/eph.go +++ b/x-pack/filebeat/input/azureeventhub/eph.go @@ -16,6 +16,14 @@ import ( "github.com/Azure/go-autorest/autorest/azure" ) +// users can select from one of the already defined azure cloud envs +var environments = map[string]azure.Environment{ + azure.ChinaCloud.ResourceManagerEndpoint: azure.ChinaCloud, + azure.GermanCloud.ResourceManagerEndpoint: azure.GermanCloud, + azure.PublicCloud.ResourceManagerEndpoint: azure.PublicCloud, + azure.USGovernmentCloud.ResourceManagerEndpoint: azure.USGovernmentCloud, +} + // runWithEPH will consume ingested events using the Event Processor Host (EPH) https://github.com/Azure/azure-event-hubs-go#event-processor-host, https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-event-processor-host func (a *azureInput) runWithEPH() error { // create a new Azure Storage Leaser / Checkpointer @@ -84,13 +92,6 @@ func getAzureEnvironment(overrideResManager string) (azure.Environment, error) { if overrideResManager == "" { return azure.PublicCloud, nil } - // users can select from one of the already defined azure cloud envs - var environments = map[string]azure.Environment{ - azure.ChinaCloud.ResourceManagerEndpoint: azure.ChinaCloud, - azure.GermanCloud.ResourceManagerEndpoint: azure.GermanCloud, - azure.PublicCloud.ResourceManagerEndpoint: azure.PublicCloud, - azure.USGovernmentCloud.ResourceManagerEndpoint: azure.USGovernmentCloud, - } if env, ok := environments[overrideResManager]; ok { return env, nil } From 33dd8f335797e571f634fc989a9ebf1e3f7a88bf Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 20 Apr 2020 13:48:56 +0200 Subject: [PATCH 3/5] rename --- filebeat/docs/modules/azure.asciidoc | 14 ++++++++++---- .../docs/inputs/input-azure-eventhub.asciidoc | 10 ++++++++-- x-pack/filebeat/filebeat.reference.yml | 4 ---- x-pack/filebeat/input/azureeventhub/config.go | 2 +- x-pack/filebeat/module/azure/_meta/config.yml | 4 ---- x-pack/filebeat/module/azure/_meta/docs.asciidoc | 14 ++++++++++---- .../azure/activitylogs/config/azure-eventhub.yml | 2 +- .../module/azure/activitylogs/manifest.yml | 2 +- .../azure/auditlogs/config/azure-eventhub.yml | 2 +- .../filebeat/module/azure/auditlogs/manifest.yml | 2 +- .../azure/signinlogs/config/azure-eventhub.yml | 2 +- .../filebeat/module/azure/signinlogs/manifest.yml | 2 +- x-pack/filebeat/modules.d/azure.yml.disabled | 4 ---- 13 files changed, 35 insertions(+), 29 deletions(-) diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 8fa93a584b2..b194b7c320c 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -43,7 +43,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" auditlogs: enabled: false @@ -53,7 +53,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" signinlogs: enabled: false @@ -63,7 +63,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" ``` @@ -93,9 +93,15 @@ The name of the storage account the state/offsets will be stored and updated. _string_ The storage account key, this key will be used to authorize access to data in your storage account. -`override_resource_manager_endpoint` :: +`resource_manager_endpoint` :: _string_ Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. +Ex: +https://management.chinacloudapi.cn/ for azure ChinaCloud +https://management.microsoftazure.de/ for azure GermanCloud +https://management.azure.com/ for azure PublicCloud +https://management.usgovcloudapi.net/ for azure USGovernmentCloud +Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints. include::../include/what-happens.asciidoc[] diff --git a/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc b/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc index f5954de0c52..ac91fb476d6 100644 --- a/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc @@ -28,7 +28,7 @@ Example configuration: storage_account: "azureeph" storage_account_key: "....." storage_account_container: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" ---- @@ -62,7 +62,13 @@ The storage account key, this key will be used to authorize access to data in yo Optional, the name of the storage account container you would like to store the offset information in. -==== `override_resource_manager_endpoint` +==== `resource_manager_endpoint` Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. +Ex: +https://management.chinacloudapi.cn/ for azure ChinaCloud +https://management.microsoftazure.de/ for azure GermanCloud +https://management.azure.com/ for azure PublicCloud +https://management.usgovcloudapi.net/ for azure USGovernmentCloud +Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 1674d6d5e60..7a522fdd41e 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -298,8 +298,6 @@ filebeat.modules: storage_account: "" # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" - # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint - override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -309,7 +307,6 @@ filebeat.modules: # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -318,7 +315,6 @@ filebeat.modules: # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" #--------------------------------- CEF Module --------------------------------- - module: cef diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index bfd60b99a3f..0521d3a76e6 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -18,7 +18,7 @@ type azureInputConfig struct { SAKey string `config:"storage_account_key"` SAContainer string `config:"storage_account_container"` // by default the azure public environment is used, to override, users can provide a specific resource manager endpoint - OverrideEnvironment string `config:"override_resource_manager_endpoint"` + OverrideEnvironment string `config:"resource_manager_endpoint"` } const ephContainerName = "filebeat" diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index 557c96664b8..ab7f477b8bb 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -13,8 +13,6 @@ storage_account: "" # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" - # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint - override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -24,7 +22,6 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -33,4 +30,3 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index fe5b3235ab3..eea82995532 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -38,7 +38,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" auditlogs: enabled: false @@ -48,7 +48,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" signinlogs: enabled: false @@ -58,7 +58,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" - override_resource_manager_endpoint: "" + resource_manager_endpoint: "" ``` @@ -88,9 +88,15 @@ The name of the storage account the state/offsets will be stored and updated. _string_ The storage account key, this key will be used to authorize access to data in your storage account. -`override_resource_manager_endpoint` :: +`resource_manager_endpoint` :: _string_ Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. +Ex: +https://management.chinacloudapi.cn/ for azure ChinaCloud +https://management.microsoftazure.de/ for azure GermanCloud +https://management.azure.com/ for azure PublicCloud +https://management.usgovcloudapi.net/ for azure USGovernmentCloud +Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints. include::../include/what-happens.asciidoc[] diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index e4f8da96b8a..9b747e1092d 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -4,4 +4,4 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} -override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml index e7078b075f3..4d5c20a7271 100644 --- a/x-pack/filebeat/module/azure/activitylogs/manifest.yml +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -10,7 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key - - name: override_resource_manager_endpoint + - name: resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index b01a4d7f8ed..3c2ea50cf8b 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -4,5 +4,5 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} -override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/auditlogs/manifest.yml b/x-pack/filebeat/module/azure/auditlogs/manifest.yml index 0f193cf6b0d..095371bff16 100644 --- a/x-pack/filebeat/module/azure/auditlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/auditlogs/manifest.yml @@ -10,7 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key - - name: override_resource_manager_endpoint + - name: resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index e4f8da96b8a..9b747e1092d 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -4,4 +4,4 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} -override_resource_manager_endpoint: {{ .override_resource_manager_endpoint }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/signinlogs/manifest.yml b/x-pack/filebeat/module/azure/signinlogs/manifest.yml index 7f82837828b..97fddae51e9 100644 --- a/x-pack/filebeat/module/azure/signinlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/signinlogs/manifest.yml @@ -10,7 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key - - name: override_resource_manager_endpoint + - name: resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index 25beff24479..0c7eb3d6e01 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -16,8 +16,6 @@ storage_account: "" # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" - # by default the azure public environment is used, to override, users can provide a specific resource manager endpoint - override_resource_manager_endpoint: "" auditlogs: enabled: false @@ -27,7 +25,6 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" signinlogs: enabled: false # var: @@ -36,4 +33,3 @@ # connection_string: "" # storage_account: "" # storage_account_key: "" - # override_resource_manager_endpoint: "" From 2fafce9521f3c0431314a616b88e063dbfc1caf9 Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 20 Apr 2020 14:14:17 +0200 Subject: [PATCH 4/5] add test --- x-pack/filebeat/input/azureeventhub/eph_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/x-pack/filebeat/input/azureeventhub/eph_test.go b/x-pack/filebeat/input/azureeventhub/eph_test.go index 3a0ac99db7d..be6db60b9fc 100644 --- a/x-pack/filebeat/input/azureeventhub/eph_test.go +++ b/x-pack/filebeat/input/azureeventhub/eph_test.go @@ -5,6 +5,7 @@ package azureeventhub import ( + "github.com/Azure/go-autorest/autorest/azure" "testing" "github.com/stretchr/testify/assert" @@ -26,3 +27,17 @@ func TestRunWithEPH(t *testing.T) { err := input.runWithEPH() assert.Error(t, err, '7') } + +func TestGetAzureEnvironment(t *testing.T) { + resMan := "" + env, err := getAzureEnvironment(resMan) + assert.NoError(t, err) + assert.Equal(t, env, azure.PublicCloud) + resMan = "https://management.microsoftazure.de/" + env, err = getAzureEnvironment(resMan) + assert.NoError(t, err) + assert.Equal(t, env, azure.GermanCloud) + resMan = "http://management.invalidhybrid.com/" + env, err = getAzureEnvironment(resMan) + assert.Errorf(t, err, "invalid character 'F' looking for beginning of value") +} From c244182bf5462e3445e7dc7bbe03103d51090c2a Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 20 Apr 2020 15:00:56 +0200 Subject: [PATCH 5/5] mage fmt update --- x-pack/filebeat/input/azureeventhub/eph_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/azureeventhub/eph_test.go b/x-pack/filebeat/input/azureeventhub/eph_test.go index be6db60b9fc..b48499eb7c4 100644 --- a/x-pack/filebeat/input/azureeventhub/eph_test.go +++ b/x-pack/filebeat/input/azureeventhub/eph_test.go @@ -5,9 +5,10 @@ package azureeventhub import ( - "github.com/Azure/go-autorest/autorest/azure" "testing" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/stretchr/testify/assert" )