From 1344e4f0cd2457342ac422f575143cd6c97623f8 Mon Sep 17 00:00:00 2001 From: yeer Date: Thu, 27 Apr 2017 21:27:42 +0800 Subject: [PATCH] Support Alibaba Cloud provider for add_cloud_metadata proccessor (#4111) * Support Alibaba Cloud provider for add_cloud_metadata processor * Add note for Alibaba Cloud provider in add_cloud_metadata processor documentation. --- CHANGELOG.asciidoc | 1 + libbeat/docs/processors-config.asciidoc | 19 +++++ .../add_cloud_metadata/add_cloud_metadata.go | 5 ++ .../provider_alibaba_cloud.go | 41 +++++++++++ .../provider_alibaba_cloud_test.go | 69 +++++++++++++++++++ .../provider_tencent_cloud.go | 3 +- 6 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 1edfbb55bf9..a077f61861b 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -294,6 +294,7 @@ https://github.com/elastic/beats/compare/v5.0.2...v5.1.1[View commits] - Add add_cloud_metadata processor for collecting cloud provider metadata. {pull}2728[2728] - Added decode_json_fields processor for decoding fields containing JSON strings. {pull}2605[2605] - Add Tencent Cloud provider for add_cloud_metadata processor. {pull}4023[4023] +- Add Alibaba Cloud provider for add_cloud_metadata processor. {pull}4111[4111] *Metricbeat* diff --git a/libbeat/docs/processors-config.asciidoc b/libbeat/docs/processors-config.asciidoc index b7d3cc79e9a..119cd809bb6 100644 --- a/libbeat/docs/processors-config.asciidoc +++ b/libbeat/docs/processors-config.asciidoc @@ -273,6 +273,7 @@ The following cloud providers are supported: - Digital Ocean - Google Compute Engine (GCE) - https://www.qcloud.com/?lang=en[Tencent Cloud] (QCloud) +- Alibaba Cloud (ECS) The simple configuration below enables the processor. @@ -359,6 +360,24 @@ _Tencent Cloud_ } ------------------------------------------------------------------------------- +_Alibaba Cloud_ + +This metadata is only available when VPC is selected as the network type of the ECS instance. + +[source,json] +------------------------------------------------------------------------------- +{ + "meta": { + "cloud": { + "availability_zone": "cn-shenzhen", + "instance_id": "i-wz9g2hqiikg0aliyun2b", + "provider": "ecs", + "region": "cn-shenzhen-a" + } + } +} +------------------------------------------------------------------------------- + [[add-locale]] === add_locale diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index d1b86e150fe..630cb8502a8 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -254,12 +254,17 @@ func setupFetchers(c common.Config) ([]*metadataFetcher, error) { if err != nil { return fetchers, err } + ecsFetcher, err := newAlibabaCloudMetadataFetcher(c) + if err != nil { + return fetchers, err + } fetchers = []*metadataFetcher{ doFetcher, ec2Fetcher, gceFetcher, qcloudFetcher, + ecsFetcher, } return fetchers, nil } diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go new file mode 100644 index 00000000000..f981c403322 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go @@ -0,0 +1,41 @@ +package add_cloud_metadata + +import "github.com/elastic/beats/libbeat/common" + +// Alibaba Cloud Metadata Service +// Document https://help.aliyun.com/knowledge_detail/49122.html +func newAlibabaCloudMetadataFetcher(c common.Config) (*metadataFetcher, error) { + ecsMetadataHost := "100.100.100.200" + ecsMetadataInstanceIDURI := "/latest/meta-data/instance-id" + ecsMetadataRegionURI := "/latest/meta-data/region-id" + ecsMetadataZoneURI := "/latest/meta-data/zone-id" + + ecsSchema := func(m map[string]interface{}) common.MapStr { + return common.MapStr(m) + } + + urls, err := getMetadataURLs(c, ecsMetadataHost, []string{ + ecsMetadataInstanceIDURI, + ecsMetadataRegionURI, + ecsMetadataZoneURI, + }) + if err != nil { + return nil, err + } + responseHandlers := map[string]responseHandler{ + urls[0]: func(all []byte, result *result) error { + result.metadata["instance_id"] = string(all) + return nil + }, + urls[1]: func(all []byte, result *result) error { + result.metadata["region"] = string(all) + return nil + }, + urls[2]: func(all []byte, result *result) error { + result.metadata["availability_zone"] = string(all) + return nil + }, + } + fetcher := &metadataFetcher{"ecs", nil, responseHandlers, ecsSchema} + return fetcher, nil +} diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go new file mode 100644 index 00000000000..5d4569aab4f --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go @@ -0,0 +1,69 @@ +package add_cloud_metadata + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/stretchr/testify/assert" +) + +func initECSTestServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/latest/meta-data/instance-id" { + w.Write([]byte("i-wz9g2hqiikg0aliyun2b")) + return + } + if r.RequestURI == "/latest/meta-data/region-id" { + w.Write([]byte("cn-shenzhen")) + return + } + if r.RequestURI == "/latest/meta-data/zone-id" { + w.Write([]byte("cn-shenzhen-a")) + return + } + + http.Error(w, "not found", http.StatusNotFound) + })) +} + +func TestRetrieveAlibabaCloudMetadata(t *testing.T) { + if testing.Verbose() { + logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) + } + + server := initECSTestServer() + defer server.Close() + + config, err := common.NewConfigFrom(map[string]interface{}{ + "host": server.Listener.Addr().String(), + }) + + if err != nil { + t.Fatal(err) + } + + p, err := newCloudMetadata(*config) + if err != nil { + t.Fatal(err) + } + + actual, err := p.Run(common.MapStr{}) + if err != nil { + t.Fatal(err) + } + + expected := common.MapStr{ + "meta": common.MapStr{ + "cloud": common.MapStr{ + "provider": "ecs", + "instance_id": "i-wz9g2hqiikg0aliyun2b", + "region": "cn-shenzhen", + "availability_zone": "cn-shenzhen-a", + }, + }, + } + assert.Equal(t, expected, actual) +} diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go index aec4d9022e8..d182a7e5336 100644 --- a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go @@ -2,7 +2,8 @@ package add_cloud_metadata import "github.com/elastic/beats/libbeat/common" -// Tenccent Cloud Metadata Service +// Tencent Cloud Metadata Service +// Document https://www.qcloud.com/document/product/213/4934 func newQcloudMetadataFetcher(c common.Config) (*metadataFetcher, error) { qcloudMetadataHost := "metadata.tencentyun.com" qcloudMetadataInstanceIDURI := "/meta-data/instance-id"