From 2f48b2d879a4409b57e00645e34d1e25d0cfcbd7 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 28 Feb 2020 07:31:29 -0700 Subject: [PATCH] [Metricbeat] Add vpc metricset for aws module (#16111) (#16677) * Add vpn into vpc metricset * Add NATGateway and TrasitGateway (cherry picked from commit 96100df8bec05a23506d7fcf37151b35f913c077) --- CHANGELOG.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 6 + metricbeat/docs/modules/aws.asciidoc | 4 + metricbeat/docs/modules/aws/vpc.asciidoc | 24 ++++ metricbeat/docs/modules_list.asciidoc | 3 +- x-pack/metricbeat/module/aws/fields.go | 2 +- x-pack/metricbeat/module/aws/module.yml | 1 + .../metricbeat/module/aws/vpc/_meta/data.json | 74 +++++++++++++ .../module/aws/vpc/_meta/docs.asciidoc | 103 ++++++++++++++++++ .../module/aws/vpc/_meta/fields.yml | 6 + x-pack/metricbeat/module/aws/vpc/manifest.yml | 44 ++++++++ .../module/aws/vpc/vpc_integration_test.go | 46 ++++++++ x-pack/metricbeat/module/aws/vpc/vpc_test.go | 21 ++++ 13 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 metricbeat/docs/modules/aws/vpc.asciidoc create mode 100644 x-pack/metricbeat/module/aws/vpc/_meta/data.json create mode 100644 x-pack/metricbeat/module/aws/vpc/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/aws/vpc/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/aws/vpc/manifest.yml create mode 100644 x-pack/metricbeat/module/aws/vpc/vpc_integration_test.go create mode 100644 x-pack/metricbeat/module/aws/vpc/vpc_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 1e82aa3bec7..907229fb82d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -179,6 +179,7 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit - Release cloudwatch, s3_daily_storage, s3_request, sqs and rds metricset as GA. {pull}14114[14114] {issue}14059[14059] - Add `elasticsearch/enrich` metricset. {pull}14243[14243] {issue}14221[14221] - Add new dashboards for Azure vms, vm guest metrics, vm scale sets {pull}14000[14000] +- Add vpc metricset for aws module. {pull}16111[16111] {issue}14854[14854] *Functionbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 61d7d568316..b3e0d9c0b3e 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3084,6 +3084,12 @@ type: keyword `usage` contains the metrics from Cloudwatch to track usage of some AWS resources. +[float] +=== vpc + +`vpc` contains the metrics from Cloudwatch to track usage of VPC related resources. + + [[exported-fields-azure]] == azure fields diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 6f157672faf..23ed3c60150 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -330,6 +330,8 @@ The following metricsets are available: * <> +* <> + include::aws/billing.asciidoc[] include::aws/cloudwatch.asciidoc[] @@ -356,3 +358,5 @@ include::aws/sqs.asciidoc[] include::aws/usage.asciidoc[] +include::aws/vpc.asciidoc[] + diff --git a/metricbeat/docs/modules/aws/vpc.asciidoc b/metricbeat/docs/modules/aws/vpc.asciidoc new file mode 100644 index 00000000000..ceb8f93f2cb --- /dev/null +++ b/metricbeat/docs/modules/aws/vpc.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-vpc]] +=== aws vpc metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/vpc/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/vpc/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 53abbe84bbe..9e849355c67 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -16,7 +16,7 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.13+| .13+| |<> beta[] +.14+| .14+| |<> beta[] |<> |<> beta[] |<> @@ -29,6 +29,7 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |<> |<> beta[] +|<> beta[] |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | .4+| .4+| |<> beta[] |<> beta[] diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index 6b6e1185766..294bebcb31a 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzsXUtz27iy3udXdM1mkpStc8887iKLW+XEvue4yjOT2JmaJQcEWhSOQYDBw7JS8+Nv4UGKop6USDlTdbOYcZES8H2NRqPR3YAu4REX74DMzSsAy63Ad/AdmZvvXgEwNFTzynIl38H/vAIA+JPMzZ9QKuYEAlVCILUGrv54gFJJbpXmsoASrebUwFSrMrz7IJRjc2LpbPIKQKNAYvAdFOQVwJSjYOZdaP0SJCnxHVD/+QmhVDlpJ/5ZeA1gFxW+84jnSrP0bANK/+/zDGM7kNoJbYPSQAQnBpxBBlYBZygtny6A8ekUNUoL/oHlaIBLIFA6YfmlRUnCqyeulSxR2skK5CjAJcZCK1ftQtjm3W7IksJM3jaP6/ZU/h+ktvU4Psg2SaTzOitJVXFZpM9+9/a71ue2SC9IkBS+YXgiwiFUhOs0pGRuQKNRTlM0kzUG5sdJ7ugjrozcttHbg+HXMGZTIPDwI6RW1zpkvERpuJLfiOB+CfrfhrUG+fu3kzRLJm8nb7/viZoplwscA7QBOyMWNFqnJbI43svpC1cfb+GLQ71Yp5RzIbgs1qi0Z8IeDH+mNv4EqqQlXHo4CGgsL4lFBnRGdIEGpkrDQjkdrEs9v7nsGJr6X2NwcrSk9XzbFKRNKyeRWTbT4VO2RT1HjWCoJlUt7sZi/hFEPp9xOls2sMHOGm+08rYF8zxMRVamZ9fwbpNCWxJNOytvt8/kPSKBZJebZsFUSPmUI4P5DGVUrZb8gVR8w3xfSFIqlp80OnUjZxob/8Xr0OX1+1N0E3NzEm3MzRkZ37x/6K+ADVX6w2lU6Q/npPrhh9PmGq3cxCpLxKRasfxL8oYSgSybCkW6Hzhg0lWoKUpLirigCqFosKk3H34AqsrKWQQnuU3iIRqBOu3NiVh42+oMgpJBjlwaSyTFyVYiVCPjNnOGFJtth1ArS8WBHKQrc9Qe/4ePv0PsxHgjEsehjS2sEf5TznLBvxLf7F68ORH+u6MgRhJW1DbwKGi5xDwjxi9n2iEDw/0TbmFODAjiJJ0h8/6rsURbZNvJGKcr4Ux2BlKpq1VGM/KEkCPK5cgQCU4KXnKvcQ3dYPP91z58/P1DaOF9xJp8Tm7gK2p1KFOTRf+guyINRDVw2UjYzxWprPeRGTA1l57y+nhfAJEsmRU7c35/QZ32siGMcY+CiOTibKYs0c6VfpxwOamI94XNKExT26CRIn/ySie9vai7By4t6qn3LrqT7lDYWYU6M0hHhV+hBoNUSRYNtXJ2KCbK2bOMwIi4/+5DwOUkX1g8WP5TpUti38GmL/XiFhoYY26Ehkcdlgi9NShDs/D69ZKjMsZ8eYFhGYoG4+aRq4lGws42LO/T9CDJqfb4mwXfWKURnpRwJRogT4QLkgsEq45hM/CgtJC3xmIcEnPNLZ55THyfFqXHOSafUUalxt4amDFoBN1S1VCL+gdVVgK9yxu0SlWowz7EHK9VMSa9DJtUqLli3opYXh5GbuAB2k5yiFl0At+ok2OMZmi5zfRIXRyC3GijuUby9Ll3DF9jiXVmQmdIH7Mp4WKw7d09Vkpb43ehdoZ6FanfiVfEGGSQKztbfRkxQcAU9nT+rVkYi+XqOx7jJYIYCyWXzh5OMovtnZnrGETqfl6AyuYRO5RMs2RQpf1/nNwcl/NuWYH65HCW0im1sX+9at7ykhQ44ZvnxNEB+tvrMCk9DN9+kyyNYag++JZR04kfgwETCbeScUqCc5A0gaENGtcO1XIDKL0t2hIva4BWmj8RixMmTdbJWw4g0NQ6XP/6kPLQUbxrnv2BKHm1WRO7j3tAu/349BMQxjQaA8QYRXmID895Mn+9sbpccDqWQEPja/I8UCsTtAGlWAsu4bjxxoVTuP3YvHntBfwGcuXiAnqMSMMUmlDFNkvzaEMU2u3K8AKIAQL//O/LnFtw0vBChuht6OQgpMOP+0ak8LpCyfx0/wu0kzL+ZWbOWi6LyxCR/Qss6pLLoNN/eY8lpMnrP5G92cPIzrx/G/0tb6rHWgpSP8HdqpeF9RwoitPSnyjOmfm8uduc9DwoDyhImTNyEtvYxBkJ34UOT0n0anZaolezcyZ676+PSPTCUNnPOjSSUpxHrCYrudG+WcP/z3KeO8vJiCU5MZhRJSXSsD8dhU7dEbQ6SsnwLcjyZr8zIVoOt/hdleSrknCf6u5CZdzrq/tf3wQVQEJn3mTsB0UFMZtldRSsD20L0/bE6pICvz0usVR6AZRUhHK7gICh/uD1+32xuRb6VK3J15bYISgQP6z60riqEhzZcvCXvU7g84yb1gO/wfAsnORfHIZ6yaDvzSd8s70oxq3qcPQeUrgl4kwlHW0/ipuG6faQU/bFocOMYWVnG7EdWZWynGrKWS+B4Mbd/mbgtXeD/hGjUBq/ODTWvIE54d6nCxEoSr1f7Vl5hJux18GULyIzqJ9QZ6RAabP/qHwcixE7hIdPd/AQOoQr3yH4DoG5sIIeFH2YakS/cc3i7DlrXo2UoaJSTVuxPE0kU2Ut9QRqK/LMWKVJcb4cxzbYCQeEesPNeEvyzEtXZs4gy6wm0pBg6TPOhtSR1A20eoDb67pkxsSKGY9hAlfBAoW48kdlbKHx4dPdZvBKMDQ201gJTsP6nxmhbCZIMSnzAeELUhReeQ3/2hj51GvzLriZyoRaXL/dCkb+j6u7YGCabHMvft4KZFztDXUfaX/IEwb1aC353DzGVMbtP37bHP/ehjQII0i+R0C+1nnmYkenqL3lJQKBe4/+Po1Na/Hx4+T1bMbrmHX0JdrrU3tsflk8fLq7gF+I5uT6fSxfWo7XSjdbPA8zJ1X0j1/IEHgAce7HIGaqYFxhHJb0uKvxy7lUtmU/vHe1NOabWbZthlCFyQqUuHE0T5mAQTFbVPxWoGVKfMe9ZlZYWs8/teKK3nNufXGo+eHqcxS61AfgM1Jnke0FxZAwoejjuLCaXuq8ReOW7sMX83FhWXup2ZcW31pf45kNp5VesUsXnlrobBeRmMDmPRK6w1UXcSHqBHdHc2NmG6hwxqJOUC/8YqD8/hWIhZ8vo58Xw7xPROymGVO7L8Izzs0wTTs0U8B6CJrBPRSKEvHCTmKtnavG3mJZKU30Aqx/ZsKq543rPi0VquAyJD2dHtlUpU1G6BGI9ZDtfiNqZ1q5YlY5O6GqLPnmONtg1j720cfKtwAyFLglWzjcchT6aOx+H3RMjAvt+vqu2fT2AlaODIxLg9qaC3AVIxZTTXuUZC+ksaFzgD1mgFNiblB4jd2ps37L/mKNSVONG9cU76N7/67k1sb0OxUcpTXxdAGdrdTVeOucVtbgtvv1NVnrpeE6QgRZQjWkKLikqvT7xdf3sfE3S5loMp1yusFP9yyocCE+FMRFnbGqRL10iOove9HV8dLrh+Zx8EK8iW8lM0iogm72zgdLpR6ZIcWinC1UEMvn1PrfRy7eNRpjMnf2tpY8pirQdSdlL0aDArcklwYzObGPY0xONKjjoot9HIMueIbjgsu7VcphiPdhFMSipIu+Hs2QUZcEIUyhNacnGN+SC8Eji23OY6LRx7MYi0MI1jGccsljZIHIwvmxen19ffem8Uv6MuvhmozFbKf30pNPTwdmXEr1lO7JoZfVHoDBUEa9xt/Too81BqtGv+cY9LT7Y3FYXRp6cui3OnyDitRzuzma5V3ZkR44CCE9m2LsPASgXyie0gpQK0pdxWPQL+eS6EUIodTua0n8vmQ91xAjbHpnSqFFt5v0GjbhtSHe3uoQfIcw5QL7Rd1b8Ltpg9Hhn5QuaH3ZTPz/t+wJh4px1ZUK7X5TbN5vUJQEIusd77JSo94R73Vt22xyoejjoNcGrNNZodGN5De3CCQk+1MPrYIRlmdpo5+NUR5zZMFLHSlOp30oESLauLQBXWYB0if3E9VKDFhNfP0efIMGBH9E+OP+9vPNPSgN9zdX1zf3F0MCR1lwiQNXwd8QOltJ7monk+xjfxeRWTeJ20rgeucXLd1MgASeWVpSslZ2e8h50k1d62XWutaguqJ7KftwJCEuGFSVFbE854LbxY789s6xSlQLoXIiMpY3CwuyrMmS9lpT955daRmvf4Vu4ToZg4tYStfJyXTSMUuAMVho40GO0i+0oR4XC2/kN2Zt0im8YF1WP3+gdLzZigGwKeozy2WpMBqZ8qtY3K7WcHRbItHN6AjkJOptjyNU2AzF/H+VPpy6IEW8O6eBI4t6S7tLHw50KBPr1PhkRJ6peOQ0fitZ5GPYZSV5Ho5hu9RrlVKOdo4oN4KPttib9PX0eO0udCL6x1HlcmCqXH4LVHNCH60m9DGjMyILzDRSpZmZUI1xuuptu+yTr++ou4bYNaSuIXSNDNQTapjyJ0z1nq37K/etTFtphRPXg3qs1DoiDqG1UsxxOIE5l0zNJ7GfQfc58dZLiitaZ4ku0LZYxP6b49qJb/f9oSzEttjfqdrk/aB0ZmIHTO+Fm5IIEQ5Ak92UvbYRKPhTDI4ccOQ+1EWkwiFCH12VabTev1cyiy0Muux7CYTDPy0rEvttajSaDGZ9FNm4qlI6CqlSXNpLLi+DE6kxXkcwRWKdxuAtriZIl0r7vak7agjuVIQV0RhJKjNTPa88GlAWVEnjyjgbiRA1vRpXtDNkw5YlFNtzhuHyvV4CoITOMJtxmwVXdJI7P/sG5L56FKupgWh2yKGKn9XnoGL3EdVhgDUaJ2xmcMjp2w/0fYBg0O7CnfaMrgrzdNirp7ph09rYrJzQCuXoae8VFuGd669mJrMqSx5HFfeY5ovIjqyK7hlCLVoAe7iO99cP7f1ww98qUOFKAakYNuGavQudq+qKtixWDGbxSONL2Qc//eM5zoVyMb4UCxnbK8KB8Yw0sqmmVODUjkROY0l42PC3DnGEMGZ9NUa3CLFEYpwOR9O79XnLO+kzRrhY1OPzqou1z9HabmOdc7bhXTMYY566ffjxtEO36UJ9w7++VAlm2Ls3+hqd2hif6F7238YdvaVMTbN48f3wc6t1LC32sAFbnEVCNEMdjjVu0b60Jpyqd6mZlsalJ9+0ntULYjzhPeJg/fvz54/L5bckLJhy7wHF2G3zIxIXoLEgmon6zo5FtWUdbrAXg3oMHcz/uvncwe2Vq9Y9Ljdx2IO3ciPi/fj74Hh3pGAHgXx9c3fz+WZo1LNtFRSDYP73zdX1Qfq8TxeUGVMZfnvoasNRKHdUc5yKc4nk4ebu5sNn+C0Mejj77Q3dwFoRmWSGEinPfPimW09XL7IJS8ydHCyOU9jXPyLzTdBvftHmDPwFH3O2re4ufV/pvoUA3cRfS9qFk6m5FIqwlxmZOCxLDGGyHbZkz2ferYnnjk2lZMj3U+FYyDnnim05j+6ql6ZbI4hjltyukO2MzpvHftHfcqLWSpvJT8/P46nbT8/P6dxB7K65SlExPGjc4owj6Scd1BSQh631f4HS8M+dxH4ek9jPz88xLqPPSKyuN5tybWzmlaNHNub0qrMK9WWtcyH000REaLr3damS6HcDzZGU+BseayKwKkZbViZluLon1BTl2Bje3fIIjny9uzmrSFCQysSKmy2iCWMVJvJSHCmxHi7xCG/Cdmnf3G32g/K0u72MPOfdXg+/br7b68CLzMyXE8l+OSvZTydeZJbu4yjRGFJgRopewdsBCkurSqvn8Ht5kOLRXmYRFkglL+NGi0GCWEc3wwU/Wy5IiZ8MezSyGC7tuGqV615WAC1j6KnvkMBbv6JBIwknoHhZIuPEotjiDDRcpLLZEzc831JUduoi09BpGHAJU8GL2ZbVvEF2FlRd8VnN8YmIpdk7UB+8Ko2LtNbXXshqSz0utGZXkS+AEiGac/LpXOMvaYrFus89kM361YVDjzljce0iu2SIZWUX9bHPcS7J6ojn6uNtLT4/VxiPMzxKF0hNYEtKFuXS3J49lF3fhtRTxvHVsAWhD58eks1cabfZBZlT8x6hhS1rcfeHYv08CDUI4UvhZKQqMSyxG346eL9X8X8BAAD//waVdZg=" + return "eJzsXVtz4zayfp9f0ZWXzEzZ2rO5nId5OFUey2fXVU7isScnjwwEtCisQYCDi2VN5cefwoUURV0pkfJs1c5D4iIl4PsajUajuwFdwhMuPgCZmzcAlluBH+A7MjffvQFgaKjmpeVKfoD/eQMA8CeZmz+hUMwJBKqEQGoNXP3xCIWS3CrNZQ4FWs2pgalWRXh3LZRjc2LpbPQGQKNAYvAD5OQNwJSjYOZDaP0SJCnwA1D/+RGhVDlpR/5ZeA1gFyV+8IjnSrP0bANK/+/zDGM7kNoJbYPSQAQnBpxBBlYBZygtny6A8ekUNUoL/oHlaIBLIFA4YfmlRUnCq2eulSxQ2tEK5CjAJcZcK1fuQtjk3WzIktyM3tePq/bU5F9IbeNxfJBtkkjrdVaQsuQyT5/97v13jc9tkV6QIMl9w/BMhEMoCddpSMncgEajnKZoRmsMzI+jiaNPuDJy20ZvD4Zfw5hNgcDjj5BaXeuQ8QKl4Up+I4L7Jeh/E9Ya5O/fj9IsGb0fvf++I2qm3ETgEKAN2BmxoNE6LZHF8V5OX7i6v4UvDvVindKEC8FlvkalORP2YPgztfEnUCUt4dLDQUBjeUEsMqAzonM0MFUaFsrpYF2q+c1ly9BU/2qDM0FLGs+3TUFat3ISmWUzLT5FU9Rz1AiGalJW4q4t5h9B5PMZp7NlAxvsrPFGa9K0YJ6HKcnK9Gwb3m1SaEqibmfl7faZvEckkOxy3SyYEimfcmQwn6GMqtWQP5CSb5jvC0kKxSYnjU7VyJnGxn9xHLocfzxFN3FiTqKNE3NGxjcfH7srYE2V/nAaVfrDOale/3DaXKOlG1lliRiVK5Z/Sd5QIpBlU6FI+wMHTLoSNUVpSR4XVCEUDTb15voHoKoonUVwktskHqIRqNPenIiFt63OICgZ5MilsURSHG0lQjUybjNnSL7Zdgi1slQcyEG6YoLa47++/x1iJ8YbkTgOTWxhjfCfcpYL/pX4ZvfinRDhvzsIYiRhRW0Cj4KWS8wzYvxyph0yMNw/4RbmxIAgTtIZMu+/Gku0RbadjHG6FM5kZyCVulplNCPPCBNEuRwZIsFJwQvuNa6mG2y+/9r1/e/XoYWPEWvyObmBr6jVoUxNFv2D9orUE9XAZSNhP1ekst5HZsDUXHrK6+N9AUSyZFbszPn9BXXay4Ywxj0KIpKLs5myRDtX+mnE5agk3hc2gzBNbYNGivzZK5309qLqHri0qKfeu2hPukNhZyXqzCAdFH6JGgxSJVk01MrZvpgoZ88yAgPi/ncfAi5Hk4XFg+U/Vbog9gNs+lInbqGBIeZGaHjQYYnQG4PSNwuvX685KkPMl1cYlr5oMG6euBppJOxsw/IxTQ+SnGqPv17wjVUa4VkJV6AB8ky4IBOBYNUxbHoelAbyxlgMQ2KuucUzj4nv06L0OIfkM8ioVNgbAzMEjaBbquxrUb9WRSnQu7xBq1SJOuxDzPFaFWPSy7BJiZor5q2I5cVh5HoeoO0k+5hFJ/CNOjnEaIaWm0yP1MU+yA02mmskT597x/A1llhnRnSG9CmbEi562949YKm0NX4XameoV5H6nXhJjEEGE2Vnqy8jJgiYwp7OvzULY7FYfcdjvEQQY6Hg0tnDSWaxvTNzHYJI1c8rUNk8YoeSqZcMqrT/j5Ob43LeLctRnxzOUjqlNvavV/VbXpAcR3zznDg6QH87DpPSw/Dt18nSGIbqgm8ZNR35MegxkXArGackOAdJExjaoHHNUC03gNLboi3xshpoqfkzsThi0mStvGUPAk2tw/jXx5SHjuJd8+wPRMnLzZrYftwB2u39809AGNNoDBBjFOUhPjznyfx1xuomgtOhBBoaX5PngVqZoPUoxUpwCceNNy6cwu19/eatF/A7mCgXF9BjRBqm0IgqtlmaRxui0G5bhhdADBD4+39fTrgFJw3PZYjehk4OQtr/uG9ECm9LlMxP979AOynjX2bmrOUyvwwR2b/Aoi64DDr9l/dYQpq8+hPZuz2M7Mz7t9Hf8qZ6qKUg9RPcrWpZWM+Bojgt/YninJnPm7vNSc+D8oCCFBNGTmIbmzgj4bvQ4SmJXs1OS/Rqds5E78P4iEQv9JX9rEIjKcV5xGqykhvtmjX8T5bz3FlORiyZEIMZVVIiDfvTQehUHUGjo5QM34JsUu93RkTL/ha/q4J8VRIeUt1dqIx7e/Xw67ugAkjozJuM/aCoIGazrI6Cdd20ME1PrCop8NvjAgulF0BJSSi3CwgYqg+OP+6LzTXQp2pNvrbE9kGB+GHVl8aVpeDIloO/7HUEn2fcNB74DYZn4ST/4jDUSwZ9rz/hm+1EMW5V+6P3mMItEWcq6Wj6UdzUTLeHnLIvDh1mDEs724jtyKqU5VRTznoJBDfu9jcDb70b9LcYhdL4xaGx5h3MCfc+XYhAUer9as/KI9yMvQqmfBGZQf2MOiM5Spv9S02GsRixQ3j8dAePoUO48h2C7xCYCyvoQdGHqUb0G9cszp6z5tVIESoq1bQRy9NEMlVUUk+gtiLPjFWa5OfLcWyDnXBAqDfcjLcgL7xwReYMssxqIg0Jlj7jrE8dSd1Aowe4HVclMyZWzHgMI7gKFijEle+VsbnGx093m8ErwdDYTGMpOA3rf2aEspkg+aiY9AhfkDz3ymv419rIp17rd8HNVCbU4vrtVjDyf1zdBQNTZ5s78fNWIONqb6j7SPtDnjGoR2PJ5+YppjJu//bb5vj3NqRBGEHyHQLylc4zFzs6Re0tLxAIPHj0D2lsGouPHyevZzNexayjL9Fcn5pj88vi8dPdBfxCNCfjj7F8aTleK91s8TzMnJTRP34lQ+ABxLkfg5ipgnGFcVjS467GL+dS2Yb98N7V0phvZtm0GULlJstR4sbRPGUCBsVsUPFbgYYp8R13mllhaT3/1Iorese59cWh5oerz1HoUh+AL0idRbYXFEPChKJPw8Kqe6nyFrVbug9fzMeFZe21Zl9afCt9jWc2nFZ6xS5deGqhs11EYgKbd0jo9lddxIWoEtwtzY2ZbaDCGYs6Qb3wi4Hy+1cgFn6+jH5eDPM+E7GbZkztvgrPODfDNG3RTAHrPmgG91AoSsQrO4mVdq4ae4tFqTTRC7D+mQmrnjeu+7RUqJzLkPR0emBTlTYZoUcg1kO2+42onWnl8lnp7IiqouCb42y9WfvYRxcr3wDIUOCWbGF/y1Hoo7b7XdAxMSy08fiu3vR2AlYMDIxLg9qaC3AlIxZTTXuUZCeksaFzgD1mgFNirld4td2psn7L/mKNSV2NG9cU76N7/67g1sb0OxUcpTXxdAGdrdTVeOucVtbgtvv1NVnrpeE6QgRZQtWnKLikqvD7xbcPsfF3S5loMp1yusFP9yyocCE+FMRFnbGqQL10iKove9FV8dLxY/04eCHexDeSGSRUQdd754OlUo1Mn2JRzuYqiOVzav3fRy7eNRpiMrf2tpY8pSrQdSdlL0aDArckl3ozObGPY0xONKjDoot9HIMueIbDgpu0q5TDEO/DKIhFSRddPZo+oy4JQphCa05PML4FF4JHFtucx0Sji2cxFIcQrGM45ZLHyAKRufNj9XY8vntX+yVdmXVwTYZittN76cinowMzLKVqSnfk0Mlq98CgL6Ne4e9o0Ycag1Wj33EMOtr9oTisLg0dOXRbHb5BReq43RzM8q7sSA8chJCeTTF2HgLQrxRPaQSoFaWu5DHoN+GS6EUIoVTua0H8vmQ91xAjbHpnSqFBt5306jfhtSHe3ugQfIcw5QK7Rd0b8Ntpg8Hhn5QuaHzZjPz/t+wJ+4pxVZUKzX5TbN5vUJQEIqsd77JSo9oR73Vtm2wmQtGnXq8NWKezQqMdya9vEUhI9qceGgUjbJKljX42RHnMkQUvVaQ4nfahRIho49IGdJkFSJ/cT1Qr0WM18fgj+AYNCP6E8MfD7eebB1AaHm6uxjcPF30CR5lziT1Xwd8QOltJ7monk+xjfxeRWTuJ20jgeucXLd1MgASeWVpSskZ2u8950k5d62XWutKgqqJ7KftwJCEuGFQVJbF8wgW3ix357Z1jlajmQk2IyNikXliQZXWWtNOauvfsSsN4/SN0C+NkDC5iKV0rJ9NKxywBxmChjQc5Cr/QhnpczL2R35i1SafwgnVZ/fyB0vFmKwbApqjPLJelwmhkyq9icbtawdFNiUQ3oyWQk6g3PY5QYdMX8/9V+nDqguTx7pwajsyrLe0ufTjQoUysU+OjAXmm4pHT+K1kkY9hlxXkpT+GzVKvVUoTtHNEuRF8tMXepK+nxyt3oRXRP44qlz1T5fJboDoh9MlqQp8yOiMyx0wjVZqZEdUYp6vetss++fqOqmuIXUPqGkLXyEA9o4Ypf8ZU79m4v3LfyrSVVjhx3avHSq0j4hBaK8UchxOYc8nUfBT76XWfE2+9pLiidZboHG2DRey/Pq6d+LbfH8pCbIv9napN3g9KZyZ2wPReuCmIEOEANNlN2WsbgZw/x+DIAUfuQ11EKhwi9MmVmUbr/Xsls9hCr8u+l0A4/NOwIrHfukajzmBWR5GNK0ulo5BKxaW95PIyOJEa43UEUyTWaQze4mqCdKm035uqo5rgTkVYEY2RpDQz1fHKox5lQZU0roizkQhR0atwRTtDNmxZQrE9Zxgu3+skAEroDLMZt1lwRUcT52dfj9xXj2LVNRD1DjlU8bPqHFTsPqI6DLBG44TNDPY5fbuBfggQDNpduNOe0ZVhnvZ79VQ7bFoZm5UTWqEcPe29wiK8c/3VzGRWZcnjKOMe03wR2ZFV0R1DqHkDYAfX8WH82NwP1/ytAhWuFJCKYR2u2bvQubKqaMtixWAWjzS+ln3w0z+e41woF+NLsZCxuSIcGM9II5tqSgVO7UDkNBaEhw1/4xBHCGNWV2O0ixALJMbpcDS9XZ+3vJM+Y4SLRTU+b9pYuxytbTfWOmcb3tWDMeSp28cfTzt0my7UN/zra5Vghr17ra/RqY3xifZl/03c0VvK1DSLF9/3P7cax9JiDxuwxVkkRD3U4VjjFu1La8KpepeaaWhcevJN61m1IMYT3gMO1j8/f75fLr8FYcGUew8oxm7rH5G4AI050UxUd3Ysyi3rcI0979VjaGH+x83nFm6vXJXucbmJwx68pRsQ7/3vvePdkYLtBfL45u7m803fqGfbKih6wfzPm6vxQfq8TxeUGVIZfntsa8NRKHdUc5yKc4nk8ebu5voz/BYGPZz99oauZ62ITDJDiZRnPnzTrqerFtmEJeZODhbHKeyrH5H5JujXv2hzBv6CDznbVneXvq9030KAbuKvJe3CydRcCkXY64xMHJYlhjDZDluy5zPv1sRzx6ZUMuT7qXAs5Jwnim05j+7K16ZbIYhjltyukO2MzpvHftHdcqLWSpvRTy8vw6nbTy8v6dxB7K6+SlExPGjc4owj6Scd1BSQh631f4HS8PedxH4ektjPLy8xLqPPSKyqN5tybWzmlaNDNub0qrMS9WWlcyH0U0dEaLr3damS6HcD9ZGU+BseayKwKkZbViZluLon1BRNsDa8u+URHPlqd3NWkaAgpYkVN1tEE8YqTOSlOFJiPVziEd6E7dK+uVvvB+Vpd3sZec67vR5/3Xy314EXmZkvJ5L9clayn068yCzdx1GgMSTHjOSdgrc9FJaWpVYv4ffyIMWjvcwiLJBKXsaNFoMEsYpuhgt+tlyQEj8Z9mhk0V/acdUqV72sAFrG0FPfIYG3fkWDRhJOQPGiQMaJRbHFGai5SGWzZ274ZEtR2amLTE2nZsAlTAXPZ1tW8xrZWVC1xWc1x2cilmbvQH3wqjQs0kpfOyGrLPWw0OpdxWQBlAhRn5NP5xp/SVMs1n3ugWzWry7se8wZi2sX2SVDLEq7qI59DnNJVks8V/e3lfj8XGE8zvAoXSAVgS0pWZRLc3v2UHZ1G1JHGcdX/RaEPn56TDZzpd16F2ROzXuEFrasxe0fivXzINQghC+Fk5GqwLDEbvjp4O5exXNJT+LyXNKjmfzf/bXHS+JvXhxB5v8DAAD//wwxsIk=" } diff --git a/x-pack/metricbeat/module/aws/module.yml b/x-pack/metricbeat/module/aws/module.yml index d750f2bedd5..70c002a83d5 100644 --- a/x-pack/metricbeat/module/aws/module.yml +++ b/x-pack/metricbeat/module/aws/module.yml @@ -7,3 +7,4 @@ metricsets: - sns - lambda - dynamodb + - vpc diff --git a/x-pack/metricbeat/module/aws/vpc/_meta/data.json b/x-pack/metricbeat/module/aws/vpc/_meta/data.json new file mode 100644 index 00000000000..c07599fbce1 --- /dev/null +++ b/x-pack/metricbeat/module/aws/vpc/_meta/data.json @@ -0,0 +1,74 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "aws": { + "cloudwatch": { + "namespace": "AWS/NATGateway" + }, + "dimensions": { + "NatGatewayId": "nat-0a5cb7b9807908cc0" + }, + "natgateway": { + "metrics": { + "ActiveConnectionCount": { + "max": 0 + }, + "BytesInFromDestination": { + "sum": 0 + }, + "BytesInFromSource": { + "sum": 0 + }, + "BytesOutToDestination": { + "sum": 0 + }, + "BytesOutToSource": { + "sum": 0 + }, + "ConnectionAttemptCount": { + "sum": 0 + }, + "ConnectionEstablishedCount": { + "sum": 0 + }, + "ErrorPortAllocation": { + "sum": 0 + }, + "PacketsDropCount": { + "sum": 0 + }, + "PacketsInFromDestination": { + "sum": 0 + }, + "PacketsInFromSource": { + "sum": 0 + }, + "PacketsOutToDestination": { + "sum": 0 + }, + "PacketsOutToSource": { + "sum": 0 + } + } + } + }, + "cloud": { + "account": { + "id": "627959692251", + "name": "elastic-test" + }, + "provider": "aws", + "region": "us-west-2" + }, + "event": { + "dataset": "aws.vpc", + "duration": 115000, + "module": "aws" + }, + "metricset": { + "name": "vpc", + "period": 10000 + }, + "service": { + "type": "aws" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/vpc/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/vpc/_meta/docs.asciidoc new file mode 100644 index 00000000000..8ede70730b9 --- /dev/null +++ b/x-pack/metricbeat/module/aws/vpc/_meta/docs.asciidoc @@ -0,0 +1,103 @@ +The vpc metricset of aws module allows users to monitor VPC related services: NAT +gateway, transit gateway and VPN tunnels. +NAT gateway metric data can be used to monitor and troubleshoot NAT gateways and +the data is provided at 1-minute intervals to CloudWatch. +Transit gateway metrics are sent to CloudWatch by VPC only when requests are +flowing through the gateway. If there are requests flowing through the transit +gateway, Amazon VPC measures and sends its metrics in 1-minute intervals. +VPN metric data is automatically sent to CloudWatch as it becomes available. Users +can use these metrics to gain a better perspective on how the web application or +service is performing. + +[float] +=== AWS Permissions +Some specific AWS permissions are required for IAM user to collect usage metrics. +---- +ec2:DescribeRegions +cloudwatch:GetMetricData +cloudwatch:ListMetrics +tag:getResources +sts:GetCallerIdentity +iam:ListAccountAliases +---- + +[float] +=== Configuration example +[source,yaml] +---- +- module: aws + period: 1m + metricsets: + - vpc + # This module uses the aws cloudwatch metricset, all + # the options for this metricset are also available here. +---- + +[float] +=== Metrics and Dimensions for NAT gateway +Metrics: +|=== +|Metric Name|Statistic Method | Description +|ActiveConnectionCount | Max | The total number of concurrent active TCP connections through the NAT gateway. +|BytesInFromDestination | Sum | The number of bytes received by the NAT gateway from the destination. +|BytesInFromSource | Sum | The number of bytes received by the NAT gateway from clients in your VPC. +|BytesOutToDestination | Sum | The number of bytes sent out through the NAT gateway to the destination. +|BytesOutToSource | Sum | The number of bytes sent through the NAT gateway to the clients in your VPC. +|ConnectionAttemptCount | Sum | The number of connection attempts made through the NAT gateway. +|ConnectionEstablishedCount | Sum | The number of connections established through the NAT gateway. +|ErrorPortAllocation | Sum | The number of times the NAT gateway could not allocate a source port. +|IdleTimeoutCount | Sum | The number of connections that transitioned from the active state to the idle state. +|PacketsDropCount | Sum | The number of packets dropped by the NAT gateway. +|PacketsInFromDestination | Sum | The number of packets received by the NAT gateway from the destination. +|PacketsInFromSource | Sum | The number of packets received by the NAT gateway from clients in your VPC. +|PacketsOutToDestination | Sum | The number of packets sent out through the NAT gateway to the destination. +|PacketsOutToSource | Sum | The number of packets sent through the NAT gateway to the clients in your VPC. +|=== + +Dimensions: +|=== +|Dimension Name| Description +|NatGatewayId | Filter the metric data by the NAT gateway ID. +|=== + +Please see https://docs.aws.amazon.com/vpc/latest/userguide/vpc-nat-gateway-cloudwatch.html[NAT Gateway CloudWatch Metrics] for more details. + +[float] +=== Metrics and Dimensions for Transit gateway +Metrics: +|=== +|Metric Name|Statistic Method | Description +|BytesIn | Sum | The number of bytes received by the transit gateway. +|BytesOut | Sum | The number of bytes sent from the transit gateway. +|PacketsIn | Sum | The number of packets received by the transit gateway. +|PacketsOut | Sum | The number of packets sent by the transit gateway. +|PacketDropCountBlackhole | Sum | The number of packets dropped because they matched a blackhole route. +|PacketDropCountNoRoute | Sum | The number of packets dropped because they did not match a route. +|=== + +Dimensions: +|=== +|Dimension Name| Description +|TransitGateway | Filters the metric data by transit gateway. +|=== + +Please see https://docs.aws.amazon.com/vpc/latest/tgw/transit-gateway-cloudwatch-metrics.html[Transit Gateway Metrics] for more details. + +[float] +=== Metrics and Dimensions for VPN +Metrics: +|=== +|Metric Name|Statistic Method | Description +|TunnelState | Max | The state of the tunnel. For static VPNs, 0 indicates DOWN and 1 indicates UP. For BGP VPNs, 1 indicates ESTABLISHED and 0 is used for all other states. +|TunnelDataIn| Sum | The bytes received through the VPN tunnel. +|TunnelDataOut| Sum | The bytes sent through the VPN tunnel. +|=== + +Dimensions: +|=== +|Dimension Name| Description +|VpnId | Filters the metric data by the Site-to-Site VPN connection ID. +|TunnelIpAddress | Filters the metric data by the IP address of the tunnel for the virtual private gateway. +|=== + +Please see https://docs.aws.amazon.com/vpn/latest/s2svpn/monitoring-cloudwatch-vpn.html[VPN Tunnel CloudWatch Metrics] for more details. diff --git a/x-pack/metricbeat/module/aws/vpc/_meta/fields.yml b/x-pack/metricbeat/module/aws/vpc/_meta/fields.yml new file mode 100644 index 00000000000..dadb3adc5f7 --- /dev/null +++ b/x-pack/metricbeat/module/aws/vpc/_meta/fields.yml @@ -0,0 +1,6 @@ +- name: vpc + type: group + description: > + `vpc` contains the metrics from Cloudwatch to track usage of VPC related resources. + release: beta + fields: diff --git a/x-pack/metricbeat/module/aws/vpc/manifest.yml b/x-pack/metricbeat/module/aws/vpc/manifest.yml new file mode 100644 index 00000000000..71f9c2e9361 --- /dev/null +++ b/x-pack/metricbeat/module/aws/vpc/manifest.yml @@ -0,0 +1,44 @@ +default: true +input: + module: aws + metricset: cloudwatch + defaults: + metrics: + - namespace: AWS/VPN + statistic: ["Average"] + name: + - TunnelState + - namespace: AWS/VPN + statistic: ["Sum"] + name: + - TunnelDataIn + - TunnelDataOut + - namespace: AWS/TransitGateway + statistic: ["Sum"] + name: + - BytesIn + - BytesOut + - PacketsIn + - PacketsOut + - PacketDropCountBlackhole + - PacketDropCountNoRoute + - namespace: AWS/NATGateway + statistic: ["Sum"] + name: + - BytesInFromDestination + - BytesInFromSource + - BytesOutToDestination + - BytesOutToSource + - ConnectionAttemptCount + - ConnectionEstablishedCount + - ErrorPortAllocation + - IdleTimeoutCount + - PacketsDropCount + - PacketsInFromDestination + - PacketsInFromSource + - PacketsOutToDestination + - PacketsOutToSource + - namespace: AWS/NATGateway + statistic: ["Maximum"] + name: + - ActiveConnectionCount diff --git a/x-pack/metricbeat/module/aws/vpc/vpc_integration_test.go b/x-pack/metricbeat/module/aws/vpc/vpc_integration_test.go new file mode 100644 index 00000000000..c3bc08bdeda --- /dev/null +++ b/x-pack/metricbeat/module/aws/vpc/vpc_integration_test.go @@ -0,0 +1,46 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package vpc + +import ( + "fmt" + "testing" + + "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/x-pack/metricbeat/module/aws/mtest" +) + +func TestData(t *testing.T) { + namespaceIs := func(namespace string) func(e common.MapStr) bool { + return func(e common.MapStr) bool { + v, err := e.GetValue("aws.cloudwatch.namespace") + return err == nil && v == namespace + } + } + + dataFiles := []struct { + namespace string + path string + }{ + {"AWS/NATGateway", "./_meta/data.json"}, + {"AWS/VPN", "./_meta/data_vpn.json"}, + {"AWS/TransitGateway", "./_meta/data_transit_gateway.json"}, + } + + config, info := mtest.GetConfigForTest("vpc", "300s") + if info != "" { + t.Skip("Skipping TestData: " + info) + } + + for _, df := range dataFiles { + metricSet := mbtest.NewFetcher(t, config) + t.Run(fmt.Sprintf("namespace: %s", df.namespace), func(t *testing.T) { + metricSet.WriteEventsCond(t, df.path, namespaceIs(df.namespace)) + }) + } +} diff --git a/x-pack/metricbeat/module/aws/vpc/vpc_test.go b/x-pack/metricbeat/module/aws/vpc/vpc_test.go new file mode 100644 index 00000000000..634c4ce09cd --- /dev/null +++ b/x-pack/metricbeat/module/aws/vpc/vpc_test.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package vpc + +import ( + "os" + + "github.com/elastic/beats/metricbeat/mb" + + // Register input module and metricset + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/cloudwatch" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +}