From 67d2e665f40cdcbbbcd7b2a83ae0a344515ab54b Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Tue, 27 Apr 2021 14:40:02 -0700 Subject: [PATCH 01/10] add detailed network summary stats to docker/memory --- go.mod | 2 +- go.sum | 4 +- .../metric/system/network/helpers.go | 10 +- metricbeat/docs/fields.asciidoc | 57 ++++ metricbeat/docs/modules/docker.asciidoc | 2 + metricbeat/metricbeat.reference.yml | 2 + .../module/docker/_meta/config.reference.yml | 2 + metricbeat/module/docker/_meta/config.yml | 2 + metricbeat/module/docker/fields.go | 2 +- .../module/docker/network/_meta/data.json | 283 ++++++++++++++++-- .../module/docker/network/_meta/fields.yml | 25 ++ metricbeat/module/docker/network/config.go | 37 +++ metricbeat/module/docker/network/data.go | 75 +++-- metricbeat/module/docker/network/helper.go | 128 +++++++- metricbeat/module/docker/network/network.go | 24 +- .../network/network_integration_test.go | 7 +- .../system/network_summary/network_summary.go | 3 +- metricbeat/modules.d/docker.yml.disabled | 2 + 18 files changed, 588 insertions(+), 79 deletions(-) rename metricbeat/module/system/network_summary/data.go => libbeat/metric/system/network/helpers.go (86%) create mode 100644 metricbeat/module/docker/network/config.go diff --git a/go.mod b/go.mod index 72e071aea01..e2405ee10c2 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 github.com/elastic/go-seccomp-bpf v1.1.0 github.com/elastic/go-structform v0.0.8 - github.com/elastic/go-sysinfo v1.6.0 + github.com/elastic/go-sysinfo v1.7.0 github.com/elastic/go-txfile v0.0.7 github.com/elastic/go-ucfg v0.8.3 github.com/elastic/go-windows v1.0.1 // indirect diff --git a/go.sum b/go.sum index ee234fb91d8..d32e37ea280 100644 --- a/go.sum +++ b/go.sum @@ -301,6 +301,8 @@ github.com/elastic/go-structform v0.0.8/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZN github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-sysinfo v1.6.0 h1:u0QbU8eWSwKRPcFQancnSY4Zi0COksCJXkUgPHxE5Tw= github.com/elastic/go-sysinfo v1.6.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= +github.com/elastic/go-sysinfo v1.7.0 h1:4vVvcfi255+8+TyQ7TYUTEK3A+G8v5FLE+ZKYL1z1Dg= +github.com/elastic/go-sysinfo v1.7.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-txfile v0.0.7 h1:Yn28gclW7X0Qy09nSMSsx0uOAvAGMsp6XHydbiLVe2s= github.com/elastic/go-txfile v0.0.7/go.mod h1:H0nCoFae0a4ga57apgxFsgmRjevNCsEaT6g56JoeKAE= github.com/elastic/go-ucfg v0.7.0/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= @@ -767,9 +769,9 @@ github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b h1:X/8hkb4rQq3+QuOxpJK7gWmAXmZucF0EI1s1BfBLq6U= github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b/go.mod h1:jAqhj/JBVC1PwcLTWd6rjQyGyItxxrhpiBl8LSuAGmw= diff --git a/metricbeat/module/system/network_summary/data.go b/libbeat/metric/system/network/helpers.go similarity index 86% rename from metricbeat/module/system/network_summary/data.go rename to libbeat/metric/system/network/helpers.go index 95b530c9325..e13ad0e2ad7 100644 --- a/metricbeat/module/system/network_summary/data.go +++ b/libbeat/metric/system/network/helpers.go @@ -15,15 +15,15 @@ // specific language governing permissions and limitations // under the License. -package network_summary +package network import ( "github.com/elastic/beats/v7/libbeat/common" sysinfotypes "github.com/elastic/go-sysinfo/types" ) -// eventMapping maps the network counters to a MapStr that wil be sent to report.Event -func eventMapping(raw *sysinfotypes.NetworkCountersInfo) common.MapStr { +// MapProcNetCounters converts the NetworkCountersInfo struct into a MapStr acceptable for sending upstream +func MapProcNetCounters(raw *sysinfotypes.NetworkCountersInfo) common.MapStr { eventByProto := common.MapStr{ "ip": combineMap(raw.Netstat.IPExt, raw.SNMP.IP), @@ -33,10 +33,6 @@ func eventMapping(raw *sysinfotypes.NetworkCountersInfo) common.MapStr { "icmp": combineMap(raw.SNMP.ICMPMsg, raw.SNMP.ICMP), } - // if value, ok := raw.SNMP.TCP["MaxConn"]; ok && value != 0 { - // eventByProto["tcp"].(map) - // } - return eventByProto } diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 4c2278437ab..fed5e2508ff 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -9915,6 +9915,63 @@ type: long -- +[float] +=== network_summary + +Summed network summary statistics of every PID associated with the container + + + +*`docker.network.network_summary.ip.*`*:: ++ +-- +IP counters + + +type: object + +-- + +*`docker.network.network_summary.tcp.*`*:: ++ +-- +TCP counters + + +type: object + +-- + +*`docker.network.network_summary.udp.*`*:: ++ +-- +UDP counters + + +type: object + +-- + +*`docker.network.network_summary.udp_lite.*`*:: ++ +-- +UDP Lite counters + + +type: object + +-- + +*`docker.network.network_summary.icmp.*`*:: ++ +-- +ICMP counters + + +type: object + +-- + [[exported-fields-dropwizard]] == Dropwizard fields diff --git a/metricbeat/docs/modules/docker.asciidoc b/metricbeat/docs/modules/docker.asciidoc index 56783db3868..e7ea865e04b 100644 --- a/metricbeat/docs/modules/docker.asciidoc +++ b/metricbeat/docs/modules/docker.asciidoc @@ -50,6 +50,8 @@ metricbeat.modules: # If set to true, replace dots in labels with `_`. #labels.dedot: false + # Set to true to enable low-level network stats for the processes in the container + # network.network_summary: true # If set to true, collects metrics per core. #cpu.cores: true diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 93d75700a69..26ed39d6f11 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -241,6 +241,8 @@ metricbeat.modules: # If set to true, replace dots in labels with `_`. #labels.dedot: false + # Set to true to enable low-level network stats for the processes in the container + # network.network_summary: true # If set to true, collects metrics per core. #cpu.cores: true diff --git a/metricbeat/module/docker/_meta/config.reference.yml b/metricbeat/module/docker/_meta/config.reference.yml index f651b1385af..d8c3f1b27f2 100644 --- a/metricbeat/module/docker/_meta/config.reference.yml +++ b/metricbeat/module/docker/_meta/config.reference.yml @@ -15,6 +15,8 @@ # If set to true, replace dots in labels with `_`. #labels.dedot: false + # Set to true to enable low-level network stats for the processes in the container + # network.network_summary: true # If set to true, collects metrics per core. #cpu.cores: true diff --git a/metricbeat/module/docker/_meta/config.yml b/metricbeat/module/docker/_meta/config.yml index 340f2106bfc..aa8fa058405 100644 --- a/metricbeat/module/docker/_meta/config.yml +++ b/metricbeat/module/docker/_meta/config.yml @@ -13,6 +13,8 @@ # If set to true, replace dots in labels with `_`. #labels.dedot: false + # Set to true to enable low-level network stats for the processes in the container + # network.network_summary: true # To connect to Docker over TLS you must specify a client and CA certificate. #ssl: diff --git a/metricbeat/module/docker/fields.go b/metricbeat/module/docker/fields.go index 994f6bd7cb1..9760b1fac67 100644 --- a/metricbeat/module/docker/fields.go +++ b/metricbeat/module/docker/fields.go @@ -32,5 +32,5 @@ func init() { // AssetDocker returns asset data. // This is the base64 encoded gzipped contents of module/docker. func AssetDocker() string { - return "eJzsXEuP47gRvvevKHQOARbTNhIscvAhwKZngmkks9OYR3IIAi9NlW3GFKkhKXs8vz7gQ7IsUZIfsrt70T6aVvFjPb4qkiXfwQq3E0gkXaG6ATDMcJzA7Vv3xe0NQIKaKpYZJsUE/noDAOAHQRtiNFDJOVKDCcyVTMPY6AZAIUeicQILcgOg0RgmFnoC/7nVmt/+1363lMpMqRRztpjAnHCNNwBzhjzREzfTHQiSYgWf/ZhtZqUqmWfhmwhG+3kQc6lSYr8GIhIHmGnDqAYyk7kJYv+oQeVCMLEAKoUhTKDSoyCliqaKqPxlORID1gGuoshSFqRoFKPl5Pazr8biU4e1Dy1NiUj2xgpwK9xupKqPdUC0n3svEMySGNgQDfgdaW5NzgSYJTbWMYrjUkgMxnElxOBxoN4Sg7BZokewU6HFF2aKw7BekOshtVNM7SXHZ2XZlCSJQq0xPjfLTp324RFK0S1LZj/q2o376nHLZT8w5rHQ4p9VREpKM53XNbEDxqVYRAZ7sLnPF2kI9+DkHAjnzkHmjKMu/LXFUfcAbi6B7XNAtUPkYmpJ1ggzRFF4LkgFdEnEAhPQTFD0A0yKuIENWQzo0Q8pWaCTOWryXpafw3ifcmFYinD/+HUYsluhEshHGTXR9WtKOCbTOZek/gOfGyaQoaIo6qM9Knr0D1k9WXPaJTERwIDOCMW4oQJcIVX6DDGDxUU4+4EJzLbOS0WezlDZB6zJqFRtHBNWZhhdxV0xEjZ9VPP4FZy8w3Srt9rgk6vVsY9H7hVstRigdcF+Di7Rgf0c1wgrHNo1AjAnNj5xrlE9tU6DJi2ULud1UJ+DDzTwnmN5t6qLUUKfTp0/X12fX8ooyjVZdEJ7EnvX8J1jXjs0+ql1BXL2P2wM+S+n1/TpfUZj2uPuWlGnYZ71st4MYM/2gO1f+vEh/eseujK4I4YqTwOYXjF51sab6RU8jD8OU4MqJPFd7Qnbq18ozdOcu02AlashyRUTC2dIzubl9iF2ANEGtApWZpfYde2MeBLoHbzZ1jQ2yL0Ai6Bqe/iABfzNPurAn45dNQ8xeqEfpVuaK4XCBB1nNvshlY2jnkrlhWrNKE4tTVwAmU8ljoOMLCaDh4+g8FuO2ug3NpIFEdLjbNqmALohzAyAEvpgFsBAZ1aRdlpraybgW445autKxUIOBu8ebRphKP3u6NtPVC6ilYzi5N2RlRLMFFJLOhP4y+jnUwn8IP8sTa5YI1wGoU0n+MXx5mmonwtxWvQGxQvgzqDnV/J8Jc+IJp1zPDF7dnto6Z15mhK1vVzdScRLpVJb2MsMlTswf7GU6mrRwggvg1srSn/l11d+bYJxx15PRK8NVou4aIET1/tnM6fe7NflHH+2MPRt9TuLKCa1vKgesHHAT8YSf09P1oRxMuMYnXeuZDr4MmWuaHw6K3u46b4s0T1u/cwfhwGmzHW8OI6u+8EOB6FW5mWQdM4q6/nijNKhKayvBGh4Wd+yD8BRLP/hbZEaDzPFnmKMUWyWd+X96Bko1M9Bz1rFv4hiMtdWyHhNeI4VXPtre2PZEUViVycFMKP3PbtY1xIJN0u6RLoagNYq0ponqHsPvK/8MiGGwIZxDlLwLcxwxwi+dyyptRFpyxsK7XL3hIbf/fb+3S///PL+/v27+3/8Bkxoo3IXTbAk2rdT5BoTm/1nOeOJU1t4lqW1q5njmXlOGGdioY1CsorGEhMGF426rMf+VAqau2rKToAJ1I12udxQNZaXDVQmcf6MhdHJDOKEBWUf20mEIplGusegq7XsAEh1faBI4pIqxlDmGkjcRN1YZG6yPEZRA3BTFUrLPKVpvjMzbXhQFUk8Qk5TSsNdy1xjY30A1nNyhrkxaimyTggdm/A8ME62ljJZgsKwOWs2t/VFUtjFXcZt4Ktg3/IC6w4kLNjaEnUWsle8z60KMyMXRPmwAxbyrAP8BtgcmLEe7TaOLl1tlowu/RY87H/94hKmkBq+dROiqFPaBdthXZOu3UmWfbEeUX9P7IANor570PVfepc81g/XTJm8sU2EofsvGyXAPgqFi5yTGDUN3KFqsRDOgRK6xMTD0kC0lpS54zgjm07WUm4V4DmZIT/1Cv+MnlE/bw+46zWrMjE/q03gQcxlQfgwI7aYtNWlMZmejMeJpHrk68kRlekYxYIJHCuco0JBcUwyNvbjU4WpNDglGZuu/zT688/jP4wTpjNOtne+je1uwxK8Y7s3Fs59B6CooYcK649rVM5N99rdjw7ujNia/AJRVT+O8hNF3uhoYgpvf1wBVPt7Jk1U2sgsu4qqwkwHoYqd4F0Ck8u0Xaq6xHlVqFGKXa7UJlpOxXE43o5iOb4nqlUbfpYm06WYyr3LoKO57oOTMEx565nhpxPzT0RdlfFpSrKMiUX48e1Pt8ep9hPZBG2FF9hcLecSrNOWDqMjO+o2KGpOWg4RqUxTNtgu+N5JM65vzx30CPg3E4nc1L2qj2NPitEB7q2818YFlPzfPCy5BrRHJKtgrl4Fl1AVWxOD041UK+twGs2o/QIjgr0Ldw/mMDeEuUGj6cU7J4yPqMxbzmU6b1g6wfydMJv3cxsLcRLmrC0OhlVLICk3XRyJ0oNVPJ8+f95jimNLnacNw4BcoXYpzHqQ23J0bKyjp9q9zgO9Xc0H4v7QgriQ2tpM7/rYh7L6V+1PeU63e0q+P4HVP5DvBerIewfw/Ozs3z5osy08t0CqKbVRgQk0lqzPKcF+9SJOr8HihWm0hDmrTi6AlqLdVC31efxyMx6ZZ1z0PwgqU5sqgyFCdbe75D82jJ+q+6de+7NiYU5me5Qkqnu/2BPaJyALM+4QZoSusEmYlRsBpWTjSAIG2z568e4i9GBI4QdX2NJ2Y6rc3VwnYD7mZiF/jwEji4U924ApET6fgDkc0vUCphvTLsHMZN7ydyenXF/E84j/F4b9vxpxN7H1K5WXEydDJZbhDP6aUC6TUAYNkJa88TsMkKESyfAB8ppATkwg/w8AAP//JPFoyQ==" + return "eJzsXEuP47gRvvevKHQOAQYzbiRY5OBDgE3PBGNkZ6cxj+QQBF6aKtsVS6SGpOzx/PoFSUmWJUryQ3Z3L9pH06r6WI+PRbLkN7DC7RgiyVeobgAMmRjHcPvWfXF7AxCh5opSQ1KM4e83AAB+ELRhRgOXcYzcYARzJZN8bHQDoDBGpnEMC3YDoNEYEgs9hv/eah3f/s9+t5TKTLkUc1qMYc5ijTcAc8I40mOn6Q0IlmAFn/2YbWqlKpml+TcBjPYzEXOpEma/BiYiB5i0Ia6BzWRmcrF/1qAyIUgsgEthGAlUepRLqaKpIip/WY6EgHWAqxiylAUJGkW8VG4/+2YsPnVY+9CShIlob6wAt8LtRqr6WAdE+7n3AsEsmYEN04DfkWfW5STALLExj1EYl0JmMIwrYgaPA/WWGYTNEj2CnQktvlxTGIaNgkwPaZ1CtZcc1krplEWRQq0xrJvSU9VOHqAU3TJl+lG3bjhWj5su/cBQxEJLfFYRKSnNdF63xA5YLMUiMNiDzX2+SMNiD07OgcWxC5A5xaiLeG0J1D2Am0tg+5yj2iFyObVka4QZoigiF6QCvmRigRFoEhz9AEkRdrBhiwEjepKwBTqZoybvpdk5jPcpE4YShPuHr8OQ3QqVwHiUchOcv+Ysxmg6jyWr/8CvDWNIUXEU9dEeEz34h6ydrDvtlEjkYECnjGPYUTlcIVXyBDGDxcVi+oERzLYuSkWWzFDZB6zLuFRtHJPPzBBfhUMxkDZ9VPPwFZy8w2yrt9rgo5vVsY9H7g1srZhD64L9FEKiA/s5oZHPcOjQyIE5sWHFmUb12DbNLWmhdAWvg/oUYqCB9xzPu1ldjBL6bOri+er2/FJmUabZohPao/i7hu8c99qh0avWGcjZ/7Ex5L+cXjOm9xmNtMfdNaNOxzzpab0ewJ/tCds/9eNT+tc9dGVyBxxVngaQXpE8a+NNegWTu4/D1KAKWXhXe8L26mfOsySL3SbAytUQZYrEwjkypnm5fQgdQLQBrYKV6SV2XTsnngR6B2+2NY0Nci/AIqnaHj5gAv+wjzrwp2NXzUOMXuhH2ZZnSqEwuY1Tu/ohl42jnkrlhWpNHKeWJi6AzC8ljoOMLJTB5CMo/JahNvq1zWTBhPQ4m74pgG4YmQFQQh/MAhjo1BrSqrW+JgHfMsxQ21AqJnIwePdo0wlD2XdH315ROYlWMgqTd8eqFGGqkFvSGcPfRj+dSuAHxWfpckWNdBmENp3gZ8ebp6F+KsRp0RsUz4A7czu/kOcLeQYs6YLjkdmzO0LL6MyShKnt5epOJp4rldrCXqao3IH5s6VUV4sWTnge3Fox+gu/vvBrE4w79nokem2wWiBEC5y43j+bOfVmvy7n+LOFoW+r31lEIanlRfWAjQNeGUX+np6tGcVsFmNQ71zJZPBpykzxsDorezh1X5boHrdx5o/DABNyHS+Oo+txsMPBuJV5GSSdWmV9vTijdGgK6ysBGlHWN+0DcBTTn7wtlsbDXLFnGGMUzbKudT94Bgr1c9CzZvFvpkhm2gq5W7M4wwqu/bm9tuyIIrKzkwLI6P3ILua1RBabJV8iXw1AaxVpzRPUvQfeV34ZMcNgQ3EMUsRbmOGOEXzvWFRrI9KWNxTa6e4JzX/32/t3P//y5f39+3f3//oNSGijMpdNsGTat1NkGiO7+s8yiiNntvxZSmpXM8cz85xRTGKhjUK2CuYSCYOLRl3W438uBc9cNWUVYAR1p11ubag6y8sGLqMwf4bS6GQGccJyYx/bSYQimga6x6CrtewASHV7oIjCkirOUOYaSJyibiwyM2kWoqgBuKkKpUVP6ZrvZKaNCKoiCWfIaUZphGu51thcH4D1nJxhboxaiqwTUscueB5YzLaWMilCYWhOzea2vkzKd3GXCRv4KuhbVmDdgYQFrS1Rp/nqFe5zq8JM2QVRTnbA8nXWAX4NNAcyNqLdxtEtV5sl8aXfguf7Xz+5iBRyE2+dQhR1SrtgO6xr0rU7ybIv1iPq74kdsEHUdw+6/ksfksfG4ZqUyRrbRBi6/7JRAuyjULjIYhaipoE7VC0WFsfAGV9i5GFpYFpLTu44zshmkLWUWwX4mM0wPvUK/4yeUa+3B9z1mlVJzM9qE5iIuSwIH2bMFpO2ujQm1eO7u0hyPfL15IjL5A7FggTeKZyjQsHxjqV058enChNpcMpSmq7/MvrrT3d/uotIpzHbvvFtbG82FOEb2r2xcO47AEUNPVRaf1yjcmG61+5+dHKnzNbkF8iq+nGUVxR4o6OJKX/74wqg2t8zaaLSRqbpVUyVazoIVegE7xKY3ErbZapLnFflNUqxy5XaBMupMA7H20Esx/dEtVrDa2kyXYKJ3LsMOprrPjgJw5S3nhlenbj+BMxVGZ8mLE1JLPIf3766Pc60n9gmt1b+Apur5dwC66yl89GRHXUbFDVnLYeIXCYJDbYLvnfSjOvbcwc9Av5DIpKbelT1cexJOTrAvZWP2rCAkv+bhyXXgPaAbJW7q9fAJVRFa2ZwupFqZQNOoxm1X2AEsHfh7sGc64ZcN2g0vXjnjOIRl1nLuUznDUsnmH8ysut+ZnMhTMIxteXBsGbJScqpCyNRerCK59Pnz3tMcWyp87hpmCNXqN0SZiPIbTk6NtbBU+3e4IHeruYDcX9oQVxIbW2md33sQ3n9q/anPKf7PWHfH8HrH9j3AnXgvQN4en72bx+0+RaeWiLVjNqowAQaS9bnlGC/ehGn12DhwjRYwpxVJxdAS9FOVUt9Hr7cDGfmGRf9E8FlYpfK3BF5dbe75D82jR+r+6de+1MxMSezPUsi1b1f7EntE5DlGncIU8ZX2CTMyo2AUrJxJAGDbR+9eHcRejCk/AdX2NJ2Y6rc3VwnYT5mZiH/iAkji4k92YQpET6dhDkc0vUSphvTboGZyazl705Oub4IryP+Xxj2/2rE3cTWr1SeT54MtbAM5/CXBeUyC8qgCdKybvwBE2SohWT4BHlZQAZcQPI4ng78RsXnLEkw2mWJl179EzLfoqm28DB5W73w3ZBZnvWCBaWNw3fo7188wMCTh+IcsP2E1/BLaf9yf4D6LLqU+q9vD1M/jcngBTH8Qgb7gRBPLhYF9x8qlvg9AAD//4xkL+0=" } diff --git a/metricbeat/module/docker/network/_meta/data.json b/metricbeat/module/docker/network/_meta/data.json index fba2fae388c..f9adc23a8f6 100644 --- a/metricbeat/module/docker/network/_meta/data.json +++ b/metricbeat/module/docker/network/_meta/data.json @@ -1,36 +1,48 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "container": { - "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a", + "id": "b1c3118fb2087e499c421aa20e28847ad4c185be6c1864166b1a03cc2affaf80", "image": { - "name": "metricbeat_elasticsearch" + "name": "docker.elastic.co/beats/elastic-agent:7.13.0-SNAPSHOT" }, - "name": "metricbeat_elasticsearch_1_df866b3a7b3d", + "name": "elastic-package-stack_elastic-agent_1", "runtime": "docker" }, "docker": { "container": { "labels": { - "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78", + "com_docker_compose_config-hash": "fdad7f67fafe5a242b9b9d27bff78bef4b96c683a2bb1412d91c32fb5981f2b3", "com_docker_compose_container-number": "1", "com_docker_compose_oneoff": "False", - "com_docker_compose_project": "metricbeat", - "com_docker_compose_service": "elasticsearch", - "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d", - "com_docker_compose_version": "1.23.1", + "com_docker_compose_project": "elastic-package-stack", + "com_docker_compose_project_config_files": "/home/alexk/.elastic-package/stack/openport/snapshot.yml", + "com_docker_compose_project_working_dir": "/home/alexk/.elastic-package/stack/openport", + "com_docker_compose_service": "elastic-agent", + "com_docker_compose_version": "1.27.4", + "description": "Agent manages other beats based on configuration provided.", + "io_k8s_description": "Agent manages other beats based on configuration provided.", + "io_k8s_display-name": "Elastic-Agent image", "license": "Elastic License", - "org_label-schema_build-date": "20181006", - "org_label-schema_license": "GPLv2", - "org_label-schema_name": "elasticsearch", + "maintainer": "infra@elastic.co", + "name": "elastic-agent", + "org_label-schema_build-date": "2021-04-21T06:45:43Z", + "org_label-schema_license": "Elastic License", + "org_label-schema_name": "elastic-agent", "org_label-schema_schema-version": "1.0", - "org_label-schema_url": "https://www.elastic.co/products/elasticsearch", - "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker", + "org_label-schema_url": "https://www.elastic.co/beats/elastic-agent", + "org_label-schema_vcs-ref": "c52c43963ca8416dc92c7d3dbf6cb6e89dd00acf", + "org_label-schema_vcs-url": "github.com/elastic/beats/v7", "org_label-schema_vendor": "Elastic", - "org_label-schema_version": "6.5.1" + "org_label-schema_version": "7.13.0-SNAPSHOT", + "org_opencontainers_image_created": "2021-04-21T06:45:43Z", + "org_opencontainers_image_licenses": "Elastic License", + "org_opencontainers_image_title": "Elastic-Agent", + "org_opencontainers_image_vendor": "Elastic", + "release": "1", + "summary": "elastic-agent", + "url": "https://www.elastic.co/beats/elastic-agent", + "vendor": "Elastic", + "version": "7.13.0-SNAPSHOT" } }, "network": { @@ -41,12 +53,236 @@ "packets": 0 }, "inbound": { - "bytes": 23047, + "bytes": 92092613, "dropped": 0, "errors": 0, - "packets": 241 + "packets": 424322 }, "interface": "eth0", + "network_summary": { + "icmp": { + "InAddrMaskReps": 0, + "InAddrMasks": 0, + "InCsumErrors": 0, + "InDestUnreachs": 0, + "InEchoReps": 0, + "InEchos": 0, + "InErrors": 0, + "InMsgs": 0, + "InParmProbs": 0, + "InRedirects": 0, + "InSrcQuenchs": 0, + "InTimeExcds": 0, + "InTimestampReps": 0, + "InTimestamps": 0, + "OutAddrMaskReps": 0, + "OutAddrMasks": 0, + "OutDestUnreachs": 0, + "OutEchoReps": 0, + "OutEchos": 0, + "OutErrors": 0, + "OutMsgs": 0, + "OutParmProbs": 0, + "OutRedirects": 0, + "OutSrcQuenchs": 0, + "OutTimeExcds": 0, + "OutTimestampReps": 0, + "OutTimestamps": 0 + }, + "ip": { + "ForwDatagrams": 0, + "FragCreates": 0, + "FragFails": 0, + "FragOKs": 0, + "InAddrErrors": 0, + "InBcastOctets": 0, + "InBcastPkts": 0, + "InCEPkts": 0, + "InCsumErrors": 0, + "InDelivers": 5108328, + "InDiscards": 0, + "InECT0Pkts": 0, + "InECT1Pkts": 0, + "InHdrErrors": 0, + "InMcastOctets": 0, + "InMcastPkts": 0, + "InNoECTPkts": 5108412, + "InNoRoutes": 0, + "InOctets": 702988830, + "InReceives": 5108328, + "InTruncatedPkts": 0, + "InUnknownProtos": 0, + "OutBcastOctets": 0, + "OutBcastPkts": 0, + "OutDiscards": 0, + "OutMcastOctets": 0, + "OutMcastPkts": 0, + "OutNoRoutes": 0, + "OutOctets": 11397546792, + "OutRequests": 6353418, + "ReasmFails": 0, + "ReasmOKs": 0, + "ReasmOverlaps": 0, + "ReasmReqds": 0, + "ReasmTimeout": 0 + }, + "tcp": { + "ActiveOpens": 3108, + "ArpFilter": 0, + "AttemptFails": 0, + "BusyPollRxPackets": 0, + "CurrEstab": 66, + "DelayedACKLocked": 0, + "DelayedACKLost": 6, + "DelayedACKs": 96156, + "EmbryonicRsts": 0, + "EstabResets": 0, + "IPReversePathFilter": 0, + "InCsumErrors": 0, + "InErrs": 0, + "InSegs": 5096568, + "ListenDrops": 0, + "ListenOverflows": 0, + "LockDroppedIcmps": 0, + "OfoPruned": 0, + "OutOfWindowIcmps": 0, + "OutRsts": 2916, + "OutSegs": 12028800, + "PAWSActive": 0, + "PAWSEstab": 60, + "PFMemallocDrop": 0, + "PassiveOpens": 24, + "PruneCalled": 0, + "RcvPruned": 0, + "RetransSegs": 6972, + "SyncookiesFailed": 0, + "SyncookiesRecv": 0, + "SyncookiesSent": 0, + "TCPACKSkippedChallenge": 0, + "TCPACKSkippedFinWait2": 0, + "TCPACKSkippedPAWS": 0, + "TCPACKSkippedSeq": 0, + "TCPACKSkippedSynRecv": 0, + "TCPACKSkippedTimeWait": 0, + "TCPAbortFailed": 0, + "TCPAbortOnClose": 0, + "TCPAbortOnData": 2892, + "TCPAbortOnLinger": 0, + "TCPAbortOnMemory": 0, + "TCPAbortOnTimeout": 0, + "TCPAckCompressed": 0, + "TCPAutoCorking": 3300, + "TCPBacklogCoalesce": 5928, + "TCPBacklogDrop": 0, + "TCPChallengeACK": 0, + "TCPDSACKIgnoredDubious": 0, + "TCPDSACKIgnoredNoUndo": 6420, + "TCPDSACKIgnoredOld": 0, + "TCPDSACKOfoRecv": 0, + "TCPDSACKOfoSent": 0, + "TCPDSACKOldSent": 6, + "TCPDSACKRecv": 6816, + "TCPDSACKRecvSegs": 6828, + "TCPDSACKUndo": 6, + "TCPDeferAcceptDrop": 0, + "TCPDelivered": 9321756, + "TCPDeliveredCE": 0, + "TCPFastOpenActive": 0, + "TCPFastOpenActiveFail": 0, + "TCPFastOpenBlackhole": 0, + "TCPFastOpenCookieReqd": 0, + "TCPFastOpenListenOverflow": 0, + "TCPFastOpenPassive": 0, + "TCPFastOpenPassiveAltKey": 0, + "TCPFastOpenPassiveFail": 0, + "TCPFastRetrans": 24, + "TCPFromZeroWindowAdv": 0, + "TCPFullUndo": 6, + "TCPHPAcks": 2200992, + "TCPHPHits": 590586, + "TCPHystartDelayCwnd": 0, + "TCPHystartDelayDetect": 0, + "TCPHystartTrainCwnd": 216, + "TCPHystartTrainDetect": 12, + "TCPKeepAlive": 702798, + "TCPLossFailures": 0, + "TCPLossProbeRecovery": 0, + "TCPLossProbes": 9390, + "TCPLossUndo": 0, + "TCPLostRetransmit": 0, + "TCPMD5Failure": 0, + "TCPMD5NotFound": 0, + "TCPMD5Unexpected": 0, + "TCPMTUPFail": 0, + "TCPMTUPSuccess": 0, + "TCPMemoryPressures": 0, + "TCPMemoryPressuresChrono": 0, + "TCPMinTTLDrop": 0, + "TCPOFODrop": 0, + "TCPOFOMerge": 0, + "TCPOFOQueue": 0, + "TCPOrigDataSent": 9311964, + "TCPPartialUndo": 6, + "TCPPureAcks": 885618, + "TCPRcvCoalesce": 60, + "TCPRcvCollapsed": 0, + "TCPRcvQDrop": 0, + "TCPRenoFailures": 0, + "TCPRenoRecovery": 0, + "TCPRenoRecoveryFail": 0, + "TCPRenoReorder": 0, + "TCPReqQFullDoCookies": 0, + "TCPReqQFullDrop": 0, + "TCPRetransFail": 0, + "TCPSACKDiscard": 0, + "TCPSACKReneging": 0, + "TCPSACKReorder": 912, + "TCPSYNChallenge": 0, + "TCPSackFailures": 0, + "TCPSackMerged": 30, + "TCPSackRecovery": 18, + "TCPSackRecoveryFail": 0, + "TCPSackShiftFallback": 594, + "TCPSackShifted": 0, + "TCPSlowStartRetrans": 0, + "TCPSpuriousRTOs": 0, + "TCPSpuriousRtxHostQueues": 12, + "TCPSynRetrans": 144, + "TCPTSReorder": 6, + "TCPTimeWaitOverflow": 0, + "TCPTimeouts": 144, + "TCPToZeroWindowAdv": 0, + "TCPWantZeroWindowAdv": 0, + "TCPWinProbe": 0, + "TCPWqueueTooBig": 0, + "TCPZeroWindowDrop": 0, + "TW": 18, + "TWKilled": 0, + "TWRecycled": 0, + "TcpDuplicateDataRehash": 0, + "TcpTimeoutRehash": 144 + }, + "udp": { + "IgnoredMulti": 0, + "InCsumErrors": 0, + "InDatagrams": 11760, + "InErrors": 0, + "NoPorts": 0, + "OutDatagrams": 11760, + "RcvbufErrors": 0, + "SndbufErrors": 0 + }, + "udp_lite": { + "IgnoredMulti": 0, + "InCsumErrors": 0, + "InDatagrams": 0, + "InErrors": 0, + "NoPorts": 0, + "OutDatagrams": 0, + "RcvbufErrors": 0, + "SndbufErrors": 0 + } + }, "out": { "bytes": 0, "dropped": 0, @@ -54,10 +290,10 @@ "packets": 0 }, "outbound": { - "bytes": 0, + "bytes": 1877416014, "dropped": 0, "errors": 0, - "packets": 0 + "packets": 631723 } } }, @@ -67,7 +303,8 @@ "module": "docker" }, "metricset": { - "name": "network" + "name": "network", + "period": 10000 }, "service": { "address": "/var/run/docker.sock", diff --git a/metricbeat/module/docker/network/_meta/fields.yml b/metricbeat/module/docker/network/_meta/fields.yml index 035047eb091..a4ba1807d7c 100644 --- a/metricbeat/module/docker/network/_meta/fields.yml +++ b/metricbeat/module/docker/network/_meta/fields.yml @@ -99,3 +99,28 @@ type: long description: > Total number of outgoing packets. + - name: network_summary + type: group + description: > + Summed network summary statistics of every PID associated with the container + fields: + - name: ip.* + type: object + description: > + IP counters + - name: tcp.* + type: object + description: > + TCP counters + - name: udp.* + type: object + description: > + UDP counters + - name: udp_lite.* + type: object + description: > + UDP Lite counters + - name: icmp.* + type: object + description: > + ICMP counters \ No newline at end of file diff --git a/metricbeat/module/docker/network/config.go b/metricbeat/module/docker/network/config.go new file mode 100644 index 00000000000..51ffc03e787 --- /dev/null +++ b/metricbeat/module/docker/network/config.go @@ -0,0 +1,37 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux darwin windows + +package network + +import "github.com/elastic/beats/v7/metricbeat/module/docker" + +//Config is the config object for docker/network +type Config struct { + TLS *docker.TLSConfig `config:"ssl"` + DeDot bool `config:"labels.dedot"` + NetworkSummary bool `config:"network.network_summary"` +} + +// DefaultConfig returns default module config +func DefaultConfig() Config { + return Config{ + DeDot: true, + NetworkSummary: false, + } +} diff --git a/metricbeat/module/docker/network/data.go b/metricbeat/module/docker/network/data.go index b4a2b90c405..f1a17ba7bdf 100644 --- a/metricbeat/module/docker/network/data.go +++ b/metricbeat/module/docker/network/data.go @@ -19,46 +19,55 @@ package network import ( "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/metric/system/network" "github.com/elastic/beats/v7/metricbeat/mb" ) -func eventsMapping(r mb.ReporterV2, netsStatsList []NetStats) { +func eventsMapping(r mb.ReporterV2, netsStatsList []NetStats, reportNetstat bool) { for i := range netsStatsList { - eventMapping(r, &netsStatsList[i]) + eventMapping(r, &netsStatsList[i], reportNetstat) } } -func eventMapping(r mb.ReporterV2, stats *NetStats) { - r.Event(mb.Event{ - RootFields: stats.Container.ToMapStr(), - MetricSetFields: common.MapStr{ - "interface": stats.NameInterface, - // Deprecated - "in": common.MapStr{ - "bytes": stats.RxBytes, - "dropped": stats.RxDropped, - "errors": stats.RxErrors, - "packets": stats.RxPackets, - }, - // Deprecated - "out": common.MapStr{ - "bytes": stats.TxBytes, - "dropped": stats.TxDropped, - "errors": stats.TxErrors, - "packets": stats.TxPackets, - }, - "inbound": common.MapStr{ - "bytes": stats.Total.RxBytes, - "dropped": stats.Total.RxDropped, - "errors": stats.Total.RxErrors, - "packets": stats.Total.RxPackets, - }, - "outbound": common.MapStr{ - "bytes": stats.Total.TxBytes, - "dropped": stats.Total.TxDropped, - "errors": stats.Total.TxErrors, - "packets": stats.Total.TxPackets, - }, +func eventMapping(r mb.ReporterV2, stats *NetStats, reportNetstat bool) { + + fields := common.MapStr{ + "interface": stats.NameInterface, + // Deprecated + "in": common.MapStr{ + "bytes": stats.RxBytes, + "dropped": stats.RxDropped, + "errors": stats.RxErrors, + "packets": stats.RxPackets, + }, + // Deprecated + "out": common.MapStr{ + "bytes": stats.TxBytes, + "dropped": stats.TxDropped, + "errors": stats.TxErrors, + "packets": stats.TxPackets, + }, + "inbound": common.MapStr{ + "bytes": stats.Total.RxBytes, + "dropped": stats.Total.RxDropped, + "errors": stats.Total.RxErrors, + "packets": stats.Total.RxPackets, + }, + "outbound": common.MapStr{ + "bytes": stats.Total.TxBytes, + "dropped": stats.Total.TxDropped, + "errors": stats.Total.TxErrors, + "packets": stats.Total.TxPackets, }, + } + + if reportNetstat { + summary := network.MapProcNetCounters(stats.Netstat) + fields["network_summary"] = summary + } + + r.Event(mb.Event{ + RootFields: stats.Container.ToMapStr(), + MetricSetFields: fields, }) } diff --git a/metricbeat/module/docker/network/helper.go b/metricbeat/module/docker/network/helper.go index 0f980fc2ab9..a8db7f518fa 100644 --- a/metricbeat/module/docker/network/helper.go +++ b/metricbeat/module/docker/network/helper.go @@ -18,17 +18,26 @@ package network import ( + "context" + "strconv" "time" "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/pkg/errors" "github.com/elastic/beats/v7/metricbeat/module/docker" + + sysinfo "github.com/elastic/go-sysinfo" + sysinfotypes "github.com/elastic/go-sysinfo/types" ) +// NetService represents maps out the interface-level stats per-container type NetService struct { NetworkStatPerContainer map[string]map[string]NetRaw } +// NetworkCalculator is the interface that reports per-second stats type NetworkCalculator interface { getRxBytesPerSecond(newStats *NetRaw, oldStats *NetRaw) float64 getRxDroppedPerSecond(newStats *NetRaw, oldStats *NetRaw) float64 @@ -40,6 +49,7 @@ type NetworkCalculator interface { getTxPacketsPerSecond(newStats *NetRaw, oldStats *NetRaw) float64 } +// NetRaw represents the raw network stats from docker type NetRaw struct { Time time.Time RxBytes uint64 @@ -52,6 +62,7 @@ type NetRaw struct { TxPackets uint64 } +// NetStats represents the network counters for a given network interface type NetStats struct { Time time.Time Container *docker.Container @@ -64,18 +75,129 @@ type NetStats struct { TxDropped float64 TxErrors float64 TxPackets float64 + Netstat *sysinfotypes.NetworkCountersInfo Total *types.NetworkStats } -func (n *NetService) getNetworkStatsPerContainer(rawStats []docker.Stat, dedot bool) []NetStats { +// NoSumStats lists "stats", often config/state values, that can't be safely summed across PIDs +var NoSumStats = []string{ + "RtoAlgorithm", + "RtoMin", + "RtoMax", + "MaxConn", + "Forwarding", + "DefaultTTL", +} + +func (n *NetService) getNetworkStatsPerContainer(client *client.Client, timeout time.Duration, rawStats []docker.Stat, cfg Config) ([]NetStats, error) { formattedStats := []NetStats{} for _, myStats := range rawStats { + + var stats *sysinfotypes.NetworkCountersInfo + var err error + if cfg.NetworkSummary { + stats, err = fetchContainerNetStats(client, timeout, myStats.Container.ID) + if err != nil { + return nil, errors.Wrap(err, "error fetching per-PID stats") + } + } + for nameInterface, rawnNetStats := range myStats.Stats.Networks { - formattedStats = append(formattedStats, n.getNetworkStats(nameInterface, rawnNetStats, myStats, dedot)) + singleStat := n.getNetworkStats(nameInterface, rawnNetStats, myStats, cfg.DeDot) + if cfg.NetworkSummary { + singleStat.Netstat = stats + } + formattedStats = append(formattedStats, singleStat) } } - return formattedStats + return formattedStats, nil +} + +// fetchContainerNetStats gathers the PIDs associated with a container, and then uses go-sysinfo to grab the /proc/[pid]/net counters and sum them across PIDs. +func fetchContainerNetStats(client *client.Client, timeout time.Duration, container string) (*sysinfotypes.NetworkCountersInfo, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + body, err := client.ContainerTop(ctx, container, []string{}) + if err != nil { + return &sysinfotypes.NetworkCountersInfo{}, errors.Wrap(err, "error fetching container pids") + } + + var pidPos int + for pos := range body.Titles { + if body.Titles[pos] == "PID" { + pidPos = pos + break + } + } + + summedMetrics := &sysinfotypes.NetworkCountersInfo{} + for _, pid := range body.Processes { + strPID := pid[pidPos] + intPID, err := strconv.Atoi(strPID) + if err != nil { + return &sysinfotypes.NetworkCountersInfo{}, errors.Wrap(err, "error converting PID to int") + } + + proc, err := sysinfo.Process(intPID) + procNet, ok := proc.(sysinfotypes.NetworkCounters) + if !ok { + break + } + + counters, err := procNet.NetworkCounters() + if err != nil { + return &sysinfotypes.NetworkCountersInfo{}, errors.Wrapf(err, "error fetching network counters for PID %d", intPID) + } + + summedMetrics = sumCounter(summedMetrics, counters) + + } + + return summedMetrics, nil + +} + +func sumCounter(totals, new *sysinfotypes.NetworkCountersInfo) *sysinfotypes.NetworkCountersInfo { + + newTotal := &sysinfotypes.NetworkCountersInfo{} + + newTotal.Netstat.IPExt = sumMapStr(new.Netstat.IPExt, totals.Netstat.IPExt) + newTotal.Netstat.TCPExt = sumMapStr(new.Netstat.TCPExt, totals.Netstat.TCPExt) + + newTotal.SNMP.IP = sumMapStr(new.SNMP.IP, totals.SNMP.IP) + newTotal.SNMP.ICMP = sumMapStr(new.SNMP.ICMP, totals.SNMP.ICMP) + newTotal.SNMP.ICMPMsg = sumMapStr(new.SNMP.ICMPMsg, totals.SNMP.ICMPMsg) + newTotal.SNMP.TCP = sumMapStr(new.SNMP.TCP, totals.SNMP.TCP) + newTotal.SNMP.UDP = sumMapStr(new.SNMP.UDP, totals.SNMP.UDP) + newTotal.SNMP.UDPLite = sumMapStr(new.SNMP.UDPLite, totals.SNMP.UDPLite) + return newTotal + +} + +func sumMapStr(m1, m2 map[string]uint64) map[string]uint64 { + + final := make(map[string]uint64) + for key, val := range m1 { + // skip over values that aren't counters and can't be summed across interfaces + // Most of these values are config settings, and it doesn't make sense to report them along with counters that aren't per-pid/per-interface + var skip = false + for _, name := range NoSumStats { + if key == name { + skip = true + } + } + if skip { + continue + } + // safety, make sure the field exists in m1 + if _, ok := m2[key]; !ok { + final[key] = val + } + final[key] = m2[key] + val + + } + return final } func (n *NetService) getNetworkStats(nameInterface string, rawNetStats types.NetworkStats, myRawstats docker.Stat, dedot bool) NetStats { diff --git a/metricbeat/module/docker/network/network.go b/metricbeat/module/docker/network/network.go index 04e67c95412..0ad1c6ca828 100644 --- a/metricbeat/module/docker/network/network.go +++ b/metricbeat/module/docker/network/network.go @@ -20,6 +20,8 @@ package network import ( + "runtime" + "github.com/docker/docker/client" "github.com/pkg/errors" @@ -34,32 +36,39 @@ func init() { ) } +// MetricSet for fetching docker network metrics. type MetricSet struct { mb.BaseMetricSet netService *NetService dockerClient *client.Client - dedot bool + cfg Config } // New creates a new instance of the docker network MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := docker.DefaultConfig() + config := DefaultConfig() if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - client, err := docker.NewDockerClient(base.HostData().URI, config) + client, err := docker.NewDockerClient(base.HostData().URI, docker.Config{DeDot: config.DeDot, TLS: config.TLS}) if err != nil { return nil, err } + // Network summary requres a linux procfs system under it to read from the cgroups. Disable reporting otherwise. + if runtime.GOOS != "linux" { + base.Logger().Debug("Not running on linux, docker network detailed stats disabled.") + config.NetworkSummary = false + } + return &MetricSet{ BaseMetricSet: base, dockerClient: client, netService: &NetService{ NetworkStatPerContainer: make(map[string]map[string]NetRaw), }, - dedot: config.DeDot, + cfg: config, }, nil } @@ -70,8 +79,11 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return errors.Wrap(err, "failed to get docker stats") } - formattedStats := m.netService.getNetworkStatsPerContainer(stats, m.dedot) - eventsMapping(r, formattedStats) + formattedStats, err := m.netService.getNetworkStatsPerContainer(m.dockerClient, m.Module().Config().Timeout, stats, m.cfg) + if err != nil { + return errors.Wrap(err, "error fetching container network stats") + } + eventsMapping(r, formattedStats, m.cfg.NetworkSummary) return nil } diff --git a/metricbeat/module/docker/network/network_integration_test.go b/metricbeat/module/docker/network/network_integration_test.go index 418d057e247..c9454aefa72 100644 --- a/metricbeat/module/docker/network/network_integration_test.go +++ b/metricbeat/module/docker/network/network_integration_test.go @@ -35,8 +35,9 @@ func TestData(t *testing.T) { func getConfig() map[string]interface{} { return map[string]interface{}{ - "module": "docker", - "metricsets": []string{"network"}, - "hosts": []string{"unix:///var/run/docker.sock"}, + "module": "docker", + "metricsets": []string{"network"}, + "network.network_summary": true, + "hosts": []string{"unix:///var/run/docker.sock"}, } } diff --git a/metricbeat/module/system/network_summary/network_summary.go b/metricbeat/module/system/network_summary/network_summary.go index bbb27d589b0..ff08d6731e0 100644 --- a/metricbeat/module/system/network_summary/network_summary.go +++ b/metricbeat/module/system/network_summary/network_summary.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/metric/system/network" "github.com/elastic/beats/v7/metricbeat/mb" sysinfo "github.com/elastic/go-sysinfo" sysinfotypes "github.com/elastic/go-sysinfo/types" @@ -70,7 +71,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } report.Event(mb.Event{ - MetricSetFields: eventMapping(counterInfo), + MetricSetFields: network.MapProcNetCounters(counterInfo), }) return nil diff --git a/metricbeat/modules.d/docker.yml.disabled b/metricbeat/modules.d/docker.yml.disabled index f414279857e..b3558345931 100644 --- a/metricbeat/modules.d/docker.yml.disabled +++ b/metricbeat/modules.d/docker.yml.disabled @@ -16,6 +16,8 @@ # If set to true, replace dots in labels with `_`. #labels.dedot: false + # Set to true to enable low-level network stats for the processes in the container + # network.network_summary: true # To connect to Docker over TLS you must specify a client and CA certificate. #ssl: From 11d3048d0e7091912f3c8daff5ea51323ca4aae4 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Tue, 27 Apr 2021 14:54:28 -0700 Subject: [PATCH 02/10] add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c8b400a05b5..db641563108 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -984,6 +984,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Apache: convert status.total_kbytes to status.total_bytes in fleet mode. {pull}23022[23022] - Release MSSQL as GA {pull}23146[23146] - Add support for SASL/SCRAM authentication to the Kafka module. {pull}24810[24810] +- Add additional network metrics to docker/network {pull}25354[25354] *Packetbeat* From 9ab1223fa2ee7dcc68c3622eec8db10712645b90 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Wed, 28 Apr 2021 08:28:59 -0700 Subject: [PATCH 03/10] update deps, notice --- NOTICE.txt | 4 ++-- go.sum | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index bbbc50ec0d9..f9893b43ddd 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -7695,11 +7695,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-sysinfo -Version: v1.6.0 +Version: v1.7.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.6.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.7.0/LICENSE.txt: Apache License diff --git a/go.sum b/go.sum index d32e37ea280..67010c69948 100644 --- a/go.sum +++ b/go.sum @@ -299,8 +299,6 @@ github.com/elastic/go-seccomp-bpf v1.1.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE github.com/elastic/go-structform v0.0.8 h1:U0qnb9Zqig7w+FhF+sLI3VZPPi/+2aJ0bIEW6R1z6Tk= github.com/elastic/go-structform v0.0.8/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= -github.com/elastic/go-sysinfo v1.6.0 h1:u0QbU8eWSwKRPcFQancnSY4Zi0COksCJXkUgPHxE5Tw= -github.com/elastic/go-sysinfo v1.6.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-sysinfo v1.7.0 h1:4vVvcfi255+8+TyQ7TYUTEK3A+G8v5FLE+ZKYL1z1Dg= github.com/elastic/go-sysinfo v1.7.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-txfile v0.0.7 h1:Yn28gclW7X0Qy09nSMSsx0uOAvAGMsp6XHydbiLVe2s= @@ -767,7 +765,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= From 15111b1604cd5f40d046631ed16a5e0d1ef266f7 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Wed, 28 Apr 2021 09:23:58 -0700 Subject: [PATCH 04/10] update ref yml --- x-pack/metricbeat/metricbeat.reference.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 4b4d7f76e48..ffdf73d9720 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -466,6 +466,8 @@ metricbeat.modules: # If set to true, replace dots in labels with `_`. #labels.dedot: false + # Set to true to enable low-level network stats for the processes in the container + # network.network_summary: true # If set to true, collects metrics per core. #cpu.cores: true From fbe6b79d536a099720cbfc57cb33b52052f0812f Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Wed, 28 Apr 2021 09:41:41 -0700 Subject: [PATCH 05/10] fix test --- .../module/system/network_summary/network_summary_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/system/network_summary/network_summary_test.go b/metricbeat/module/system/network_summary/network_summary_test.go index bd2ba95d006..bac14f51921 100644 --- a/metricbeat/module/system/network_summary/network_summary_test.go +++ b/metricbeat/module/system/network_summary/network_summary_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/metric/system/network" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/go-sysinfo/types" ) @@ -50,7 +51,7 @@ func TestMapping(t *testing.T) { "udp": map[string]uint64{"IgnoredMulti": 10}, "udp_lite": map[string]uint64{"IgnoredMulti": 0}} - out := eventMapping(example) + out := network.MapProcNetCounters(example) assert.Equal(t, exampleOut, out) } From 022b2d4045c6e564763dad44d5ef89fa114035d0 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Thu, 29 Apr 2021 12:12:24 -0700 Subject: [PATCH 06/10] move to new metricset, refactor --- metricbeat/docs/fields.asciidoc | 32 +- metricbeat/docs/modules/docker.asciidoc | 7 +- .../modules/docker/network_summary.asciidoc | 23 ++ metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/include/list_docker.go | 1 + metricbeat/metricbeat.reference.yml | 3 +- .../module/docker/_meta/config.reference.yml | 3 +- metricbeat/module/docker/_meta/config.yml | 3 +- metricbeat/module/docker/fields.go | 2 +- .../module/docker/network/_meta/data.json | 277 ++-------------- .../module/docker/network/_meta/fields.yml | 25 -- metricbeat/module/docker/network/config.go | 8 +- metricbeat/module/docker/network/data.go | 12 +- metricbeat/module/docker/network/helper.go | 118 +------ metricbeat/module/docker/network/network.go | 12 +- .../network/network_integration_test.go | 7 +- .../docker/network_summary/_meta/data.json | 296 ++++++++++++++++++ .../network_summary/_meta/docs.asciidoc | 4 + .../docker/network_summary/_meta/fields.yml | 34 ++ .../module/docker/network_summary/config.go | 37 +++ .../module/docker/network_summary/doc.go | 18 ++ .../module/docker/network_summary/helper.go | 94 ++++++ .../docker/network_summary/network_summary.go | 129 ++++++++ .../network_summary/network_summary_test.go | 42 +++ metricbeat/modules.d/docker.yml.disabled | 3 +- 25 files changed, 752 insertions(+), 441 deletions(-) create mode 100644 metricbeat/docs/modules/docker/network_summary.asciidoc create mode 100644 metricbeat/module/docker/network_summary/_meta/data.json create mode 100644 metricbeat/module/docker/network_summary/_meta/docs.asciidoc create mode 100644 metricbeat/module/docker/network_summary/_meta/fields.yml create mode 100644 metricbeat/module/docker/network_summary/config.go create mode 100644 metricbeat/module/docker/network_summary/doc.go create mode 100644 metricbeat/module/docker/network_summary/helper.go create mode 100644 metricbeat/module/docker/network_summary/network_summary.go create mode 100644 metricbeat/module/docker/network_summary/network_summary_test.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3c7f0d90c2d..1ffeba521bf 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -12457,11 +12457,11 @@ type: long [float] === network_summary -Summed network summary statistics of every PID associated with the container +network_summary -*`docker.network.network_summary.ip.*`*:: +*`docker.network_summary.ip.*`*:: + -- IP counters @@ -12471,7 +12471,7 @@ type: object -- -*`docker.network.network_summary.tcp.*`*:: +*`docker.network_summary.tcp.*`*:: + -- TCP counters @@ -12481,7 +12481,7 @@ type: object -- -*`docker.network.network_summary.udp.*`*:: +*`docker.network_summary.udp.*`*:: + -- UDP counters @@ -12491,7 +12491,7 @@ type: object -- -*`docker.network.network_summary.udp_lite.*`*:: +*`docker.network_summary.udp_lite.*`*:: + -- UDP Lite counters @@ -12501,7 +12501,7 @@ type: object -- -*`docker.network.network_summary.icmp.*`*:: +*`docker.network_summary.icmp.*`*:: + -- ICMP counters @@ -12511,6 +12511,26 @@ type: object -- +*`docker.network_summary.namespace.pid`*:: ++ +-- +The root PID of the container, corresponding to /proc/[pid]/net + + +type: long + +-- + +*`docker.network_summary.namespace.id`*:: ++ +-- +The ID of the network namespace used by the container, corresponding to /proc/[pid]/ns/net + + +type: long + +-- + [[exported-fields-dropwizard]] == Dropwizard fields diff --git a/metricbeat/docs/modules/docker.asciidoc b/metricbeat/docs/modules/docker.asciidoc index e7ea865e04b..bafd02ff5d5 100644 --- a/metricbeat/docs/modules/docker.asciidoc +++ b/metricbeat/docs/modules/docker.asciidoc @@ -44,14 +44,13 @@ metricbeat.modules: #- "image" - "memory" - "network" + #- "network_summary" hosts: ["unix:///var/run/docker.sock"] period: 10s enabled: true # If set to true, replace dots in labels with `_`. #labels.dedot: false - # Set to true to enable low-level network stats for the processes in the container - # network.network_summary: true # If set to true, collects metrics per core. #cpu.cores: true @@ -88,6 +87,8 @@ The following metricsets are available: * <> +* <> + include::docker/container.asciidoc[] include::docker/cpu.asciidoc[] @@ -106,3 +107,5 @@ include::docker/memory.asciidoc[] include::docker/network.asciidoc[] +include::docker/network_summary.asciidoc[] + diff --git a/metricbeat/docs/modules/docker/network_summary.asciidoc b/metricbeat/docs/modules/docker/network_summary.asciidoc new file mode 100644 index 00000000000..3d0510ac68f --- /dev/null +++ b/metricbeat/docs/modules/docker/network_summary.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-docker-network_summary]] +=== Docker network_summary metricset + +beta[] + +include::../../../module/docker/network_summary/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/network_summary/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 688f9235d07..cd1e738c9e4 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -80,7 +80,7 @@ This file is generated! See scripts/mage/docs_collector.go |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.9+| .9+| |<> +.10+| .10+| |<> |<> |<> |<> @@ -89,6 +89,7 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |<> +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | diff --git a/metricbeat/include/list_docker.go b/metricbeat/include/list_docker.go index 6a9201efc72..2e6e65d84fc 100644 --- a/metricbeat/include/list_docker.go +++ b/metricbeat/include/list_docker.go @@ -32,6 +32,7 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/docker/info" _ "github.com/elastic/beats/v7/metricbeat/module/docker/memory" _ "github.com/elastic/beats/v7/metricbeat/module/docker/network" + _ "github.com/elastic/beats/v7/metricbeat/module/docker/network_summary" _ "github.com/elastic/beats/v7/metricbeat/module/kubernetes/apiserver" _ "github.com/elastic/beats/v7/metricbeat/module/kubernetes/container" _ "github.com/elastic/beats/v7/metricbeat/module/kubernetes/controllermanager" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 26ed39d6f11..09aa5d402d4 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -235,14 +235,13 @@ metricbeat.modules: #- "image" - "memory" - "network" + #- "network_summary" hosts: ["unix:///var/run/docker.sock"] period: 10s enabled: true # If set to true, replace dots in labels with `_`. #labels.dedot: false - # Set to true to enable low-level network stats for the processes in the container - # network.network_summary: true # If set to true, collects metrics per core. #cpu.cores: true diff --git a/metricbeat/module/docker/_meta/config.reference.yml b/metricbeat/module/docker/_meta/config.reference.yml index d8c3f1b27f2..f34e21c7fed 100644 --- a/metricbeat/module/docker/_meta/config.reference.yml +++ b/metricbeat/module/docker/_meta/config.reference.yml @@ -9,14 +9,13 @@ #- "image" - "memory" - "network" + #- "network_summary" hosts: ["unix:///var/run/docker.sock"] period: 10s enabled: true # If set to true, replace dots in labels with `_`. #labels.dedot: false - # Set to true to enable low-level network stats for the processes in the container - # network.network_summary: true # If set to true, collects metrics per core. #cpu.cores: true diff --git a/metricbeat/module/docker/_meta/config.yml b/metricbeat/module/docker/_meta/config.yml index aa8fa058405..ffe51d651fa 100644 --- a/metricbeat/module/docker/_meta/config.yml +++ b/metricbeat/module/docker/_meta/config.yml @@ -8,13 +8,12 @@ # - info # - memory # - network + # - network_summary period: 10s hosts: ["unix:///var/run/docker.sock"] # If set to true, replace dots in labels with `_`. #labels.dedot: false - # Set to true to enable low-level network stats for the processes in the container - # network.network_summary: true # To connect to Docker over TLS you must specify a client and CA certificate. #ssl: diff --git a/metricbeat/module/docker/fields.go b/metricbeat/module/docker/fields.go index 9760b1fac67..ef99e2d135d 100644 --- a/metricbeat/module/docker/fields.go +++ b/metricbeat/module/docker/fields.go @@ -32,5 +32,5 @@ func init() { // AssetDocker returns asset data. // This is the base64 encoded gzipped contents of module/docker. func AssetDocker() string { - return "eJzsXEuP47gRvvevKHQOAQYzbiRY5OBDgE3PBGNkZ6cxj+QQBF6aKtsVS6SGpOzx/PoFSUmWJUryQ3Z3L9pH06r6WI+PRbLkN7DC7RgiyVeobgAMmRjHcPvWfXF7AxCh5opSQ1KM4e83AAB+ELRhRgOXcYzcYARzJZN8bHQDoDBGpnEMC3YDoNEYEgs9hv/eah3f/s9+t5TKTLkUc1qMYc5ijTcAc8I40mOn6Q0IlmAFn/2YbWqlKpml+TcBjPYzEXOpEma/BiYiB5i0Ia6BzWRmcrF/1qAyIUgsgEthGAlUepRLqaKpIip/WY6EgHWAqxiylAUJGkW8VG4/+2YsPnVY+9CShIlob6wAt8LtRqr6WAdE+7n3AsEsmYEN04DfkWfW5STALLExj1EYl0JmMIwrYgaPA/WWGYTNEj2CnQktvlxTGIaNgkwPaZ1CtZcc1krplEWRQq0xrJvSU9VOHqAU3TJl+lG3bjhWj5su/cBQxEJLfFYRKSnNdF63xA5YLMUiMNiDzX2+SMNiD07OgcWxC5A5xaiLeG0J1D2Am0tg+5yj2iFyObVka4QZoigiF6QCvmRigRFoEhz9AEkRdrBhiwEjepKwBTqZoybvpdk5jPcpE4YShPuHr8OQ3QqVwHiUchOcv+Ysxmg6jyWr/8CvDWNIUXEU9dEeEz34h6ydrDvtlEjkYECnjGPYUTlcIVXyBDGDxcVi+oERzLYuSkWWzFDZB6zLuFRtHJPPzBBfhUMxkDZ9VPPwFZy8w2yrt9rgo5vVsY9H7g1srZhD64L9FEKiA/s5oZHPcOjQyIE5sWHFmUb12DbNLWmhdAWvg/oUYqCB9xzPu1ldjBL6bOri+er2/FJmUabZohPao/i7hu8c99qh0avWGcjZ/7Ex5L+cXjOm9xmNtMfdNaNOxzzpab0ewJ/tCds/9eNT+tc9dGVyBxxVngaQXpE8a+NNegWTu4/D1KAKWXhXe8L26mfOsySL3SbAytUQZYrEwjkypnm5fQgdQLQBrYKV6SV2XTsnngR6B2+2NY0Nci/AIqnaHj5gAv+wjzrwp2NXzUOMXuhH2ZZnSqEwuY1Tu/ohl42jnkrlhWpNHKeWJi6AzC8ljoOMLJTB5CMo/JahNvq1zWTBhPQ4m74pgG4YmQFQQh/MAhjo1BrSqrW+JgHfMsxQ21AqJnIwePdo0wlD2XdH315ROYlWMgqTd8eqFGGqkFvSGcPfRj+dSuAHxWfpckWNdBmENp3gZ8ebp6F+KsRp0RsUz4A7czu/kOcLeQYs6YLjkdmzO0LL6MyShKnt5epOJp4rldrCXqao3IH5s6VUV4sWTnge3Fox+gu/vvBrE4w79nokem2wWiBEC5y43j+bOfVmvy7n+LOFoW+r31lEIanlRfWAjQNeGUX+np6tGcVsFmNQ71zJZPBpykzxsDorezh1X5boHrdx5o/DABNyHS+Oo+txsMPBuJV5GSSdWmV9vTijdGgK6ysBGlHWN+0DcBTTn7wtlsbDXLFnGGMUzbKudT94Bgr1c9CzZvFvpkhm2gq5W7M4wwqu/bm9tuyIIrKzkwLI6P3ILua1RBabJV8iXw1AaxVpzRPUvQfeV34ZMcNgQ3EMUsRbmOGOEXzvWFRrI9KWNxTa6e4JzX/32/t3P//y5f39+3f3//oNSGijMpdNsGTat1NkGiO7+s8yiiNntvxZSmpXM8cz85xRTGKhjUK2CuYSCYOLRl3W438uBc9cNWUVYAR1p11ubag6y8sGLqMwf4bS6GQGccJyYx/bSYQimga6x6CrtewASHV7oIjCkirOUOYaSJyibiwyM2kWoqgBuKkKpUVP6ZrvZKaNCKoiCWfIaUZphGu51thcH4D1nJxhboxaiqwTUscueB5YzLaWMilCYWhOzea2vkzKd3GXCRv4KuhbVmDdgYQFrS1Rp/nqFe5zq8JM2QVRTnbA8nXWAX4NNAcyNqLdxtEtV5sl8aXfguf7Xz+5iBRyE2+dQhR1SrtgO6xr0rU7ybIv1iPq74kdsEHUdw+6/ksfksfG4ZqUyRrbRBi6/7JRAuyjULjIYhaipoE7VC0WFsfAGV9i5GFpYFpLTu44zshmkLWUWwX4mM0wPvUK/4yeUa+3B9z1mlVJzM9qE5iIuSwIH2bMFpO2ujQm1eO7u0hyPfL15IjL5A7FggTeKZyjQsHxjqV058enChNpcMpSmq7/MvrrT3d/uotIpzHbvvFtbG82FOEb2r2xcO47AEUNPVRaf1yjcmG61+5+dHKnzNbkF8iq+nGUVxR4o6OJKX/74wqg2t8zaaLSRqbpVUyVazoIVegE7xKY3ErbZapLnFflNUqxy5XaBMupMA7H20Esx/dEtVrDa2kyXYKJ3LsMOprrPjgJw5S3nhlenbj+BMxVGZ8mLE1JLPIf3766Pc60n9gmt1b+Apur5dwC66yl89GRHXUbFDVnLYeIXCYJDbYLvnfSjOvbcwc9Av5DIpKbelT1cexJOTrAvZWP2rCAkv+bhyXXgPaAbJW7q9fAJVRFa2ZwupFqZQNOoxm1X2AEsHfh7sGc64ZcN2g0vXjnjOIRl1nLuUznDUsnmH8ysut+ZnMhTMIxteXBsGbJScqpCyNRerCK59Pnz3tMcWyp87hpmCNXqN0SZiPIbTk6NtbBU+3e4IHeruYDcX9oQVxIbW2md33sQ3n9q/anPKf7PWHfH8HrH9j3AnXgvQN4en72bx+0+RaeWiLVjNqowAQaS9bnlGC/ehGn12DhwjRYwpxVJxdAS9FOVUt9Hr7cDGfmGRf9E8FlYpfK3BF5dbe75D82jR+r+6de+1MxMSezPUsi1b1f7EntE5DlGncIU8ZX2CTMyo2AUrJxJAGDbR+9eHcRejCk/AdX2NJ2Y6rc3VwnYT5mZiH/iAkji4k92YQpET6dhDkc0vUSphvTboGZyazl705Oub4IryP+Xxj2/2rE3cTWr1SeT54MtbAM5/CXBeUyC8qgCdKybvwBE2SohWT4BHlZQAZcQPI4ng78RsXnLEkw2mWJl179EzLfoqm28DB5W73w3ZBZnvWCBaWNw3fo7188wMCTh+IcsP2E1/BLaf9yf4D6LLqU+q9vD1M/jcngBTH8Qgb7gRBPLhYF9x8qlvg9AAD//4xkL+0=" + return "eJzsXE1v4zjSvudXFPIeXqCR2NjFYA8+LDCb7kUHOz0ddCe7h8HAQ1Nlu9YSqSYpu92/fkFSkm2JkvwhO8kgPuQQWlVPfRfJkm9hgesRRJIvUF0BGDIxjuD6vfvH9RVAhJorSg1JMYK/XwEA+EXQhhkNXMYxcoMRTJVM8rXBFYDCGJnGEczYFYBGY0jM9Ah+u9Y6vv7d/m8ulRlzKaY0G8GUxRqvAKaEcaRHjtMtCJbgFj77MevUUlUyS/P/BDDaz72YSpUw+29gInKASRviGthEZiYn+/8aVCYEiRlwKQwjgUoPcirbaLYRld8sV0LAWsBtKbKkBQkaRbxkbj+7aiw+VVi70JKEiWhnrQC3wPVKqupaC0T7ufMEwcyZgRXTgN+RZ9bkJMDMsSbHIIxLITMYxhUxg4eBes8MwmqOHsFGhRZfzikMw3pBpvvUTsHaUw5zpXTMokih1hjmTemxbO8foCTdIDL9qGo37KuHiUs/MOSx0OCf24iUlGY8rWpiAyyWYhZY7MDmPo/SsNiDk1NgcewcZEox6sJfGxx1B+DqHNi+5qg2iFxMzdkSYYIoCs8FqYDPmZhhBJoER79AUoQNbNisR4++T9gMHc1BPe+l2SkZ70smDCUIdw9P/SS7BSqB8SDlJii/5izGaDyNJat+wdeGEaSoOIrqaoeKHvxDVk/WnFYkEjkY0CnjGDZUDldIlbxAzGBxsZh+YASTtfNSkSUTVPYBazIuVVOOySUzxBdhVwyETVeqeXgCR28/3eq1NvjsanXZxyP3CrZazKG1wX4JLtGC/RTXyCXs2zVyYI5smHGmUT23TnNNWihtzuugvgQfqOE9xfJOqrOlhC6dOn++uD4fyyjKNJu1QnsWe1fwnWJeuzR41yiBnPwXa0v+n+NL+vRuRiPtcbdJ1GqYFy3WTQ/2bA7YbtEPD+lfd9CVwR0wVHkaQHpB8qSNN+kF3A8/99ODKmThXe0R26ufOc+SLHabAEtXQ5QpEjNnyJim5fYhdADRBHQbrEzPsevaGPEo0Bt4k7WpbZA7ARZB1fTwHgL8wz7qwB+PXdUPMTqhH6RbnimFwuQ6Tm31Qy5rRz1bnReqJXEc2zRxBmS+lLgcZGTBDO4/g8JvGWqjb2wkCyakx1m3TQF0xcj0gBK6YBbAQKdWkZattTUJ+JZhhtq6UiHI3uDdo3Uj9KXfTfr2jEohGpNROHm3VKUIU4XcJp0R/G3w07EJfC//LE2uqBYuvaRNR/jV5c3jUL+UxGnRGxSvIHfmen5Lnm/JM6BJ5xzPnD3bPbT0zixJmFqfr+9k4rWmUtvYyxSVOzB/tSnV9aKFEV5Hbt1S+lt+fcuvdTDu2OuZ0mstqwVctMCJy92zmWNv9qt0Dj9b6Pu2+oNFFKJaXlT3ODjgmVHk7+nZklHMJjEG+U6VTHoXU2aKh9lZ2v2xe5yje9z6mT8OA0zITby4HF31gw0Oxi3N8yBp5Sqr9eKE1qFOrKsFqHlZl9h74CjEv39flMb9TLGjGGMUTbK2uh88A4XqOehJUvybKZKZtkSGSxZnuIVrV7Ybmx1RRFY6KYCM3vXsQq45stjM+Rz5ooe0tkWtfoK688DHrW9GzDBYURyDFPEaJrjJCH52LKqMEWmbNxRacXeI5t/74+OHn395/Hj38cPdv/4AEtqozEUTzJn24xSZxshW/0lGceTUlj9LSeVq5vDMPGUUk5hpo5AtgrFEwuCs1pd12J9LwTPXTVkGGEHVaOerDdvG8rSByyicP0NhdHQGccRyZR86SYQiGgemx6BttGwPSFV9oIjClLaMocwlkDhG7VhkZtIslKJ6yE3bUBr4lKb5TmZc86BtJOEIOU4pNXcta42N9R6ynqPTz41RQ5N1ROjYgueBxWxtUyZFKAxNqT7c1hVJ+S7uPG4DT4K+ZQXWDUiY0dIm6jSvXuE5t22YKTsjyvsNsLzOOsA3QFMgYz3abRxduVrNic/9Fjzf/3rhIlLITbx2DFFUU9oZx2HdkK7dSZZzsR5R90xsjwOifnrQzV96lzzUD5ekTFbbJkLf85e1FmAXhcJZFrNQaup5QtViYXEMnPE5Rh6WBqa15OSO44ysO1lDu1WAj9kE42Ov8E+YGfV8O8BdbliVxPSkMYF7MZVFwocJs82k7S6NSfVoOIwk1wPfTw64TIYoZiRwqHCKCgXHIUtp6NfHChNpcMxSGi//MvjrT8P/G0ak05itb/0Y2+2KIrylzRsLp74DUPTQfYX15yUq56Y74+4HB3fKbE9+hqiqHkd5RoE3OuqY8rc/LgCq+T2TOiptZJpeRFU5p71QhU7wzoHJVdo2VZ3jvCrvUYpdrtQm2E6Fcbi8HcRy+ExUozY8l3qmSzCRO5dBB+e6T45CP+2tzwzvjqw/AXVtrY8TlqYkZvmXr99dH6baL2yVayt/gc31cq7AOm3pfHVgV90GRU1ZwyEil0lCve2C7xw14+b23EGPgP+QiOSq6lVdOfaoGO3h3sp7bZhAmf/rhyWXgPaAbJGbq1PBJVRFS2ZwvJJqYR1Ooxk0X2AEsLfh7sCc84acN2g0nXinjOIBl1nDuUzrDUsrmH8ysnU/s7EQTsIxNcVBv2rJk5RjF0aidG8dz5evX3cyxaGtzvOGYY5coXYlzHqQ23K0bKyDp9qdzgOdU8174v7UgLig2jhM7+bY+7L6k/anPMfbPWHfn8Hqn9j3AnXgvQN4eXb2bx802RZeWiBVlFrrwAQam6xPacF+9SSO78HCjWmwhTmpTy6AlqQdq4b+PHy5GY7MEy767wWXiS2VuSHy7m5zyX9oGD/X9E+196dCMEezOUoi1b5f7AjtI5DlHDcIU8YXWE+YWzcCSsnakQT0tn305N1F6N6Q8i9cYEvbjmnr7uYyAfM5MzP5ZwwYWQj2YgOmRPhyAmZ/SJcLmHZMmwIzkVnDz50cc30RriP+Vxh2f2rE3cRWr1ReT5z0VVj6M/hbQTlPQek1QBrqxp8wQPoqJP0HyFsBObGAVHZt4/rbFOH4KPdhEzRsv11dM4vWCZH0kPPzrpr2UBzfNYxl8z65Pd51sMuiPtk9ve9mN47JYM88fyGD7YyJJ71a8e5Th6T2r/89jLThWuyIq6g5ul/rgofNXG2Z4W+AS6VQp9KPoBoJw1RJPvwtpej3ocDw+PEGZ68wNwCL0lQy8pOg+e8h7A1fOwn+FwAA//+qunky" } diff --git a/metricbeat/module/docker/network/_meta/data.json b/metricbeat/module/docker/network/_meta/data.json index f9adc23a8f6..4ac4293b6cd 100644 --- a/metricbeat/module/docker/network/_meta/data.json +++ b/metricbeat/module/docker/network/_meta/data.json @@ -1,48 +1,43 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", "container": { - "id": "b1c3118fb2087e499c421aa20e28847ad4c185be6c1864166b1a03cc2affaf80", + "id": "b9f9e931fdf5f0d8c3d6a18f38cedd9794b6de48a9ff708c5c0e9a29d54db169", "image": { - "name": "docker.elastic.co/beats/elastic-agent:7.13.0-SNAPSHOT" + "name": "docker.elastic.co/elasticsearch/elasticsearch:7.13.0-SNAPSHOT" }, - "name": "elastic-package-stack_elastic-agent_1", + "name": "elastic-package-stack_elasticsearch_1", "runtime": "docker" }, "docker": { "container": { "labels": { - "com_docker_compose_config-hash": "fdad7f67fafe5a242b9b9d27bff78bef4b96c683a2bb1412d91c32fb5981f2b3", + "com_docker_compose_config-hash": "2a2f6b4cb4003f7043c3d76a9abfc052644fe5b8d984b5c238df25cd1d0f653e", "com_docker_compose_container-number": "1", "com_docker_compose_oneoff": "False", "com_docker_compose_project": "elastic-package-stack", "com_docker_compose_project_config_files": "/home/alexk/.elastic-package/stack/openport/snapshot.yml", "com_docker_compose_project_working_dir": "/home/alexk/.elastic-package/stack/openport", - "com_docker_compose_service": "elastic-agent", + "com_docker_compose_service": "elasticsearch", "com_docker_compose_version": "1.27.4", - "description": "Agent manages other beats based on configuration provided.", - "io_k8s_description": "Agent manages other beats based on configuration provided.", - "io_k8s_display-name": "Elastic-Agent image", - "license": "Elastic License", - "maintainer": "infra@elastic.co", - "name": "elastic-agent", - "org_label-schema_build-date": "2021-04-21T06:45:43Z", - "org_label-schema_license": "Elastic License", - "org_label-schema_name": "elastic-agent", + "org_label-schema_build-date": "2021-04-21T03:53:24.920332641Z", + "org_label-schema_license": "Elastic-License-2.0", + "org_label-schema_name": "Elasticsearch", "org_label-schema_schema-version": "1.0", - "org_label-schema_url": "https://www.elastic.co/beats/elastic-agent", - "org_label-schema_vcs-ref": "c52c43963ca8416dc92c7d3dbf6cb6e89dd00acf", - "org_label-schema_vcs-url": "github.com/elastic/beats/v7", + "org_label-schema_url": "https://www.elastic.co/products/elasticsearch", + "org_label-schema_usage": "https://www.elastic.co/guide/en/elasticsearch/reference/index.html", + "org_label-schema_vcs-ref": "4c99108262abdeb5c4d9bf23edcd99fe403f5652", + "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch", "org_label-schema_vendor": "Elastic", "org_label-schema_version": "7.13.0-SNAPSHOT", - "org_opencontainers_image_created": "2021-04-21T06:45:43Z", - "org_opencontainers_image_licenses": "Elastic License", - "org_opencontainers_image_title": "Elastic-Agent", + "org_opencontainers_image_created": "2021-04-21T03:53:24.920332641Z", + "org_opencontainers_image_documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/index.html", + "org_opencontainers_image_licenses": "Elastic-License-2.0", + "org_opencontainers_image_revision": "4c99108262abdeb5c4d9bf23edcd99fe403f5652", + "org_opencontainers_image_source": "https://github.com/elastic/elasticsearch", + "org_opencontainers_image_title": "Elasticsearch", + "org_opencontainers_image_url": "https://www.elastic.co/products/elasticsearch", "org_opencontainers_image_vendor": "Elastic", - "release": "1", - "summary": "elastic-agent", - "url": "https://www.elastic.co/beats/elastic-agent", - "vendor": "Elastic", - "version": "7.13.0-SNAPSHOT" + "org_opencontainers_image_version": "7.13.0-SNAPSHOT" } }, "network": { @@ -53,236 +48,12 @@ "packets": 0 }, "inbound": { - "bytes": 92092613, + "bytes": 10553051848, "dropped": 0, "errors": 0, - "packets": 424322 + "packets": 43569407 }, "interface": "eth0", - "network_summary": { - "icmp": { - "InAddrMaskReps": 0, - "InAddrMasks": 0, - "InCsumErrors": 0, - "InDestUnreachs": 0, - "InEchoReps": 0, - "InEchos": 0, - "InErrors": 0, - "InMsgs": 0, - "InParmProbs": 0, - "InRedirects": 0, - "InSrcQuenchs": 0, - "InTimeExcds": 0, - "InTimestampReps": 0, - "InTimestamps": 0, - "OutAddrMaskReps": 0, - "OutAddrMasks": 0, - "OutDestUnreachs": 0, - "OutEchoReps": 0, - "OutEchos": 0, - "OutErrors": 0, - "OutMsgs": 0, - "OutParmProbs": 0, - "OutRedirects": 0, - "OutSrcQuenchs": 0, - "OutTimeExcds": 0, - "OutTimestampReps": 0, - "OutTimestamps": 0 - }, - "ip": { - "ForwDatagrams": 0, - "FragCreates": 0, - "FragFails": 0, - "FragOKs": 0, - "InAddrErrors": 0, - "InBcastOctets": 0, - "InBcastPkts": 0, - "InCEPkts": 0, - "InCsumErrors": 0, - "InDelivers": 5108328, - "InDiscards": 0, - "InECT0Pkts": 0, - "InECT1Pkts": 0, - "InHdrErrors": 0, - "InMcastOctets": 0, - "InMcastPkts": 0, - "InNoECTPkts": 5108412, - "InNoRoutes": 0, - "InOctets": 702988830, - "InReceives": 5108328, - "InTruncatedPkts": 0, - "InUnknownProtos": 0, - "OutBcastOctets": 0, - "OutBcastPkts": 0, - "OutDiscards": 0, - "OutMcastOctets": 0, - "OutMcastPkts": 0, - "OutNoRoutes": 0, - "OutOctets": 11397546792, - "OutRequests": 6353418, - "ReasmFails": 0, - "ReasmOKs": 0, - "ReasmOverlaps": 0, - "ReasmReqds": 0, - "ReasmTimeout": 0 - }, - "tcp": { - "ActiveOpens": 3108, - "ArpFilter": 0, - "AttemptFails": 0, - "BusyPollRxPackets": 0, - "CurrEstab": 66, - "DelayedACKLocked": 0, - "DelayedACKLost": 6, - "DelayedACKs": 96156, - "EmbryonicRsts": 0, - "EstabResets": 0, - "IPReversePathFilter": 0, - "InCsumErrors": 0, - "InErrs": 0, - "InSegs": 5096568, - "ListenDrops": 0, - "ListenOverflows": 0, - "LockDroppedIcmps": 0, - "OfoPruned": 0, - "OutOfWindowIcmps": 0, - "OutRsts": 2916, - "OutSegs": 12028800, - "PAWSActive": 0, - "PAWSEstab": 60, - "PFMemallocDrop": 0, - "PassiveOpens": 24, - "PruneCalled": 0, - "RcvPruned": 0, - "RetransSegs": 6972, - "SyncookiesFailed": 0, - "SyncookiesRecv": 0, - "SyncookiesSent": 0, - "TCPACKSkippedChallenge": 0, - "TCPACKSkippedFinWait2": 0, - "TCPACKSkippedPAWS": 0, - "TCPACKSkippedSeq": 0, - "TCPACKSkippedSynRecv": 0, - "TCPACKSkippedTimeWait": 0, - "TCPAbortFailed": 0, - "TCPAbortOnClose": 0, - "TCPAbortOnData": 2892, - "TCPAbortOnLinger": 0, - "TCPAbortOnMemory": 0, - "TCPAbortOnTimeout": 0, - "TCPAckCompressed": 0, - "TCPAutoCorking": 3300, - "TCPBacklogCoalesce": 5928, - "TCPBacklogDrop": 0, - "TCPChallengeACK": 0, - "TCPDSACKIgnoredDubious": 0, - "TCPDSACKIgnoredNoUndo": 6420, - "TCPDSACKIgnoredOld": 0, - "TCPDSACKOfoRecv": 0, - "TCPDSACKOfoSent": 0, - "TCPDSACKOldSent": 6, - "TCPDSACKRecv": 6816, - "TCPDSACKRecvSegs": 6828, - "TCPDSACKUndo": 6, - "TCPDeferAcceptDrop": 0, - "TCPDelivered": 9321756, - "TCPDeliveredCE": 0, - "TCPFastOpenActive": 0, - "TCPFastOpenActiveFail": 0, - "TCPFastOpenBlackhole": 0, - "TCPFastOpenCookieReqd": 0, - "TCPFastOpenListenOverflow": 0, - "TCPFastOpenPassive": 0, - "TCPFastOpenPassiveAltKey": 0, - "TCPFastOpenPassiveFail": 0, - "TCPFastRetrans": 24, - "TCPFromZeroWindowAdv": 0, - "TCPFullUndo": 6, - "TCPHPAcks": 2200992, - "TCPHPHits": 590586, - "TCPHystartDelayCwnd": 0, - "TCPHystartDelayDetect": 0, - "TCPHystartTrainCwnd": 216, - "TCPHystartTrainDetect": 12, - "TCPKeepAlive": 702798, - "TCPLossFailures": 0, - "TCPLossProbeRecovery": 0, - "TCPLossProbes": 9390, - "TCPLossUndo": 0, - "TCPLostRetransmit": 0, - "TCPMD5Failure": 0, - "TCPMD5NotFound": 0, - "TCPMD5Unexpected": 0, - "TCPMTUPFail": 0, - "TCPMTUPSuccess": 0, - "TCPMemoryPressures": 0, - "TCPMemoryPressuresChrono": 0, - "TCPMinTTLDrop": 0, - "TCPOFODrop": 0, - "TCPOFOMerge": 0, - "TCPOFOQueue": 0, - "TCPOrigDataSent": 9311964, - "TCPPartialUndo": 6, - "TCPPureAcks": 885618, - "TCPRcvCoalesce": 60, - "TCPRcvCollapsed": 0, - "TCPRcvQDrop": 0, - "TCPRenoFailures": 0, - "TCPRenoRecovery": 0, - "TCPRenoRecoveryFail": 0, - "TCPRenoReorder": 0, - "TCPReqQFullDoCookies": 0, - "TCPReqQFullDrop": 0, - "TCPRetransFail": 0, - "TCPSACKDiscard": 0, - "TCPSACKReneging": 0, - "TCPSACKReorder": 912, - "TCPSYNChallenge": 0, - "TCPSackFailures": 0, - "TCPSackMerged": 30, - "TCPSackRecovery": 18, - "TCPSackRecoveryFail": 0, - "TCPSackShiftFallback": 594, - "TCPSackShifted": 0, - "TCPSlowStartRetrans": 0, - "TCPSpuriousRTOs": 0, - "TCPSpuriousRtxHostQueues": 12, - "TCPSynRetrans": 144, - "TCPTSReorder": 6, - "TCPTimeWaitOverflow": 0, - "TCPTimeouts": 144, - "TCPToZeroWindowAdv": 0, - "TCPWantZeroWindowAdv": 0, - "TCPWinProbe": 0, - "TCPWqueueTooBig": 0, - "TCPZeroWindowDrop": 0, - "TW": 18, - "TWKilled": 0, - "TWRecycled": 0, - "TcpDuplicateDataRehash": 0, - "TcpTimeoutRehash": 144 - }, - "udp": { - "IgnoredMulti": 0, - "InCsumErrors": 0, - "InDatagrams": 11760, - "InErrors": 0, - "NoPorts": 0, - "OutDatagrams": 11760, - "RcvbufErrors": 0, - "SndbufErrors": 0 - }, - "udp_lite": { - "IgnoredMulti": 0, - "InCsumErrors": 0, - "InDatagrams": 0, - "InErrors": 0, - "NoPorts": 0, - "OutDatagrams": 0, - "RcvbufErrors": 0, - "SndbufErrors": 0 - } - }, "out": { "bytes": 0, "dropped": 0, @@ -290,10 +61,10 @@ "packets": 0 }, "outbound": { - "bytes": 1877416014, + "bytes": 4106659715, "dropped": 0, "errors": 0, - "packets": 631723 + "packets": 40961196 } } }, diff --git a/metricbeat/module/docker/network/_meta/fields.yml b/metricbeat/module/docker/network/_meta/fields.yml index a4ba1807d7c..035047eb091 100644 --- a/metricbeat/module/docker/network/_meta/fields.yml +++ b/metricbeat/module/docker/network/_meta/fields.yml @@ -99,28 +99,3 @@ type: long description: > Total number of outgoing packets. - - name: network_summary - type: group - description: > - Summed network summary statistics of every PID associated with the container - fields: - - name: ip.* - type: object - description: > - IP counters - - name: tcp.* - type: object - description: > - TCP counters - - name: udp.* - type: object - description: > - UDP counters - - name: udp_lite.* - type: object - description: > - UDP Lite counters - - name: icmp.* - type: object - description: > - ICMP counters \ No newline at end of file diff --git a/metricbeat/module/docker/network/config.go b/metricbeat/module/docker/network/config.go index 51ffc03e787..4be453ac125 100644 --- a/metricbeat/module/docker/network/config.go +++ b/metricbeat/module/docker/network/config.go @@ -23,15 +23,13 @@ import "github.com/elastic/beats/v7/metricbeat/module/docker" //Config is the config object for docker/network type Config struct { - TLS *docker.TLSConfig `config:"ssl"` - DeDot bool `config:"labels.dedot"` - NetworkSummary bool `config:"network.network_summary"` + TLS *docker.TLSConfig `config:"ssl"` + DeDot bool `config:"labels.dedot"` } // DefaultConfig returns default module config func DefaultConfig() Config { return Config{ - DeDot: true, - NetworkSummary: false, + DeDot: true, } } diff --git a/metricbeat/module/docker/network/data.go b/metricbeat/module/docker/network/data.go index f1a17ba7bdf..eb89cd50fee 100644 --- a/metricbeat/module/docker/network/data.go +++ b/metricbeat/module/docker/network/data.go @@ -19,17 +19,16 @@ package network import ( "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/metric/system/network" "github.com/elastic/beats/v7/metricbeat/mb" ) -func eventsMapping(r mb.ReporterV2, netsStatsList []NetStats, reportNetstat bool) { +func eventsMapping(r mb.ReporterV2, netsStatsList []NetStats) { for i := range netsStatsList { - eventMapping(r, &netsStatsList[i], reportNetstat) + eventMapping(r, &netsStatsList[i]) } } -func eventMapping(r mb.ReporterV2, stats *NetStats, reportNetstat bool) { +func eventMapping(r mb.ReporterV2, stats *NetStats) { fields := common.MapStr{ "interface": stats.NameInterface, @@ -61,11 +60,6 @@ func eventMapping(r mb.ReporterV2, stats *NetStats, reportNetstat bool) { }, } - if reportNetstat { - summary := network.MapProcNetCounters(stats.Netstat) - fields["network_summary"] = summary - } - r.Event(mb.Event{ RootFields: stats.Container.ToMapStr(), MetricSetFields: fields, diff --git a/metricbeat/module/docker/network/helper.go b/metricbeat/module/docker/network/helper.go index a8db7f518fa..7e1a2fbfde7 100644 --- a/metricbeat/module/docker/network/helper.go +++ b/metricbeat/module/docker/network/helper.go @@ -18,18 +18,11 @@ package network import ( - "context" - "strconv" "time" "github.com/docker/docker/api/types" - "github.com/docker/docker/client" - "github.com/pkg/errors" "github.com/elastic/beats/v7/metricbeat/module/docker" - - sysinfo "github.com/elastic/go-sysinfo" - sysinfotypes "github.com/elastic/go-sysinfo/types" ) // NetService represents maps out the interface-level stats per-container @@ -75,38 +68,15 @@ type NetStats struct { TxDropped float64 TxErrors float64 TxPackets float64 - Netstat *sysinfotypes.NetworkCountersInfo Total *types.NetworkStats } -// NoSumStats lists "stats", often config/state values, that can't be safely summed across PIDs -var NoSumStats = []string{ - "RtoAlgorithm", - "RtoMin", - "RtoMax", - "MaxConn", - "Forwarding", - "DefaultTTL", -} - -func (n *NetService) getNetworkStatsPerContainer(client *client.Client, timeout time.Duration, rawStats []docker.Stat, cfg Config) ([]NetStats, error) { +func (n *NetService) getNetworkStatsPerContainer(rawStats []docker.Stat, cfg Config) ([]NetStats, error) { formattedStats := []NetStats{} for _, myStats := range rawStats { - var stats *sysinfotypes.NetworkCountersInfo - var err error - if cfg.NetworkSummary { - stats, err = fetchContainerNetStats(client, timeout, myStats.Container.ID) - if err != nil { - return nil, errors.Wrap(err, "error fetching per-PID stats") - } - } - for nameInterface, rawnNetStats := range myStats.Stats.Networks { singleStat := n.getNetworkStats(nameInterface, rawnNetStats, myStats, cfg.DeDot) - if cfg.NetworkSummary { - singleStat.Netstat = stats - } formattedStats = append(formattedStats, singleStat) } } @@ -114,92 +84,6 @@ func (n *NetService) getNetworkStatsPerContainer(client *client.Client, timeout return formattedStats, nil } -// fetchContainerNetStats gathers the PIDs associated with a container, and then uses go-sysinfo to grab the /proc/[pid]/net counters and sum them across PIDs. -func fetchContainerNetStats(client *client.Client, timeout time.Duration, container string) (*sysinfotypes.NetworkCountersInfo, error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - body, err := client.ContainerTop(ctx, container, []string{}) - if err != nil { - return &sysinfotypes.NetworkCountersInfo{}, errors.Wrap(err, "error fetching container pids") - } - - var pidPos int - for pos := range body.Titles { - if body.Titles[pos] == "PID" { - pidPos = pos - break - } - } - - summedMetrics := &sysinfotypes.NetworkCountersInfo{} - for _, pid := range body.Processes { - strPID := pid[pidPos] - intPID, err := strconv.Atoi(strPID) - if err != nil { - return &sysinfotypes.NetworkCountersInfo{}, errors.Wrap(err, "error converting PID to int") - } - - proc, err := sysinfo.Process(intPID) - procNet, ok := proc.(sysinfotypes.NetworkCounters) - if !ok { - break - } - - counters, err := procNet.NetworkCounters() - if err != nil { - return &sysinfotypes.NetworkCountersInfo{}, errors.Wrapf(err, "error fetching network counters for PID %d", intPID) - } - - summedMetrics = sumCounter(summedMetrics, counters) - - } - - return summedMetrics, nil - -} - -func sumCounter(totals, new *sysinfotypes.NetworkCountersInfo) *sysinfotypes.NetworkCountersInfo { - - newTotal := &sysinfotypes.NetworkCountersInfo{} - - newTotal.Netstat.IPExt = sumMapStr(new.Netstat.IPExt, totals.Netstat.IPExt) - newTotal.Netstat.TCPExt = sumMapStr(new.Netstat.TCPExt, totals.Netstat.TCPExt) - - newTotal.SNMP.IP = sumMapStr(new.SNMP.IP, totals.SNMP.IP) - newTotal.SNMP.ICMP = sumMapStr(new.SNMP.ICMP, totals.SNMP.ICMP) - newTotal.SNMP.ICMPMsg = sumMapStr(new.SNMP.ICMPMsg, totals.SNMP.ICMPMsg) - newTotal.SNMP.TCP = sumMapStr(new.SNMP.TCP, totals.SNMP.TCP) - newTotal.SNMP.UDP = sumMapStr(new.SNMP.UDP, totals.SNMP.UDP) - newTotal.SNMP.UDPLite = sumMapStr(new.SNMP.UDPLite, totals.SNMP.UDPLite) - return newTotal - -} - -func sumMapStr(m1, m2 map[string]uint64) map[string]uint64 { - - final := make(map[string]uint64) - for key, val := range m1 { - // skip over values that aren't counters and can't be summed across interfaces - // Most of these values are config settings, and it doesn't make sense to report them along with counters that aren't per-pid/per-interface - var skip = false - for _, name := range NoSumStats { - if key == name { - skip = true - } - } - if skip { - continue - } - // safety, make sure the field exists in m1 - if _, ok := m2[key]; !ok { - final[key] = val - } - final[key] = m2[key] + val - - } - return final -} - func (n *NetService) getNetworkStats(nameInterface string, rawNetStats types.NetworkStats, myRawstats docker.Stat, dedot bool) NetStats { newNetworkStats := createNetRaw(myRawstats.Stats.Read, &rawNetStats) oldNetworkStat, exist := n.NetworkStatPerContainer[myRawstats.Container.ID][nameInterface] diff --git a/metricbeat/module/docker/network/network.go b/metricbeat/module/docker/network/network.go index 0ad1c6ca828..660282e8a49 100644 --- a/metricbeat/module/docker/network/network.go +++ b/metricbeat/module/docker/network/network.go @@ -20,8 +20,6 @@ package network import ( - "runtime" - "github.com/docker/docker/client" "github.com/pkg/errors" @@ -56,12 +54,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } - // Network summary requres a linux procfs system under it to read from the cgroups. Disable reporting otherwise. - if runtime.GOOS != "linux" { - base.Logger().Debug("Not running on linux, docker network detailed stats disabled.") - config.NetworkSummary = false - } - return &MetricSet{ BaseMetricSet: base, dockerClient: client, @@ -79,11 +71,11 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return errors.Wrap(err, "failed to get docker stats") } - formattedStats, err := m.netService.getNetworkStatsPerContainer(m.dockerClient, m.Module().Config().Timeout, stats, m.cfg) + formattedStats, err := m.netService.getNetworkStatsPerContainer(stats, m.cfg) if err != nil { return errors.Wrap(err, "error fetching container network stats") } - eventsMapping(r, formattedStats, m.cfg.NetworkSummary) + eventsMapping(r, formattedStats) return nil } diff --git a/metricbeat/module/docker/network/network_integration_test.go b/metricbeat/module/docker/network/network_integration_test.go index c9454aefa72..418d057e247 100644 --- a/metricbeat/module/docker/network/network_integration_test.go +++ b/metricbeat/module/docker/network/network_integration_test.go @@ -35,9 +35,8 @@ func TestData(t *testing.T) { func getConfig() map[string]interface{} { return map[string]interface{}{ - "module": "docker", - "metricsets": []string{"network"}, - "network.network_summary": true, - "hosts": []string{"unix:///var/run/docker.sock"}, + "module": "docker", + "metricsets": []string{"network"}, + "hosts": []string{"unix:///var/run/docker.sock"}, } } diff --git a/metricbeat/module/docker/network_summary/_meta/data.json b/metricbeat/module/docker/network_summary/_meta/data.json new file mode 100644 index 00000000000..769fa82dd3a --- /dev/null +++ b/metricbeat/module/docker/network_summary/_meta/data.json @@ -0,0 +1,296 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "container": { + "id": "b1c3118fb2087e499c421aa20e28847ad4c185be6c1864166b1a03cc2affaf80", + "image": { + "name": "docker.elastic.co/beats/elastic-agent:7.13.0-SNAPSHOT" + }, + "name": "elastic-package-stack_elastic-agent_1", + "runtime": "docker" + }, + "docker": { + "container": { + "labels": { + "com_docker_compose_config-hash": "fdad7f67fafe5a242b9b9d27bff78bef4b96c683a2bb1412d91c32fb5981f2b3", + "com_docker_compose_container-number": "1", + "com_docker_compose_oneoff": "False", + "com_docker_compose_project": "elastic-package-stack", + "com_docker_compose_project_config_files": "/home/alexk/.elastic-package/stack/openport/snapshot.yml", + "com_docker_compose_project_working_dir": "/home/alexk/.elastic-package/stack/openport", + "com_docker_compose_service": "elastic-agent", + "com_docker_compose_version": "1.27.4", + "description": "Agent manages other beats based on configuration provided.", + "io_k8s_description": "Agent manages other beats based on configuration provided.", + "io_k8s_display-name": "Elastic-Agent image", + "license": "Elastic License", + "maintainer": "infra@elastic.co", + "name": "elastic-agent", + "org_label-schema_build-date": "2021-04-21T06:45:43Z", + "org_label-schema_license": "Elastic License", + "org_label-schema_name": "elastic-agent", + "org_label-schema_schema-version": "1.0", + "org_label-schema_url": "https://www.elastic.co/beats/elastic-agent", + "org_label-schema_vcs-ref": "c52c43963ca8416dc92c7d3dbf6cb6e89dd00acf", + "org_label-schema_vcs-url": "github.com/elastic/beats/v7", + "org_label-schema_vendor": "Elastic", + "org_label-schema_version": "7.13.0-SNAPSHOT", + "org_opencontainers_image_created": "2021-04-21T06:45:43Z", + "org_opencontainers_image_licenses": "Elastic License", + "org_opencontainers_image_title": "Elastic-Agent", + "org_opencontainers_image_vendor": "Elastic", + "release": "1", + "summary": "elastic-agent", + "url": "https://www.elastic.co/beats/elastic-agent", + "vendor": "Elastic", + "version": "7.13.0-SNAPSHOT" + } + }, + "network_summary": { + "icmp": { + "InAddrMaskReps": 0, + "InAddrMasks": 0, + "InCsumErrors": 0, + "InDestUnreachs": 0, + "InEchoReps": 0, + "InEchos": 0, + "InErrors": 0, + "InMsgs": 0, + "InParmProbs": 0, + "InRedirects": 0, + "InSrcQuenchs": 0, + "InTimeExcds": 0, + "InTimestampReps": 0, + "InTimestamps": 0, + "OutAddrMaskReps": 0, + "OutAddrMasks": 0, + "OutDestUnreachs": 0, + "OutEchoReps": 0, + "OutEchos": 0, + "OutErrors": 0, + "OutMsgs": 0, + "OutParmProbs": 0, + "OutRedirects": 0, + "OutSrcQuenchs": 0, + "OutTimeExcds": 0, + "OutTimestampReps": 0, + "OutTimestamps": 0 + }, + "ip": { + "DefaultTTL": 64, + "ForwDatagrams": 0, + "Forwarding": 1, + "FragCreates": 0, + "FragFails": 0, + "FragOKs": 0, + "InAddrErrors": 0, + "InBcastOctets": 0, + "InBcastPkts": 0, + "InCEPkts": 0, + "InCsumErrors": 0, + "InDelivers": 1246323, + "InDiscards": 0, + "InECT0Pkts": 0, + "InECT1Pkts": 0, + "InHdrErrors": 0, + "InMcastOctets": 0, + "InMcastPkts": 0, + "InNoECTPkts": 1246337, + "InNoRoutes": 0, + "InOctets": 172967508, + "InReceives": 1246323, + "InTruncatedPkts": 0, + "InUnknownProtos": 0, + "OutBcastOctets": 0, + "OutBcastPkts": 0, + "OutDiscards": 0, + "OutMcastOctets": 0, + "OutMcastPkts": 0, + "OutNoRoutes": 0, + "OutOctets": 2783525210, + "OutRequests": 1550542, + "ReasmFails": 0, + "ReasmOKs": 0, + "ReasmOverlaps": 0, + "ReasmReqds": 0, + "ReasmTimeout": 0 + }, + "namespace": { + "ID": 4026532414, + "pid": 2240728 + }, + "tcp": { + "ActiveOpens": 742, + "ArpFilter": 0, + "AttemptFails": 0, + "BusyPollRxPackets": 0, + "CurrEstab": 12, + "DelayedACKLocked": 0, + "DelayedACKLost": 1, + "DelayedACKs": 23466, + "EmbryonicRsts": 0, + "EstabResets": 0, + "IPReversePathFilter": 0, + "InCsumErrors": 0, + "InErrs": 0, + "InSegs": 1243467, + "ListenDrops": 0, + "ListenOverflows": 0, + "LockDroppedIcmps": 0, + "MaxConn": -1, + "OfoPruned": 0, + "OutOfWindowIcmps": 0, + "OutRsts": 709, + "OutSegs": 2936174, + "PAWSActive": 0, + "PAWSEstab": 19, + "PFMemallocDrop": 0, + "PassiveOpens": 4, + "PruneCalled": 0, + "RcvPruned": 0, + "RetransSegs": 1613, + "RtoAlgorithm": 1, + "RtoMax": 120000, + "RtoMin": 200, + "SyncookiesFailed": 0, + "SyncookiesRecv": 0, + "SyncookiesSent": 0, + "TCPACKSkippedChallenge": 0, + "TCPACKSkippedFinWait2": 0, + "TCPACKSkippedPAWS": 0, + "TCPACKSkippedSeq": 0, + "TCPACKSkippedSynRecv": 0, + "TCPACKSkippedTimeWait": 0, + "TCPAbortFailed": 0, + "TCPAbortOnClose": 0, + "TCPAbortOnData": 705, + "TCPAbortOnLinger": 0, + "TCPAbortOnMemory": 0, + "TCPAbortOnTimeout": 0, + "TCPAckCompressed": 0, + "TCPAutoCorking": 763, + "TCPBacklogCoalesce": 1358, + "TCPBacklogDrop": 0, + "TCPChallengeACK": 0, + "TCPDSACKIgnoredDubious": 0, + "TCPDSACKIgnoredNoUndo": 1498, + "TCPDSACKIgnoredOld": 0, + "TCPDSACKOfoRecv": 0, + "TCPDSACKOfoSent": 0, + "TCPDSACKOldSent": 1, + "TCPDSACKRecv": 1586, + "TCPDSACKRecvSegs": 1589, + "TCPDSACKUndo": 2, + "TCPDeferAcceptDrop": 0, + "TCPDelivered": 2275375, + "TCPDeliveredCE": 0, + "TCPFastOpenActive": 0, + "TCPFastOpenActiveFail": 0, + "TCPFastOpenBlackhole": 0, + "TCPFastOpenCookieReqd": 0, + "TCPFastOpenListenOverflow": 0, + "TCPFastOpenPassive": 0, + "TCPFastOpenPassiveAltKey": 0, + "TCPFastOpenPassiveFail": 0, + "TCPFastRetrans": 7, + "TCPFromZeroWindowAdv": 0, + "TCPFullUndo": 1, + "TCPHPAcks": 537452, + "TCPHPHits": 144087, + "TCPHystartDelayCwnd": 0, + "TCPHystartDelayDetect": 0, + "TCPHystartTrainCwnd": 36, + "TCPHystartTrainDetect": 2, + "TCPKeepAlive": 171573, + "TCPLossFailures": 0, + "TCPLossProbeRecovery": 0, + "TCPLossProbes": 2183, + "TCPLossUndo": 0, + "TCPLostRetransmit": 0, + "TCPMD5Failure": 0, + "TCPMD5NotFound": 0, + "TCPMD5Unexpected": 0, + "TCPMTUPFail": 0, + "TCPMTUPSuccess": 0, + "TCPMemoryPressures": 0, + "TCPMemoryPressuresChrono": 0, + "TCPMinTTLDrop": 0, + "TCPOFODrop": 0, + "TCPOFOMerge": 0, + "TCPOFOQueue": 0, + "TCPOrigDataSent": 2273068, + "TCPPartialUndo": 1, + "TCPPureAcks": 215589, + "TCPRcvCoalesce": 10, + "TCPRcvCollapsed": 0, + "TCPRcvQDrop": 0, + "TCPRenoFailures": 0, + "TCPRenoRecovery": 0, + "TCPRenoRecoveryFail": 0, + "TCPRenoReorder": 0, + "TCPReqQFullDoCookies": 0, + "TCPReqQFullDrop": 0, + "TCPRetransFail": 0, + "TCPSACKDiscard": 0, + "TCPSACKReneging": 0, + "TCPSACKReorder": 256, + "TCPSYNChallenge": 0, + "TCPSackFailures": 0, + "TCPSackMerged": 5, + "TCPSackRecovery": 4, + "TCPSackRecoveryFail": 0, + "TCPSackShiftFallback": 160, + "TCPSackShifted": 0, + "TCPSlowStartRetrans": 0, + "TCPSpuriousRTOs": 0, + "TCPSpuriousRtxHostQueues": 2, + "TCPSynRetrans": 24, + "TCPTSReorder": 1, + "TCPTimeWaitOverflow": 0, + "TCPTimeouts": 24, + "TCPToZeroWindowAdv": 0, + "TCPWantZeroWindowAdv": 0, + "TCPWinProbe": 0, + "TCPWqueueTooBig": 0, + "TCPZeroWindowDrop": 0, + "TW": 3, + "TWKilled": 0, + "TWRecycled": 0, + "TcpDuplicateDataRehash": 0, + "TcpTimeoutRehash": 24 + }, + "udp": { + "IgnoredMulti": 0, + "InCsumErrors": 0, + "InDatagrams": 2856, + "InErrors": 0, + "NoPorts": 0, + "OutDatagrams": 2856, + "RcvbufErrors": 0, + "SndbufErrors": 0 + }, + "udp_lite": { + "IgnoredMulti": 0, + "InCsumErrors": 0, + "InDatagrams": 0, + "InErrors": 0, + "NoPorts": 0, + "OutDatagrams": 0, + "RcvbufErrors": 0, + "SndbufErrors": 0 + } + } + }, + "event": { + "dataset": "docker.network_summary", + "duration": 115000, + "module": "docker" + }, + "metricset": { + "name": "network_summary", + "period": 10000 + }, + "service": { + "address": "unix:///var/run/docker.sock", + "type": "docker" + } +} \ No newline at end of file diff --git a/metricbeat/module/docker/network_summary/_meta/docs.asciidoc b/metricbeat/module/docker/network_summary/_meta/docs.asciidoc new file mode 100644 index 00000000000..3d3597e8e12 --- /dev/null +++ b/metricbeat/module/docker/network_summary/_meta/docs.asciidoc @@ -0,0 +1,4 @@ +The `network_summary` metricset collects detailed network metrics from the processes associated with a container. +These stats come from `/proc/[PID]/net`, and are summed across the different namespaces found across the PIDs. + +Because this metricset will try to access network counters from procfs, it is only available on linux. \ No newline at end of file diff --git a/metricbeat/module/docker/network_summary/_meta/fields.yml b/metricbeat/module/docker/network_summary/_meta/fields.yml new file mode 100644 index 00000000000..7533a77e5b4 --- /dev/null +++ b/metricbeat/module/docker/network_summary/_meta/fields.yml @@ -0,0 +1,34 @@ +- name: network_summary + type: group + release: beta + description: > + network_summary + fields: + - name: ip.* + type: object + description: > + IP counters + - name: tcp.* + type: object + description: > + TCP counters + - name: udp.* + type: object + description: > + UDP counters + - name: udp_lite.* + type: object + description: > + UDP Lite counters + - name: icmp.* + type: object + description: > + ICMP counters + - name: namespace.pid + type: long + description: > + The root PID of the container, corresponding to /proc/[pid]/net + - name: namespace.id + type: long + description: > + The ID of the network namespace used by the container, corresponding to /proc/[pid]/ns/net \ No newline at end of file diff --git a/metricbeat/module/docker/network_summary/config.go b/metricbeat/module/docker/network_summary/config.go new file mode 100644 index 00000000000..62e355113b6 --- /dev/null +++ b/metricbeat/module/docker/network_summary/config.go @@ -0,0 +1,37 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux + +package network_summary + +import "github.com/elastic/beats/v7/metricbeat/module/docker" + +//Config is the config object for docker/network +type Config struct { + TLS *docker.TLSConfig `config:"ssl"` + DeDot bool `config:"labels.dedot"` + NetworkSummary bool `config:"network.network_summary"` +} + +// DefaultConfig returns default module config +func DefaultConfig() Config { + return Config{ + DeDot: true, + NetworkSummary: false, + } +} diff --git a/metricbeat/module/docker/network_summary/doc.go b/metricbeat/module/docker/network_summary/doc.go new file mode 100644 index 00000000000..917cff9bf6c --- /dev/null +++ b/metricbeat/module/docker/network_summary/doc.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package network_summary diff --git a/metricbeat/module/docker/network_summary/helper.go b/metricbeat/module/docker/network_summary/helper.go new file mode 100644 index 00000000000..adb0fdcc75c --- /dev/null +++ b/metricbeat/module/docker/network_summary/helper.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux + +package network_summary + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "time" + + "github.com/docker/docker/client" + "github.com/pkg/errors" + + "github.com/elastic/go-sysinfo" + + sysinfotypes "github.com/elastic/go-sysinfo/types" +) + +// NoSumStats lists "stats", often config/state values, that can't be safely summed across PIDs +var NoSumStats = []string{ + "RtoAlgorithm", + "RtoMin", + "RtoMax", + "MaxConn", + "Forwarding", + "DefaultTTL", +} + +var nsRegex = regexp.MustCompile(`\d+`) + +// fetchContainerNetStats gathers the PIDs associated with a container, and then uses go-sysinfo to grab the /proc/[pid]/net counters and sum them across PIDs. +func fetchContainerNetStats(client *client.Client, timeout time.Duration, container string) (*sysinfotypes.NetworkCountersInfo, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + inspect, err := client.ContainerInspect(ctx, container) + if err != nil { + return nil, errors.Wrapf(err, "error fetching stats for container %s", container) + } + rootPID := inspect.ContainerJSONBase.State.Pid + + proc, err := sysinfo.Process(rootPID) + procNet, ok := proc.(sysinfotypes.NetworkCounters) + if !ok { + return nil, errors.Wrapf(err, "cannot fetch network counters for PID %d", rootPID) + } + + counters, err := procNet.NetworkCounters() + if err != nil { + return &sysinfotypes.NetworkCountersInfo{}, errors.Wrapf(err, "error fetching network counters for PID %d", rootPID) + } + + return counters, nil + +} + +// fetch the network namespace associated with the PID. +func fetchNamespace(pid int) (int, error) { + nsLink, err := os.Readlink(filepath.Join("/proc/", fmt.Sprintf("%d", pid), "/ns/net")) + if err != nil { + return 0, errors.Wrap(err, "error reading network namespace link") + } + nsidString := nsRegex.FindString(nsLink) + // This is minor metadata, so don't consider it an error + if nsidString == "" { + return 0, nil + } + + nsID, err := strconv.Atoi(nsidString) + if err != nil { + return 0, errors.Wrapf(err, "error converting %s to int", nsidString) + } + return nsID, nil +} diff --git a/metricbeat/module/docker/network_summary/network_summary.go b/metricbeat/module/docker/network_summary/network_summary.go new file mode 100644 index 00000000000..c60d6a16919 --- /dev/null +++ b/metricbeat/module/docker/network_summary/network_summary.go @@ -0,0 +1,129 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux + +package network_summary + +import ( + "context" + "runtime" + + "github.com/docker/docker/client" + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/metric/system/network" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/docker" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("docker", "network_summary", New) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + dockerClient *client.Client + cfg Config +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The docker network_summary metricset is beta.") + + config := DefaultConfig() + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + client, err := docker.NewDockerClient(base.HostData().URI, docker.Config{DeDot: config.DeDot, TLS: config.TLS}) + if err != nil { + return nil, err + } + + // Network summary requres a linux procfs system under it to read from the cgroups. Disable reporting otherwise. + if runtime.GOOS != "linux" { + base.Logger().Debug("Not running on linux, docker network detailed stats disabled.") + config.NetworkSummary = false + } + + return &MetricSet{ + BaseMetricSet: base, + dockerClient: client, + cfg: config, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + + stats, err := docker.FetchStats(m.dockerClient, m.Module().Config().Timeout) + if err != nil { + return errors.Wrap(err, "failed to get docker stats") + } + + for _, myStats := range stats { + + ctx, cancel := context.WithTimeout(context.Background(), m.Module().Config().Timeout) + defer cancel() + + inspect, err := m.dockerClient.ContainerInspect(ctx, myStats.Container.ID) + if err != nil { + return errors.Wrapf(err, "error fetching stats for container %s", myStats.Container.ID) + } + + rootPID := inspect.ContainerJSONBase.State.Pid + + netNS, err := fetchNamespace(rootPID) + if err != nil { + return errors.Wrapf(err, "error fetching namespace for PID %d", rootPID) + } + + networkStats, err := fetchContainerNetStats(m.dockerClient, m.Module().Config().Timeout, myStats.Container.ID) + if err != nil { + return errors.Wrap(err, "error fetching per-PID stats") + } + + summary := network.MapProcNetCounters(networkStats) + // attach metadata associated with the network counters + summary["namespace"] = common.MapStr{ + "ID": netNS, + "pid": rootPID, + } + + report.Event(mb.Event{ + RootFields: docker.NewContainer(myStats.Container, m.cfg.DeDot).ToMapStr(), + MetricSetFields: summary, + }) + + } + + return nil +} diff --git a/metricbeat/module/docker/network_summary/network_summary_test.go b/metricbeat/module/docker/network_summary/network_summary_test.go new file mode 100644 index 00000000000..cadccf5b5d4 --- /dev/null +++ b/metricbeat/module/docker/network_summary/network_summary_test.go @@ -0,0 +1,42 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package network_summary + +import ( + "testing" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + ms := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + err := mbtest.WriteEventsReporterV2Error(ms, t, "") + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "docker", + "metricsets": []string{"network_summary"}, + "hosts": []string{"unix:///var/run/docker.sock"}, + } +} diff --git a/metricbeat/modules.d/docker.yml.disabled b/metricbeat/modules.d/docker.yml.disabled index b3558345931..cd2b5a8aaa6 100644 --- a/metricbeat/modules.d/docker.yml.disabled +++ b/metricbeat/modules.d/docker.yml.disabled @@ -11,13 +11,12 @@ # - info # - memory # - network + # - network_summary period: 10s hosts: ["unix:///var/run/docker.sock"] # If set to true, replace dots in labels with `_`. #labels.dedot: false - # Set to true to enable low-level network stats for the processes in the container - # network.network_summary: true # To connect to Docker over TLS you must specify a client and CA certificate. #ssl: From 71cf0a75b959d2807cdf6222e8c745c60a3d3ce9 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Thu, 29 Apr 2021 12:20:26 -0700 Subject: [PATCH 07/10] revert changes to docker/network --- .../module/docker/network/_meta/data.json | 52 +++++++-------- metricbeat/module/docker/network/data.go | 63 +++++++++---------- metricbeat/module/docker/network/helper.go | 12 +--- metricbeat/module/docker/network/network.go | 14 ++--- 4 files changed, 60 insertions(+), 81 deletions(-) diff --git a/metricbeat/module/docker/network/_meta/data.json b/metricbeat/module/docker/network/_meta/data.json index 4ac4293b6cd..fba2fae388c 100644 --- a/metricbeat/module/docker/network/_meta/data.json +++ b/metricbeat/module/docker/network/_meta/data.json @@ -1,43 +1,36 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, "container": { - "id": "b9f9e931fdf5f0d8c3d6a18f38cedd9794b6de48a9ff708c5c0e9a29d54db169", + "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a", "image": { - "name": "docker.elastic.co/elasticsearch/elasticsearch:7.13.0-SNAPSHOT" + "name": "metricbeat_elasticsearch" }, - "name": "elastic-package-stack_elasticsearch_1", + "name": "metricbeat_elasticsearch_1_df866b3a7b3d", "runtime": "docker" }, "docker": { "container": { "labels": { - "com_docker_compose_config-hash": "2a2f6b4cb4003f7043c3d76a9abfc052644fe5b8d984b5c238df25cd1d0f653e", + "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78", "com_docker_compose_container-number": "1", "com_docker_compose_oneoff": "False", - "com_docker_compose_project": "elastic-package-stack", - "com_docker_compose_project_config_files": "/home/alexk/.elastic-package/stack/openport/snapshot.yml", - "com_docker_compose_project_working_dir": "/home/alexk/.elastic-package/stack/openport", + "com_docker_compose_project": "metricbeat", "com_docker_compose_service": "elasticsearch", - "com_docker_compose_version": "1.27.4", - "org_label-schema_build-date": "2021-04-21T03:53:24.920332641Z", - "org_label-schema_license": "Elastic-License-2.0", - "org_label-schema_name": "Elasticsearch", + "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d", + "com_docker_compose_version": "1.23.1", + "license": "Elastic License", + "org_label-schema_build-date": "20181006", + "org_label-schema_license": "GPLv2", + "org_label-schema_name": "elasticsearch", "org_label-schema_schema-version": "1.0", "org_label-schema_url": "https://www.elastic.co/products/elasticsearch", - "org_label-schema_usage": "https://www.elastic.co/guide/en/elasticsearch/reference/index.html", - "org_label-schema_vcs-ref": "4c99108262abdeb5c4d9bf23edcd99fe403f5652", - "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch", + "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker", "org_label-schema_vendor": "Elastic", - "org_label-schema_version": "7.13.0-SNAPSHOT", - "org_opencontainers_image_created": "2021-04-21T03:53:24.920332641Z", - "org_opencontainers_image_documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/index.html", - "org_opencontainers_image_licenses": "Elastic-License-2.0", - "org_opencontainers_image_revision": "4c99108262abdeb5c4d9bf23edcd99fe403f5652", - "org_opencontainers_image_source": "https://github.com/elastic/elasticsearch", - "org_opencontainers_image_title": "Elasticsearch", - "org_opencontainers_image_url": "https://www.elastic.co/products/elasticsearch", - "org_opencontainers_image_vendor": "Elastic", - "org_opencontainers_image_version": "7.13.0-SNAPSHOT" + "org_label-schema_version": "6.5.1" } }, "network": { @@ -48,10 +41,10 @@ "packets": 0 }, "inbound": { - "bytes": 10553051848, + "bytes": 23047, "dropped": 0, "errors": 0, - "packets": 43569407 + "packets": 241 }, "interface": "eth0", "out": { @@ -61,10 +54,10 @@ "packets": 0 }, "outbound": { - "bytes": 4106659715, + "bytes": 0, "dropped": 0, "errors": 0, - "packets": 40961196 + "packets": 0 } } }, @@ -74,8 +67,7 @@ "module": "docker" }, "metricset": { - "name": "network", - "period": 10000 + "name": "network" }, "service": { "address": "/var/run/docker.sock", diff --git a/metricbeat/module/docker/network/data.go b/metricbeat/module/docker/network/data.go index eb89cd50fee..b4a2b90c405 100644 --- a/metricbeat/module/docker/network/data.go +++ b/metricbeat/module/docker/network/data.go @@ -29,39 +29,36 @@ func eventsMapping(r mb.ReporterV2, netsStatsList []NetStats) { } func eventMapping(r mb.ReporterV2, stats *NetStats) { - - fields := common.MapStr{ - "interface": stats.NameInterface, - // Deprecated - "in": common.MapStr{ - "bytes": stats.RxBytes, - "dropped": stats.RxDropped, - "errors": stats.RxErrors, - "packets": stats.RxPackets, - }, - // Deprecated - "out": common.MapStr{ - "bytes": stats.TxBytes, - "dropped": stats.TxDropped, - "errors": stats.TxErrors, - "packets": stats.TxPackets, - }, - "inbound": common.MapStr{ - "bytes": stats.Total.RxBytes, - "dropped": stats.Total.RxDropped, - "errors": stats.Total.RxErrors, - "packets": stats.Total.RxPackets, - }, - "outbound": common.MapStr{ - "bytes": stats.Total.TxBytes, - "dropped": stats.Total.TxDropped, - "errors": stats.Total.TxErrors, - "packets": stats.Total.TxPackets, - }, - } - r.Event(mb.Event{ - RootFields: stats.Container.ToMapStr(), - MetricSetFields: fields, + RootFields: stats.Container.ToMapStr(), + MetricSetFields: common.MapStr{ + "interface": stats.NameInterface, + // Deprecated + "in": common.MapStr{ + "bytes": stats.RxBytes, + "dropped": stats.RxDropped, + "errors": stats.RxErrors, + "packets": stats.RxPackets, + }, + // Deprecated + "out": common.MapStr{ + "bytes": stats.TxBytes, + "dropped": stats.TxDropped, + "errors": stats.TxErrors, + "packets": stats.TxPackets, + }, + "inbound": common.MapStr{ + "bytes": stats.Total.RxBytes, + "dropped": stats.Total.RxDropped, + "errors": stats.Total.RxErrors, + "packets": stats.Total.RxPackets, + }, + "outbound": common.MapStr{ + "bytes": stats.Total.TxBytes, + "dropped": stats.Total.TxDropped, + "errors": stats.Total.TxErrors, + "packets": stats.Total.TxPackets, + }, + }, }) } diff --git a/metricbeat/module/docker/network/helper.go b/metricbeat/module/docker/network/helper.go index 7e1a2fbfde7..0f980fc2ab9 100644 --- a/metricbeat/module/docker/network/helper.go +++ b/metricbeat/module/docker/network/helper.go @@ -25,12 +25,10 @@ import ( "github.com/elastic/beats/v7/metricbeat/module/docker" ) -// NetService represents maps out the interface-level stats per-container type NetService struct { NetworkStatPerContainer map[string]map[string]NetRaw } -// NetworkCalculator is the interface that reports per-second stats type NetworkCalculator interface { getRxBytesPerSecond(newStats *NetRaw, oldStats *NetRaw) float64 getRxDroppedPerSecond(newStats *NetRaw, oldStats *NetRaw) float64 @@ -42,7 +40,6 @@ type NetworkCalculator interface { getTxPacketsPerSecond(newStats *NetRaw, oldStats *NetRaw) float64 } -// NetRaw represents the raw network stats from docker type NetRaw struct { Time time.Time RxBytes uint64 @@ -55,7 +52,6 @@ type NetRaw struct { TxPackets uint64 } -// NetStats represents the network counters for a given network interface type NetStats struct { Time time.Time Container *docker.Container @@ -71,17 +67,15 @@ type NetStats struct { Total *types.NetworkStats } -func (n *NetService) getNetworkStatsPerContainer(rawStats []docker.Stat, cfg Config) ([]NetStats, error) { +func (n *NetService) getNetworkStatsPerContainer(rawStats []docker.Stat, dedot bool) []NetStats { formattedStats := []NetStats{} for _, myStats := range rawStats { - for nameInterface, rawnNetStats := range myStats.Stats.Networks { - singleStat := n.getNetworkStats(nameInterface, rawnNetStats, myStats, cfg.DeDot) - formattedStats = append(formattedStats, singleStat) + formattedStats = append(formattedStats, n.getNetworkStats(nameInterface, rawnNetStats, myStats, dedot)) } } - return formattedStats, nil + return formattedStats } func (n *NetService) getNetworkStats(nameInterface string, rawNetStats types.NetworkStats, myRawstats docker.Stat, dedot bool) NetStats { diff --git a/metricbeat/module/docker/network/network.go b/metricbeat/module/docker/network/network.go index 660282e8a49..04e67c95412 100644 --- a/metricbeat/module/docker/network/network.go +++ b/metricbeat/module/docker/network/network.go @@ -34,22 +34,21 @@ func init() { ) } -// MetricSet for fetching docker network metrics. type MetricSet struct { mb.BaseMetricSet netService *NetService dockerClient *client.Client - cfg Config + dedot bool } // New creates a new instance of the docker network MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := DefaultConfig() + config := docker.DefaultConfig() if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - client, err := docker.NewDockerClient(base.HostData().URI, docker.Config{DeDot: config.DeDot, TLS: config.TLS}) + client, err := docker.NewDockerClient(base.HostData().URI, config) if err != nil { return nil, err } @@ -60,7 +59,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { netService: &NetService{ NetworkStatPerContainer: make(map[string]map[string]NetRaw), }, - cfg: config, + dedot: config.DeDot, }, nil } @@ -71,10 +70,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return errors.Wrap(err, "failed to get docker stats") } - formattedStats, err := m.netService.getNetworkStatsPerContainer(stats, m.cfg) - if err != nil { - return errors.Wrap(err, "error fetching container network stats") - } + formattedStats := m.netService.getNetworkStatsPerContainer(stats, m.dedot) eventsMapping(r, formattedStats) return nil From 69450b10059da1a3f1e86e3428c94bff41ee72f8 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Thu, 29 Apr 2021 12:22:49 -0700 Subject: [PATCH 08/10] remove config file --- metricbeat/module/docker/network/config.go | 35 ---------------------- 1 file changed, 35 deletions(-) delete mode 100644 metricbeat/module/docker/network/config.go diff --git a/metricbeat/module/docker/network/config.go b/metricbeat/module/docker/network/config.go deleted file mode 100644 index 4be453ac125..00000000000 --- a/metricbeat/module/docker/network/config.go +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build linux darwin windows - -package network - -import "github.com/elastic/beats/v7/metricbeat/module/docker" - -//Config is the config object for docker/network -type Config struct { - TLS *docker.TLSConfig `config:"ssl"` - DeDot bool `config:"labels.dedot"` -} - -// DefaultConfig returns default module config -func DefaultConfig() Config { - return Config{ - DeDot: true, - } -} From 3f9eb2866bc37d69b2d70bcef5614806ef3a8ff8 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Thu, 29 Apr 2021 13:25:10 -0700 Subject: [PATCH 09/10] update xpack --- x-pack/metricbeat/metricbeat.reference.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index ffdf73d9720..c26eb34ffc2 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -460,14 +460,13 @@ metricbeat.modules: #- "image" - "memory" - "network" + #- "network_summary" hosts: ["unix:///var/run/docker.sock"] period: 10s enabled: true # If set to true, replace dots in labels with `_`. #labels.dedot: false - # Set to true to enable low-level network stats for the processes in the container - # network.network_summary: true # If set to true, collects metrics per core. #cpu.cores: true From c4f31e589b09350a24a98d5c6fc90ce60df32d70 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Fri, 30 Apr 2021 09:06:10 -0700 Subject: [PATCH 10/10] small fixes --- .../module/docker/network/_meta/data.json | 59 +++++++++++-------- .../docker/network_summary/network_summary.go | 6 +- .../network_summary/network_summary_test.go | 9 +-- 3 files changed, 42 insertions(+), 32 deletions(-) diff --git a/metricbeat/module/docker/network/_meta/data.json b/metricbeat/module/docker/network/_meta/data.json index fba2fae388c..a671884a103 100644 --- a/metricbeat/module/docker/network/_meta/data.json +++ b/metricbeat/module/docker/network/_meta/data.json @@ -1,36 +1,48 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "container": { - "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a", + "id": "b1c3118fb2087e499c421aa20e28847ad4c185be6c1864166b1a03cc2affaf80", "image": { - "name": "metricbeat_elasticsearch" + "name": "docker.elastic.co/beats/elastic-agent:7.13.0-SNAPSHOT" }, - "name": "metricbeat_elasticsearch_1_df866b3a7b3d", + "name": "elastic-package-stack_elastic-agent_1", "runtime": "docker" }, "docker": { "container": { "labels": { - "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78", + "com_docker_compose_config-hash": "fdad7f67fafe5a242b9b9d27bff78bef4b96c683a2bb1412d91c32fb5981f2b3", "com_docker_compose_container-number": "1", "com_docker_compose_oneoff": "False", - "com_docker_compose_project": "metricbeat", - "com_docker_compose_service": "elasticsearch", - "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d", - "com_docker_compose_version": "1.23.1", + "com_docker_compose_project": "elastic-package-stack", + "com_docker_compose_project_config_files": "/home/alexk/.elastic-package/stack/openport/snapshot.yml", + "com_docker_compose_project_working_dir": "/home/alexk/.elastic-package/stack/openport", + "com_docker_compose_service": "elastic-agent", + "com_docker_compose_version": "1.27.4", + "description": "Agent manages other beats based on configuration provided.", + "io_k8s_description": "Agent manages other beats based on configuration provided.", + "io_k8s_display-name": "Elastic-Agent image", "license": "Elastic License", - "org_label-schema_build-date": "20181006", - "org_label-schema_license": "GPLv2", - "org_label-schema_name": "elasticsearch", + "maintainer": "infra@elastic.co", + "name": "elastic-agent", + "org_label-schema_build-date": "2021-04-21T06:45:43Z", + "org_label-schema_license": "Elastic License", + "org_label-schema_name": "elastic-agent", "org_label-schema_schema-version": "1.0", - "org_label-schema_url": "https://www.elastic.co/products/elasticsearch", - "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker", + "org_label-schema_url": "https://www.elastic.co/beats/elastic-agent", + "org_label-schema_vcs-ref": "c52c43963ca8416dc92c7d3dbf6cb6e89dd00acf", + "org_label-schema_vcs-url": "github.com/elastic/beats/v7", "org_label-schema_vendor": "Elastic", - "org_label-schema_version": "6.5.1" + "org_label-schema_version": "7.13.0-SNAPSHOT", + "org_opencontainers_image_created": "2021-04-21T06:45:43Z", + "org_opencontainers_image_licenses": "Elastic License", + "org_opencontainers_image_title": "Elastic-Agent", + "org_opencontainers_image_vendor": "Elastic", + "release": "1", + "summary": "elastic-agent", + "url": "https://www.elastic.co/beats/elastic-agent", + "vendor": "Elastic", + "version": "7.13.0-SNAPSHOT" } }, "network": { @@ -41,10 +53,10 @@ "packets": 0 }, "inbound": { - "bytes": 23047, + "bytes": 156469392, "dropped": 0, "errors": 0, - "packets": 241 + "packets": 710866 }, "interface": "eth0", "out": { @@ -54,10 +66,10 @@ "packets": 0 }, "outbound": { - "bytes": 0, + "bytes": 3150916813, "dropped": 0, "errors": 0, - "packets": 0 + "packets": 1059447 } } }, @@ -67,7 +79,8 @@ "module": "docker" }, "metricset": { - "name": "network" + "name": "network", + "period": 10000 }, "service": { "address": "/var/run/docker.sock", diff --git a/metricbeat/module/docker/network_summary/network_summary.go b/metricbeat/module/docker/network_summary/network_summary.go index c60d6a16919..8ed3d4c87ef 100644 --- a/metricbeat/module/docker/network_summary/network_summary.go +++ b/metricbeat/module/docker/network_summary/network_summary.go @@ -82,7 +82,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). -func (m *MetricSet) Fetch(report mb.ReporterV2) error { +func (m *MetricSet) Fetch(ctx context.Context, report mb.ReporterV2) error { stats, err := docker.FetchStats(m.dockerClient, m.Module().Config().Timeout) if err != nil { @@ -91,7 +91,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, myStats := range stats { - ctx, cancel := context.WithTimeout(context.Background(), m.Module().Config().Timeout) + ctx, cancel := context.WithTimeout(ctx, m.Module().Config().Timeout) defer cancel() inspect, err := m.dockerClient.ContainerInspect(ctx, myStats.Container.ID) @@ -114,7 +114,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { summary := network.MapProcNetCounters(networkStats) // attach metadata associated with the network counters summary["namespace"] = common.MapStr{ - "ID": netNS, + "id": netNS, "pid": rootPID, } diff --git a/metricbeat/module/docker/network_summary/network_summary_test.go b/metricbeat/module/docker/network_summary/network_summary_test.go index cadccf5b5d4..fdf6d100f72 100644 --- a/metricbeat/module/docker/network_summary/network_summary_test.go +++ b/metricbeat/module/docker/network_summary/network_summary_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build integration +// +build integration linux package network_summary @@ -26,11 +26,8 @@ import ( ) func TestData(t *testing.T) { - ms := mbtest.NewReportingMetricSetV2Error(t, getConfig()) - err := mbtest.WriteEventsReporterV2Error(ms, t, "") - if err != nil { - t.Fatal("write", err) - } + ms := mbtest.NewFetcher(t, getConfig()) + ms.WriteEvents(t, "") } func getConfig() map[string]interface{} {