Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add detailed docker network summary stats #25354

Merged
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
@@ -972,6 +972,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Apache: convert status.total_kbytes to status.total_bytes in fleet mode. {pull}23022[23022]
- Release MSSQL as GA {pull}23146[23146]
- Add support for SASL/SCRAM authentication to the Kafka module. {pull}24810[24810]
- Add additional network metrics to docker/network {pull}25354[25354]

*Packetbeat*

4 changes: 2 additions & 2 deletions NOTICE.txt
Original file line number Diff line number Diff line change
@@ -7695,11 +7695,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v

--------------------------------------------------------------------------------
Dependency : github.com/elastic/go-sysinfo
Version: v1.6.0
Version: v1.7.0
Licence type (autodetected): Apache-2.0
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.6.0/LICENSE.txt:
Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.7.0/LICENSE.txt:


Apache License
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -70,7 +70,7 @@ require (
github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595
github.com/elastic/go-seccomp-bpf v1.1.0
github.com/elastic/go-structform v0.0.8
github.com/elastic/go-sysinfo v1.6.0
github.com/elastic/go-sysinfo v1.7.0
github.com/elastic/go-txfile v0.0.7
github.com/elastic/go-ucfg v0.8.3
github.com/elastic/go-windows v1.0.1 // indirect
7 changes: 3 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
@@ -299,8 +299,8 @@ github.com/elastic/go-seccomp-bpf v1.1.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE
github.com/elastic/go-structform v0.0.8 h1:U0qnb9Zqig7w+FhF+sLI3VZPPi/+2aJ0bIEW6R1z6Tk=
github.com/elastic/go-structform v0.0.8/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
github.com/elastic/go-sysinfo v1.6.0 h1:u0QbU8eWSwKRPcFQancnSY4Zi0COksCJXkUgPHxE5Tw=
github.com/elastic/go-sysinfo v1.6.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
github.com/elastic/go-sysinfo v1.7.0 h1:4vVvcfi255+8+TyQ7TYUTEK3A+G8v5FLE+ZKYL1z1Dg=
github.com/elastic/go-sysinfo v1.7.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
github.com/elastic/go-txfile v0.0.7 h1:Yn28gclW7X0Qy09nSMSsx0uOAvAGMsp6XHydbiLVe2s=
github.com/elastic/go-txfile v0.0.7/go.mod h1:H0nCoFae0a4ga57apgxFsgmRjevNCsEaT6g56JoeKAE=
github.com/elastic/go-ucfg v0.7.0/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
@@ -765,11 +765,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b h1:X/8hkb4rQq3+QuOxpJK7gWmAXmZucF0EI1s1BfBLq6U=
github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b/go.mod h1:jAqhj/JBVC1PwcLTWd6rjQyGyItxxrhpiBl8LSuAGmw=
Original file line number Diff line number Diff line change
@@ -15,15 +15,15 @@
// specific language governing permissions and limitations
// under the License.

package network_summary
package network

import (
"github.com/elastic/beats/v7/libbeat/common"
sysinfotypes "github.com/elastic/go-sysinfo/types"
)

// eventMapping maps the network counters to a MapStr that wil be sent to report.Event
func eventMapping(raw *sysinfotypes.NetworkCountersInfo) common.MapStr {
// MapProcNetCounters converts the NetworkCountersInfo struct into a MapStr acceptable for sending upstream
func MapProcNetCounters(raw *sysinfotypes.NetworkCountersInfo) common.MapStr {

eventByProto := common.MapStr{
"ip": combineMap(raw.Netstat.IPExt, raw.SNMP.IP),
@@ -33,10 +33,6 @@ func eventMapping(raw *sysinfotypes.NetworkCountersInfo) common.MapStr {
"icmp": combineMap(raw.SNMP.ICMPMsg, raw.SNMP.ICMP),
}

// if value, ok := raw.SNMP.TCP["MaxConn"]; ok && value != 0 {
// eventByProto["tcp"].(map)
// }

return eventByProto
}

77 changes: 77 additions & 0 deletions metricbeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
@@ -12376,6 +12376,83 @@ type: long
Total number of outgoing packets.


type: long

--

[float]
=== network_summary

network_summary



*`docker.network_summary.ip.*`*::
+
--
IP counters


type: object

--

*`docker.network_summary.tcp.*`*::
+
--
TCP counters


type: object

--

*`docker.network_summary.udp.*`*::
+
--
UDP counters


type: object

--

*`docker.network_summary.udp_lite.*`*::
+
--
UDP Lite counters


type: object

--

*`docker.network_summary.icmp.*`*::
+
--
ICMP counters


type: object

--

*`docker.network_summary.namespace.pid`*::
+
--
The root PID of the container, corresponding to /proc/[pid]/net


type: long

--

*`docker.network_summary.namespace.id`*::
+
--
The ID of the network namespace used by the container, corresponding to /proc/[pid]/ns/net


type: long

--
5 changes: 5 additions & 0 deletions metricbeat/docs/modules/docker.asciidoc
Original file line number Diff line number Diff line change
@@ -44,6 +44,7 @@ metricbeat.modules:
#- "image"
- "memory"
- "network"
#- "network_summary"
hosts: ["unix:///var/run/docker.sock"]
period: 10s
enabled: true
@@ -86,6 +87,8 @@ The following metricsets are available:

* <<metricbeat-metricset-docker-network,network>>

* <<metricbeat-metricset-docker-network_summary,network_summary>>

include::docker/container.asciidoc[]

include::docker/cpu.asciidoc[]
@@ -104,3 +107,5 @@ include::docker/memory.asciidoc[]

include::docker/network.asciidoc[]

include::docker/network_summary.asciidoc[]

23 changes: 23 additions & 0 deletions metricbeat/docs/modules/docker/network_summary.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
////
This file is generated! See scripts/mage/docs_collector.go
////

[[metricbeat-metricset-docker-network_summary]]
=== Docker network_summary metricset

beta[]

include::../../../module/docker/network_summary/_meta/docs.asciidoc[]


==== Fields

For a description of each field in the metricset, see the
<<exported-fields-docker,exported fields>> section.

Here is an example document generated by this metricset:

[source,json]
----
include::../../../module/docker/network_summary/_meta/data.json[]
----
3 changes: 2 additions & 1 deletion metricbeat/docs/modules_list.asciidoc
Original file line number Diff line number Diff line change
@@ -80,7 +80,7 @@ This file is generated! See scripts/mage/docs_collector.go
|<<metricbeat-module-couchdb,CouchDB>> |image:./images/icon-yes.png[Prebuilt dashboards are available] |
.1+| .1+| |<<metricbeat-metricset-couchdb-server,server>>
|<<metricbeat-module-docker,Docker>> |image:./images/icon-yes.png[Prebuilt dashboards are available] |
.9+| .9+| |<<metricbeat-metricset-docker-container,container>>
.10+| .10+| |<<metricbeat-metricset-docker-container,container>>
|<<metricbeat-metricset-docker-cpu,cpu>>
|<<metricbeat-metricset-docker-diskio,diskio>>
|<<metricbeat-metricset-docker-event,event>>
@@ -89,6 +89,7 @@ This file is generated! See scripts/mage/docs_collector.go
|<<metricbeat-metricset-docker-info,info>>
|<<metricbeat-metricset-docker-memory,memory>>
|<<metricbeat-metricset-docker-network,network>>
|<<metricbeat-metricset-docker-network_summary,network_summary>> beta[]
|<<metricbeat-module-dropwizard,Dropwizard>> |image:./images/icon-no.png[No prebuilt dashboards] |
.1+| .1+| |<<metricbeat-metricset-dropwizard-collector,collector>>
|<<metricbeat-module-elasticsearch,Elasticsearch>> |image:./images/icon-no.png[No prebuilt dashboards] |
1 change: 1 addition & 0 deletions metricbeat/include/list_docker.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions metricbeat/metricbeat.reference.yml
Original file line number Diff line number Diff line change
@@ -235,6 +235,7 @@ metricbeat.modules:
#- "image"
- "memory"
- "network"
#- "network_summary"
hosts: ["unix:///var/run/docker.sock"]
period: 10s
enabled: true
1 change: 1 addition & 0 deletions metricbeat/module/docker/_meta/config.reference.yml
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@
#- "image"
- "memory"
- "network"
#- "network_summary"
hosts: ["unix:///var/run/docker.sock"]
period: 10s
enabled: true
1 change: 1 addition & 0 deletions metricbeat/module/docker/_meta/config.yml
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@
# - info
# - memory
# - network
# - network_summary
period: 10s
hosts: ["unix:///var/run/docker.sock"]

2 changes: 1 addition & 1 deletion metricbeat/module/docker/fields.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

59 changes: 36 additions & 23 deletions metricbeat/module/docker/network/_meta/data.json
Original file line number Diff line number Diff line change
@@ -1,36 +1,48 @@
{
"@timestamp": "2017-10-12T08:05:34.853Z",
"agent": {
"hostname": "host.example.com",
"name": "host.example.com"
},
"container": {
"id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a",
"id": "b1c3118fb2087e499c421aa20e28847ad4c185be6c1864166b1a03cc2affaf80",
"image": {
"name": "metricbeat_elasticsearch"
"name": "docker.elastic.co/beats/elastic-agent:7.13.0-SNAPSHOT"
},
"name": "metricbeat_elasticsearch_1_df866b3a7b3d",
"name": "elastic-package-stack_elastic-agent_1",
"runtime": "docker"
},
"docker": {
"container": {
"labels": {
"com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78",
"com_docker_compose_config-hash": "fdad7f67fafe5a242b9b9d27bff78bef4b96c683a2bb1412d91c32fb5981f2b3",
"com_docker_compose_container-number": "1",
"com_docker_compose_oneoff": "False",
"com_docker_compose_project": "metricbeat",
"com_docker_compose_service": "elasticsearch",
"com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d",
"com_docker_compose_version": "1.23.1",
"com_docker_compose_project": "elastic-package-stack",
"com_docker_compose_project_config_files": "/home/alexk/.elastic-package/stack/openport/snapshot.yml",
"com_docker_compose_project_working_dir": "/home/alexk/.elastic-package/stack/openport",
"com_docker_compose_service": "elastic-agent",
"com_docker_compose_version": "1.27.4",
"description": "Agent manages other beats based on configuration provided.",
"io_k8s_description": "Agent manages other beats based on configuration provided.",
"io_k8s_display-name": "Elastic-Agent image",
"license": "Elastic License",
"org_label-schema_build-date": "20181006",
"org_label-schema_license": "GPLv2",
"org_label-schema_name": "elasticsearch",
"maintainer": "infra@elastic.co",
"name": "elastic-agent",
"org_label-schema_build-date": "2021-04-21T06:45:43Z",
"org_label-schema_license": "Elastic License",
"org_label-schema_name": "elastic-agent",
"org_label-schema_schema-version": "1.0",
"org_label-schema_url": "https://www.elastic.co/products/elasticsearch",
"org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker",
"org_label-schema_url": "https://www.elastic.co/beats/elastic-agent",
"org_label-schema_vcs-ref": "c52c43963ca8416dc92c7d3dbf6cb6e89dd00acf",
"org_label-schema_vcs-url": "github.com/elastic/beats/v7",
"org_label-schema_vendor": "Elastic",
"org_label-schema_version": "6.5.1"
"org_label-schema_version": "7.13.0-SNAPSHOT",
"org_opencontainers_image_created": "2021-04-21T06:45:43Z",
"org_opencontainers_image_licenses": "Elastic License",
"org_opencontainers_image_title": "Elastic-Agent",
"org_opencontainers_image_vendor": "Elastic",
"release": "1",
"summary": "elastic-agent",
"url": "https://www.elastic.co/beats/elastic-agent",
"vendor": "Elastic",
"version": "7.13.0-SNAPSHOT"
}
},
"network": {
@@ -41,10 +53,10 @@
"packets": 0
},
"inbound": {
"bytes": 23047,
"bytes": 156469392,
"dropped": 0,
"errors": 0,
"packets": 241
"packets": 710866
},
"interface": "eth0",
fearful-symmetry marked this conversation as resolved.
Show resolved Hide resolved
"out": {
@@ -54,10 +66,10 @@
"packets": 0
},
"outbound": {
"bytes": 0,
"bytes": 3150916813,
"dropped": 0,
"errors": 0,
"packets": 0
"packets": 1059447
}
}
},
@@ -67,7 +79,8 @@
"module": "docker"
},
"metricset": {
"name": "network"
"name": "network",
"period": 10000
},
"service": {
"address": "/var/run/docker.sock",
296 changes: 296 additions & 0 deletions metricbeat/module/docker/network_summary/_meta/data.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,296 @@
{
"@timestamp": "2017-10-12T08:05:34.853Z",
"container": {
"id": "b1c3118fb2087e499c421aa20e28847ad4c185be6c1864166b1a03cc2affaf80",
"image": {
"name": "docker.elastic.co/beats/elastic-agent:7.13.0-SNAPSHOT"
},
"name": "elastic-package-stack_elastic-agent_1",
"runtime": "docker"
},
"docker": {
"container": {
"labels": {
"com_docker_compose_config-hash": "fdad7f67fafe5a242b9b9d27bff78bef4b96c683a2bb1412d91c32fb5981f2b3",
"com_docker_compose_container-number": "1",
"com_docker_compose_oneoff": "False",
"com_docker_compose_project": "elastic-package-stack",
"com_docker_compose_project_config_files": "/home/alexk/.elastic-package/stack/openport/snapshot.yml",
"com_docker_compose_project_working_dir": "/home/alexk/.elastic-package/stack/openport",
"com_docker_compose_service": "elastic-agent",
"com_docker_compose_version": "1.27.4",
"description": "Agent manages other beats based on configuration provided.",
"io_k8s_description": "Agent manages other beats based on configuration provided.",
"io_k8s_display-name": "Elastic-Agent image",
"license": "Elastic License",
"maintainer": "infra@elastic.co",
"name": "elastic-agent",
"org_label-schema_build-date": "2021-04-21T06:45:43Z",
"org_label-schema_license": "Elastic License",
"org_label-schema_name": "elastic-agent",
"org_label-schema_schema-version": "1.0",
"org_label-schema_url": "https://www.elastic.co/beats/elastic-agent",
"org_label-schema_vcs-ref": "c52c43963ca8416dc92c7d3dbf6cb6e89dd00acf",
"org_label-schema_vcs-url": "github.com/elastic/beats/v7",
"org_label-schema_vendor": "Elastic",
"org_label-schema_version": "7.13.0-SNAPSHOT",
"org_opencontainers_image_created": "2021-04-21T06:45:43Z",
"org_opencontainers_image_licenses": "Elastic License",
"org_opencontainers_image_title": "Elastic-Agent",
"org_opencontainers_image_vendor": "Elastic",
"release": "1",
"summary": "elastic-agent",
"url": "https://www.elastic.co/beats/elastic-agent",
"vendor": "Elastic",
"version": "7.13.0-SNAPSHOT"
}
},
"network_summary": {
"icmp": {
"InAddrMaskReps": 0,
"InAddrMasks": 0,
"InCsumErrors": 0,
"InDestUnreachs": 0,
"InEchoReps": 0,
"InEchos": 0,
"InErrors": 0,
"InMsgs": 0,
"InParmProbs": 0,
"InRedirects": 0,
"InSrcQuenchs": 0,
"InTimeExcds": 0,
"InTimestampReps": 0,
"InTimestamps": 0,
"OutAddrMaskReps": 0,
"OutAddrMasks": 0,
"OutDestUnreachs": 0,
"OutEchoReps": 0,
"OutEchos": 0,
"OutErrors": 0,
"OutMsgs": 0,
"OutParmProbs": 0,
"OutRedirects": 0,
"OutSrcQuenchs": 0,
"OutTimeExcds": 0,
"OutTimestampReps": 0,
"OutTimestamps": 0
},
"ip": {
"DefaultTTL": 64,
"ForwDatagrams": 0,
"Forwarding": 1,
"FragCreates": 0,
"FragFails": 0,
"FragOKs": 0,
"InAddrErrors": 0,
"InBcastOctets": 0,
"InBcastPkts": 0,
"InCEPkts": 0,
"InCsumErrors": 0,
"InDelivers": 1246323,
"InDiscards": 0,
"InECT0Pkts": 0,
"InECT1Pkts": 0,
"InHdrErrors": 0,
"InMcastOctets": 0,
"InMcastPkts": 0,
"InNoECTPkts": 1246337,
"InNoRoutes": 0,
"InOctets": 172967508,
"InReceives": 1246323,
"InTruncatedPkts": 0,
"InUnknownProtos": 0,
"OutBcastOctets": 0,
"OutBcastPkts": 0,
"OutDiscards": 0,
"OutMcastOctets": 0,
"OutMcastPkts": 0,
"OutNoRoutes": 0,
"OutOctets": 2783525210,
"OutRequests": 1550542,
"ReasmFails": 0,
"ReasmOKs": 0,
"ReasmOverlaps": 0,
"ReasmReqds": 0,
"ReasmTimeout": 0
},
"namespace": {
"ID": 4026532414,
"pid": 2240728
},
"tcp": {
"ActiveOpens": 742,
"ArpFilter": 0,
"AttemptFails": 0,
"BusyPollRxPackets": 0,
"CurrEstab": 12,
"DelayedACKLocked": 0,
"DelayedACKLost": 1,
"DelayedACKs": 23466,
"EmbryonicRsts": 0,
"EstabResets": 0,
"IPReversePathFilter": 0,
"InCsumErrors": 0,
"InErrs": 0,
"InSegs": 1243467,
"ListenDrops": 0,
"ListenOverflows": 0,
"LockDroppedIcmps": 0,
"MaxConn": -1,
"OfoPruned": 0,
"OutOfWindowIcmps": 0,
"OutRsts": 709,
"OutSegs": 2936174,
"PAWSActive": 0,
"PAWSEstab": 19,
"PFMemallocDrop": 0,
"PassiveOpens": 4,
"PruneCalled": 0,
"RcvPruned": 0,
"RetransSegs": 1613,
"RtoAlgorithm": 1,
"RtoMax": 120000,
"RtoMin": 200,
"SyncookiesFailed": 0,
"SyncookiesRecv": 0,
"SyncookiesSent": 0,
"TCPACKSkippedChallenge": 0,
"TCPACKSkippedFinWait2": 0,
"TCPACKSkippedPAWS": 0,
"TCPACKSkippedSeq": 0,
"TCPACKSkippedSynRecv": 0,
"TCPACKSkippedTimeWait": 0,
"TCPAbortFailed": 0,
"TCPAbortOnClose": 0,
"TCPAbortOnData": 705,
"TCPAbortOnLinger": 0,
"TCPAbortOnMemory": 0,
"TCPAbortOnTimeout": 0,
"TCPAckCompressed": 0,
"TCPAutoCorking": 763,
"TCPBacklogCoalesce": 1358,
"TCPBacklogDrop": 0,
"TCPChallengeACK": 0,
"TCPDSACKIgnoredDubious": 0,
"TCPDSACKIgnoredNoUndo": 1498,
"TCPDSACKIgnoredOld": 0,
"TCPDSACKOfoRecv": 0,
"TCPDSACKOfoSent": 0,
"TCPDSACKOldSent": 1,
"TCPDSACKRecv": 1586,
"TCPDSACKRecvSegs": 1589,
"TCPDSACKUndo": 2,
"TCPDeferAcceptDrop": 0,
"TCPDelivered": 2275375,
"TCPDeliveredCE": 0,
"TCPFastOpenActive": 0,
"TCPFastOpenActiveFail": 0,
"TCPFastOpenBlackhole": 0,
"TCPFastOpenCookieReqd": 0,
"TCPFastOpenListenOverflow": 0,
"TCPFastOpenPassive": 0,
"TCPFastOpenPassiveAltKey": 0,
"TCPFastOpenPassiveFail": 0,
"TCPFastRetrans": 7,
"TCPFromZeroWindowAdv": 0,
"TCPFullUndo": 1,
"TCPHPAcks": 537452,
"TCPHPHits": 144087,
"TCPHystartDelayCwnd": 0,
"TCPHystartDelayDetect": 0,
"TCPHystartTrainCwnd": 36,
"TCPHystartTrainDetect": 2,
"TCPKeepAlive": 171573,
"TCPLossFailures": 0,
"TCPLossProbeRecovery": 0,
"TCPLossProbes": 2183,
"TCPLossUndo": 0,
"TCPLostRetransmit": 0,
"TCPMD5Failure": 0,
"TCPMD5NotFound": 0,
"TCPMD5Unexpected": 0,
"TCPMTUPFail": 0,
"TCPMTUPSuccess": 0,
"TCPMemoryPressures": 0,
"TCPMemoryPressuresChrono": 0,
"TCPMinTTLDrop": 0,
"TCPOFODrop": 0,
"TCPOFOMerge": 0,
"TCPOFOQueue": 0,
"TCPOrigDataSent": 2273068,
"TCPPartialUndo": 1,
"TCPPureAcks": 215589,
"TCPRcvCoalesce": 10,
"TCPRcvCollapsed": 0,
"TCPRcvQDrop": 0,
"TCPRenoFailures": 0,
"TCPRenoRecovery": 0,
"TCPRenoRecoveryFail": 0,
"TCPRenoReorder": 0,
"TCPReqQFullDoCookies": 0,
"TCPReqQFullDrop": 0,
"TCPRetransFail": 0,
"TCPSACKDiscard": 0,
"TCPSACKReneging": 0,
"TCPSACKReorder": 256,
"TCPSYNChallenge": 0,
"TCPSackFailures": 0,
"TCPSackMerged": 5,
"TCPSackRecovery": 4,
"TCPSackRecoveryFail": 0,
"TCPSackShiftFallback": 160,
"TCPSackShifted": 0,
"TCPSlowStartRetrans": 0,
"TCPSpuriousRTOs": 0,
"TCPSpuriousRtxHostQueues": 2,
"TCPSynRetrans": 24,
"TCPTSReorder": 1,
"TCPTimeWaitOverflow": 0,
"TCPTimeouts": 24,
"TCPToZeroWindowAdv": 0,
"TCPWantZeroWindowAdv": 0,
"TCPWinProbe": 0,
"TCPWqueueTooBig": 0,
"TCPZeroWindowDrop": 0,
"TW": 3,
"TWKilled": 0,
"TWRecycled": 0,
"TcpDuplicateDataRehash": 0,
"TcpTimeoutRehash": 24
},
"udp": {
"IgnoredMulti": 0,
"InCsumErrors": 0,
"InDatagrams": 2856,
"InErrors": 0,
"NoPorts": 0,
"OutDatagrams": 2856,
"RcvbufErrors": 0,
"SndbufErrors": 0
},
"udp_lite": {
"IgnoredMulti": 0,
"InCsumErrors": 0,
"InDatagrams": 0,
"InErrors": 0,
"NoPorts": 0,
"OutDatagrams": 0,
"RcvbufErrors": 0,
"SndbufErrors": 0
}
}
},
"event": {
"dataset": "docker.network_summary",
"duration": 115000,
"module": "docker"
},
"metricset": {
"name": "network_summary",
"period": 10000
},
"service": {
"address": "unix:///var/run/docker.sock",
"type": "docker"
}
}
4 changes: 4 additions & 0 deletions metricbeat/module/docker/network_summary/_meta/docs.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
The `network_summary` metricset collects detailed network metrics from the processes associated with a container.
These stats come from `/proc/[PID]/net`, and are summed across the different namespaces found across the PIDs.

Because this metricset will try to access network counters from procfs, it is only available on linux.
34 changes: 34 additions & 0 deletions metricbeat/module/docker/network_summary/_meta/fields.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
- name: network_summary
type: group
release: beta
description: >
network_summary
fields:
- name: ip.*
type: object
description: >
IP counters
- name: tcp.*
type: object
description: >
TCP counters
- name: udp.*
type: object
description: >
UDP counters
- name: udp_lite.*
type: object
description: >
UDP Lite counters
- name: icmp.*
type: object
description: >
ICMP counters
- name: namespace.pid
type: long
description: >
The root PID of the container, corresponding to /proc/[pid]/net
- name: namespace.id
type: long
description: >
The ID of the network namespace used by the container, corresponding to /proc/[pid]/ns/net
37 changes: 37 additions & 0 deletions metricbeat/module/docker/network_summary/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// +build linux

package network_summary

import "github.com/elastic/beats/v7/metricbeat/module/docker"

//Config is the config object for docker/network
type Config struct {
TLS *docker.TLSConfig `config:"ssl"`
DeDot bool `config:"labels.dedot"`
NetworkSummary bool `config:"network.network_summary"`
}

// DefaultConfig returns default module config
func DefaultConfig() Config {
return Config{
DeDot: true,
NetworkSummary: false,
}
}
18 changes: 18 additions & 0 deletions metricbeat/module/docker/network_summary/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package network_summary
94 changes: 94 additions & 0 deletions metricbeat/module/docker/network_summary/helper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// +build linux

package network_summary

import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"time"

"github.com/docker/docker/client"
"github.com/pkg/errors"

"github.com/elastic/go-sysinfo"

sysinfotypes "github.com/elastic/go-sysinfo/types"
)

// NoSumStats lists "stats", often config/state values, that can't be safely summed across PIDs
var NoSumStats = []string{
"RtoAlgorithm",
"RtoMin",
"RtoMax",
"MaxConn",
"Forwarding",
"DefaultTTL",
}

var nsRegex = regexp.MustCompile(`\d+`)

// fetchContainerNetStats gathers the PIDs associated with a container, and then uses go-sysinfo to grab the /proc/[pid]/net counters and sum them across PIDs.
func fetchContainerNetStats(client *client.Client, timeout time.Duration, container string) (*sysinfotypes.NetworkCountersInfo, error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()

inspect, err := client.ContainerInspect(ctx, container)
if err != nil {
return nil, errors.Wrapf(err, "error fetching stats for container %s", container)
}
rootPID := inspect.ContainerJSONBase.State.Pid

proc, err := sysinfo.Process(rootPID)
procNet, ok := proc.(sysinfotypes.NetworkCounters)
if !ok {
return nil, errors.Wrapf(err, "cannot fetch network counters for PID %d", rootPID)
}

counters, err := procNet.NetworkCounters()
if err != nil {
return &sysinfotypes.NetworkCountersInfo{}, errors.Wrapf(err, "error fetching network counters for PID %d", rootPID)
}

return counters, nil

}

// fetch the network namespace associated with the PID.
func fetchNamespace(pid int) (int, error) {
nsLink, err := os.Readlink(filepath.Join("/proc/", fmt.Sprintf("%d", pid), "/ns/net"))
if err != nil {
return 0, errors.Wrap(err, "error reading network namespace link")
}
nsidString := nsRegex.FindString(nsLink)
// This is minor metadata, so don't consider it an error
if nsidString == "" {
return 0, nil
}

nsID, err := strconv.Atoi(nsidString)
if err != nil {
return 0, errors.Wrapf(err, "error converting %s to int", nsidString)
}
return nsID, nil
}
129 changes: 129 additions & 0 deletions metricbeat/module/docker/network_summary/network_summary.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// +build linux

package network_summary

import (
"context"
"runtime"

"github.com/docker/docker/client"
"github.com/pkg/errors"

"github.com/elastic/beats/v7/libbeat/common"
"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
"github.com/elastic/beats/v7/libbeat/metric/system/network"
"github.com/elastic/beats/v7/metricbeat/mb"
"github.com/elastic/beats/v7/metricbeat/module/docker"
)

// init registers the MetricSet with the central registry as soon as the program
// starts. The New function will be called later to instantiate an instance of
// the MetricSet for each host defined in the module's configuration. After the
// MetricSet has been created then Fetch will begin to be called periodically.
func init() {
mb.Registry.MustAddMetricSet("docker", "network_summary", New)
}

// MetricSet holds any configuration or state information. It must implement
// the mb.MetricSet interface. And this is best achieved by embedding
// mb.BaseMetricSet because it implements all of the required mb.MetricSet
// interface methods except for Fetch.
type MetricSet struct {
mb.BaseMetricSet
dockerClient *client.Client
cfg Config
}

// New creates a new instance of the MetricSet. New is responsible for unpacking
// any MetricSet specific configuration options if there are any.
func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
cfgwarn.Beta("The docker network_summary metricset is beta.")

config := DefaultConfig()
if err := base.Module().UnpackConfig(&config); err != nil {
return nil, err
}

client, err := docker.NewDockerClient(base.HostData().URI, docker.Config{DeDot: config.DeDot, TLS: config.TLS})
if err != nil {
return nil, err
}

// Network summary requres a linux procfs system under it to read from the cgroups. Disable reporting otherwise.
if runtime.GOOS != "linux" {
base.Logger().Debug("Not running on linux, docker network detailed stats disabled.")
config.NetworkSummary = false
}

return &MetricSet{
BaseMetricSet: base,
dockerClient: client,
cfg: config,
}, nil
}

// Fetch methods implements the data gathering and data conversion to the right
// format. It publishes the event which is then forwarded to the output. In case
// of an error set the Error field of mb.Event or simply call report.Error().
func (m *MetricSet) Fetch(ctx context.Context, report mb.ReporterV2) error {

stats, err := docker.FetchStats(m.dockerClient, m.Module().Config().Timeout)
if err != nil {
return errors.Wrap(err, "failed to get docker stats")
}

for _, myStats := range stats {

ctx, cancel := context.WithTimeout(ctx, m.Module().Config().Timeout)
defer cancel()

inspect, err := m.dockerClient.ContainerInspect(ctx, myStats.Container.ID)
if err != nil {
return errors.Wrapf(err, "error fetching stats for container %s", myStats.Container.ID)
}

rootPID := inspect.ContainerJSONBase.State.Pid

netNS, err := fetchNamespace(rootPID)
if err != nil {
return errors.Wrapf(err, "error fetching namespace for PID %d", rootPID)
}

networkStats, err := fetchContainerNetStats(m.dockerClient, m.Module().Config().Timeout, myStats.Container.ID)
if err != nil {
return errors.Wrap(err, "error fetching per-PID stats")
}

summary := network.MapProcNetCounters(networkStats)
// attach metadata associated with the network counters
summary["namespace"] = common.MapStr{
"id": netNS,
"pid": rootPID,
}

report.Event(mb.Event{
RootFields: docker.NewContainer(myStats.Container, m.cfg.DeDot).ToMapStr(),
MetricSetFields: summary,
})

}

return nil
}
39 changes: 39 additions & 0 deletions metricbeat/module/docker/network_summary/network_summary_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// +build integration linux

package network_summary

import (
"testing"

mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
)

func TestData(t *testing.T) {
ms := mbtest.NewFetcher(t, getConfig())
ms.WriteEvents(t, "")
}

func getConfig() map[string]interface{} {
return map[string]interface{}{
"module": "docker",
"metricsets": []string{"network_summary"},
"hosts": []string{"unix:///var/run/docker.sock"},
}
}
3 changes: 2 additions & 1 deletion metricbeat/module/system/network_summary/network_summary.go
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@ import (
"github.com/pkg/errors"

"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
"github.com/elastic/beats/v7/libbeat/metric/system/network"
"github.com/elastic/beats/v7/metricbeat/mb"
sysinfo "github.com/elastic/go-sysinfo"
sysinfotypes "github.com/elastic/go-sysinfo/types"
@@ -70,7 +71,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
}

report.Event(mb.Event{
MetricSetFields: eventMapping(counterInfo),
MetricSetFields: network.MapProcNetCounters(counterInfo),
})

return nil
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@ import (
"github.com/stretchr/testify/assert"

"github.com/elastic/beats/v7/libbeat/common"
"github.com/elastic/beats/v7/libbeat/metric/system/network"
mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
"github.com/elastic/go-sysinfo/types"
)
@@ -50,7 +51,7 @@ func TestMapping(t *testing.T) {
"udp": map[string]uint64{"IgnoredMulti": 10},
"udp_lite": map[string]uint64{"IgnoredMulti": 0}}

out := eventMapping(example)
out := network.MapProcNetCounters(example)
assert.Equal(t, exampleOut, out)
}

1 change: 1 addition & 0 deletions metricbeat/modules.d/docker.yml.disabled
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@
# - info
# - memory
# - network
# - network_summary
period: 10s
hosts: ["unix:///var/run/docker.sock"]

1 change: 1 addition & 0 deletions x-pack/metricbeat/metricbeat.reference.yml
Original file line number Diff line number Diff line change
@@ -460,6 +460,7 @@ metricbeat.modules:
#- "image"
- "memory"
- "network"
#- "network_summary"
hosts: ["unix:///var/run/docker.sock"]
period: 10s
enabled: true