diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 43e286b37885..280c184e54dc 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -104,6 +104,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `nats` module with `stats` metricset. {pull}9825[9825] - Add `connections` metricset to the `nats` module. {pull}10095[10095] - Add `routes` metricset to the `nats` module. {pull}10095[10095] +- Add remaining memory metrics of pods in Kubernetes metricbeat module {pull}10157[10157] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 7276d6f94924..632ced1b868b 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -11876,6 +11876,65 @@ format: percentage Memory usage as a percentage of the defined limit for the pod containers (or total node allocatable memory if unlimited) +-- + + +*`kubernetes.pod.memory.available.bytes`*:: ++ +-- +type: long + +format: bytes + +Total memory available + + +-- + + +*`kubernetes.pod.memory.workingSet.bytes`*:: ++ +-- +type: long + +format: bytes + +Total working set memory + + +-- + + +*`kubernetes.pod.memory.rss.bytes`*:: ++ +-- +type: long + +format: bytes + +Total resident set size memory + + +-- + +*`kubernetes.pod.memory.pageFaults`*:: ++ +-- +type: long + +Total page faults + + +-- + +*`kubernetes.pod.memory.majorPageFaults`*:: ++ +-- +type: long + +Total major page faults + + -- [float] diff --git a/metricbeat/module/kubernetes/event/event_test.go b/metricbeat/module/kubernetes/event/event_test.go index 5fe869e3694f..5201f1cf95dc 100644 --- a/metricbeat/module/kubernetes/event/event_test.go +++ b/metricbeat/module/kubernetes/event/event_test.go @@ -20,7 +20,7 @@ package event import ( "testing" - "github.com/ericchiang/k8s/apis/core/v1" + v1 "github.com/ericchiang/k8s/apis/core/v1" k8s_io_apimachinery_pkg_apis_meta_v1 "github.com/ericchiang/k8s/apis/meta/v1" "github.com/stretchr/testify/assert" diff --git a/metricbeat/module/kubernetes/fields.go b/metricbeat/module/kubernetes/fields.go index 75053e2ad790..c4932629092d 100644 --- a/metricbeat/module/kubernetes/fields.go +++ b/metricbeat/module/kubernetes/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsXU1v4zjSvvevKOSUBjw5vXgPOSwwm9nBBv0xgTs9c1gsAloq25xIpIaknPG/X1AflCyRlGzTjtORTt2yXc/DYrFYZBWZn+AZt7fwnC9QMFQoPwAoqhK8hatP5uXVB4AYZSRopihnt/CPDwAAzRcgRSVopH8tMEEi8RZW5AOARKUoW8lb+M+VlMnVDK7WSmVX/9WfrblQTxFnS7q6hSVJJH4AWFJMYnlbAPwEjKTYoacftc00guB5Vr2x0NPPPVtykRL9GgiLQSqiqFQ0ksCXkPFYQkoYWWEMi20L56aS0GbTZkQyKlFsUJhPbKQ8xDr6+/nhHkqBLVXWj1HpAhVpve+SaxMU+FeOUt1ECUWmdr5SM33G7QsXceczD1/93BXyIOaUrUCtsQaSXhYCJc9FhOF4zEvJGINVdpeAzBen5OAS36MR8Sw8ASjEwnWU5FKhmBWgMiMRzox2Pnp5bVAswtP69+PjA/RE9yyU5w4DTThb7Yf8yBVJgOXpAoUe4KOMMyEKWbS9kXkaiEalAAmV6BnIPNV8yv9TlEAZpDQSXGLEWTyOYEhN1X1kGB6otEUePaOdFF/8iVH3o/LlUyDasKZS8ZUgKZREZM9TR5wpQtlxnrqZGBp5Pke9GuumpSJCPSma2r1CTFT3gwEFfdMCoSfQaCPLrUBdXYxAunv4DrkkK7QowtXsNpXit71PfYR8UncayYVN8LDwIYA2COu2tw9jMe/2M6Df9nNnjE5r/Y4LrFTPCLO6kB5bwrhWi4v0IOGRZEujwHgA0NDiMd5kPSexy0pGJMH4aZlw4vpiGeTdQoYiQqbshrV3M7SCiQTSEqv9o456VDnR8BiBJAmPiCKLBPXvvO1NaErVm2xwjEvKMC5boOGLt40zvNZvnEoBuoScFb/F2B6KJHzVtZWDXdNnvtIz7JLv6ZLIhtBEcz6JW1ps1eHjr+5wn5CRfV1oxzQVIpKRiKqtDkns0o1frb7542untOTxmtEu78fXSuHYxyuFak9gww0xw/cj4V3px09l5VqiGSfO5jS0lgL9gUcoVhpoDCGHXYYnVJiGhVBNJMWUi67jcNvB5KjBYoFWJZ4joL4shZRqcDb38qPLL60G7BlgOkwA3kKMOabZR4SZlVm4I822koQ8zcR0KSNl/u2bf5zUhF+4eKZsJXt7OPBD6eOPspkgUY3TS0ZWuCR5otx24mA+gtFXs9emYcCBY+ZO8icXZ+JTYDlZmdHDuVqOX60NzebvYl0x51zBkiYot1JhuvcS432EPHYttYPw974Ss2uoir9fb0V2hpXGd8sao4bHzW6Wc+8d/sc1tvOxhTyT1kYFEU8SjJT5RK2JAiIQVshQEFUmkMvshgSRM0Y77aVM0rgIdD5109mwV5LXvQh2aNqr3zstpUQBgREXsSxiriYfpGiK5buMCEWjPCGiVAOsiQQeRbkQO71fMyx+qUiaWVj2jc2XJ1lSIdVTBcUcSdz9syWPNUHdzgIDGgz9rmtXrTCbnJyQhhjg06yvZS+ccWdwvSS+lKIqY8DYROErukHmZCCQSM5CEJgXkvbF12Ah0B+3mVmN+BFTVCQmO4PVbdwDKi8lAZGSR7TwJi9UrT0k/MPFPvD2j9GMqxGoSfXM1WnmI5z6jqkXAJQzv+bb+SxHxvSggoWvJDV97scsqirCAhci9dz9sqbRunKsL0Q2M4s9Bq8KO542KCTtjLyjSP1eCtxRiL/KJqddiCPgvzP6V45AY2SKLinqBX+LiKWqwCTTMVk+JZQ9ByQz/wwCM4FSs6lKnlwOgbINTzYYP1k4nsov1Jg2vfg8BMloeMv5+eEeNrvW4+muZ8oCmo3G1hJHAId1HqzlPDygpxuvteQ9VB92wH6//2UAu70je0yU3qrDKfb/phKcqQTH8YQuwfmq7e1tV99MyTjbMyXjOk+4ZNyUbekQnrItduJTtsWTbWGotN0E89fi7x/a+OYYId0U+7EuWWbXWAguTj0pz/924Zjdmh+7Qx4FYTKlSl1Onzxa+8RsNk+pzfIZqc1fp6zmngqaEprN01POe8hlNjGAq3KyS+ocJa8Nq8sodm34uApeTUyTM+cOziF+m6Y6AjxR8bJ7ThgGGAKBkSMcxm6RjBnpsN9Wyn1aRLz7zxowcuaA96zGEXML7OPs3qEK7TOQWazytsKO2cPOePwmt7CnFWn5TCvS5nlLHfLmVqTvImd0IVmSHq0LPUWyzxnl93YuWU+s5tCI7J4aGXcgOXCWbEoIdWhf6riaTmcFHWyjj2i9jQtbDi6c6U/Ehhe1V/hJRVQebis6WxPpdkD2BnQb4fPUpjkFEFxXNfAzeCFUFf9QKFLKiP9gHpLYvVu+4DxB0q2dGsmyYViA2PW7U7ClV0HuPSDKFK52zPRAMiWOY4fPU1/dJnNU//1R9hBcG1Z3RT2u7rQ7QeT6M+fZP0n0zJfLGfxLiGLd/JAnyQzMP6vP+12rH+0Tqt6nnGmgNEtQYTxrNHFHGONqnrMCgosZ/Pbbl080STD+WDX/5ujoeGiUlP7ZFRUem/rdvcenQCkRPd1e35J2DkbC3Ghnx9vVki9+HuCVCYy0I7iF/7/5vxDMDZeR+vRxH6Z3Kq2ftW6r7ERX4Odt4lDUuJcKqqigDCoG8z51B74+76bb6rjGtWMYY5bwbXrkEbVWTNMIDBLUhC2I/mTl2cNoan8sW9C+6f4g+BLFNuc3hpUlNCLBLqGy86hRDrmeKkZJhScbfFRA8kurq0zarUJsWF/LDCN39Daczg/FsclyOPqttfJm56PVwhpBLIutp6qCkypx+oQuvCw/5OrHv644KnouKsPbSwq4ViLHWXmltQ59c/bM+Atzj5ucyWiNce430qNWPwXLHRyfMwwZUrd2AAbCWNd+x9jm6WCqvd/gD2LrfPPJYmvDyWS2z1eT39L5a0VKX13bP2PvGXxd5hVbf1WCPScMwfqu2Eg7lWm2+ybjljMsvQ45KZ1iH7FbG3O5SXdqP1zdez0+TNTM7h+sYGsu1dNpELVoF+yek/B+wNVkeVga8oS7mR2a1XbmvN7OfEAWU7a6ubk5dBczJLvj4o4qGvDEoCG5GjQb31mfbXdlhqGWz5XA6nzKBa+f20SdC+gTLlzb+O4VdIgjgofPHY87l9WYlWqGAublf75ZjluNXVO/Fi+/BwnHSnuPfbnxRfGnak6ltOrei+I0eYUEi22RamzIFWk9wZPEsj42G5xkgT7fFkqLyzxJtjXaoDbbcysu8yScW6slXr5f22HqdGz2W2ecfTd0tdIam2tmzAU5cI0Zj9Yfi2z2t4pW1/jP4Gl3NGJM6CBne+Lh2di9GZ07Ju9SIryC1+3tX/oI1uQa/3Pqfm55Otr8obLL6m7TyS2yl9HNdeeOIGZ8blHlHcrdliXjrdKXED7XVv4CxzneJmXldLXTJSi9Z7oEpf9M959YxE2VndNVH33C01UfduLTVR8dtJrNhid5GioNWwq7wEXg7yUxZyAy3b1QPdPdC9PdC/YvTHcvHNdo2z3yNipnuODg15F/x+t8f++sIvO/AAAA//8TSARs" + return "eJzsXU9v6zgOv79PQfTUB2R6WuyhhwVmO/uwxfszRV/fzGGxKBSbSTS1JY8kp5P99AvJtuzYkuwkTpq21qm1E/IniqJIkVJ+gifcXMNTPkfBUKH8AKCoSvAaLj7bhxcfAGKUkaCZopxdwz8+AADUH4AUlaCR/rbABInEa1iSDwASlaJsKa/hPxdSJhczuFgplV38V79bcaEeI84WdHkNC5JI/ACwoJjE8tow+AkYSbEFTze1yTQHwfOsfOKAp9stW3CREv0YCItBKqKoVDSSwBeQ8VhCShhZYgzzTYPPVUmhiaaJiGRUolijsG9coALAWvL7+e4WCoINUVbNinSOijSet8E1AQr8M0eprqKEIlNbH6mQPuHmmYu49S6AV7cbQw9iTtkS1AorRjKIQqDkuYhwPBz3BWWMwUm7DUDm82Ni8JHvwIh4Nj4AMGThMkpyqVDMDFOZkQhnVjofg7jWKObjw/r3w8MddEh3NJTnHgVNOFvuxvmBK5IAy9M5Cj3BBylnQhSyaHMl83QkGKUAJJSkZyDzVOMp/qcogTJIaSS4xIizeBjAMSVVjZFFuKfQ5nn0hG5QfP4HRu1XxcPHkWDDikrFl4KkUACRHUsdcaYIZYdZ6nphqOmFDPVyqJmWigj1qGjqtgoxUe0XPQL6rglCh6CVRpY7GbVlMYDTzd0PyCVZokMQvm43oZjvdt6GAIWobnWSCxfhfuJ9DJpMWLu/XTYO9W62Hvk2241VOi31Gy6wFD0jzGlCOmgJ41osPtC9gAeCLZQC4x6GFhaP8SrrGIltVDIiCcaPi4QT3wcLJ+8aMhQRMuVWrJ27oQVMJJAGWW0ftdejioWGxwgkSXhEFJknqL8X7G9CU6peZYdjXFCGcdEDzd48rY3hpX7iFQrQBeTMfBdjtyuS8GVbV/Y2TV/4Uq+wC76jSSJrQhON+Shmab5R+8+/asBDRAaOtZGO7SpEJCMRVRvtkripW7tafvLtS6fQ5OGS0Sbv7UvFGPbhQqHaErj4jrHCdz3hbeqHL2VFLFHPE293algLgWHHYyxUmtEQQB69HB+QUQ0HoApIiikXbcPh14PJUINDA51CPIVDfV4CKcTg7e75e5dfGx3Y0cH0qAC8Bh9zSLcPcDNLtfB7mk0hCXmchelcZsr99+/heVIBfubiibKl7OzhwJuSx+9FN0GiGiaXjCxxQfJE+fXEg3wAom92r02zAQ8fu3aSP7g4ER7Dy4vKzh7O1WJ4tNa3mr+LuOKecwULmqDcSIXpziHG+3B53FJqOuHvPRJzS6j0v18uIjtBpPHDEWNU7HG9neXceYf/YYXNfKyhZ9PaqCDiSYKRsm/UiiggAmGJDAVRRQK5yG5IEDljtNVfyiSNjaPzuZ3Ohp2SvP4g2CPpoHxvNJWCCwiMuIil8bnqfJCiKRbPMiIUjfKEiEIMsCISeBTlQmyNfoXQfFORNHOg7CpbKE+yoEKqx5IV8yRxd8+WPFQAdT8ND6h56GdtvWq42eTogDSLHjx1fC077ow/gxsE8bUgVSoDxtYLX9I1Mi8CgURyNgaAe0NpV/6a2RjcHzaZjUbCHFNUJCZbk9Wv3D0iLygBkZJH1FiTZ6pWARDh6eKeeLv7aNbUCNSgOurqVfMBRn1L1Q0DyllY8s18lidjulfBwjeS2jEP8zRVFeMyNiT12v28otGqNKzPRNYri9sHLws7HtcoJG3NvINA/VYQ3BJIuMomp20WB7D/weifOQKNkSm6oKgD/gYQR1WBTaZjsnhMKHsaEcz9FxCYCZQaTVny5DMIlK15ssb40YHxWHah4umSS8hCkIyOrzk/393Celt7AsP1RNmIaqN5a4oDGI9rPFjDeASYHm++VpR3EP24E/bH7S89vJs7sod46Y06HLP/N5XgTCU4njZ2Cc43rW+vu/pmSsa52pSMa7XxknFTtqUFeMq2uIFP2ZZAtoWh0nozmr0Wf71p5bvHCOna7Mf6aNldYyG4OPaifP+Xj4/drXnbA/IgCJMpVep8xuTBOSZ2s3lKbRZtoDQ/TVnNHQU0JTTr1hHOe8hl1j6Ar3KyDeoUJa81qvModq3x+AperU+TM+8Ozj52m6baAzxS8bJ/Tehn0McEBs5wGLpFMmSmw25bKbep8Xh3XzVg4MoB71mMA9YW2MXYvUMRulcgG6zypsAO2cPOePwqt7CniLRoU0Rat9c0IK8uIn0XOaMzyZJ0YJ3pKZJdzii/t3PJemG1h0Zk+9TIsAPJI2fJpoRQC/a5zqvpdNaok23vI1rvY2twa9L4u9xKIH5/4wnEQirPnTSiP4J44xnmQiACpSmLNBKR9H+9VQgZWeKnY+UxC0yDc6p3x8fiz6fWu6jnfBvU3lV5XS/f4qLu8mGpiMrHy3NlKyL9htrdgXYnQm6g7Y5hBJflAZsZPBOqzB8KRUoZCZ/6RRL7U3FzzhMk7cLMgShrhIaJW75b1aCKiMBUoEzhcktN9wRT8PGkDwKHN5pgDhq/34sRgkuL6sYU++tBuxFErr5wnv2TRE98sZjBv4Qwm3J3eZLMwP5Zvu8OrW7a4ShHn3KmGaVZggrjWS2JG8IYV/c5Myy4mMGvv379TJME449l968ODr37Zknh/PlCzkNt4PYlYYZLwTEw7NUVjKdAJOx1mW5+21IKBec9uDKBkTYE1/D3q7+NgdxiGSjPEPZ+eMeS+kmLQotB9Hk+wS72uU07iaD0rIuIpTepXA3gy+Ouh60KmnzpiBizhG/SA8+/NnyamuAoTs24py0+O3F2eNQOsCO/FVru92JfcHGt+bViZQmNyHCHay8cFZd97r6LUVIRKDU5yCH5pTFUNqdfcqxRX8oMo0M2BMbCWKdQPePW2NZjp4PV4DUAWBY7j2yODqrg0wV05md+xox+wnHFQd6zOXbSDCngUokcZ8V9+dr1zdkT48/MP29yJqMVxnlYSQ+KfgzKLT4hYzimS93YXuxxY32bqUO7p52p5mZm2ImtilmO5ltbTLZs5nQHfhoyfylP6Ztvb3noJaYvi7xEGy55checwGhjZ3bpj6WazbHJuOOAXGdAjgrHJCnahXfnW9FD3Tc3dB4PdxM1sts7J7MVl+rxOBw1aR/bHRfh3RiXi+V+NQ5H3M1swSy3M++r7cw7ZDFly6urq313McdEd5jfUXoDAR90TKyWmwvvrIu2HZnhWOFzSbDMWp1x/NwE6g2gjxi4Nvn7I+gxzh8fkFfaugnLRqoZCrgv/nGlYofG1C+FK2xBxkOlrceu2Pjc/A7WsYRWXqpjrqooOcF8Y+oYanAmrSd4kjjiY7vBSeYYsm1jSXGRJ8mm4tYrzebaios8Gc+sVRTP365tIfUaNveVVt6x67u3bYX1HVb29i24xIxHq4+mVOZ7Caut/CewtFsSsSq0l7E98vSs9d7Ozi2V9wkRXsDqdvYvQwArcLX9OfY4NywdrX8F8byG2w5yA+x5DHM1uAOAWZtrjpCMZW6L8yiNuroxbK6r/AUOM7x1yspraqcbljpthFOEb+6GpelyJQe5qWz8HVV5TvcIbbfpHqFD7xGq0Kx5kqdjpWELYmcYBP5WAPM6ItPFLmWbLnaZLnZxf2C62OWwTrt+pMIF5QS3p3wa+COBp/sxxRLM/wMAAP///ZX97w==" } diff --git a/metricbeat/module/kubernetes/pod/_meta/fields.yml b/metricbeat/module/kubernetes/pod/_meta/fields.yml index fc2b4d62e6e3..f449f367045c 100644 --- a/metricbeat/module/kubernetes/pod/_meta/fields.yml +++ b/metricbeat/module/kubernetes/pod/_meta/fields.yml @@ -78,3 +78,35 @@ format: percentage description: > Memory usage as a percentage of the defined limit for the pod containers (or total node allocatable memory if unlimited) + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total memory available + - name: workingSet + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total working set memory + - name: rss + type: group + fields: + - name: bytes + type: long + format: bytes + description: > + Total resident set size memory + - name: pageFaults + type: long + description: > + Total page faults + - name: majorPageFaults + type: long + description: > + Total major page faults diff --git a/metricbeat/module/kubernetes/pod/data.go b/metricbeat/module/kubernetes/pod/data.go index 1619b172ef97..c1370487474d 100644 --- a/metricbeat/module/kubernetes/pod/data.go +++ b/metricbeat/module/kubernetes/pod/data.go @@ -33,20 +33,26 @@ func eventMapping(content []byte, perfMetrics *util.PerfMetricsCache) ([]common. var summary kubernetes.Summary err := json.Unmarshal(content, &summary) if err != nil { - return nil, fmt.Errorf("Cannot unmarshal json response: %s", err) + return nil, fmt.Errorf("cannot unmarshal json response: %s", err) } node := summary.Node nodeCores := perfMetrics.NodeCoresAllocatable.Get(node.NodeName) nodeMem := perfMetrics.NodeMemAllocatable.Get(node.NodeName) for _, pod := range summary.Pods { - var usageNanoCores, usageMem int64 + var usageNanoCores, usageMem, availMem, rss, workingSet, pageFaults, majorPageFaults int64 var coresLimit, memLimit float64 for _, cont := range pod.Containers { cuid := util.ContainerUID(pod.PodRef.Namespace, pod.PodRef.Name, cont.Name) usageNanoCores += cont.CPU.UsageNanoCores usageMem += cont.Memory.UsageBytes + availMem += cont.Memory.AvailableBytes + rss += cont.Memory.RssBytes + workingSet += cont.Memory.WorkingSetBytes + pageFaults += cont.Memory.PageFaults + majorPageFaults += cont.Memory.MajorPageFaults + coresLimit += perfMetrics.ContainerCoresLimit.GetWithDefault(cuid, nodeCores) memLimit += perfMetrics.ContainerMemLimit.GetWithDefault(cuid, nodeMem) } @@ -70,6 +76,17 @@ func eventMapping(content []byte, perfMetrics *util.PerfMetricsCache) ([]common. "usage": common.MapStr{ "bytes": usageMem, }, + "available": common.MapStr{ + "bytes": availMem, + }, + "workingSet": common.MapStr{ + "bytes": workingSet, + }, + "rss": common.MapStr{ + "bytes": rss, + }, + "pageFaults": pageFaults, + "majorPageFaults": majorPageFaults, }, "network": common.MapStr{ diff --git a/metricbeat/module/kubernetes/util/kubernetes_test.go b/metricbeat/module/kubernetes/util/kubernetes_test.go index d2ef22efa72b..6e51e94ffc52 100644 --- a/metricbeat/module/kubernetes/util/kubernetes_test.go +++ b/metricbeat/module/kubernetes/util/kubernetes_test.go @@ -20,7 +20,7 @@ package util import ( "testing" - "github.com/ericchiang/k8s/apis/meta/v1" + v1 "github.com/ericchiang/k8s/apis/meta/v1" "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/common"