Docker CGroups v2 Utilization Support

newrelic/common/utilization.py

@@ -265,21 +265,42 @@ def get_values(cls, response):
 class DockerUtilization(CommonUtilization):
     VENDOR_NAME = 'docker'
     EXPECTED_KEYS = ('id',)
-    METADATA_FILE = '/proc/self/cgroup'
-    DOCKER_RE = re.compile(r'([0-9a-f]{64,})')
+
+    METADATA_FILE_CGROUPS_V1 = '/proc/self/cgroup'
+    METADATA_RE_CGROUPS_V1 = re.compile(r'([0-9a-f]{64,})')
+
+    METADATA_FILE_CGROUPS_V2 = '/proc/self/mountinfo'
+    METADATA_RE_CGROUPS_V2 = re.compile(r'^.*/docker/containers/([0-9a-f]{64,})/.*$')
 
     @classmethod
     def fetch(cls):
+        # Try to read from cgroups
         try:
-            with open(cls.METADATA_FILE, 'rb') as f:
+            with open(cls.METADATA_FILE_CGROUPS_V1, 'rb') as f:
                 for line in f:
                     stripped = line.decode('utf-8').strip()
                     cgroup = stripped.split(':')
                     if len(cgroup) != 3:
                         continue
                     subsystems = cgroup[1].split(',')
                     if 'cpu' in subsystems:
-                        return cgroup[2]
+                        contents = cgroup[2].split('/')[-1]
+                        match = cls.METADATA_RE_CGROUPS_V1.search(contents)
+                        if match:
+                            return match.group(1)
+        except:
+            # There are all sorts of exceptions that can occur here
+            # (i.e. permissions, non-existent file, etc)
+            pass
+
+        # Fallback to reading from mountinfo
+        try:
+            with open(cls.METADATA_FILE_CGROUPS_V2, 'rb') as f:
+                for line in f:
+                    stripped = line.decode('utf-8').strip()
+                    match = cls.METADATA_RE_CGROUPS_V2.match(stripped)
+                    if match:
+                        return match.group(1)
         except:
             # There are all sorts of exceptions that can occur here
             # (i.e. permissions, non-existent file, etc)
@@ -290,11 +311,7 @@ def get_values(cls, contents):
         if contents is None:
             return
 
-        value = contents.split('/')[-1]
-        match = cls.DOCKER_RE.search(value)
-        if match:
-            value = match.group(0)
-            return {'id': value}
+        return {'id': contents}
 
     @classmethod
     def valid_chars(cls, data):
@@ -315,11 +332,7 @@ def valid_length(cls, data):
             return False
 
         # Must be exactly 64 characters
-        valid = len(data) == 64
-        if valid:
-            return True
-
-        return False
+        return bool(len(data) == 64)
 
 
 class KubernetesUtilization(CommonUtilization):

tests/cross_agent/fixtures/docker_container_id_v2/README.md

@@ -0,0 +1,6 @@
+These tests cover parsing of Docker container IDs on Linux hosts out of
+`/proc/self/mountinfo` (or `/proc/<pid>/mountinfo` more generally).
+
+The `cases.json` file lists each filename in this directory containing
+example `/proc/self/mountinfo` content, and the expected Docker container ID that
+should be parsed from that file.

tests/cross_agent/fixtures/docker_container_id_v2/cases.json

@@ -0,0 +1,36 @@
+[
+  {
+    "filename": "docker-20.10.16.txt",
+    "containerId": "84cf3472a20d1bfb4b50e48b6ff50d96dfcd812652d76dd907951e6f98997bce",
+    "expectedMetrics": null
+  },
+  {
+    "filename": "docker-24.0.2.txt",
+    "containerId": "b0a24eed1b031271d8ba0784b8f354b3da892dfd08bbcf14dd7e8a1cf9292f65",
+    "expectedMetrics": null
+  },
+  {
+    "filename": "empty.txt",
+    "containerId": null,
+    "expectedMetrics": null
+  },
+  {
+    "filename": "invalid-characters.txt",
+    "containerId": null,
+    "expectedMetrics": null
+  },
+  {
+    "filename": "docker-too-long.txt",
+    "containerId": null,
+    "expectedMetrics": null
+  },
+  {
+    "filename": "invalid-length.txt",
+    "containerId": null,
+    "expectedMetrics": {
+      "Supportability/utilization/docker/error": {
+        "callCount": 1
+      }
+    }
+  }
+]

tests/cross_agent/fixtures/docker_container_id_v2/docker-20.10.16.txt

@@ -0,0 +1,24 @@
+519 413 0:152 / / rw,relatime master:180 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/YCID3333O5VYPYDNTQRZX4GI67:/var/lib/docker/overlay2/l/G7H4TULAFM2UBFRL7QFQPUNXY5:/var/lib/docker/overlay2/l/RLC4GCL75VGXXXYJJO57STHIYN:/var/lib/docker/overlay2/l/YOZKNWFAP6YX74XEKPHX4KG4UN:/var/lib/docker/overlay2/l/46EQ6YX5PQQZ4Z3WCSMQ6Z4YWI:/var/lib/docker/overlay2/l/KGKX3Z5ZMOCDWOFKBS2FSHMQMQ:/var/lib/docker/overlay2/l/CKFYAF4TXZD4RCE6RG6UNL5WVI,upperdir=/var/lib/docker/overlay2/358c429f7b04ee5a228b94efaebe3413a98fcc676b726f078fe875727e3bddd2/diff,workdir=/var/lib/docker/overlay2/358c429f7b04ee5a228b94efaebe3413a98fcc676b726f078fe875727e3bddd2/work
+520 519 0:155 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
+521 519 0:156 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+522 521 0:157 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+523 519 0:158 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro
+524 523 0:30 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw
+525 521 0:154 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw
+526 521 0:159 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k
+527 519 254:1 /docker/volumes/3237dea4f8022f1addd7b6f072a9c847eb3e5b8df0d599f462ba7040884d4618/_data /data rw,relatime master:28 - ext4 /dev/vda1 rw
+528 519 254:1 /docker/containers/84cf3472a20d1bfb4b50e48b6ff50d96dfcd812652d76dd907951e6f98997bce/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw
+529 519 254:1 /docker/containers/84cf3472a20d1bfb4b50e48b6ff50d96dfcd812652d76dd907951e6f98997bce/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw
+530 519 254:1 /docker/containers/84cf3472a20d1bfb4b50e48b6ff50d96dfcd812652d76dd907951e6f98997bce/hosts /etc/hosts rw,relatime - ext4 /dev/vda1 rw
+414 521 0:157 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+415 520 0:155 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw
+416 520 0:155 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw
+417 520 0:155 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw
+418 520 0:155 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw
+419 520 0:155 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw
+420 520 0:160 / /proc/acpi ro,relatime - tmpfs tmpfs ro
+421 520 0:156 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+422 520 0:156 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+423 520 0:156 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+424 520 0:156 /null /proc/sched_debug rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+425 523 0:161 / /sys/firmware ro,relatime - tmpfs tmpfs ro

tests/cross_agent/fixtures/docker_container_id_v2/docker-24.0.2.txt

@@ -0,0 +1,21 @@
+1014 1013 0:269 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
+1019 1013 0:270 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+1020 1019 0:271 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+1021 1013 0:272 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro
+1022 1021 0:30 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw
+1023 1019 0:268 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw
+1024 1019 0:273 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k
+1025 1013 254:1 /docker/containers/b0a24eed1b031271d8ba0784b8f354b3da892dfd08bbcf14dd7e8a1cf9292f65/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw,discard
+1026 1013 254:1 /docker/containers/b0a24eed1b031271d8ba0784b8f354b3da892dfd08bbcf14dd7e8a1cf9292f65/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw,discard
+1027 1013 254:1 /docker/containers/b0a24eed1b031271d8ba0784b8f354b3da892dfd08bbcf14dd7e8a1cf9292f65/hosts /etc/hosts rw,relatime - ext4 /dev/vda1 rw,discard
+717 1019 0:271 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+718 1014 0:269 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw
+719 1014 0:269 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw
+720 1014 0:269 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw
+721 1014 0:269 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw
+723 1014 0:269 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw
+726 1014 0:274 / /proc/acpi ro,relatime - tmpfs tmpfs ro
+727 1014 0:270 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+728 1014 0:270 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+729 1014 0:270 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+730 1021 0:275 / /sys/firmware ro,relatime - tmpfs tmpfs ro

tests/cross_agent/fixtures/docker_container_id_v2/docker-too-long.txt

@@ -0,0 +1,21 @@
+1014 1013 0:269 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
+1019 1013 0:270 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+1020 1019 0:271 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+1021 1013 0:272 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro
+1022 1021 0:30 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw
+1023 1019 0:268 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw
+1024 1019 0:273 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k
+1025 1013 254:1 /docker/containers/3ccfa00432798ff38f85839de1e396f771b4acbe9f4ddea0a761c39b9790a7821/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw,discard
+1026 1013 254:1 /docker/containers/3ccfa00432798ff38f85839de1e396f771b4acbe9f4ddea0a761c39b9790a7821/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw,discard
+1027 1013 254:1 /docker/containers/3ccfa00432798ff38f85839de1e396f771b4acbe9f4ddea0a761c39b9790a7821/hosts /etc/hosts rw,relatime - ext4 /dev/vda1 rw,discard
+717 1019 0:271 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+718 1014 0:269 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw
+719 1014 0:269 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw
+720 1014 0:269 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw
+721 1014 0:269 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw
+723 1014 0:269 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw
+726 1014 0:274 / /proc/acpi ro,relatime - tmpfs tmpfs ro
+727 1014 0:270 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+728 1014 0:270 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+729 1014 0:270 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+730 1021 0:275 / /sys/firmware ro,relatime - tmpfs tmpfs ro

tests/cross_agent/fixtures/docker_container_id_v2/invalid-characters.txt

@@ -0,0 +1,21 @@
+1014 1013 0:269 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
+1019 1013 0:270 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+1020 1019 0:271 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+1021 1013 0:272 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro
+1022 1021 0:30 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw
+1023 1019 0:268 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw
+1024 1019 0:273 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k
+1025 1013 254:1 /docker/containers/WRONGINCORRECTINVALIDCHARSERRONEOUSBADPHONYBROKEN2TERRIBLENOPE55/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw,discard
+1026 1013 254:1 /docker/containers/WRONGINCORRECTINVALIDCHARSERRONEOUSBADPHONYBROKEN2TERRIBLENOPE55/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw,discard
+1027 1013 254:1 /docker/containers/WRONGINCORRECTINVALIDCHARSERRONEOUSBADPHONYBROKEN2TERRIBLENOPE55/hosts /etc/hosts rw,relatime - ext4 /dev/vda1 rw,discard
+717 1019 0:271 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+718 1014 0:269 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw
+719 1014 0:269 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw
+720 1014 0:269 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw
+721 1014 0:269 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw
+723 1014 0:269 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw
+726 1014 0:274 / /proc/acpi ro,relatime - tmpfs tmpfs ro
+727 1014 0:270 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+728 1014 0:270 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+729 1014 0:270 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+730 1021 0:275 / /sys/firmware ro,relatime - tmpfs tmpfs ro

tests/cross_agent/fixtures/docker_container_id_v2/invalid-length.txt

@@ -0,0 +1,21 @@
+1014 1013 0:269 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
+1019 1013 0:270 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+1020 1019 0:271 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+1021 1013 0:272 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro
+1022 1021 0:30 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw
+1023 1019 0:268 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw
+1024 1019 0:273 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k
+1025 1013 254:1 /docker/containers/47cbd16b77c5/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw,discard
+1026 1013 254:1 /docker/containers/47cbd16b77c5/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw,discard
+1027 1013 254:1 /docker/containers/47cbd16b77c5/hosts /etc/hosts rw,relatime - ext4 /dev/vda1 rw,discard
+717 1019 0:271 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+718 1014 0:269 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw
+719 1014 0:269 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw
+720 1014 0:269 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw
+721 1014 0:269 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw
+723 1014 0:269 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw
+726 1014 0:274 / /proc/acpi ro,relatime - tmpfs tmpfs ro
+727 1014 0:270 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+728 1014 0:270 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+729 1014 0:270 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+730 1021 0:275 / /sys/firmware ro,relatime - tmpfs tmpfs ro

tests/cross_agent/test_docker.py → tests/cross_agent/test_docker_container_id.py

@@ -39,13 +39,23 @@ def _load_docker_test_attributes():
     return docker_test_attributes
 
 
+def mock_open(mock_file):
+    def _mock_open(filename, mode):
+        if filename == "/proc/self/mountinfo":
+            raise FileNotFoundError()
+        elif filename == "/proc/self/cgroup":
+            return mock_file
+        raise RuntimeError()
+    return _mock_open
+
+
 @pytest.mark.parametrize('filename, containerId',
                           _load_docker_test_attributes())
-def test_docker_container_id(filename, containerId):
+def test_docker_container_id_v1(monkeypatch, filename, containerId):
     path = os.path.join(DOCKER_FIXTURE, filename)
     with open(path, 'rb') as f:
-        with mock.patch.object(u, 'open', create=True, return_value=f):
-            if containerId is not None:
-                assert u.DockerUtilization.detect() == {'id': containerId}
-            else:
-                assert u.DockerUtilization.detect() is None
+        monkeypatch.setattr(u, "open", mock_open(f), raising=False)
+        if containerId is not None:
+            assert u.DockerUtilization.detect() == {'id': containerId}
+        else:
+            assert u.DockerUtilization.detect() is None

tests/cross_agent/test_docker_container_id_v2.py

@@ -0,0 +1,61 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import mock
+import os
+import pytest
+
+import newrelic.common.utilization as u
+
+CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
+DOCKER_FIXTURE = os.path.join(CURRENT_DIR, 'fixtures', 'docker_container_id_v2')
+
+
+def _load_docker_test_attributes():
+    """Returns a list of docker test attributes in the form:
+       [(<filename>, <containerId>), ...]
+
+    """
+    docker_test_attributes = []
+    test_cases = os.path.join(DOCKER_FIXTURE, 'cases.json')
+    with open(test_cases, 'r') as fh:
+        js = fh.read()
+    json_list = json.loads(js)
+    for json_record in json_list:
+        docker_test_attributes.append(
+            (json_record['filename'], json_record['containerId']))
+    return docker_test_attributes
+
+
+def mock_open(mock_file):
+    def _mock_open(filename, mode):
+        if filename == "/proc/self/cgroup":
+            raise FileNotFoundError()
+        elif filename == "/proc/self/mountinfo":
+            return mock_file
+        raise RuntimeError()
+    return _mock_open
+
+
+@pytest.mark.parametrize('filename, containerId',
+                          _load_docker_test_attributes())
+def test_docker_container_id_v2(monkeypatch, filename, containerId):
+    path = os.path.join(DOCKER_FIXTURE, filename)
+    with open(path, 'rb') as f:
+        monkeypatch.setattr(u, "open", mock_open(f), raising=False)
+        if containerId is not None:
+            assert u.DockerUtilization.detect() == {'id': containerId}
+        else:
+            assert u.DockerUtilization.detect() is None