[disk] Add include_all_devices option and improve error logs

disk/assets/configuration/spec.yaml

@@ -42,6 +42,9 @@ files:
               start matching from the beginning and therefore to match anywhere you
               must prepend `.*`. For exact matches append `$`.
 
+              Some common Linux special file systems can be excluded by enabling the
+              `exclude_special_file_systems` option.
+
               When conflicts arise, this will override `file_system_whitelist`.
             value:
               example:
@@ -51,6 +54,31 @@ files:
               type: array
               items:
                 type: string
+          - name: exclude_special_file_systems
+            description: |
+              Instruct the check to not collect from some common special file systems.
+              Works on Linux only.
+
+              The special file systems excluded with this option are:
+                - binfmt_misc
+                - configfs
+                - debugfs
+                - devtmpfs
+                - overlay
+                - proc
+                - rootfs
+                - securityfs
+                - sysfs
+                - tmpfs
+                - tracefs
+
+              These are added to the `file_system_blacklist`.
+
+              For more fine-grained control, use `file_system_blacklist` and `file_system_whitelist`.
+            value:
+              example: true
+              type: boolean
+              default: false
           - name: device_whitelist
             description: |
                 Instruct the check to only collect from matching devices.

disk/datadog_checks/disk/data/conf.yaml.default

@@ -44,13 +44,39 @@ instances:
     ## start matching from the beginning and therefore to match anywhere you
     ## must prepend `.*`. For exact matches append `$`.
     ##
+    ## Some common Linux special file systems can be excluded by enabling the
+    ## `exclude_special_file_systems` option.
+    ##
     ## When conflicts arise, this will override `file_system_whitelist`.
     #
     # file_system_blacklist:
     #   - tmpfs$
     #   - rootfs$
     #   - autofs$
 
+    ## @param exclude_special_file_systems - boolean - optional - default: False
+    ## Instruct the check to not collect from some common special file systems.
+    ## Works on Linux only.
+    ##
+    ## The special file systems excluded with this option are:
+    ##   - binfmt_misc
+    ##   - configfs
+    ##   - debugfs
+    ##   - devtmpfs
+    ##   - overlay
+    ##   - proc
+    ##   - rootfs
+    ##   - securityfs
+    ##   - sysfs
+    ##   - tmpfs
+    ##   - tracefs
+    ##
+    ## These are added to the `file_system_blacklist`.
+    ##
+    ## For more fine-grained control, use `file_system_blacklist` and `file_system_whitelist`.
+    #
+    # exclude_special_file_systems: true
+
     ## @param device_whitelist - list of strings - optional
     ## Instruct the check to only collect from matching devices.
     ##

disk/datadog_checks/disk/disk.py

@@ -20,13 +20,42 @@
     # See: https://github.com/DataDog/integrations-core/pull/1109#discussion_r167133580
     IGNORE_CASE = re.I
 
+    KNOWN_SPECIAL_FILE_SYSTEMS = set()
+
     def _base_device_name(device):
         return device.strip('\\').lower()
 
 
+elif platform.system() == "Linux":
+    IGNORE_CASE = 0
+
+    # Remember to update the configuration specification
+    # if modifying the list of special file systems.
+    KNOWN_SPECIAL_FILE_SYSTEMS = set(
+        [
+            'binfmt_misc',
+            'configfs',
+            'debugfs',
+            'devtmpfs',
+            'overlay',
+            'proc',
+            'rootfs',
+            'securityfs',
+            'sysfs',
+            'tmpfs',
+            'tracefs',
+        ]
+    )
+
+    def _base_device_name(device):
+        return os.path.basename(device)
+
+
 else:
     IGNORE_CASE = 0
 
+    KNOWN_SPECIAL_FILE_SYSTEMS = set()
+
     def _base_device_name(device):
         return os.path.basename(device)
 
@@ -47,6 +76,8 @@ def __init__(self, name, init_config, instances):
         self._all_partitions = is_affirmative(instance.get('all_partitions', False))
         self._file_system_whitelist = instance.get('file_system_whitelist', [])
         self._file_system_blacklist = instance.get('file_system_blacklist', [])
+        # FIXME (8.X): Exclude special file systems by default
+        self._exclude_special_file_systems = instance.get('exclude_special_file_systems', False)
         self._device_whitelist = instance.get('device_whitelist', [])
         self._device_blacklist = instance.get('device_blacklist', [])
         self._mount_point_whitelist = instance.get('mount_point_whitelist', [])
@@ -88,7 +119,21 @@ def check(self, instance):
                 )
                 continue
             except Exception as e:
-                self.log.warning('Unable to get disk metrics for %s: %s', part.mountpoint, e)
+                if not self._exclude_special_file_systems and part.fstype in KNOWN_SPECIAL_FILE_SYSTEMS:
+                    self.log.warning(
+                        u'Unable to get disk metrics for %s with special file system %s: %s. '
+                        u'Enable `exclude_special_file_systems` to ignore common special file systems.',
+                        part.mountpoint,
+                        part.fstype,
+                        e,
+                    )
+                else:
+                    self.log.warning(
+                        u'Unable to get disk metrics for %s: %s. '
+                        u'You can exclude this mountpoint in the settings if it is invalid.',
+                        part.mountpoint,
+                        e,
+                    )
                 continue
 
             # Exclude disks with size less than min_disk_size
@@ -147,7 +192,6 @@ def _exclude_disk(self, device, file_system, mount_point):
         """
         Return True for disks we don't want or that match regex in the config file
         """
-        self.log.debug('_exclude_disk: %s, %s, %s', device, file_system, mount_point)
 
         if not device or device == 'none':
             device = None
@@ -190,7 +234,9 @@ def _file_system_blacklisted(self, file_system):
         if self._file_system_blacklist is None:
             return False
 
-        return not not self._file_system_blacklist.match(file_system)
+        return (self._exclude_special_file_systems and file_system in KNOWN_SPECIAL_FILE_SYSTEMS) or bool(
+            self._file_system_blacklist.match(file_system)
+        )
 
     def _device_whitelisted(self, device):
         if not device or self._device_whitelist is None:
@@ -245,7 +291,12 @@ def _collect_inodes_metrics(self, mountpoint):
             )
             return metrics
         except Exception as e:
-            self.log.warning('Unable to get disk metrics for %s: %s', mountpoint, e)
+            self.log.warning(
+                u'Unable to get disk metrics for %s: %s. '
+                u'You can exclude this mountpoint in the settings if it is invalid.',
+                mountpoint,
+                e,
+            )
             return metrics
 
         if inodes.f_files != 0:

disk/tests/test_filter.py

@@ -7,7 +7,7 @@
 from datadog_checks.disk.disk import IGNORE_CASE, Disk
 
 from .mocks import MockPart
-from .utils import assert_regex_equal, requires_windows
+from .utils import assert_regex_equal, requires_linux, requires_windows
 
 
 def test_default_casing():
@@ -104,6 +104,16 @@ def test_file_system_whitelist_blacklist():
     assert c.exclude_disk(MockPart(fstype='NTFS')) is True
 
 
+@requires_linux
+def test_exclude_special_file_system():
+    instance = {'exclude_special_file_systems': True}
+    c = Disk('disk', {}, [instance])
+
+    assert c.exclude_disk(MockPart(fstype='debugfs')) is True
+    assert c.exclude_disk(MockPart(fstype='tmpfs')) is True
+    assert c.exclude_disk(MockPart(fstype='ext4')) is False
+
+
 def test_device_whitelist():
     instance = {'device_whitelist': ['/dev/sda[1-3]', 'c:']}
     c = Disk('disk', {}, [instance])