Move "FQDNs" grains calculation to "network" module and allow disabled them

salt/grains/core.py

@@ -23,6 +23,9 @@
 from errno import EACCES, EPERM
 import datetime
 import warnings
+import salt.modules.network
+
+from salt.utils.network import _get_interfaces
 
 # pylint: disable=import-error
 try:
@@ -78,6 +81,7 @@ def linux_distribution(**kwargs):
     'cmd.run_all': salt.modules.cmdmod._run_all_quiet,
     'smbios.records': salt.modules.smbios.records,
     'smbios.get': salt.modules.smbios.get,
+    'network.fqdns': salt.modules.network.fqdns,
 }
 log = logging.getLogger(__name__)
 
@@ -101,8 +105,6 @@ def linux_distribution(**kwargs):
 if not hasattr(os, 'uname'):
     HAS_UNAME = False
 
-_INTERFACES = {}
-
 
 def _windows_cpudata():
     '''
@@ -1526,17 +1528,6 @@ def _linux_bin_exists(binary):
         return False
 
 
-def _get_interfaces():
-    '''
-    Provide a dict of the connected interfaces and their ip addresses
-    '''
-
-    global _INTERFACES
-    if not _INTERFACES:
-        _INTERFACES = salt.utils.network.interfaces()
-    return _INTERFACES
-
-
 def _parse_lsb_release():
     ret = {}
     try:
@@ -2198,31 +2189,7 @@ def fqdns():
     Return all known FQDNs for the system by enumerating all interfaces and
     then trying to reverse resolve them (excluding 'lo' interface).
     '''
-    # Provides:
-    # fqdns
-
-    grains = {}
-    fqdns = set()
-
-    addresses = salt.utils.network.ip_addrs(include_loopback=False,
-                                            interface_data=_INTERFACES)
-    addresses.extend(salt.utils.network.ip_addrs6(include_loopback=False,
-                                                  interface_data=_INTERFACES))
-    err_message = 'Exception during resolving address: %s'
-    for ip in addresses:
-        try:
-            fqdns.add(socket.getfqdn(socket.gethostbyaddr(ip)[0]))
-        except socket.herror as err:
-            if err.errno == 0:
-                # No FQDN for this IP address, so we don't need to know this all the time.
-                log.debug("Unable to resolve address %s: %s", ip, err)
-            else:
-                log.error(err_message, err)
-        except (socket.error, socket.gaierror, socket.timeout) as err:
-            log.error(err_message, err)
-
-    grains['fqdns'] = sorted(list(fqdns))
-    return grains
+    return __salt__['network.fqdns']()
 
 
 def ip_fqdn():

salt/modules/network.py

@@ -11,6 +11,10 @@
 import re
 import os
 import socket
+import time
+
+from multiprocessing.pool import ThreadPool
+
 
 # Import salt libs
 import salt.utils.decorators.path
@@ -1887,3 +1891,59 @@ def iphexval(ip):
     a = ip.split('.')
     hexval = ['%02X' % int(x) for x in a]  # pylint: disable=E1321
     return ''.join(hexval)
+
+
+def fqdns():
+    '''
+    Return all known FQDNs for the system by enumerating all interfaces and
+    then trying to reverse resolve them (excluding 'lo' interface).
+    '''
+    # Provides:
+    # fqdns
+
+    # Possible value for h_errno defined in netdb.h
+    HOST_NOT_FOUND = 1
+    NO_DATA = 4
+
+    grains = {}
+    fqdns = set()
+
+    def _lookup_fqdn(ip):
+        try:
+            return [socket.getfqdn(socket.gethostbyaddr(ip)[0])]
+        except socket.herror as err:
+            if err.errno in (0, HOST_NOT_FOUND, NO_DATA):
+                # No FQDN for this IP address, so we don't need to know this all the time.
+                log.debug("Unable to resolve address %s: %s", ip, err)
+            else:
+                log.error(err_message, err)
+        except (socket.error, socket.gaierror, socket.timeout) as err:
+            log.error(err_message, err)
+
+    start = time.time()
+
+    addresses = salt.utils.network.ip_addrs(include_loopback=False, interface_data=salt.utils.network._get_interfaces())
+    addresses.extend(salt.utils.network.ip_addrs6(include_loopback=False, interface_data=salt.utils.network._get_interfaces()))
+    err_message = 'Exception during resolving address: %s'
+
+    # Create a ThreadPool to process the underlying calls to 'socket.gethostbyaddr' in parallel.
+    # This avoid blocking the execution when the "fqdn" is not defined for certains IP addresses, which was causing
+    # that "socket.timeout" was reached multiple times secuencially, blocking execution for several seconds.
+
+    results = []
+    try:
+        pool = ThreadPool(8)
+        results = pool.map(_lookup_fqdn, addresses)
+        pool.close()
+        pool.join()
+    except Exception as exc:
+        log.error("Exception while creating a ThreadPool for resolving FQDNs: %s", exc)
+
+    for item in results:
+        if item:
+            fqdns.update(item)
+
+    elapsed = time.time() - start
+    log.debug('Elapsed time getting FQDNs: {} seconds'.format(elapsed))
+
+    return {"fqdns": sorted(list(fqdns))}

salt/utils/network.py

@@ -55,6 +55,20 @@
 # pylint: disable=C0103
 
 
+_INTERFACES = {}
+
+
+def _get_interfaces():
+    '''
+    Provide a dict of the connected interfaces and their ip addresses
+    '''
+
+    global _INTERFACES
+    if not _INTERFACES:
+        _INTERFACES = interfaces()
+    return _INTERFACES
+
+
 def sanitize_host(host):
     '''
     Sanitize host string.

tests/unit/grains/test_core.py

@@ -33,6 +33,7 @@
 import salt.utils.platform
 import salt.utils.path
 import salt.grains.core as core
+import salt.modules.network
 
 # Import 3rd-party libs
 from salt.ext import six
@@ -992,11 +993,12 @@ def test_fqdns_return(self):
                                ('foo.bar.baz', [], ['fe80::a8b2:93ff:fe00:0']),
                                ('bluesniff.foo.bar', [], ['fe80::a8b2:93ff:dead:beef'])]
         ret = {'fqdns': ['bluesniff.foo.bar', 'foo.bar.baz', 'rinzler.evil-corp.com']}
-        with patch.object(socket, 'gethostbyaddr', side_effect=reverse_resolv_mock):
-            fqdns = core.fqdns()
-            self.assertIn('fqdns', fqdns)
-            self.assertEqual(len(fqdns['fqdns']), len(ret['fqdns']))
-            self.assertEqual(set(fqdns['fqdns']), set(ret['fqdns']))
+        with patch.dict(core.__salt__, {'network.fqdns': salt.modules.network.fqdns}):
+            with patch.object(socket, 'gethostbyaddr', side_effect=reverse_resolv_mock):
+                fqdns = core.fqdns()
+                assert "fqdns" in fqdns
+                assert len(fqdns['fqdns']) == len(ret['fqdns'])
+                assert set(fqdns['fqdns']) == set(ret['fqdns'])
 
     def test_core_virtual(self):
         '''