Merge branch 'master_proc_file' of https://github.com/bloomberg/salt …

…into master_proc_file

doc/ref/modules/all/index.rst

@@ -423,6 +423,7 @@ execution modules
     ssh
     ssh_package
     ssh_service
+    snap
     snapper
     state
     status

doc/ref/modules/all/salt.modules.snap.rst

@@ -0,0 +1,6 @@
+=================
+salt.modules.snap
+=================
+
+.. automodule:: salt.modules.snap
+    :members:

doc/ref/states/all/index.rst

@@ -262,6 +262,7 @@ state modules
     slack
     smartos
     smtp
+    snap
     snapper
     solrcloud
     splunk

doc/ref/states/all/salt.states.snap.rst

@@ -0,0 +1,6 @@
+================
+salt.states.snap
+================
+
+.. automodule:: salt.states.snap
+    :members:

salt/cloud/__init__.py

@@ -76,6 +76,11 @@ def _call(queue, args, kwargs):
             queue.put('ERROR')
             queue.put('Exception')
             queue.put('{0}\n{1}\n'.format(ex, trace))
+        except SystemExit as ex:
+            trace = traceback.format_exc()
+            queue.put('ERROR')
+            queue.put('System exit')
+            queue.put('{0}\n{1}\n'.format(ex, trace))
         return ret
     return _call
 

salt/cloud/clouds/gce.py

@@ -2521,6 +2521,27 @@ def request_instance(vm_):
                 '\'pd-standard\', \'pd-ssd\''
             )
 
+    # GCE accelerator options are only supported as of libcloud >= 2.3.0
+    # and Python 3+ is required so that libcloud will detect a type of
+    # 'string' rather than 'unicode'
+    if LIBCLOUD_VERSION_INFO >= (2, 3, 0) and isinstance(u'test', str):
+
+        kwargs.update({
+            'ex_accelerator_type': config.get_cloud_config_value(
+                'ex_accelerator_type', vm_, __opts__, default=None),
+            'ex_accelerator_count': config.get_cloud_config_value(
+                'ex_accelerator_count', vm_, __opts__, default=None)
+        })
+        if kwargs.get('ex_accelerator_type'):
+            log.warning(
+                'An accelerator is being attached to this instance, '
+                'the ex_on_host_maintenance setting is being set to '
+                '\'TERMINATE\' as a result'
+            )
+            kwargs.update({
+                'ex_on_host_maintenance': 'TERMINATE'
+            })
+
     log.info(
         'Creating GCE instance %s in %s',
         vm_['name'], kwargs['location'].name

salt/grains/core.py

@@ -2188,14 +2188,13 @@ def fqdns():
     grains = {}
     fqdns = set()
 
-    addresses = salt.utils.network.ip_addrs(include_loopback=False,
-                                            interface_data=_INTERFACES)
-    addresses.extend(salt.utils.network.ip_addrs6(include_loopback=False,
-                                                  interface_data=_INTERFACES))
+    addresses = salt.utils.network.ip_addrs(include_loopback=False, interface_data=_get_interfaces())
+    addresses.extend(salt.utils.network.ip_addrs6(include_loopback=False, interface_data=_get_interfaces()))
     err_message = 'Exception during resolving address: %s'
     for ip in addresses:
         try:
-            fqdns.add(socket.getfqdn(socket.gethostbyaddr(ip)[0]))
+            name, aliaslist, addresslist = socket.gethostbyaddr(ip)
+            fqdns.update([socket.getfqdn(name)] + [als for als in aliaslist if salt.utils.network.is_fqdn(als)])
         except socket.herror as err:
             if err.errno == 0:
                 # No FQDN for this IP address, so we don't need to know this all the time.
@@ -2205,8 +2204,7 @@ def fqdns():
         except (socket.error, socket.gaierror, socket.timeout) as err:
             log.error(err_message, err)
 
-    grains['fqdns'] = sorted(list(fqdns))
-    return grains
+    return {"fqdns": sorted(list(fqdns))}
 
 
 def ip_fqdn():

salt/modules/aptpkg.py

@@ -2304,7 +2304,7 @@ def mod_repo(repo, saltenv='base', **kwargs):
         # and the resulting source line.  The idea here is to ensure
         # we are not returning bogus data because the source line
         # has already been modified on a previous run.
-        repo_matches = source.type == repo_type and source.uri == repo_uri and source.dist == repo_dist
+        repo_matches = source.type == repo_type and source.uri.rstrip('/') == repo_uri.rstrip('/') and source.dist == repo_dist
         kw_matches = source.dist == kw_dist and source.type == kw_type
 
         if repo_matches or kw_matches:

salt/modules/deb_postgres.py

@@ -34,7 +34,10 @@ def cluster_create(version,
                    port=None,
                    locale=None,
                    encoding=None,
-                   datadir=None):
+                   datadir=None,
+                   allow_group_access=None,
+                   data_checksums=None,
+                   wal_segsize=None):
     '''
     Adds a cluster to the Postgres server.
 
@@ -52,6 +55,8 @@ def cluster_create(version,
 
         salt '*' postgres.cluster_create '9.3' locale='fr_FR'
 
+        salt '*' postgres.cluster_create '11' data_checksums=True wal_segsize='32'
+
     '''
     cmd = [salt.utils.path.which('pg_createcluster')]
     if port:
@@ -63,6 +68,15 @@ def cluster_create(version,
     if datadir:
         cmd += ['--datadir', datadir]
     cmd += [version, name]
+    # initdb-specific options are passed after '--'
+    if allow_group_access or data_checksums or wal_segsize:
+        cmd += ['--']
+    if allow_group_access is True:
+        cmd += ['--allow-group-access']
+    if data_checksums is True:
+        cmd += ['--data-checksums']
+    if wal_segsize:
+        cmd += ['--wal-segsize', wal_segsize]
     cmdstr = ' '.join([pipes.quote(c) for c in cmd])
     ret = __salt__['cmd.run_all'](cmdstr, python_shell=False)
     if ret.get('retcode', 0) != 0:

salt/modules/saltsupport.py

@@ -57,7 +57,7 @@ class LogCollector(object):
 
     class MessagesList(list):
         def append(self, obj):
-            list.append(self, '{} - {}'.format(datetime.datetime.utcnow().strftime('%T.%f')[:-3], obj))
+            list.append(self, '{} - {}'.format(datetime.datetime.utcnow().strftime('%H:%M:%S.%f')[:-3], obj))
         __call__ = append
 
     def __init__(self):

salt/modules/saltutil.py

@@ -1132,7 +1132,7 @@ def running():
     return salt.utils.minion.running(__opts__)
 
 
-def clear_cache():
+def clear_cache(days=-1):
     '''
     Forcibly removes all caches on a minion.
 
@@ -1145,12 +1145,16 @@ def clear_cache():
 
     .. code-block:: bash
 
-        salt '*' saltutil.clear_cache
+        salt '*' saltutil.clear_cache days=7
     '''
+    threshold = time.time() - days * 24 * 60 * 60
     for root, dirs, files in salt.utils.files.safe_walk(__opts__['cachedir'], followlinks=False):
         for name in files:
             try:
-                os.remove(os.path.join(root, name))
+                file = os.path.join(root, name)
+                mtime = os.path.getmtime(file)
+                if mtime < threshold:
+                    os.remove(file)
             except OSError as exc:
                 log.error(
                     'Attempt to clear cache with saltutil.clear_cache '
@@ -1175,7 +1179,7 @@ def clear_job_cache(hours=24):
 
         salt '*' saltutil.clear_job_cache hours=12
     '''
-    threshold = time.time() - hours * 3600
+    threshold = time.time() - hours * 60 * 60
     for root, dirs, files in salt.utils.files.safe_walk(os.path.join(__opts__['cachedir'], 'minion_jobs'),
                                                   followlinks=False):
         for name in dirs:

salt/modules/snap.py

@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+'''
+Manage snap packages via Salt
+
+:depends: snapd for distribution
+
+'''
+
+from __future__ import absolute_import, print_function, unicode_literals
+import subprocess
+
+import salt.utils.path
+
+SNAP_BINARY_NAME = 'snap'
+
+__virtualname__ = 'snap'
+
+
+def __virtual__():
+    if salt.utils.path.which('snap'):
+        return __virtualname__
+
+    return (False, 'The snap execution module cannot be loaded: the "snap" binary is not in the path.')
+
+
+def install(pkg, channel=None, refresh=False):
+    '''
+    Install the specified snap package from the specified channel.
+    Returns a dictionary of "result" and "output".
+
+    pkg
+        The snap package name
+
+    channel
+        Optional. The snap channel to install from, eg "beta"
+
+    refresh : False
+        If True, use "snap refresh" instead of "snap install".
+        This allows changing the channel of a previously installed package.
+    '''
+    args = []
+    ret = {'result': None, 'output': ""}
+
+    if refresh:
+        cmd = 'refresh'
+    else:
+        cmd = 'install'
+
+    if channel:
+        args.append('--channel=' + channel)
+
+    try:
+        # Try to run it, merging stderr into output
+        ret['output'] = subprocess.check_output([SNAP_BINARY_NAME, cmd, pkg] + args, stderr=subprocess.STDOUT)
+        ret['result'] = True
+    except subprocess.CalledProcessError as e:
+        ret['output'] = e.output
+        ret['result'] = False
+
+    return ret
+
+
+def is_installed(pkg):
+    '''
+    Returns True if there is any version of the specified package installed.
+
+    pkg
+        The package name
+    '''
+    return bool(versions_installed(pkg))
+
+
+def remove(pkg):
+    '''
+    Remove the specified snap package. Returns a dictionary of "result" and "output".
+
+    pkg
+        The package name
+    '''
+    ret = {'result': None, 'output': ""}
+    try:
+        ret['output'] = subprocess.check_output([SNAP_BINARY_NAME, 'remove', pkg])
+        ret['result'] = True
+    except subprocess.CalledProcessError as e:
+        ret['output'] = e.output
+        ret['result'] = False
+
+
+# Parse 'snap list' into a dict
+def versions_installed(pkg):
+    '''
+    Query which version(s) of the specified snap package are installed.
+    Returns a list of 0 or more dictionaries.
+
+    pkg
+        The package name
+    '''
+
+    try:
+        # Try to run it, merging stderr into output
+        output = subprocess.check_output([SNAP_BINARY_NAME, 'list', pkg], stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError:
+        return []
+
+    lines = output.splitlines()[1:]
+    ret = []
+    for item in lines:
+        # If fields contain spaces this will break.
+        i = item.split()
+        # Ignore 'Notes' field
+        ret.append({
+            'name':         i[0],
+            'version':      i[1],
+            'rev':          i[2],
+            'tracking':     i[3],
+            'publisher':    i[4]
+            })
+
+    return ret

salt/modules/ssh.py

@@ -16,7 +16,9 @@
 import logging
 import os
 import re
+import shutil
 import subprocess
+import tempfile
 
 # Import salt libs
 import salt.utils.decorators.path
@@ -927,15 +929,35 @@ def recv_known_host_entries(hostname,
         cmd.extend(['-t', enc])
     if not enc and __grains__.get('osfinger') in need_dash_t:
         cmd.extend(['-t', 'rsa'])
-    if hash_known_hosts:
-        cmd.append('-H')
     cmd.extend(['-T', six.text_type(timeout)])
     cmd.append(hostname)
     lines = None
     attempts = 5
     while not lines and attempts > 0:
         attempts = attempts - 1
-        lines = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
+        output = __salt__['cmd.run'](cmd, python_shell=False)
+
+    # This is a workaround because ssh-keyscan hashing is broken for
+    # non-standard SSH ports on basically every platform. See
+    # https://github.com/saltstack/salt/issues/40152 for more info.
+    if hash_known_hosts:
+        # Use a tempdir, because ssh-keygen creates a backup .old file
+        # and we want to get rid of that. We won't need our temp keys
+        # file, either.
+        tempdir = tempfile.mkdtemp()
+        try:
+            filename = os.path.join(tempdir, 'keys')
+            with salt.utils.files.fopen(filename, mode='w') as f:
+                f.write(output)
+            cmd = ['ssh-keygen', '-H', '-f', filename]
+            __salt__['cmd.run'](cmd, python_shell=False)
+            # ssh-keygen moves the old file, so we have to re-open to
+            # read the hashed entries
+            with salt.utils.files.fopen(filename, mode='r') as f:
+                output = f.read()
+        finally:
+            shutil.rmtree(tempdir, ignore_errors=True)
+    lines = output.splitlines()
     known_host_entries = list(_parse_openssh_output(lines,
                                              fingerprint_hash_type=fingerprint_hash_type))
     return known_host_entries if known_host_entries else None
@@ -1096,11 +1118,6 @@ def set_known_host(user=None,
         return {'status': 'error',
                 'error': 'hostname argument required'}
 
-    if port is not None and port != DEFAULT_SSH_PORT and hash_known_hosts:
-        return {'status': 'error',
-                'error': 'argument port can not be used in '
-                'conjunction with argument hash_known_hosts'}
-
     update_required = False
     check_required = False
     stored_host_entries = get_known_host_entries(user,