Merge pull request #49815 from isbm/isbm-ipv6-scope-errors-2018.3

Bugfix/rework: IPv6 scope errors (bp)

salt/_compat.py

@@ -2,18 +2,21 @@
 '''
 Salt compatibility code
 '''
-# pylint: disable=import-error,unused-import,invalid-name
+# pylint: disable=import-error,unused-import,invalid-name,W0231,W0233
 
 # Import python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals, print_function
 import sys
 import types
+import logging
 
 # Import 3rd-party libs
-from salt.ext.six import binary_type, string_types, text_type
+from salt.exceptions import SaltException
+from salt.ext.six import binary_type, string_types, text_type, integer_types
 from salt.ext.six.moves import cStringIO, StringIO
 
-HAS_XML = True
+log = logging.getLogger(__name__)
+
 try:
     # Python >2.5
     import xml.etree.cElementTree as ElementTree
@@ -31,11 +34,10 @@
                 import elementtree.ElementTree as ElementTree
             except Exception:
                 ElementTree = None
-                HAS_XML = False
 
 
 # True if we are running on Python 3.
-PY3 = sys.version_info[0] == 3
+PY3 = sys.version_info.major == 3
 
 
 if PY3:
@@ -45,13 +47,12 @@
     import exceptions
 
 
-if HAS_XML:
+if ElementTree is not None:
     if not hasattr(ElementTree, 'ParseError'):
         class ParseError(Exception):
             '''
             older versions of ElementTree do not have ParseError
             '''
-            pass
 
         ElementTree.ParseError = ParseError
 
@@ -61,67 +62,45 @@ def text_(s, encoding='latin-1', errors='strict'):
     If ``s`` is an instance of ``binary_type``, return
     ``s.decode(encoding, errors)``, otherwise return ``s``
     '''
-    if isinstance(s, binary_type):
-        return s.decode(encoding, errors)
-    return s
+    return s.decode(encoding, errors) if isinstance(s, binary_type) else s
 
 
 def bytes_(s, encoding='latin-1', errors='strict'):
     '''
     If ``s`` is an instance of ``text_type``, return
     ``s.encode(encoding, errors)``, otherwise return ``s``
     '''
-    if isinstance(s, text_type):
-        return s.encode(encoding, errors)
-    return s
+    return s.encode(encoding, errors) if isinstance(s, text_type) else s
 
 
-if PY3:
-    def ascii_native_(s):
-        if isinstance(s, text_type):
-            s = s.encode('ascii')
-        return str(s, 'ascii', 'strict')
-else:
-    def ascii_native_(s):
-        if isinstance(s, text_type):
-            s = s.encode('ascii')
-        return str(s)
+def ascii_native_(s):
+    '''
+    Python 3: If ``s`` is an instance of ``text_type``, return
+    ``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')``
 
-ascii_native_.__doc__ = '''
-Python 3: If ``s`` is an instance of ``text_type``, return
-``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')``
+    Python 2: If ``s`` is an instance of ``text_type``, return
+    ``s.encode('ascii')``, otherwise return ``str(s)``
+    '''
+    if isinstance(s, text_type):
+        s = s.encode('ascii')
 
-Python 2: If ``s`` is an instance of ``text_type``, return
-``s.encode('ascii')``, otherwise return ``str(s)``
-'''
+    return str(s, 'ascii', 'strict') if PY3 else s
 
 
-if PY3:
-    def native_(s, encoding='latin-1', errors='strict'):
-        '''
-        If ``s`` is an instance of ``text_type``, return
-        ``s``, otherwise return ``str(s, encoding, errors)``
-        '''
-        if isinstance(s, text_type):
-            return s
-        return str(s, encoding, errors)
-else:
-    def native_(s, encoding='latin-1', errors='strict'):
-        '''
-        If ``s`` is an instance of ``text_type``, return
-        ``s.encode(encoding, errors)``, otherwise return ``str(s)``
-        '''
-        if isinstance(s, text_type):
-            return s.encode(encoding, errors)
-        return str(s)
+def native_(s, encoding='latin-1', errors='strict'):
+    '''
+    Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise
+    return ``str(s, encoding, errors)``
 
-native_.__doc__ = '''
-Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise
-return ``str(s, encoding, errors)``
+    Python 2: If ``s`` is an instance of ``text_type``, return
+    ``s.encode(encoding, errors)``, otherwise return ``str(s)``
+    '''
+    if PY3:
+        out = s if isinstance(s, text_type) else str(s, encoding, errors)
+    else:
+        out = s.encode(encoding, errors) if isinstance(s, text_type) else str(s)
 
-Python 2: If ``s`` is an instance of ``text_type``, return
-``s.encode(encoding, errors)``, otherwise return ``str(s)``
-'''
+    return out
 
 
 def string_io(data=None):  # cStringIO can't handle unicode
@@ -133,7 +112,199 @@ def string_io(data=None):  # cStringIO can't handle unicode
     except (UnicodeEncodeError, TypeError):
         return StringIO(data)
 
-if PY3:
-    import ipaddress
-else:
-    import salt.ext.ipaddress as ipaddress
+
+try:
+    if PY3:
+        import ipaddress
+    else:
+        import salt.ext.ipaddress as ipaddress
+except ImportError:
+    ipaddress = None
+
+
+class IPv6AddressScoped(ipaddress.IPv6Address):
+    '''
+    Represent and manipulate single IPv6 Addresses.
+    Scope-aware version
+    '''
+    def __init__(self, address):
+        '''
+        Instantiate a new IPv6 address object. Scope is moved to an attribute 'scope'.
+
+        Args:
+            address: A string or integer representing the IP
+
+              Additionally, an integer can be passed, so
+              IPv6Address('2001:db8::') == IPv6Address(42540766411282592856903984951653826560)
+              or, more generally
+              IPv6Address(int(IPv6Address('2001:db8::'))) == IPv6Address('2001:db8::')
+
+        Raises:
+            AddressValueError: If address isn't a valid IPv6 address.
+
+        :param address:
+        '''
+        # pylint: disable-all
+        if not hasattr(self, '_is_packed_binary'):
+            # This method (below) won't be around for some Python 3 versions
+            # and we need check this differently anyway
+            self._is_packed_binary = lambda p: isinstance(p, bytes)
+        # pylint: enable-all
+
+        if isinstance(address, string_types) and '%' in address:
+            buff = address.split('%')
+            if len(buff) != 2:
+                raise SaltException('Invalid IPv6 address: "{}"'.format(address))
+            address, self.__scope = buff
+        else:
+            self.__scope = None
+
+        if sys.version_info.major == 2:
+            ipaddress._BaseAddress.__init__(self, address)
+            ipaddress._BaseV6.__init__(self, address)
+        else:
+            # Python 3.4 fix. Versions higher are simply not affected
+            # https://github.com/python/cpython/blob/3.4/Lib/ipaddress.py#L543-L544
+            self._version = 6
+            self._max_prefixlen = ipaddress.IPV6LENGTH
+
+        # Efficient constructor from integer.
+        if isinstance(address, integer_types):
+            self._check_int_address(address)
+            self._ip = address
+        elif self._is_packed_binary(address):
+            self._check_packed_address(address, 16)
+            self._ip = ipaddress._int_from_bytes(address, 'big')
+        else:
+            address = str(address)
+            if '/' in address:
+                raise ipaddress.AddressValueError("Unexpected '/' in {}".format(address))
+            self._ip = self._ip_int_from_string(address)
+
+    def _is_packed_binary(self, data):
+        '''
+        Check if data is hexadecimal packed
+
+        :param data:
+        :return:
+        '''
+        packed = False
+        if len(data) == 16 and ':' not in data:
+            try:
+                packed = bool(int(str(bytearray(data)).encode('hex'), 16))
+            except ValueError:
+                pass
+
+        return packed
+
+    @property
+    def scope(self):
+        '''
+        Return scope of IPv6 address.
+
+        :return:
+        '''
+        return self.__scope
+
+    def __str__(self):
+        return text_type(self._string_from_ip_int(self._ip) +
+                         ('%' + self.scope if self.scope is not None else ''))
+
+
+class IPv6InterfaceScoped(ipaddress.IPv6Interface, IPv6AddressScoped):
+    '''
+    Update
+    '''
+    def __init__(self, address):
+        if isinstance(address, (bytes, int)):
+            IPv6AddressScoped.__init__(self, address)
+            self.network = ipaddress.IPv6Network(self._ip)
+            self._prefixlen = self._max_prefixlen
+            return
+
+        addr = ipaddress._split_optional_netmask(address)
+        IPv6AddressScoped.__init__(self, addr[0])
+        self.network = ipaddress.IPv6Network(address, strict=False)
+        self.netmask = self.network.netmask
+        self._prefixlen = self.network._prefixlen
+        self.hostmask = self.network.hostmask
+
+
+def ip_address(address):
+    """Take an IP string/int and return an object of the correct type.
+
+    Args:
+        address: A string or integer, the IP address.  Either IPv4 or
+          IPv6 addresses may be supplied; integers less than 2**32 will
+          be considered to be IPv4 by default.
+
+    Returns:
+        An IPv4Address or IPv6Address object.
+
+    Raises:
+        ValueError: if the *address* passed isn't either a v4 or a v6
+          address
+
+    """
+    try:
+        return ipaddress.IPv4Address(address)
+    except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err:
+        log.debug('Error while parsing IPv4 address: %s', address)
+        log.debug(err)
+
+    try:
+        return IPv6AddressScoped(address)
+    except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err:
+        log.debug('Error while parsing IPv6 address: %s', address)
+        log.debug(err)
+
+    if isinstance(address, bytes):
+        raise ipaddress.AddressValueError('{} does not appear to be an IPv4 or IPv6 address. '
+                                          'Did you pass in a bytes (str in Python 2) instead '
+                                          'of a unicode object?'.format(repr(address)))
+
+    raise ValueError('{} does not appear to be an IPv4 or IPv6 address'.format(repr(address)))
+
+
+def ip_interface(address):
+    """Take an IP string/int and return an object of the correct type.
+
+    Args:
+        address: A string or integer, the IP address.  Either IPv4 or
+          IPv6 addresses may be supplied; integers less than 2**32 will
+          be considered to be IPv4 by default.
+
+    Returns:
+        An IPv4Interface or IPv6Interface object.
+
+    Raises:
+        ValueError: if the string passed isn't either a v4 or a v6
+          address.
+
+    Notes:
+        The IPv?Interface classes describe an Address on a particular
+        Network, so they're basically a combination of both the Address
+        and Network classes.
+
+    """
+    try:
+        return ipaddress.IPv4Interface(address)
+    except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err:
+        log.debug('Error while getting IPv4 interface for address %s', address)
+        log.debug(err)
+
+    try:
+        return ipaddress.IPv6Interface(address)
+    except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err:
+        log.debug('Error while getting IPv6 interface for address %s', address)
+        log.debug(err)
+
+    raise ValueError('{} does not appear to be an IPv4 or IPv6 interface'.format(address))
+
+
+if ipaddress:
+    ipaddress.IPv6Address = IPv6AddressScoped
+    if sys.version_info.major == 2:
+        ipaddress.IPv6Interface = IPv6InterfaceScoped
+    ipaddress.ip_address = ip_address
+    ipaddress.ip_interface = ip_interface

salt/cloud/clouds/saltify.py

@@ -27,10 +27,7 @@
 import salt.config as config
 import salt.client
 import salt.ext.six as six
-if six.PY3:
-    import ipaddress
-else:
-    import salt.ext.ipaddress as ipaddress
+from salt._compat import ipaddress
 
 from salt.exceptions import SaltCloudException, SaltCloudSystemExit
 

salt/cloud/clouds/vagrant.py

@@ -25,13 +25,8 @@
 import salt.utils
 import salt.config as config
 import salt.client
-import salt.ext.six as six
-if six.PY3:
-    import ipaddress
-else:
-    import salt.ext.ipaddress as ipaddress
-from salt.exceptions import SaltCloudException, SaltCloudSystemExit, \
-    SaltInvocationError
+from salt._compat import ipaddress
+from salt.exceptions import SaltCloudException, SaltCloudSystemExit, SaltInvocationError
 
 # Get logging started
 log = logging.getLogger(__name__)

salt/ext/win_inet_pton.py

@@ -9,7 +9,7 @@
 import socket
 import ctypes
 import os
-import ipaddress
+from salt._compat import ipaddress
 import salt.ext.six as six
 
 

salt/minion.py

@@ -26,10 +26,7 @@
 # Import Salt Libs
 # pylint: disable=import-error,no-name-in-module,redefined-builtin
 from salt.ext import six
-if six.PY3:
-    import ipaddress
-else:
-    import salt.ext.ipaddress as ipaddress
+from salt._compat import ipaddress
 from salt.ext.six.moves import range
 from salt.utils.zeromq import zmq, ZMQDefaultLoop, install_zmq, ZMQ_VERSION_INFO