improve encoding handling for setup.cfg

setuptools/__init__.py

@@ -5,12 +5,14 @@
 import functools
 import distutils.core
 import distutils.filelist
+import re
+from distutils.errors import DistutilsOptionError
 from distutils.util import convert_path
 from fnmatch import fnmatchcase
 
 from ._deprecation_warning import SetuptoolsDeprecationWarning
 
-from setuptools.extern.six import PY3
+from setuptools.extern.six import PY3, string_types
 from setuptools.extern.six.moves import filter, map
 
 import setuptools.version
@@ -161,6 +163,37 @@ def __init__(self, dist, **kw):
         _Command.__init__(self, dist)
         vars(self).update(kw)
 
+    def _ensure_stringlike(self, option, what, default=None):
+        val = getattr(self, option)
+        if val is None:
+            setattr(self, option, default)
+            return default
+        elif not isinstance(val, string_types):
+            raise DistutilsOptionError("'%s' must be a %s (got `%s`)"
+                                       % (option, what, val))
+        return val
+
+    def ensure_string_list(self, option):
+        r"""Ensure that 'option' is a list of strings.  If 'option' is
+        currently a string, we split it either on /,\s*/ or /\s+/, so
+        "foo bar baz", "foo,bar,baz", and "foo,   bar baz" all become
+        ["foo", "bar", "baz"].
+        """
+        val = getattr(self, option)
+        if val is None:
+            return
+        elif isinstance(val, string_types):
+            setattr(self, option, re.split(r',\s*|\s+', val))
+        else:
+            if isinstance(val, list):
+                ok = all(isinstance(v, string_types) for v in val)
+            else:
+                ok = False
+            if not ok:
+                raise DistutilsOptionError(
+                      "'%s' must be a list of strings (got %r)"
+                      % (option, val))
+
     def reinitialize_command(self, command, reinit_subcommands=0, **kw):
         cmd = _Command.reinitialize_command(self, command, reinit_subcommands)
         vars(cmd).update(kw)

setuptools/dist.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 __all__ = ['Distribution']
 
+import io
+import sys
 import re
 import os
 import warnings
@@ -9,9 +11,11 @@
 import distutils.core
 import distutils.cmd
 import distutils.dist
+from distutils.errors import DistutilsOptionError
+from distutils.util import strtobool
+from distutils.debug import DEBUG
 import itertools
 
-
 from collections import defaultdict
 from email import message_from_file
 
@@ -31,8 +35,8 @@
 from setuptools import windows_support
 from setuptools.monkey import get_unpatched
 from setuptools.config import parse_configuration
+from .unicode_utils import detect_encoding
 import pkg_resources
-from .py36compat import Distribution_parse_config_files
 
 __import__('setuptools.extern.packaging.specifiers')
 __import__('setuptools.extern.packaging.version')
@@ -332,7 +336,7 @@ def check_packages(dist, attr, value):
 _Distribution = get_unpatched(distutils.core.Distribution)
 
 
-class Distribution(Distribution_parse_config_files, _Distribution):
+class Distribution(_Distribution):
     """Distribution with support for features, tests, and package data
 
     This is an enhanced version of 'distutils.dist.Distribution' that
@@ -556,12 +560,78 @@ def _clean_req(self, req):
         req.marker = None
         return req
 
+    def _parse_config_files(self, filenames=None):
+        """
+        Adapted from distutils.dist.Distribution.parse_config_files,
+        this method provides the same functionality in subtly-improved
+        ways.
+        """
+        from setuptools.extern.six.moves.configparser import ConfigParser
+
+        # Ignore install directory options if we have a venv
+        if six.PY3 and sys.prefix != sys.base_prefix:
+            ignore_options = [
+                'install-base', 'install-platbase', 'install-lib',
+                'install-platlib', 'install-purelib', 'install-headers',
+                'install-scripts', 'install-data', 'prefix', 'exec-prefix',
+                'home', 'user', 'root']
+        else:
+            ignore_options = []
+
+        ignore_options = frozenset(ignore_options)
+
+        if filenames is None:
+            filenames = self.find_config_files()
+
+        if DEBUG:
+            self.announce("Distribution.parse_config_files():")
+
+        parser = ConfigParser()
+        for filename in filenames:
+            with io.open(filename, 'rb') as fp:
+                encoding = detect_encoding(fp)
+                if DEBUG:
+                    self.announce("  reading %s [%s]" % (
+                        filename, encoding or 'locale')
+                    )
+                reader = io.TextIOWrapper(fp, encoding=encoding)
+                (parser.read_file if six.PY3 else parser.readfp)(reader)
+            for section in parser.sections():
+                options = parser.options(section)
+                opt_dict = self.get_option_dict(section)
+
+                for opt in options:
+                    if opt != '__name__' and opt not in ignore_options:
+                        val = parser.get(section, opt)
+                        opt = opt.replace('-', '_')
+                        opt_dict[opt] = (filename, val)
+
+            # Make the ConfigParser forget everything (so we retain
+            # the original filenames that options come from)
+            parser.__init__()
+
+        # If there was a "global" section in the config file, use it
+        # to set Distribution options.
+
+        if 'global' in self.command_options:
+            for (opt, (src, val)) in self.command_options['global'].items():
+                alias = self.negative_opt.get(opt)
+                try:
+                    if alias:
+                        setattr(self, alias, not strtobool(val))
+                    elif opt in ('verbose', 'dry_run'):  # ugh!
+                        setattr(self, opt, strtobool(val))
+                    else:
+                        setattr(self, opt, val)
+                except ValueError as msg:
+                    raise DistutilsOptionError(msg)
+
     def parse_config_files(self, filenames=None, ignore_option_errors=False):
         """Parses configuration files from various levels
         and loads configuration.
 
         """
-        _Distribution.parse_config_files(self, filenames=filenames)
+        self._parse_config_files(filenames=filenames)
 
         parse_configuration(self, self.command_options,
                             ignore_option_errors=ignore_option_errors)

setuptools/tests/test_config.py

@@ -1,10 +1,15 @@
+# -*- coding: UTF-8 -*-
+from __future__ import unicode_literals
+
 import contextlib
 import pytest
 
 from distutils.errors import DistutilsOptionError, DistutilsFileError
 from mock import patch
 from setuptools.dist import Distribution, _Distribution
 from setuptools.config import ConfigHandler, read_configuration
+from setuptools.extern.six.moves.configparser import InterpolationMissingOptionError
+from setuptools.tests import is_ascii
 from . import py2_only, py3_only
 from .textwrap import DALS
 
@@ -23,7 +28,7 @@ def make_package_dir(name, base_dir, ns=False):
     return dir_package, init_file
 
 
-def fake_env(tmpdir, setup_cfg, setup_py=None, package_path='fake_package'):
+def fake_env(tmpdir, setup_cfg, setup_py=None, encoding='ascii', package_path='fake_package'):
 
     if setup_py is None:
         setup_py = (
@@ -33,7 +38,7 @@ def fake_env(tmpdir, setup_cfg, setup_py=None, package_path='fake_package'):
 
     tmpdir.join('setup.py').write(setup_py)
     config = tmpdir.join('setup.cfg')
-    config.write(setup_cfg)
+    config.write(setup_cfg.encode(encoding), mode='wb')
 
     package_dir, init_file = make_package_dir(package_path, tmpdir)
 
@@ -428,6 +433,72 @@ def test_deprecated_config_handlers(self, tmpdir):
                 assert metadata.description == 'Some description'
                 assert metadata.requires == ['some', 'requirement']
 
+    def test_interpolation(self, tmpdir):
+        fake_env(
+            tmpdir,
+            '[metadata]\n'
+            'description = %(message)s\n'
+        )
+        with pytest.raises(InterpolationMissingOptionError):
+            with get_dist(tmpdir):
+                pass
+
+    skip_if_not_ascii = pytest.mark.skipif(not is_ascii, reason='Test not supported with this locale')
+
+    @skip_if_not_ascii
+    def test_non_ascii_1(self, tmpdir):
+        fake_env(
+            tmpdir,
+            '[metadata]\n'
+            'description = éàïôñ\n',
+            encoding='utf-8'
+        )
+        with pytest.raises(UnicodeDecodeError):
+            with get_dist(tmpdir):
+                pass
+
+    def test_non_ascii_2(self, tmpdir):
+        fake_env(
+            tmpdir,
+            '# -*- coding: invalid\n'
+        )
+        with pytest.raises(LookupError):
+            with get_dist(tmpdir):
+                pass
+
+    def test_non_ascii_3(self, tmpdir):
+        fake_env(
+            tmpdir,
+            '\n'
+            '# -*- coding: invalid\n'
+        )
+        with get_dist(tmpdir):
+            pass
+
+    @skip_if_not_ascii
+    def test_non_ascii_4(self, tmpdir):
+        fake_env(
+            tmpdir,
+            '# -*- coding: utf-8\n'
+            '[metadata]\n'
+            'description = éàïôñ\n',
+            encoding='utf-8'
+        )
+        with get_dist(tmpdir) as dist:
+            assert dist.metadata.description == 'éàïôñ'
+
+    @skip_if_not_ascii
+    def test_non_ascii_5(self, tmpdir):
+        fake_env(
+            tmpdir,
+            '# vim: set fileencoding=iso-8859-15 :\n'
+            '[metadata]\n'
+            'description = éàïôñ\n',
+            encoding='iso-8859-15'
+        )
+        with get_dist(tmpdir) as dist:
+            assert dist.metadata.description == 'éàïôñ'
+
 
 class TestOptions: