Add License-File field to package metadata

pypa#2645

changelog.d/2645.breaking.rst

@@ -0,0 +1,3 @@
+License files excluded via the ``MANIFEST.in`` but matched by either
+the ``license_file`` (deprecated) or ``license_files`` options,
+will be nevertheless included in the source distribution. - by :user:`cdce8p`

changelog.d/2645.change.rst

@@ -0,0 +1,4 @@
+Added ``License-File`` (multiple) to the output package metadata.
+The field will contain the path of a license file, matched by the
+``license_file`` (deprecated) and ``license_files`` options,
+relative to ``.dist-info``. - by :user:`cdce8p`

setuptools/command/egg_info.py

@@ -541,6 +541,7 @@ def run(self):
         self.add_defaults()
         if os.path.exists(self.template):
             self.read_template()
+        self.add_license_files()
         self.prune_file_list()
         self.filelist.sort()
         self.filelist.remove_duplicates()
@@ -575,7 +576,6 @@ def _should_suppress_warning(msg):
 
     def add_defaults(self):
         sdist.add_defaults(self)
-        self.check_license()
         self.filelist.append(self.template)
         self.filelist.append(self.manifest)
         rcfiles = list(walk_revctrl())
@@ -592,6 +592,13 @@ def add_defaults(self):
         ei_cmd = self.get_finalized_command('egg_info')
         self.filelist.graft(ei_cmd.egg_info)
 
+    def add_license_files(self):
+        license_files = self.distribution.metadata.license_files or []
+        for lf in license_files:
+            log.info("adding license file '%s'", lf)
+            pass
+        self.filelist.extend(license_files)
+
     def prune_file_list(self):
         build = self.get_finalized_command('build')
         base_dir = self.distribution.get_fullname()

setuptools/command/sdist.py

@@ -4,9 +4,6 @@
 import sys
 import io
 import contextlib
-from glob import iglob
-
-from setuptools.extern import ordered_set
 
 from .py36compat import sdist_add_defaults
 
@@ -190,46 +187,3 @@ def read_manifest(self):
                 continue
             self.filelist.append(line)
         manifest.close()
-
-    def check_license(self):
-        """Checks if license_file' or 'license_files' is configured and adds any
-        valid paths to 'self.filelist'.
-        """
-        opts = self.distribution.get_option_dict('metadata')
-
-        files = ordered_set.OrderedSet()
-        try:
-            license_files = self.distribution.metadata.license_files
-        except TypeError:
-            log.warn("warning: 'license_files' option is malformed")
-            license_files = ordered_set.OrderedSet()
-        patterns = license_files if isinstance(license_files, ordered_set.OrderedSet) \
-            else ordered_set.OrderedSet(license_files)
-
-        if 'license_file' in opts:
-            log.warn(
-                "warning: the 'license_file' option is deprecated, "
-                "use 'license_files' instead")
-            patterns.append(opts['license_file'][1])
-
-        if 'license_file' not in opts and 'license_files' not in opts:
-            # Default patterns match the ones wheel uses
-            # See https://wheel.readthedocs.io/en/stable/user_guide.html
-            # -> 'Including license files in the generated wheel file'
-            patterns = ('LICEN[CS]E*', 'COPYING*', 'NOTICE*', 'AUTHORS*')
-
-        for pattern in patterns:
-            for path in iglob(pattern):
-                if path.endswith('~'):
-                    log.debug(
-                        "ignoring license file '%s' as it looks like a backup",
-                        path)
-                    continue
-
-                if path not in files and os.path.isfile(path):
-                    log.info(
-                        "adding license file '%s' (matched pattern '%s')",
-                        path, pattern)
-                    files.add(path)
-
-        self.filelist.extend(sorted(files))

setuptools/config.py

@@ -520,6 +520,11 @@ def parsers(self):
             'obsoletes': parse_list,
             'classifiers': self._get_parser_compound(parse_file, parse_list),
             'license': exclude_files_parser('license'),
+            'license_file': self._deprecated_config_handler(
+                exclude_files_parser('license_file'),
+                "The license_file parameter is deprecated, "
+                "use license_files instead.",
+                DeprecationWarning),
             'license_files': parse_list,
             'description': parse_file,
             'long_description': parse_file,

setuptools/dist.py

@@ -14,6 +14,7 @@
 from distutils.util import strtobool
 from distutils.debug import DEBUG
 from distutils.fancy_getopt import translate_longopt
+from glob import iglob
 import itertools
 
 from collections import defaultdict
@@ -117,6 +118,8 @@ def _read_list(name):
         self.provides = None
         self.obsoletes = None
 
+    self.license_files = _read_list('license-file')
+
 
 def single_line(val):
     # quick and dirty validation for description pypa/setuptools#1390
@@ -199,6 +202,7 @@ def write_field(key, value):
         for extra in self.provides_extras:
             write_field('Provides-Extra', extra)
 
+    self._write_list(file, 'License-File', self.license_files or [])
 
 sequence = tuple, list
 
@@ -398,7 +402,8 @@ class Distribution(_Distribution):
         'long_description_content_type': None,
         'project_urls': dict,
         'provides_extras': ordered_set.OrderedSet,
-        'license_files': ordered_set.OrderedSet,
+        'license_file': lambda: None,
+        'license_files': lambda: None,
     }
 
     _patched_dist = None
@@ -557,6 +562,34 @@ def _clean_req(self, req):
         req.marker = None
         return req
 
+    def _finalize_license_files(self):
+        """Compute names of all license files which should be included."""
+        license_files: Optional[List[str]] = self.metadata.license_files
+        patterns: List[str] = license_files if license_files else []
+
+        license_file: Optional[str] = self.metadata.license_file
+        if license_file and license_file not in patterns:
+            patterns.append(license_file)
+
+        if license_files is None and license_file is None:
+            # Default patterns match the ones wheel uses
+            # See https://wheel.readthedocs.io/en/stable/user_guide.html
+            # -> 'Including license files in the generated wheel file'
+            patterns = ('LICEN[CS]E*', 'COPYING*', 'NOTICE*', 'AUTHORS*')
+
+        self.metadata.license_files = list(
+            unique_everseen(self._expand_patterns(patterns)))
+
+    @staticmethod
+    def _expand_patterns(patterns):
+        return (
+            path
+            for pattern in patterns
+            for path in iglob(pattern)
+            if not path.endswith('~')
+            and os.path.isfile(path)
+        )
+
     # FIXME: 'Distribution._parse_config_files' is too complex (14)
     def _parse_config_files(self, filenames=None):  # noqa: C901
         """
@@ -680,6 +713,7 @@ def parse_config_files(self, filenames=None, ignore_option_errors=False):
         parse_configuration(self, self.command_options,
                             ignore_option_errors=ignore_option_errors)
         self._finalize_requires()
+        self._finalize_license_files()
 
     def fetch_build_eggs(self, requires):
         """Resolve pre-setup requirements"""
@@ -1020,3 +1054,21 @@ def handle_display_options(self, option_order):
 class DistDeprecationWarning(SetuptoolsDeprecationWarning):
     """Class for warning about deprecations in dist in
     setuptools. Not ignored by default, unlike DeprecationWarning."""
+
+
+def unique_everseen(iterable, key=None):
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen = set()
+    seen_add = seen.add
+    if key is None:
+        for element in itertools.filterfalse(seen.__contains__, iterable):
+            seen_add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen_add(k)
+                yield element

setuptools/tests/test_egg_info.py

@@ -541,7 +541,7 @@ def test_doesnt_provides_extra(self, tmpdir_cwd, env):
                               """),
             'MANIFEST.in': "exclude LICENSE",
             'LICENSE': "Test license"
-        }, False),  # license file is manually excluded
+        }, True),  # manifest is overwritten by license_file
         pytest.param({
             'setup.cfg': DALS("""
                               [metadata]
@@ -637,7 +637,7 @@ def test_setup_cfg_license_file(
                               """),
             'MANIFEST.in': "exclude LICENSE",
             'LICENSE': "Test license"
-        }, [], ['LICENSE']),  # license file is manually excluded
+        }, ['LICENSE'], []),  # manifest is overwritten by license_files
         ({
             'setup.cfg': DALS("""
                               [metadata]
@@ -648,7 +648,8 @@ def test_setup_cfg_license_file(
             'MANIFEST.in': "exclude LICENSE-XYZ",
             'LICENSE-ABC': "ABC license",
             'LICENSE-XYZ': "XYZ license"
-        }, ['LICENSE-ABC'], ['LICENSE-XYZ']),  # subset is manually excluded
+            # manifest is overwritten by license_files
+        }, ['LICENSE-ABC', 'LICENSE-XYZ'], []),
         pytest.param({
             'setup.cfg': "",
             'LICENSE-ABC': "ABC license",
@@ -678,6 +679,17 @@ def test_setup_cfg_license_file(
             'NOTICE-XYZ': "XYZ notice",
             }, ['LICENSE-ABC'], ['NOTICE-XYZ'],
             id="no_default_glob_patterns"),
+        pytest.param({
+            'setup.cfg': DALS("""
+                              [metadata]
+                              license_files =
+                                  LICENSE-ABC
+                                  LICENSE*
+                              """),
+            'LICENSE-ABC': "ABC license",
+            }, ['LICENSE-ABC'], [],
+            id="files_only_added_once",
+        ),
     ])
     def test_setup_cfg_license_files(
             self, tmpdir_cwd, env, files, incl_licenses, excl_licenses):
@@ -781,8 +793,8 @@ def test_setup_cfg_license_files(
             'LICENSE-ABC': "ABC license",
             'LICENSE-PQR': "PQR license",
             'LICENSE-XYZ': "XYZ license"
-            # manually excluded
-        }, ['LICENSE-XYZ'], ['LICENSE-ABC', 'LICENSE-PQR']),
+            # manifest is overwritten
+        }, ['LICENSE-ABC', 'LICENSE-PQR', 'LICENSE-XYZ'], []),
         pytest.param({
             'setup.cfg': DALS("""
                               [metadata]
@@ -825,6 +837,38 @@ def test_setup_cfg_license_file_license_files(
         for lf in excl_licenses:
             assert sources_lines.count(lf) == 0
 
+    def test_license_file_attr_pkg_info(self, tmpdir_cwd, env):
+        """All matched license files should have a corresponding License-File."""
+        self._create_project()
+        build_files({
+            "setup.cfg": DALS("""
+                              [metadata]
+                              license_files =
+                                  NOTICE*
+                                  LICENSE*
+                              """),
+            "LICENSE-ABC": "ABC license",
+            "LICENSE-XYZ": "XYZ license",
+            "NOTICE": "included",
+            "IGNORE": "not include",
+        })
+
+        environment.run_setup_py(
+            cmd=['egg_info'],
+            pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)])
+        )
+        egg_info_dir = os.path.join('.', 'foo.egg-info')
+        with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file:
+            pkg_info_lines = pkginfo_file.read().split('\n')
+        license_file_lines = [
+            line for line in pkg_info_lines if line.startswith('License-File:')]
+
+        # Only 'NOTICE', LICENSE-ABC', and 'LICENSE-XYZ' should have been matched
+        # Also assert that order from license_files is keeped
+        assert "License-File: NOTICE" == license_file_lines[0]
+        assert "License-File: LICENSE-ABC" in license_file_lines[1:]
+        assert "License-File: LICENSE-XYZ" in license_file_lines[1:]
+
     def test_long_description_content_type(self, tmpdir_cwd, env):
         # Test that specifying a `long_description_content_type` keyword arg to
         # the `setup` function results in writing a `Description-Content-Type`

setuptools/tests/test_manifest.py

@@ -55,7 +55,6 @@ def touch(filename):
 default_files = frozenset(map(make_local_path, [
     'README.rst',
     'MANIFEST.in',
-    'LICENSE',
     'setup.py',
     'app.egg-info/PKG-INFO',
     'app.egg-info/SOURCES.txt',