Implement PEP 639, Metadata 2.4 (pypi#16949)

* run from my fork of packaging temporarily

* add License-File and License-Expression fields to Release

* store license-expression and license-files when included

* enforce that license-files exist in distributions

* enforce License/License-Expression mutual exclusion

* clarify Release.license_files contents in comment

* refactor license-file check for tar.gz sdists

* harmonize with the definition of "root license directory"/"license directory" from PEP 639

> The directory under which license files are stored in a project source tree, distribution archive or installed project. Also, the root directory that their paths recorded in the License-File Core Metadata field are relative to. Defined to be the project root directory for a project source tree or source distribution; and a subdirectory named licenses of the directory containing the built metadata— i.e., the .dist-info/licenses directory— for a Built Distribution or installed project.

* fix comment

* sync License-File(s) and License-Expression metadata to Big Query

* add license_files and license_expression to JSON API

* display License-Expression in project details if provided

* move to first commit of pypa/packaging including spdx bits!

* update to latest pypa/packaging commit

* zip file license-files check: add note to remove when 625 is implemented

* add 2 minute lock and statement timeouts to migrations

* remove redundant metadata version gate for license-files check

requirements/main.in

@@ -33,7 +33,7 @@ msgpack
 natsort
 opensearch-py
 orjson
-packaging>=23.2
+packaging>=24.2
 packaging_legacy
 paginate>=0.5.2
 paginate_sqlalchemy

tests/unit/forklift/test_legacy.py

@@ -73,13 +73,28 @@ def _get_tar_testdata(compression_type=""):
     temp_f = io.BytesIO()
     with tarfile.open(fileobj=temp_f, mode=f"w:{compression_type}") as tar:
         tar.add("/dev/null", arcname="fake_package/PKG-INFO")
+        tar.add("/dev/null", arcname="LICENSE.MIT")
+        tar.add("/dev/null", arcname="LICENSE.APACHE")
+    return temp_f.getvalue()
+
+
+def _get_zip_testdata():
+    temp_f = io.BytesIO()
+    with zipfile.ZipFile(file=temp_f, mode="w") as zfp:
+        zfp.writestr("fake_package/PKG-INFO", "Fake PKG-INFO")
+        zfp.writestr("LICENSE.MIT", "Fake License")
+        zfp.writestr("LICENSE.APACHE", "Fake License")
     return temp_f.getvalue()
 
 
 def _get_whl_testdata(name="fake_package", version="1.0"):
     temp_f = io.BytesIO()
     with zipfile.ZipFile(file=temp_f, mode="w") as zfp:
         zfp.writestr(f"{name}-{version}.dist-info/METADATA", "Fake metadata")
+        zfp.writestr(f"{name}-{version}.dist-info/licenses/LICENSE.MIT", "Fake License")
+        zfp.writestr(
+            f"{name}-{version}.dist-info/licenses/LICENSE.APACHE", "Fake License"
+        )
     return temp_f.getvalue()
 
 
@@ -98,6 +113,12 @@ def _storage_hash(data):
 _TAR_BZ2_PKG_STORAGE_HASH = _storage_hash(_TAR_BZ2_PKG_TESTDATA)
 
 
+_ZIP_PKG_TESTDATA = _get_zip_testdata()
+_ZIP_PKG_MD5 = hashlib.md5(_ZIP_PKG_TESTDATA).hexdigest()
+_ZIP_PKG_SHA256 = hashlib.sha256(_ZIP_PKG_TESTDATA).hexdigest()
+_ZIP_PKG_STORAGE_HASH = _storage_hash(_ZIP_PKG_TESTDATA)
+
+
 class TestExcWithMessage:
     def test_exc_with_message(self):
         exc = legacy._exc_with_message(HTTPBadRequest, "My Test Message.")
@@ -2877,7 +2898,7 @@ def test_upload_succeeds_pep625_normalized_filename(
         RoleFactory.create(user=user, project=project)
 
         filename = f"{filename_prefix}-{version}.tar.gz"
-        filebody = _get_whl_testdata(name=project_name, version=version)
+        filebody = _TAR_GZ_PKG_TESTDATA
 
         @pretend.call_recorder
         def storage_service_store(path, file_path, *, meta):
@@ -4708,6 +4729,261 @@ def test_upload_with_token_api_warns_if_trusted_publisher_configured(
             if not warning_already_sent:
                 assert not warning_exists
 
+    @pytest.mark.parametrize(
+        ("version", "expected_version", "filetype", "mimetype"),
+        [
+            ("1.0", "1.0", "sdist", "application/tar"),
+            ("v1.0", "1.0", "sdist", "application/tar"),
+            ("1.0", "1.0", "sdist", "application/zip"),
+            ("v1.0", "1.0", "sdist", "application/zip"),
+            ("1.0", "1.0", "bdist_wheel", "application/zip"),
+            ("v1.0", "1.0", "bdist_wheel", "application/zip"),
+        ],
+    )
+    def test_upload_succeeds_creates_release_metadata_2_4(
+        self,
+        pyramid_config,
+        db_request,
+        metrics,
+        monkeypatch,
+        version,
+        expected_version,
+        filetype,
+        mimetype,
+    ):
+        user = UserFactory.create()
+        EmailFactory.create(user=user)
+        project = ProjectFactory.create()
+        RoleFactory.create(user=user, project=project)
+
+        if filetype == "sdist":
+            if mimetype == "application/tar":
+                filename = "{}-{}.tar.gz".format(project.name, "1.0")
+                digest = _TAR_GZ_PKG_MD5
+                data = _TAR_GZ_PKG_TESTDATA
+            elif mimetype == "application/zip":
+                filename = "{}-{}.zip".format(project.name, "1.0")
+                digest = _ZIP_PKG_MD5
+                data = _ZIP_PKG_TESTDATA
+        elif filetype == "bdist_wheel":
+            filename = "{}-{}-py3-none-any.whl".format(project.name, "1.0")
+            data = _get_whl_testdata(name=project.name, version="1.0")
+            digest = hashlib.md5(data).hexdigest()
+            monkeypatch.setattr(legacy, "_is_valid_dist_file", lambda *a, **kw: True)
+
+        pyramid_config.testing_securitypolicy(identity=user)
+        db_request.user = user
+        db_request.user_agent = "warehouse-tests/6.6.6"
+        db_request.POST = MultiDict(
+            {
+                "metadata_version": "2.4",
+                "name": project.name,
+                "version": version,
+                "summary": "This is my summary!",
+                "filetype": filetype,
+                "md5_digest": digest,
+                "content": pretend.stub(
+                    filename=filename,
+                    file=io.BytesIO(data),
+                    type=mimetype,
+                ),
+            }
+        )
+        db_request.POST.extend(
+            [
+                ("license_expression", "MIT OR Apache-2.0"),
+                ("license_files", "LICENSE.APACHE"),
+                ("license_files", "LICENSE.MIT"),
+            ]
+        )
+        if filetype == "bdist_wheel":
+            db_request.POST.extend([("pyversion", "py3")])
+
+        storage_service = pretend.stub(store=lambda path, filepath, meta: None)
+        db_request.find_service = lambda svc, name=None, context=None: {
+            IFileStorage: storage_service,
+            IMetricsService: metrics,
+        }.get(svc)
+
+        resp = legacy.file_upload(db_request)
+
+        assert resp.status_code == 200
+
+        # Ensure that a Release object has been created.
+        release = (
+            db_request.db.query(Release)
+            .filter(
+                (Release.project == project) & (Release.version == expected_version)
+            )
+            .one()
+        )
+        assert release.summary == "This is my summary!"
+        assert release.version == expected_version
+        assert release.canonical_version == "1"
+        assert release.uploaded_via == "warehouse-tests/6.6.6"
+        assert release.license_expression == "MIT OR Apache-2.0"
+        assert set(release.license_files) == {
+            "LICENSE.APACHE",
+            "LICENSE.MIT",
+        }
+
+        # Ensure that a File object has been created.
+        db_request.db.query(File).filter(
+            (File.release == release) & (File.filename == filename)
+        ).one()
+
+        # Ensure that a Filename object has been created.
+        db_request.db.query(Filename).filter(Filename.filename == filename).one()
+
+    @pytest.mark.parametrize(
+        ("version", "expected_version", "filetype", "mimetype"),
+        [
+            ("1.0", "1.0", "sdist", "application/tar"),
+            ("v1.0", "1.0", "sdist", "application/tar"),
+            ("1.0", "1.0", "sdist", "application/zip"),
+            ("v1.0", "1.0", "sdist", "application/zip"),
+            ("1.0", "1.0", "bdist_wheel", "application/zip"),
+            ("v1.0", "1.0", "bdist_wheel", "application/zip"),
+        ],
+    )
+    def test_upload_fails_missing_license_file_metadata_2_4(
+        self,
+        pyramid_config,
+        db_request,
+        metrics,
+        monkeypatch,
+        version,
+        expected_version,
+        filetype,
+        mimetype,
+    ):
+        user = UserFactory.create()
+        EmailFactory.create(user=user)
+        project = ProjectFactory.create()
+        RoleFactory.create(user=user, project=project)
+
+        if filetype == "sdist":
+            if mimetype == "application/tar":
+                filename = "{}-{}.tar.gz".format(project.name, "1.0")
+                digest = _TAR_GZ_PKG_MD5
+                data = _TAR_GZ_PKG_TESTDATA
+            elif mimetype == "application/zip":
+                filename = "{}-{}.zip".format(project.name, "1.0")
+                digest = _ZIP_PKG_MD5
+                data = _ZIP_PKG_TESTDATA
+            license_filename = "LICENSE"
+        elif filetype == "bdist_wheel":
+            filename = "{}-{}-py3-none-any.whl".format(project.name, "1.0")
+            data = _get_whl_testdata(name=project.name, version="1.0")
+            digest = hashlib.md5(data).hexdigest()
+            monkeypatch.setattr(legacy, "_is_valid_dist_file", lambda *a, **kw: True)
+            license_filename = f"{project.name}-1.0.dist-info/licenses/LICENSE"
+
+        pyramid_config.testing_securitypolicy(identity=user)
+        db_request.user = user
+        db_request.user_agent = "warehouse-tests/6.6.6"
+        db_request.POST = MultiDict(
+            {
+                "metadata_version": "2.4",
+                "name": project.name,
+                "version": version,
+                "summary": "This is my summary!",
+                "filetype": filetype,
+                "md5_digest": digest,
+                "content": pretend.stub(
+                    filename=filename,
+                    file=io.BytesIO(data),
+                    type=mimetype,
+                ),
+            }
+        )
+        db_request.POST.extend(
+            [
+                ("license_expression", "MIT OR Apache-2.0"),
+                ("license_files", "LICENSE"),  # Does not exist in test data
+                ("license_files", "LICENSE.MIT"),
+            ]
+        )
+        if filetype == "bdist_wheel":
+            db_request.POST.extend([("pyversion", "py3")])
+
+        storage_service = pretend.stub(store=lambda path, filepath, meta: None)
+        db_request.find_service = lambda svc, name=None, context=None: {
+            IFileStorage: storage_service,
+            IMetricsService: metrics,
+        }.get(svc)
+
+        with pytest.raises(HTTPBadRequest) as excinfo:
+            legacy.file_upload(db_request)
+
+        resp = excinfo.value
+
+        assert resp.status_code == 400
+        assert resp.status == (
+            f"400 License-File {license_filename} does not exist "
+            f"in distribution file {filename}"
+        )
+
+    def test_upload_fails_when_license_and_license_expression_are_present(
+        self,
+        pyramid_config,
+        db_request,
+        metrics,
+    ):
+        user = UserFactory.create()
+        EmailFactory.create(user=user)
+        project = ProjectFactory.create()
+        RoleFactory.create(user=user, project=project)
+
+        filename = "{}-{}-py3-none-any.whl".format(project.name, "1.0")
+        data = _get_whl_testdata(name=project.name, version="1.0")
+        digest = hashlib.md5(data).hexdigest()
+
+        pyramid_config.testing_securitypolicy(identity=user)
+        db_request.user = user
+        db_request.user_agent = "warehouse-tests/6.6.6"
+        db_request.POST = MultiDict(
+            {
+                "metadata_version": "2.4",
+                "name": project.name,
+                "version": "1.0",
+                "summary": "This is my summary!",
+                "filetype": "bdist_wheel",
+                "md5_digest": digest,
+                "content": pretend.stub(
+                    filename=filename,
+                    file=io.BytesIO(data),
+                    type="application/zip",
+                ),
+            }
+        )
+        db_request.POST.extend(
+            [
+                ("license_expression", "MIT OR Apache-2.0"),
+                ("license", "MIT LICENSE or Apache-2.0 License"),
+            ]
+        )
+        db_request.POST.extend([("pyversion", "py3")])
+
+        storage_service = pretend.stub(store=lambda path, filepath, meta: None)
+        db_request.find_service = lambda svc, name=None, context=None: {
+            IFileStorage: storage_service,
+            IMetricsService: metrics,
+        }.get(svc)
+
+        with pytest.raises(HTTPBadRequest) as excinfo:
+            legacy.file_upload(db_request)
+
+        resp = excinfo.value
+
+        assert resp.status_code == 400
+        assert resp.status == (
+            "400 License is deprecated when License-Expression is present. "
+            "Only License-Expression should be present. "
+            "See https://packaging.python.org/specifications/core-metadata "
+            "for more information."
+        )
+
 
 def test_submit(pyramid_request):
     resp = legacy.submit(pyramid_request)

tests/unit/legacy/api/test_json.py

@@ -248,6 +248,8 @@ def test_renders(self, pyramid_config, db_request, db_session):
                 "home_page": None,
                 "keywords": None,
                 "license": None,
+                "license_expression": None,
+                "license_files": None,
                 "maintainer": None,
                 "maintainer_email": None,
                 "name": project.name,
@@ -572,6 +574,8 @@ def test_detail_renders(self, pyramid_config, db_request, db_session):
                 "home_page": None,
                 "keywords": None,
                 "license": None,
+                "license_expression": None,
+                "license_files": None,
                 "maintainer": None,
                 "maintainer_email": None,
                 "name": project.name,
@@ -664,6 +668,8 @@ def test_minimal_renders(self, pyramid_config, db_request):
                 "home_page": None,
                 "keywords": None,
                 "license": None,
+                "license_expression": None,
+                "license_files": None,
                 "maintainer": None,
                 "maintainer_email": None,
                 "name": project.name,