Improve files metadata in lock files (#1460)

Co-authored-by: adisbladis <[email protected]>

poetry/installation/pip_installer.py

@@ -65,7 +65,7 @@ def install(self, package, update=False):
         if update:
             args.append("-U")
 
-        if package.hashes and not package.source_type:
+        if package.files and not package.source_type:
             # Format as a requirements.txt
             # We need to create a requirements.txt file
             # for each package in order to check hashes.
@@ -112,8 +112,9 @@ def run(self, *args, **kwargs):  # type: (...) -> str
     def requirement(self, package, formatted=False):
         if formatted and not package.source_type:
             req = "{}=={}".format(package.name, package.version)
-            for h in package.hashes:
+            for f in package.files:
                 hash_type = "sha256"
+                h = f["hash"]
                 if ":" in h:
                     hash_type, h = h.split(":")
 

poetry/packages/locker.py

@@ -6,6 +6,9 @@
 
 from hashlib import sha256
 from tomlkit import document
+from tomlkit import inline_table
+from tomlkit import item
+from tomlkit import table
 from typing import List
 
 from poetry.utils._compat import Path
@@ -84,7 +87,15 @@ def locked_repository(
             package.description = info.get("description", "")
             package.category = info["category"]
             package.optional = info["optional"]
-            package.hashes = lock_data["metadata"]["hashes"][info["name"]]
+            if "hashes" in lock_data["metadata"]:
+                # Old lock so we create dummy files from the hashes
+                package.files = [
+                    {"name": h, "hash": h}
+                    for h in lock_data["metadata"]["hashes"][info["name"]]
+                ]
+            else:
+                package.files = lock_data["metadata"]["files"][info["name"]]
+
             package.python_versions = info["python-versions"]
             extras = info.get("extras", {})
             if extras:
@@ -135,15 +146,24 @@ def locked_repository(
         return packages
 
     def set_lock_data(self, root, packages):  # type: (...) -> bool
-        hashes = {}
+        files = table()
         packages = self._lock_packages(packages)
         # Retrieving hashes
         for package in packages:
-            if package["name"] not in hashes:
-                hashes[package["name"]] = []
+            if package["name"] not in files:
+                files[package["name"]] = []
+
+            for f in package["files"]:
+                file_metadata = inline_table()
+                for k, v in sorted(f.items()):
+                    file_metadata[k] = v
+
+                files[package["name"]].append(file_metadata)
+
+            if files[package["name"]]:
+                files[package["name"]] = item(files[package["name"]]).multiline(True)
 
-            hashes[package["name"]] += package["hashes"]
-            del package["hashes"]
+            del package["files"]
 
         lock = document()
         lock["package"] = packages
@@ -157,7 +177,7 @@ def set_lock_data(self, root, packages):  # type: (...) -> bool
         lock["metadata"] = {
             "python-versions": root.python_versions,
             "content-hash": self._content_hash,
-            "hashes": hashes,
+            "files": files,
         }
 
         if not self.is_locked() or lock != self.lock_data:
@@ -247,7 +267,7 @@ def _dump_package(self, package):  # type: (poetry.packages.Package) -> dict
             "category": package.category,
             "optional": package.optional,
             "python-versions": package.python_versions,
-            "hashes": sorted(package.hashes),
+            "files": sorted(package.files, key=lambda x: x["file"]),
         }
         if not package.marker.is_any():
             data["marker"] = str(package.marker)

poetry/packages/package.py

@@ -66,7 +66,7 @@ def __init__(self, name, version, pretty_version=None):
         self.requires_extras = []
 
         self.category = "main"
-        self.hashes = []
+        self.files = []
         self.optional = False
 
         self.classifiers = []

poetry/puzzle/provider.py

@@ -226,7 +226,9 @@ def search_for_file(self, dependency):  # type: (FileDependency) -> List[Package
             )
 
         package.source_url = dependency.path.as_posix()
-        package.hashes = [dependency.hash()]
+        package.files = [
+            {"file": dependency.path.name, "hash": "sha256:" + dependency.hash()}
+        ]
 
         for extra in dependency.extras:
             if extra in package.extras:

poetry/repositories/legacy_repository.py

@@ -333,7 +333,7 @@ def package(
             package.description = release_info.get("summary", "")
 
             # Adding hashes information
-            package.hashes = release_info["digests"]
+            package.files = release_info["files"]
 
             # Activate extra dependencies
             for extra in extras:
@@ -358,7 +358,7 @@ def _get_release_info(self, name, version):  # type: (str, str) -> dict
             "summary": "",
             "requires_dist": [],
             "requires_python": None,
-            "digests": [],
+            "files": [],
             "_cache_version": str(self.CACHE_VERSION),
         }
 
@@ -370,7 +370,7 @@ def _get_release_info(self, name, version):  # type: (str, str) -> dict
                 )
             )
         urls = defaultdict(list)
-        hashes = []
+        files = []
         for link in links:
             if link.is_wheel:
                 urls["bdist_wheel"].append(link.url)
@@ -379,13 +379,12 @@ def _get_release_info(self, name, version):  # type: (str, str) -> dict
             ):
                 urls["sdist"].append(link.url)
 
-            hash = link.hash
-            if link.hash_name == "sha256":
-                hashes.append(hash)
-            elif hash:
-                hashes.append(link.hash_name + ":" + hash)
+            h = link.hash
+            if h:
+                h = link.hash_name + ":" + link.hash
+                files.append({"file": link.filename, "hash": h})
 
-        data["digests"] = hashes
+        data["files"] = files
 
         info = self._get_info_from_urls(urls)
 

poetry/repositories/pypi_repository.py

@@ -50,7 +50,7 @@
 
 class PyPiRepository(Repository):
 
-    CACHE_VERSION = parse_constraint("0.12.0")
+    CACHE_VERSION = parse_constraint("1.0.0b2")
 
     def __init__(self, url="https://pypi.org/", disable_cache=False, fallback=True):
         self._url = url
@@ -210,7 +210,7 @@ def package(
             package.platform = release_info["platform"]
 
         # Adding hashes information
-        package.hashes = release_info["digests"]
+        package.files = release_info["files"]
 
         # Activate extra dependencies
         for extra in extras:
@@ -311,7 +311,7 @@ def _get_release_info(self, name, version):  # type: (str, str) -> dict
             "platform": info["platform"],
             "requires_dist": info["requires_dist"],
             "requires_python": info["requires_python"],
-            "digests": [],
+            "files": [],
             "_cache_version": str(self.CACHE_VERSION),
         }
 
@@ -321,7 +321,12 @@ def _get_release_info(self, name, version):  # type: (str, str) -> dict
             version_info = []
 
         for file_info in version_info:
-            data["digests"].append(file_info["digests"]["sha256"])
+            data["files"].append(
+                {
+                    "file": file_info["filename"],
+                    "hash": "sha256:" + file_info["digests"]["sha256"],
+                }
+            )
 
         if self._fallback and data["requires_dist"] is None:
             self._log("No dependencies found, downloading archives", level="debug")

poetry/utils/exporter.py

@@ -110,9 +110,10 @@ def _export_requirements_txt(
             if package.source_type == "legacy" and package.source_url:
                 indexes.append(package.source_url)
 
-            if package.hashes and with_hashes:
+            if package.files and with_hashes:
                 hashes = []
-                for h in package.hashes:
+                for f in package.files:
+                    h = f["hash"]
                     algorithm = "sha256"
                     if ":" in h:
                         algorithm, h = h.split(":")
@@ -126,7 +127,7 @@ def _export_requirements_txt(
                     line += " \\\n"
                     for i, h in enumerate(hashes):
                         line += "    --hash={}{}".format(
-                            h, " \\\n" if i < len(package.hashes) - 1 else ""
+                            h, " \\\n" if i < len(hashes) - 1 else ""
                         )
 
             line += "\n"

pyproject.toml

@@ -34,7 +34,7 @@ cachecontrol = { version = "^0.12.4", extras = ["filecache"] }
 pkginfo = "^1.4"
 html5lib = "^1.0"
 shellingham = "^1.1"
-tomlkit = "^0.5.7"
+tomlkit = "^0.5.8"
 pexpect = "^4.7.0"
 
 # The typing module is not in the stdlib in Python 2.7 and 3.4

tests/installation/fixtures/extras-with-dependencies.test

@@ -40,7 +40,7 @@ foo = ["C"]
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 A = []
 B = []
 C = []

tests/installation/fixtures/extras.test

@@ -37,7 +37,7 @@ foo = ["D"]
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 A = []
 B = []
 C = []

tests/installation/fixtures/install-no-dev.test

@@ -26,7 +26,7 @@ python-versions = "*"
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 "A" = []
 "B" = []
 "C" = []

tests/installation/fixtures/no-dependencies.test

@@ -4,4 +4,4 @@ package = []
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]

tests/installation/fixtures/remove.test

@@ -10,5 +10,5 @@ python-versions = "*"
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 "A" = []

tests/installation/fixtures/update-with-lock.test

@@ -10,5 +10,5 @@ python-versions = "*"
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 "A" = []

tests/installation/fixtures/update-with-locked-extras.test

@@ -42,7 +42,7 @@ python-versions = "*"
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 "A" = []
 "B" = []
 "C" = []

tests/installation/fixtures/with-category-change.test

@@ -21,6 +21,6 @@ A = "^1.0"
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 "A" = []
 "B" = []

tests/installation/fixtures/with-conditional-dependency.test

@@ -24,5 +24,5 @@ python = ">=3.6,<4.0"
 python-versions = "~2.7 || ^3.4"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 A = []

tests/installation/fixtures/with-dependencies-extras.test

@@ -32,7 +32,7 @@ python-versions = "*"
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 "A" = []
 "B" = []
 "C" = []

tests/installation/fixtures/with-dependencies.test

@@ -18,6 +18,6 @@ python-versions = "*"
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 "A" = []
 "B" = []

tests/installation/fixtures/with-directory-dependency-poetry-transitive.test

@@ -35,6 +35,6 @@ url = "tests/fixtures/directory/project_with_transitive_directory_dependencies"
 content-hash = "123456789"
 python-versions = "*"
 
-[metadata.hashes]
+[metadata.files]
 project-with-extras = []
 project-with-transitive-directory-dependencies = []

tests/installation/fixtures/with-directory-dependency-poetry.test

@@ -30,6 +30,6 @@ url = "tests/fixtures/project_with_extras"
 content-hash = "123456789"
 python-versions = "*"
 
-[metadata.hashes]
+[metadata.files]
 project-with-extras = []
 pendulum = []

tests/installation/fixtures/with-directory-dependency-setuptools.test

@@ -35,7 +35,7 @@ python-versions = "*"
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 cachy = []
 my-package = []
 pendulum = []

tests/installation/fixtures/with-duplicate-dependencies-update.test

@@ -32,7 +32,7 @@ python-versions = "*"
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 A = []
 B = []
 C = []

tests/installation/fixtures/with-duplicate-dependencies.test

@@ -58,7 +58,7 @@ python-versions = "*"
 python-versions = "*"
 content-hash = "123456789"
 
-[metadata.hashes]
+[metadata.files]
 A = []
 B = []
 C = []