Fix conan cache restore when restoring the same cache multiple times

conan/api/subapi/cache.py

@@ -162,32 +162,49 @@ def restore(self, path):
             the_tar.extractall(path=self.conan_api.cache_folder)
             the_tar.close()
 
+        # After unzipping the files, we need to update the DB that references these files
         out = ConanOutput()
         package_list = PackagesList.deserialize(json.loads(pkglist))
         cache = ClientCache(self.conan_api.cache_folder, self.conan_api.config.global_conf)
         for ref, ref_bundle in package_list.refs().items():
             ref.timestamp = revision_timestamp_now()
             ref_bundle["timestamp"] = ref.timestamp
-            recipe_layout = cache.get_or_create_ref_layout(ref)
+            recipe_layout = cache.get_or_create_ref_layout(ref)  # DB folder entry
             recipe_folder = ref_bundle["recipe_folder"]
             rel_path = os.path.relpath(recipe_layout.base_folder, cache.cache_folder)
             rel_path = rel_path.replace("\\", "/")
+            # In the case of recipes, they are always "in place", so just checking it
             assert rel_path == recipe_folder, f"{rel_path}!={recipe_folder}"
             out.info(f"Restore: {ref} in {recipe_folder}")
             for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items():
                 pref.timestamp = revision_timestamp_now()
                 pref_bundle["timestamp"] = pref.timestamp
-                pkg_layout = cache.get_or_create_pkg_layout(pref)
-                pkg_folder = pref_bundle["package_folder"]
-                out.info(f"Restore: {pref} in {pkg_folder}")
-                # We need to put the package in the final location in the cache
-                shutil.move(os.path.join(cache.cache_folder, pkg_folder), pkg_layout.package())
-                metadata_folder = pref_bundle.get("metadata_folder")
-                if metadata_folder:
-                    out.info(f"Restore: {pref} metadata in {metadata_folder}")
-                    # We need to put the package in the final location in the cache
-                    shutil.move(os.path.join(cache.cache_folder, metadata_folder),
-                                pkg_layout.metadata())
+                pkg_layout = cache.get_or_create_pkg_layout(pref)  # DB Folder entry
+                unzipped_pkg_folder = pref_bundle["package_folder"]
+                out.info(f"Restore: {pref} in {unzipped_pkg_folder}")
+                # If the DB folder entry is different to the disk unzipped one, we need to move it
+                # This happens for built (not downloaded) packages in the source "conan cache save"
+                db_pkg_folder = os.path.relpath(pkg_layout.package(), cache.cache_folder)
+                db_pkg_folder = db_pkg_folder.replace("\\", "/")
+                if db_pkg_folder != unzipped_pkg_folder:
+                    # If a previous package exists, like a previous restore, then remove it
+                    if os.path.exists(pkg_layout.package()):
+                        shutil.rmtree(pkg_layout.package())
+                    shutil.move(os.path.join(cache.cache_folder, unzipped_pkg_folder),
+                                pkg_layout.package())
+                    pref_bundle["package_folder"] = db_pkg_folder
+                unzipped_metadata_folder = pref_bundle.get("metadata_folder")
+                if unzipped_metadata_folder:
+                    out.info(f"Restore: {pref} metadata in {unzipped_metadata_folder}")
+                    db_metadata_folder = os.path.relpath(pkg_layout.metadata(), cache.cache_folder)
+                    db_metadata_folder = db_metadata_folder.replace("\\", "/")
+                    if db_metadata_folder != unzipped_metadata_folder:
+                        # We need to put the package in the final location in the cache
+                        if os.path.exists(pkg_layout.metadata()):
+                            shutil.rmtree(pkg_layout.metadata())
+                        shutil.move(os.path.join(cache.cache_folder, unzipped_metadata_folder),
+                                    pkg_layout.metadata())
+                        pref_bundle["metadata_folder"] = db_metadata_folder
 
         return package_list
 

conans/test/integration/command_v2/test_cache_save_restore.py

@@ -30,6 +30,36 @@ def test_cache_save_restore():
     assert "\\" not in package_list
 
 
+def test_cache_save_restore_with_package_file():
+    """If we have some sources in the root (like the CMakeLists.txt)
+    we don't declare folders.source"""
+    conan_file = GenConanfile() \
+        .with_settings("os") \
+        .with_package_file("bin/file.txt", "content!!")
+
+    client = TestClient()
+    client.save({"conanfile.py": conan_file})
+    client.run("create . --name=pkg --version=1.0 -s os=Linux")
+    client.run("cache save pkg/*:* ")
+    cache_path = os.path.join(client.current_folder, "conan_cache_save.tgz")
+    assert os.path.exists(cache_path)
+
+    c2 = TestClient()
+    shutil.copy2(cache_path, c2.current_folder)
+    c2.run("cache restore conan_cache_save.tgz")
+    c2.run("list *:*#*")
+    assert "pkg/1.0" in c2.out
+    tree = _get_directory_tree(c2.base_folder)
+
+    # Restore again, expect the tree to be unchanged
+    c2.run("cache restore conan_cache_save.tgz")
+    c2.run("list *:*#*")
+    assert "pkg/1.0" in c2.out
+    tree2 = _get_directory_tree(c2.base_folder)
+
+    assert tree2 == tree
+
+
 def test_cache_save_downloaded_restore():
     """ what happens if we save packages downloaded from server, not
     created
@@ -49,6 +79,18 @@ def test_cache_save_downloaded_restore():
     _validate_restore(cache_path)
 
 
+def _get_directory_tree(base_folder):
+    tree = []
+    for d, _, fs in os.walk(base_folder):
+        rel_d = os.path.relpath(d, base_folder) if d != base_folder else ""
+        if rel_d:
+            tree.append(rel_d)
+        for f in fs:
+            tree.append(os.path.join(rel_d, f))
+    tree.sort()
+    return tree
+
+
 def _validate_restore(cache_path):
     c2 = TestClient()
     # Create a package in the cache to check put doesn't interact badly
@@ -61,6 +103,7 @@ def _validate_restore(cache_path):
     assert "pkg/1.0" in c2.out
     assert "pkg/1.1" in c2.out
     assert "other/2.0" not in c2.out
+    tree = _get_directory_tree(c2.base_folder)
 
     # Restore again, just in case
     c2.run("cache restore conan_cache_save.tgz")
@@ -69,6 +112,8 @@ def _validate_restore(cache_path):
     assert "pkg/1.0" in c2.out
     assert "pkg/1.1" in c2.out
     assert "other/2.0" not in c2.out
+    tree2 = _get_directory_tree(c2.base_folder)
+    assert tree2 == tree
 
 
 def test_cache_save_restore_metadata():