From 4d7c955dabacf8211225d6cb54ffa4830d306c93 Mon Sep 17 00:00:00 2001 From: Noah Miller Date: Wed, 27 Mar 2024 00:29:37 +1300 Subject: [PATCH 1/2] Fix conan cache restore when restoring the same cache multiple times Fixes: #15949 --- conan/api/subapi/cache.py | 18 +++++--- .../command_v2/test_cache_save_restore.py | 45 +++++++++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 36068942d28..fa9df555f0f 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -180,14 +180,22 @@ def restore(self, path): pkg_layout = cache.get_or_create_pkg_layout(pref) pkg_folder = pref_bundle["package_folder"] out.info(f"Restore: {pref} in {pkg_folder}") - # We need to put the package in the final location in the cache - shutil.move(os.path.join(cache.cache_folder, pkg_folder), pkg_layout.package()) + if pkg_layout.package() != os.path.join(cache.cache_folder, pkg_folder): + # We need to put the package in the final location in the cache + if os.path.exists(pkg_layout.package()): + shutil.rmtree(pkg_layout.package()) + shutil.move(os.path.join(cache.cache_folder, pkg_folder), pkg_layout.package()) + pref_bundle["package_folder"] = os.path.relpath(pkg_layout.package(), cache.cache_folder) metadata_folder = pref_bundle.get("metadata_folder") if metadata_folder: out.info(f"Restore: {pref} metadata in {metadata_folder}") - # We need to put the package in the final location in the cache - shutil.move(os.path.join(cache.cache_folder, metadata_folder), - pkg_layout.metadata()) + if pkg_layout.metadata() != os.path.join(cache.cache_folder, metadata_folder): + # We need to put the package in the final location in the cache + if os.path.exists(pkg_layout.metadata()): + shutil.rmtree(pkg_layout.metadata()) + shutil.move(os.path.join(cache.cache_folder, metadata_folder), + pkg_layout.metadata()) + pref_bundle["metadata_folder"] = os.path.relpath(pkg_layout.metadata(), cache.cache_folder) return package_list diff --git a/conans/test/integration/command_v2/test_cache_save_restore.py b/conans/test/integration/command_v2/test_cache_save_restore.py index e178cf240b2..046e0fb0e96 100644 --- a/conans/test/integration/command_v2/test_cache_save_restore.py +++ b/conans/test/integration/command_v2/test_cache_save_restore.py @@ -30,6 +30,36 @@ def test_cache_save_restore(): assert "\\" not in package_list +def test_cache_save_restore_with_package_file(): + """If we have some sources in the root (like the CMakeLists.txt) + we don't declare folders.source""" + conan_file = GenConanfile() \ + .with_settings("os") \ + .with_package_file("bin/file.txt", "content!!") + + client = TestClient() + client.save({"conanfile.py": conan_file}) + client.run("create . --name=pkg --version=1.0 -s os=Linux") + client.run("cache save pkg/*:* ") + cache_path = os.path.join(client.current_folder, "conan_cache_save.tgz") + assert os.path.exists(cache_path) + + c2 = TestClient() + shutil.copy2(cache_path, c2.current_folder) + c2.run("cache restore conan_cache_save.tgz") + c2.run("list *:*#*") + assert "pkg/1.0" in c2.out + tree = _get_directory_tree(c2.base_folder) + + # Restore again, expect the tree to be unchanged + c2.run("cache restore conan_cache_save.tgz") + c2.run("list *:*#*") + assert "pkg/1.0" in c2.out + tree2 = _get_directory_tree(c2.base_folder) + + assert tree2 == tree + + def test_cache_save_downloaded_restore(): """ what happens if we save packages downloaded from server, not created @@ -49,6 +79,18 @@ def test_cache_save_downloaded_restore(): _validate_restore(cache_path) +def _get_directory_tree(base_folder): + tree = [] + for d, _, fs in os.walk(base_folder): + rel_d = os.path.relpath(d, base_folder) if d != base_folder else "" + if rel_d: + tree.append(rel_d) + for f in fs: + tree.append(os.path.join(rel_d, f)) + tree.sort() + return tree + + def _validate_restore(cache_path): c2 = TestClient() # Create a package in the cache to check put doesn't interact badly @@ -61,6 +103,7 @@ def _validate_restore(cache_path): assert "pkg/1.0" in c2.out assert "pkg/1.1" in c2.out assert "other/2.0" not in c2.out + tree = _get_directory_tree(c2.base_folder) # Restore again, just in case c2.run("cache restore conan_cache_save.tgz") @@ -69,6 +112,8 @@ def _validate_restore(cache_path): assert "pkg/1.0" in c2.out assert "pkg/1.1" in c2.out assert "other/2.0" not in c2.out + tree2 = _get_directory_tree(c2.base_folder) + assert tree2 == tree def test_cache_save_restore_metadata(): From b99b55c5039701581597b74dd34ea78b5ab89e00 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 26 Mar 2024 16:21:06 +0100 Subject: [PATCH 2/2] fix test --- conan/api/subapi/cache.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index fa9df555f0f..0875f6fb2a0 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -162,40 +162,49 @@ def restore(self, path): the_tar.extractall(path=self.conan_api.cache_folder) the_tar.close() + # After unzipping the files, we need to update the DB that references these files out = ConanOutput() package_list = PackagesList.deserialize(json.loads(pkglist)) cache = ClientCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) for ref, ref_bundle in package_list.refs().items(): ref.timestamp = revision_timestamp_now() ref_bundle["timestamp"] = ref.timestamp - recipe_layout = cache.get_or_create_ref_layout(ref) + recipe_layout = cache.get_or_create_ref_layout(ref) # DB folder entry recipe_folder = ref_bundle["recipe_folder"] rel_path = os.path.relpath(recipe_layout.base_folder, cache.cache_folder) rel_path = rel_path.replace("\\", "/") + # In the case of recipes, they are always "in place", so just checking it assert rel_path == recipe_folder, f"{rel_path}!={recipe_folder}" out.info(f"Restore: {ref} in {recipe_folder}") for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items(): pref.timestamp = revision_timestamp_now() pref_bundle["timestamp"] = pref.timestamp - pkg_layout = cache.get_or_create_pkg_layout(pref) - pkg_folder = pref_bundle["package_folder"] - out.info(f"Restore: {pref} in {pkg_folder}") - if pkg_layout.package() != os.path.join(cache.cache_folder, pkg_folder): - # We need to put the package in the final location in the cache + pkg_layout = cache.get_or_create_pkg_layout(pref) # DB Folder entry + unzipped_pkg_folder = pref_bundle["package_folder"] + out.info(f"Restore: {pref} in {unzipped_pkg_folder}") + # If the DB folder entry is different to the disk unzipped one, we need to move it + # This happens for built (not downloaded) packages in the source "conan cache save" + db_pkg_folder = os.path.relpath(pkg_layout.package(), cache.cache_folder) + db_pkg_folder = db_pkg_folder.replace("\\", "/") + if db_pkg_folder != unzipped_pkg_folder: + # If a previous package exists, like a previous restore, then remove it if os.path.exists(pkg_layout.package()): shutil.rmtree(pkg_layout.package()) - shutil.move(os.path.join(cache.cache_folder, pkg_folder), pkg_layout.package()) - pref_bundle["package_folder"] = os.path.relpath(pkg_layout.package(), cache.cache_folder) - metadata_folder = pref_bundle.get("metadata_folder") - if metadata_folder: - out.info(f"Restore: {pref} metadata in {metadata_folder}") - if pkg_layout.metadata() != os.path.join(cache.cache_folder, metadata_folder): + shutil.move(os.path.join(cache.cache_folder, unzipped_pkg_folder), + pkg_layout.package()) + pref_bundle["package_folder"] = db_pkg_folder + unzipped_metadata_folder = pref_bundle.get("metadata_folder") + if unzipped_metadata_folder: + out.info(f"Restore: {pref} metadata in {unzipped_metadata_folder}") + db_metadata_folder = os.path.relpath(pkg_layout.metadata(), cache.cache_folder) + db_metadata_folder = db_metadata_folder.replace("\\", "/") + if db_metadata_folder != unzipped_metadata_folder: # We need to put the package in the final location in the cache if os.path.exists(pkg_layout.metadata()): shutil.rmtree(pkg_layout.metadata()) - shutil.move(os.path.join(cache.cache_folder, metadata_folder), + shutil.move(os.path.join(cache.cache_folder, unzipped_metadata_folder), pkg_layout.metadata()) - pref_bundle["metadata_folder"] = os.path.relpath(pkg_layout.metadata(), cache.cache_folder) + pref_bundle["metadata_folder"] = db_metadata_folder return package_list