diff --git a/news/2839.bugfix.md b/news/2839.bugfix.md
new file mode 100644
index 0000000000..eb09f71d1b
--- /dev/null
+++ b/news/2839.bugfix.md
@@ -0,0 +1 @@
+Fixes cached packages metadata files (`.referrers`) collisions on `sync` when using a `venv` with `symlink` cache method.
diff --git a/src/pdm/installers/installers.py b/src/pdm/installers/installers.py
index 5b18511c1e..7d3563ce97 100644
--- a/src/pdm/installers/installers.py
+++ b/src/pdm/installers/installers.py
@@ -66,7 +66,7 @@ def read_dist_info(self, filename: str) -> str:
     def iter_files(self) -> Iterable[Path]:
         for root, _, files in os.walk(self.package.path):
             for file in files:
-                if Path(root) == self.package.path and file in (".checksum", ".lock"):
+                if Path(root) == self.package.path and file in CachedPackage.cache_files:
                     continue
                 yield Path(root, file)
 
diff --git a/src/pdm/models/cached_package.py b/src/pdm/models/cached_package.py
index fc2bbc7a07..418c1b3224 100644
--- a/src/pdm/models/cached_package.py
+++ b/src/pdm/models/cached_package.py
@@ -4,7 +4,7 @@
 import shutil
 from functools import cached_property
 from pathlib import Path
-from typing import Any, ContextManager
+from typing import Any, ClassVar, ContextManager
 
 from pdm.termui import logger
 
@@ -22,6 +22,9 @@ class CachedPackage:
     *Only wheel installations will be cached*
     """
 
+    cache_files: ClassVar[tuple[str, ...]] = (".lock", ".checksum", ".referrers")
+    """List of files storing cache metadata and not being part of the package"""
+
     def __init__(self, path: str | Path, original_wheel: Path | None = None) -> None:
         self.path = Path(os.path.normcase(os.path.expanduser(path))).resolve()
         self.original_wheel = original_wheel
diff --git a/tests/test_installer.py b/tests/test_installer.py
index 60501b11ee..5dc944dc5d 100644
--- a/tests/test_installer.py
+++ b/tests/test_installer.py
@@ -2,6 +2,7 @@
 
 import logging
 import os
+import venv
 from pathlib import Path
 from typing import Callable
 
@@ -9,9 +10,15 @@
 from unearth import Link
 
 from pdm import utils
+from pdm.core import Core
+from pdm.environments.base import BaseEnvironment
+from pdm.environments.local import PythonLocalEnvironment
+from pdm.environments.python import PythonEnvironment
 from pdm.installers import InstallManager
+from pdm.models.cached_package import CachedPackage
 from pdm.models.candidates import Candidate
 from pdm.models.requirements import parse_requirement
+from pdm.project.core import Project
 from tests import FIXTURES
 
 pytestmark = pytest.mark.usefixtures("local_finder")
@@ -32,6 +39,22 @@ def mocked_support(linker: str) -> bool:
     return mocked_support
 
 
+def _prepare_project_for_env(project: Project, env_cls: type[BaseEnvironment]):
+    project._saved_python = None
+    project._python = None
+    if env_cls is PythonEnvironment:
+        venv.create(project.root / ".venv", symlinks=True)
+        project.project_config["python.use_venv"] = True
+
+
+@pytest.fixture(params=(PythonEnvironment, PythonLocalEnvironment), autouse=True)
+def environment(request: pytest.RequestFixture, project: Project) -> type[BaseEnvironment]:
+    # Run all test against all environments as installation and cache behavior may differ
+    env_cls: type[BaseEnvironment] = request.param
+    _prepare_project_for_env(project, env_cls)
+    return env_cls
+
+
 def test_install_wheel_with_inconsistent_dist_info(project):
     req = parse_requirement("pyfunctional")
     candidate = Candidate(
@@ -134,6 +157,9 @@ def test_install_wheel_with_cache(project, pdm, supports_link):
         assert os.path.isfile(os.path.join(lib_path, "future_fstrings.py"))
         assert os.path.isfile(os.path.join(lib_path, "aaaaa_future_fstrings.pth"))
 
+    for file in CachedPackage.cache_files:
+        assert not os.path.exists(os.path.join(lib_path, file))
+
     cache_name = "future_fstrings-1.2.0-py2.py3-none-any.whl.cache"
     assert any(p.path.name == cache_name for p in project.package_cache.iter_packages())
     pdm(["run", "python", "-m", "site"], object=project)
@@ -152,6 +178,40 @@ def test_install_wheel_with_cache(project, pdm, supports_link):
     assert not any(p.path.name == cache_name for p in project.package_cache.iter_packages())
 
 
+@pytest.mark.parametrize("preferred", ["symlink", "hardlink", None])
+def test_can_install_wheel_with_cache_in_multiple_projects(
+    project: Project, core: Core, supports_link, tmp_path_factory, environment
+):
+    projects = []
+
+    for idx in range(3):
+        path: Path = tmp_path_factory.mktemp(f"project-{idx}")
+        p = core.create_project(path, global_config=project.global_config.config_file)
+        _prepare_project_for_env(p, environment)
+        projects.append(p)
+
+    req = parse_requirement("future-fstrings")
+    candidate = Candidate(
+        req,
+        link=Link("http://fixtures.test/artifacts/future_fstrings-1.2.0-py2.py3-none-any.whl"),
+    )
+
+    for p in projects:
+        installer = InstallManager(p.environment, use_install_cache=True)
+        installer.install(candidate)
+
+        lib_path = p.environment.get_paths()["purelib"]
+        if supports_link("symlink"):
+            assert os.path.islink(os.path.join(lib_path, "future_fstrings.py"))
+            assert os.path.islink(os.path.join(lib_path, "aaaaa_future_fstrings.pth"))
+        else:
+            assert os.path.isfile(os.path.join(lib_path, "future_fstrings.py"))
+            assert os.path.isfile(os.path.join(lib_path, "aaaaa_future_fstrings.pth"))
+
+        for file in CachedPackage.cache_files:
+            assert not os.path.exists(os.path.join(lib_path, file))
+
+
 def test_url_requirement_is_not_cached(project):
     req = parse_requirement(
         "future-fstrings @ http://fixtures.test/artifacts/future_fstrings-1.2.0-py2.py3-none-any.whl"