diff --git a/dvc/cachemgr.py b/dvc/cachemgr.py
index eeb4fe76d6a..4707bb45c05 100644
--- a/dvc/cachemgr.py
+++ b/dvc/cachemgr.py
@@ -1,3 +1,5 @@
+import os
+
 from dvc.fs import GitFileSystem, Schemes
 from dvc_data.hashfile.db import get_odb
 
@@ -28,16 +30,30 @@ def __init__(self, repo):
         self.config = config = repo.config["cache"]
         self._odb = {}
 
+        default = None
+        if repo and repo.dvc_dir:
+            if isinstance(repo.fs, GitFileSystem):
+                dvc_dir = os.path.join(
+                    repo.scm.root_dir,
+                    *repo.fs.path.relparts(repo.root_dir, "/"),
+                    repo.DVC_DIR,
+                )
+                if os.path.exists(dvc_dir):
+                    default = os.path.join(dvc_dir, self.CACHE_DIR)
+            else:
+                default = repo.fs.path.join(repo.dvc_dir, self.CACHE_DIR)
+
         local = config.get("local")
 
         if local:
             settings = {"name": local}
-        elif "dir" not in config:
+        elif "dir" not in config and not default:
             settings = None
         else:
             from dvc.config_schema import LOCAL_COMMON
 
-            settings = {"url": config["dir"]}
+            url = config.get("dir") or default
+            settings = {"url": url}
             for opt in LOCAL_COMMON:
                 if opt in config:
                     settings[str(opt)] = config.get(opt)
diff --git a/dvc/commands/cache.py b/dvc/commands/cache.py
index cab80053575..e21b2081dc4 100644
--- a/dvc/commands/cache.py
+++ b/dvc/commands/cache.py
@@ -1,4 +1,5 @@
 import argparse
+import os
 
 from dvc.cli import completion
 from dvc.cli.utils import append_doc_link, fix_subparsers
@@ -9,13 +10,21 @@
 class CmdCacheDir(CmdConfig):
     def run(self):
         if self.args.value is None and not self.args.unset:
+            from dvc.config import ConfigError
+
             if self.args.level:
                 conf = self.config.read(level=self.args.level)
             else:
                 # Use merged config with default values
                 conf = self.config
-            self._check(conf, False, "cache", "dir")
-            ui.write(conf["cache"]["dir"])
+            try:
+                self._check(conf, False, "cache", "dir")
+                path = conf["cache"]["dir"]
+            except ConfigError:
+                if not self.config.dvc_dir or self.args.level:
+                    raise
+                path = os.path.join(self.config.dvc_dir, "cache")
+            ui.write(path)
             return 0
         with self.config.edit(level=self.args.level) as conf:
             if self.args.unset:
diff --git a/dvc/config.py b/dvc/config.py
index 112caef7eee..e8524fc67ef 100644
--- a/dvc/config.py
+++ b/dvc/config.py
@@ -170,10 +170,6 @@ def load(self, validate: bool = True, config: Optional["DictStrAny"] = None):
         self.clear()
         self.update(conf)
 
-        # Add resolved default cache.dir
-        if not self["cache"].get("dir") and self.dvc_dir:
-            self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
-
     def _get_fs(self, level):
         # NOTE: this might be a Gitfs, which doesn't see things outside of
         # the repo.