diff --git a/dvc/ignore.py b/dvc/ignore.py
index b9d4de35af..eefa3bfef8 100644
--- a/dvc/ignore.py
+++ b/dvc/ignore.py
@@ -6,7 +6,6 @@
 from pathspec.patterns import GitWildMatchPattern
 
 from dvc.scm.tree import BaseTree
-from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -35,12 +34,19 @@ def __call__(self, root, dirs, files):
         return dirs, files
 
     def matches(self, dirname, basename):
-        abs_path = os.path.join(dirname, basename)
-        rel_path = relpath(abs_path, self.dirname)
-
-        if os.pardir + os.sep in rel_path:
+        # NOTE: `relpath` is too slow, so we have to assume that both
+        # `dirname` and `self.dirname` are relative or absolute together.
+        prefix = self.dirname + os.sep
+        if dirname == self.dirname:
+            path = basename
+        elif dirname.startswith(prefix):
+            rel = dirname[len(prefix) :]
+            # NOTE: `os.path.join` is ~x5.5 slower
+            path = f"{rel}{os.sep}{basename}"
+        else:
             return False
-        return self.ignore_spec.match_file(rel_path)
+
+        return self.ignore_spec.match_file(path)
 
     def __hash__(self):
         return hash(self.ignore_file_path)
@@ -135,7 +141,9 @@ def isexec(self, path):
 
     def walk(self, top, topdown=True):
         for root, dirs, files in self.tree.walk(top, topdown):
-            dirs[:], files[:] = self.dvcignore(root, dirs, files)
+            dirs[:], files[:] = self.dvcignore(
+                os.path.abspath(root), dirs, files
+            )
 
             yield root, dirs, files
 
diff --git a/tests/unit/test_ignore.py b/tests/unit/test_ignore.py
index 71672e0694..1f7c4d8fc3 100644
--- a/tests/unit/test_ignore.py
+++ b/tests/unit/test_ignore.py
@@ -69,7 +69,6 @@ def test_ignore_from_file_should_filter_dirs_and_files():
         ),
         ("dont_ignore.txt", ["dont_ignore"], False),
         ("dont_ignore.txt", ["dont*", "!dont_ignore.txt"], False),
-        ("../../../something.txt", ["**/something.txt"], False),
     ],
 )
 def test_match_ignore_from_file(