diff --git a/conda_build/post.py b/conda_build/post.py
index 7ed419eebf..28ff905314 100644
--- a/conda_build/post.py
+++ b/conda_build/post.py
@@ -73,6 +73,12 @@
     "linux": (elffile,),
 }
 
+GNU_ARCH_MAP = {
+    "ppc64le": "powerpc64le",
+    "32": "i686",
+    "64": "x86_64",
+}
+
 
 def fix_shebang(f, prefix, build_python, osx_is_app=False):
     path = join(prefix, f)
@@ -1406,8 +1412,20 @@ def check_overlinking_impl(
                     list(diffs)[1:3],
                 )
                 sysroots_files[srs] = sysroot_files
+
+    def sysroot_matches_subdir(path):
+        # The path looks like <PREFIX>/aarch64-conda-linux-gnu/sysroot/
+        # We check that the triplet "aarch64-conda-linux-gnu"
+        # matches the subdir for eg: linux-aarch64.
+        sysroot_arch = Path(path).parent.name.split("-")[0]
+        subdir_arch = subdir.split("-")[-1]
+        return sysroot_arch == GNU_ARCH_MAP.get(subdir_arch, subdir_arch)
+
     sysroots_files = OrderedDict(
-        sorted(sysroots_files.items(), key=lambda x: -len(x[1]))
+        sorted(
+            sysroots_files.items(),
+            key=lambda x: (not sysroot_matches_subdir(x[0]), -len(x[1])),
+        )
     )
 
     all_needed_dsos, needed_dsos_for_file = _collect_needed_dsos(
diff --git a/news/5404-sysroot-prefer b/news/5404-sysroot-prefer
new file mode 100644
index 0000000000..79bcd5927f
--- /dev/null
+++ b/news/5404-sysroot-prefer
@@ -0,0 +1,19 @@
+### Enhancements
+
+* <news item>
+
+### Bug fixes
+
+* On Linux platforms, prefer the sysroot matching the target_platform when cross-compiling (#5403).
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* <news item>
diff --git a/tests/test-recipes/metadata/_sysroot_detection/build.sh b/tests/test-recipes/metadata/_sysroot_detection/build.sh
new file mode 100644
index 0000000000..56c6c57c6e
--- /dev/null
+++ b/tests/test-recipes/metadata/_sysroot_detection/build.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+mkdir -p ${PREFIX}/bin
+
+# Delete the x86_64 libc.so.6 to make sure we find the powerpc libc.so.6
+rm -f ${BUILD_PREFIX}/x86_64-conda-linux-gnu/sysroot/lib64/libc.so.6
+
+${CC} ${CFLAGS} main.c -o ${PREFIX}/bin/sysroot-detection
diff --git a/tests/test-recipes/metadata/_sysroot_detection/conda_build_config.yaml b/tests/test-recipes/metadata/_sysroot_detection/conda_build_config.yaml
new file mode 100644
index 0000000000..549ed687b9
--- /dev/null
+++ b/tests/test-recipes/metadata/_sysroot_detection/conda_build_config.yaml
@@ -0,0 +1,2 @@
+target_platform:
+  - linux-ppc64le
diff --git a/tests/test-recipes/metadata/_sysroot_detection/main.c b/tests/test-recipes/metadata/_sysroot_detection/main.c
new file mode 100644
index 0000000000..1394ce82a6
--- /dev/null
+++ b/tests/test-recipes/metadata/_sysroot_detection/main.c
@@ -0,0 +1,5 @@
+#include <stdio.h>
+
+int main() {
+  return 0;
+}
diff --git a/tests/test-recipes/metadata/_sysroot_detection/meta.yaml b/tests/test-recipes/metadata/_sysroot_detection/meta.yaml
new file mode 100644
index 0000000000..bffaf71b88
--- /dev/null
+++ b/tests/test-recipes/metadata/_sysroot_detection/meta.yaml
@@ -0,0 +1,15 @@
+{% set version = "1" %}
+
+package:
+  name: sysroot_detection
+  version: {{ version }}
+
+source:
+  path: main.c
+
+build:
+  number: 0
+
+requirements:
+  build:
+    - {{ compiler('c') }}
diff --git a/tests/test_api_build.py b/tests/test_api_build.py
index 5f5e3f2263..554bc452ae 100644
--- a/tests/test_api_build.py
+++ b/tests/test_api_build.py
@@ -1776,6 +1776,18 @@ def test_overdepending_detection(testing_config, variants_conda_build_sysroot):
         api.build(recipe, config=testing_config, variants=variants_conda_build_sysroot)
 
 
+@pytest.mark.skipif(not on_linux, reason="cannot compile for linux-ppc64le")
+def test_sysroots_detection(testing_config, variants_conda_build_sysroot):
+    recipe = os.path.join(metadata_dir, "_sysroot_detection")
+    testing_config.activate = True
+    testing_config.error_overlinking = True
+    testing_config.error_overdepending = True
+    testing_config.channel_urls = [
+        "conda-forge",
+    ]
+    api.build(recipe, config=testing_config, variants=variants_conda_build_sysroot)
+
+
 @pytest.mark.skipif(sys.platform != "darwin", reason="macOS-only test (at present)")
 def test_macos_tbd_handling(testing_config, variants_conda_build_sysroot):
     """