diff --git a/cpp/src/arrow/util/cpu_info.cc b/cpp/src/arrow/util/cpu_info.cc
index 0faf1e6f8577d..fbe55aec0c157 100644
--- a/cpp/src/arrow/util/cpu_info.cc
+++ b/cpp/src/arrow/util/cpu_info.cc
@@ -296,35 +296,31 @@ void OsRetrieveCpuInfo(int64_t* hardware_flags, CpuInfo::Vendor* vendor,
 //------------------------------ LINUX ------------------------------//
 // Get cache size, return 0 on error
 int64_t LinuxGetCacheSize(int level) {
-  const struct {
-    int sysconf_name;
-    const char* sysfs_path;
-  } kCacheSizeEntries[] = {
-      {
-          _SC_LEVEL1_DCACHE_SIZE,
-          "/sys/devices/system/cpu/cpu0/cache/index0/size",  // l1d (index1 is l1i)
-      },
-      {
-          _SC_LEVEL2_CACHE_SIZE,
-          "/sys/devices/system/cpu/cpu0/cache/index2/size",  // l2
-      },
-      {
-          _SC_LEVEL3_CACHE_SIZE,
-          "/sys/devices/system/cpu/cpu0/cache/index3/size",  // l3
-      },
+  // get cache size by sysconf()
+#ifdef _SC_LEVEL1_DCACHE_SIZE
+  const int kCacheSizeConf[] = {
+      _SC_LEVEL1_DCACHE_SIZE,
+      _SC_LEVEL2_CACHE_SIZE,
+      _SC_LEVEL3_CACHE_SIZE,
   };
-  static_assert(sizeof(kCacheSizeEntries) / sizeof(kCacheSizeEntries[0]) == kCacheLevels,
-                "");
+  static_assert(sizeof(kCacheSizeConf) / sizeof(kCacheSizeConf[0]) == kCacheLevels, "");
 
-  // get cache size by sysconf()
   errno = 0;
-  const int64_t cache_size = sysconf(kCacheSizeEntries[level].sysconf_name);
+  const int64_t cache_size = sysconf(kCacheSizeConf[level]);
   if (errno == 0 && cache_size > 0) {
     return cache_size;
   }
+#endif
+
+  // get cache size from sysfs if sysconf() fails or not supported
+  const char* kCacheSizeSysfs[] = {
+      "/sys/devices/system/cpu/cpu0/cache/index0/size",  // l1d (index1 is l1i)
+      "/sys/devices/system/cpu/cpu0/cache/index2/size",  // l2
+      "/sys/devices/system/cpu/cpu0/cache/index3/size",  // l3
+  };
+  static_assert(sizeof(kCacheSizeSysfs) / sizeof(kCacheSizeSysfs[0]) == kCacheLevels, "");
 
-  // get cache size from sysfs if sysconf() fails (it does happen on Arm)
-  std::ifstream cacheinfo(kCacheSizeEntries[level].sysfs_path, std::ios::in);
+  std::ifstream cacheinfo(kCacheSizeSysfs[level], std::ios::in);
   if (!cacheinfo) {
     return 0;
   }