diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 356b805b39cc0..c16676d48c750 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -286,7 +286,7 @@ if(ARROW_USE_SIMD)
   add_definitions(-DARROW_USE_SIMD)
 endif()
 
-if(APPLE)
+if(APPLE AND "${COMPILER_FAMILY}" STREQUAL "clang")
   # Depending on the default OSX_DEPLOYMENT_TARGET (< 10.9), libstdc++ may be
   # the default standard library which does not support C++11. libc++ is the
   # default from 10.9 onward.
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index a991df5323d43..eadcfd6b5aa66 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -1668,7 +1668,7 @@ macro(build_benchmark)
     set(GBENCHMARK_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++11")
   endif()
 
-  if(APPLE)
+  if(APPLE AND "${COMPILER_FAMILY}" STREQUAL "clang")
     set(GBENCHMARK_CMAKE_CXX_FLAGS "${GBENCHMARK_CMAKE_CXX_FLAGS} -stdlib=libc++")
   endif()