diff --git a/cpp/src/arrow/acero/swiss_join.cc b/cpp/src/arrow/acero/swiss_join.cc
index ed1608e67d177..3f11b89af39de 100644
--- a/cpp/src/arrow/acero/swiss_join.cc
+++ b/cpp/src/arrow/acero/swiss_join.cc
@@ -473,6 +473,12 @@ Status RowArrayMerge::PrepareForMerge(RowArray* target,
     (*first_target_row_id)[sources.size()] = num_rows;
   }
 
+  if (num_bytes > std::numeric_limits<uint32_t>::max()) {
+    return Status::Invalid(
+        "There are more than 2^32 bytes of key data.  Acero cannot "
+        "process a join of this magnitude");
+  }
+
   // Allocate target memory
   //
   target->rows_.Clean();