diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index c8457eae8ed33..49c0fdb2774b0 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -476,7 +476,14 @@ static void AdjustIntegerStringWithScale(int32_t scale, std::string* str) {
     //               adjusted_exponent = -7
     // After inserting decimal point: *str = "-1.23"
     // After appending exponent: *str = "-1.23E-7"
-    str->insert(str->begin() + 1 + is_negative_offset, '.');
+    // Example 3:
+    // Precondition: *str = "0", is_negative_offset = 0, num_digits = 1, scale = -1,
+    //               adjusted_exponent = 1
+    // After inserting decimal point: *str = "0" // Not inserted
+    // After appending exponent: *str = "0E+1"
+    if (num_digits > 1) {
+      str->insert(str->begin() + 1 + is_negative_offset, '.');
+    }
     str->push_back('E');
     if (adjusted_exponent >= 0) {
       str->push_back('+');
diff --git a/cpp/src/arrow/util/decimal_test.cc b/cpp/src/arrow/util/decimal_test.cc
index 0a8b7a09730bf..f0760a09080ec 100644
--- a/cpp/src/arrow/util/decimal_test.cc
+++ b/cpp/src/arrow/util/decimal_test.cc
@@ -689,22 +689,22 @@ struct ToStringTestParam {
 };
 
 static const ToStringTestParam kToStringTestData[] = {
-    {0, -1, "0.E+1"},
+    {0, -1, "0E+1"},
     {0, 0, "0"},
     {0, 1, "0.0"},
     {0, 6, "0.000000"},
-    {2, 7, "2.E-7"},
-    {2, -1, "2.E+1"},
+    {2, 7, "2E-7"},
+    {2, -1, "2E+1"},
     {2, 0, "2"},
     {2, 1, "0.2"},
     {2, 6, "0.000002"},
-    {-2, 7, "-2.E-7"},
-    {-2, 7, "-2.E-7"},
-    {-2, -1, "-2.E+1"},
+    {-2, 7, "-2E-7"},
+    {-2, 7, "-2E-7"},
+    {-2, -1, "-2E+1"},
     {-2, 0, "-2"},
     {-2, 1, "-0.2"},
     {-2, 6, "-0.000002"},
-    {-2, 7, "-2.E-7"},
+    {-2, 7, "-2E-7"},
     {123, -3, "1.23E+5"},
     {123, -1, "1.23E+3"},
     {123, 1, "12.3"},