Merge #107490

107490: testutils/floatcmp: revert to base-10 normalization in FloatsMatch r=rharding6373 a=michae2 In #106552 we tried changing float normalization to use base 2 instead of base 10 (in other words, to use `math.Frexp` instead of our hand-rolled `normalize`). This appears to have broken a logic test, so revert back to the pre-existing base 10 normalization. Fixes: #107461 Release note: None Co-authored-by: Michael Erickson <[email protected]>
cockroachdb · Jul 24, 2023 · 65f591b · 65f591b
2 parents acc75f4 + aff21b6
commit 65f591b
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 2 deletions.
diff --git a/pkg/testutils/floatcmp/floatcmp.go b/pkg/testutils/floatcmp/floatcmp.go
@@ -121,9 +121,23 @@ func FloatsMatch(expectedString, actualString string) (bool, error) {
 	actual = math.Abs(actual)
 	// Check that 15 significant digits match. We do so by normalizing the
 	// numbers and then checking one digit at a time.
+	//
+	// normalize converts f to base * 10**power representation where base is in
+	// [1.0, 10.0) range.
+	normalize := func(f float64) (base float64, power int) {
+		for f >= 10 {
+			f = f / 10
+			power++
+		}
+		for f < 1 {
+			f *= 10
+			power--
+		}
+		return f, power
+	}
 	var expPower, actPower int
-	expected, expPower = math.Frexp(expected)
-	actual, actPower = math.Frexp(actual)
+	expected, expPower = normalize(expected)
+	actual, actPower = normalize(actual)
 	if expPower != actPower {
 		return false, nil
 	}

diff --git a/pkg/testutils/floatcmp/floatcmp_test.go b/pkg/testutils/floatcmp/floatcmp_test.go
@@ -178,6 +178,7 @@ func TestFloatsMatch(t *testing.T) {
 		{f1: "0.1234567890123456", f2: "0.1234567890123457", match: true},
 		{f1: "-0.1234567890123456", f2: "0.1234567890123456", match: false},
 		{f1: "-0.1234567890123456", f2: "-0.1234567890123455", match: true},
+		{f1: "0.142857142857143", f2: "0.14285714285714285", match: true},
 	} {
 		match, err := FloatsMatch(tc.f1, tc.f2)
 		if err != nil {