backupccl: create stripped crdb_internal.fingerprint overload

docs/generated/sql/functions.md

@@ -3127,6 +3127,8 @@ may increase either contention or retry errors, or both.</p>
 </span></td><td>Stable</td></tr>
 <tr><td><a name="crdb_internal.fingerprint"></a><code>crdb_internal.fingerprint(span: <a href="bytes.html">bytes</a>[], start_time: <a href="timestamp.html">timestamptz</a>, all_revisions: <a href="bool.html">bool</a>) &rarr; <a href="int.html">int</a></code></td><td><span class="funcdesc"><p>This function is used only by CockroachDB’s developers for testing purposes.</p>
 </span></td><td>Stable</td></tr>
+<tr><td><a name="crdb_internal.fingerprint"></a><code>crdb_internal.fingerprint(span: <a href="bytes.html">bytes</a>[], stripped: <a href="bool.html">bool</a>) &rarr; <a href="int.html">int</a></code></td><td><span class="funcdesc"><p>This function is used only by CockroachDB’s developers for testing purposes.</p>
+</span></td><td>Stable</td></tr>
 <tr><td><a name="crdb_internal.force_assertion_error"></a><code>crdb_internal.force_assertion_error(msg: <a href="string.html">string</a>) &rarr; <a href="int.html">int</a></code></td><td><span class="funcdesc"><p>This function is used only by CockroachDB’s developers for testing purposes.</p>
 </span></td><td>Volatile</td></tr>
 <tr><td><a name="crdb_internal.force_error"></a><code>crdb_internal.force_error(errorCode: <a href="string.html">string</a>, msg: <a href="string.html">string</a>) &rarr; <a href="int.html">int</a></code></td><td><span class="funcdesc"><p>This function is used only by CockroachDB’s developers for testing purposes.</p>

pkg/ccl/backupccl/backup_test.go

@@ -1080,6 +1080,8 @@ SELECT payload FROM "".crdb_internal.system_jobs ORDER BY created DESC LIMIT 10
 
 		sqlDB.Exec(t, incBackupQuery, queryArgs...)
 	}
+	bankTableID := sqlutils.QueryTableID(t, conn, "data", "public", "bank")
+	backupTableFingerprint := sqlutils.FingerprintTable(t, sqlDB, bankTableID)
 
 	sqlDB.Exec(t, `DROP DATABASE data CASCADE`)
 
@@ -1100,7 +1102,7 @@ SELECT payload FROM "".crdb_internal.system_jobs ORDER BY created DESC LIMIT 10
 		restoreQuery = fmt.Sprintf("%s WITH kms = %s", restoreQuery, kmsURIFmtString)
 	}
 	queryArgs := append(restoreURIArgs, kmsURIArgs...)
-	verifyRestoreData(t, sqlDB, storageSQLDB, restoreQuery, queryArgs, numAccounts)
+	verifyRestoreData(t, sqlDB, storageSQLDB, restoreQuery, queryArgs, numAccounts, backupTableFingerprint)
 }
 
 func verifyRestoreData(
@@ -1110,6 +1112,7 @@ func verifyRestoreData(
 	restoreQuery string,
 	restoreURIArgs []interface{},
 	numAccounts int,
+	bankStrippedFingerprint int,
 ) {
 	var unused string
 	var restored struct {
@@ -1160,6 +1163,8 @@ func verifyRestoreData(
 			t.Fatal("unexpected span start at primary index")
 		}
 	}
+	restorebankID := sqlutils.QueryTableID(t, sqlDB.DB, "data", "public", "bank")
+	require.Equal(t, bankStrippedFingerprint, sqlutils.FingerprintTable(t, sqlDB, restorebankID))
 }
 
 func TestBackupRestoreSystemTables(t *testing.T) {

pkg/ccl/backupccl/backuprand/BUILD.bazel

@@ -30,6 +30,7 @@ go_test(
         "//pkg/util/log",
         "//pkg/util/randutil",
         "@com_github_stretchr_testify//assert",
+        "@com_github_stretchr_testify//require",
     ],
 )
 

pkg/ccl/backupccl/backuprand/backup_rand_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // TestBackupRestoreRandomDataRoundtrips conducts backup/restore roundtrips on
@@ -97,12 +98,13 @@ database_name = 'rand' AND schema_name = 'public'`)
 	}
 
 	expectedCreateTableStmt := make(map[string]string)
-	expectedData := make(map[string][][]string)
+	expectedData := make(map[string]int)
 	for _, tableName := range tableNames {
 		expectedCreateTableStmt[tableName] = sqlDB.QueryStr(t,
 			fmt.Sprintf(`SELECT create_statement FROM [SHOW CREATE TABLE %s]`, tree.NameString(tableName)))[0][0]
 		if runSchemaOnlyExtension == "" {
-			expectedData[tableName] = sqlDB.QueryStr(t, fmt.Sprintf(`SELECT * FROM %s`, tree.NameString(tableName)))
+			tableID := sqlutils.QueryTableID(t, sqlDB.DB, "rand", "public", tableName)
+			expectedData[tableName] = sqlutils.FingerprintTable(t, sqlDB, tableID)
 		}
 	}
 
@@ -135,7 +137,8 @@ database_name = 'rand' AND schema_name = 'public'`)
 			assert.Equal(t, expectedCreateTableStmt[tableName], createStmt,
 				"SHOW CREATE %s not equal after RESTORE", tableName)
 			if runSchemaOnlyExtension == "" {
-				sqlDB.CheckQueryResults(t, fmt.Sprintf(`SELECT * FROM %s`, restoreTable), expectedData[tableName])
+				tableID := sqlutils.QueryTableID(t, sqlDB.DB, "restoredb", "public", tableName)
+				require.Equal(t, expectedData[tableName], sqlutils.FingerprintTable(t, sqlDB, tableID))
 			} else {
 				sqlDB.CheckQueryResults(t, fmt.Sprintf(`SELECT count(*) FROM %s`, restoreTable),
 					[][]string{{"0"}})

pkg/keys/sql.go

@@ -65,6 +65,54 @@ func DecodeTenantPrefixE(key roachpb.Key) ([]byte, roachpb.TenantID, error) {
 	return rem, id, nil
 }
 
+// StripTenantPrefix removes the tenant prefix from the provided key. This
+// function should be used instead of sqlDecoder.StripTenantPrefix, if the user
+// cannot instantiate a codec that operates on a single tenant.
+func StripTenantPrefix(key roachpb.Key) ([]byte, error) {
+	var err error
+	if len(key) == 0 { // key.Equal(roachpb.RKeyMin)
+		return nil, nil
+	}
+	if key[0] != tenantPrefixByte {
+		return key, nil
+	}
+	key, _, err = encoding.DecodeUvarintAscending(key[1:])
+	return key, err
+}
+
+// StripTablePrefix removes the table and tenant prefix from the provided key.
+// This function should be used instead of sqlDecoder.DecodeTablePrefix, if the
+// user cannot instantiate a codec that operates on a single tenant.
+func StripTablePrefix(key roachpb.Key) ([]byte, error) {
+	var err error
+	key, err = StripTenantPrefix(key)
+	if err != nil {
+		return nil, err
+	}
+	if encoding.PeekType(key) != encoding.Int {
+		return nil, errors.Errorf("invalid table key prefix: %q", key)
+	}
+	key, _, err = encoding.DecodeUvarintAscending(key)
+	return key, err
+}
+
+// StripIndexPrefix removes the index, table and tenant prefix from the provided
+// key. This function should be used instead of sqlDecoder.DecodeIndexPrefix, if
+// the user cannot instantiate a codec that operates on a single tenant.
+func StripIndexPrefix(key roachpb.Key) ([]byte, error) {
+	var err error
+	// StripTablePrefix automatically removes the tenant prefix as well.
+	key, err = StripTablePrefix(key)
+	if err != nil {
+		return nil, err
+	}
+	if encoding.PeekType(key) != encoding.Int {
+		return nil, errors.Errorf("invalid index key prefix: %q", key)
+	}
+	key, _, err = encoding.DecodeUvarintAscending(key)
+	return key, err
+}
+
 // SQLCodec provides methods for encoding SQL table keys bound to a given
 // tenant. The generator also provides methods for efficiently decoding keys
 // previously generated by it. The generated keys are safe to use indefinitely

pkg/kv/kvpb/api.proto

@@ -1696,7 +1696,13 @@ message ExportRequest {
 
   reserved 2, 5, 7, 8, 11;
 
-  // Next ID: 15
+  FingerprintOptions fingerprint_options = 15 [(gogoproto.nullable) = false];
+
+  // Next ID: 16
+}
+
+message FingerprintOptions {
+  bool strip_index_prefix_and_timestamp = 1;
 }
 
 // BulkOpSummary summarizes the data processed by an operation, counting the

pkg/kv/kvserver/batcheval/cmd_export.go

@@ -194,8 +194,23 @@ func evalExport(
 			// values before fingerprinting so that the fingerprint is tenant
 			// agnostic.
 			opts.FingerprintOptions = storage.MVCCExportFingerprintOptions{
-				StripTenantPrefix:  true,
-				StripValueChecksum: true,
+				StripTenantPrefix:            true,
+				StripValueChecksum:           true,
+				StripIndexPrefixAndTimestamp: args.FingerprintOptions.StripIndexPrefixAndTimestamp,
+			}
+			if opts.FingerprintOptions.StripIndexPrefixAndTimestamp && args.MVCCFilter == kvpb.MVCCFilter_All {
+				// If a key's value were updated from a to b, the xor hash without
+				// timestamps of those two mvcc values would look the same if the key
+				// were updated from b to a. In other words, the order of key value
+				// updates without timestamps does not affect the xor hash; but this
+				// order clearly presents different mvcc history, therefore, do not
+				// strip timestamps if fingerprinting all mvcc history.
+				return result.Result{}, errors.New("cannot fingerprint without mvcc timestamps and with mvcc history")
+			}
+			if opts.FingerprintOptions.StripIndexPrefixAndTimestamp && !args.StartTime.IsEmpty() {
+				// Supplying a startKey only complicates results (e.g. it surfaces
+				// tombstones), given that only the latest keys are surfaced.
+				return result.Result{}, errors.New("cannot fingerprint without mvcc timestamps and with a start time")
 			}
 			var hasRangeKeys bool
 			summary, resumeInfo, fingerprint, hasRangeKeys, err = storage.MVCCExportFingerprint(ctx,

pkg/sql/sem/builtins/BUILD.bazel

@@ -9,6 +9,7 @@ go_library(
         "all_builtins.go",
         "builtins.go",
         "compression.go",
+        "fingerprint_builtins.go",
         "fixed_oids.go",
         "generator_builtins.go",
         "generator_probe_ranges.go",