storage: Write sstables in v4 format for shared ingestions

We require sstables to be written with obsolete bits for shared sstable ingestion to work correctly. This change moves to writing sstables with the new format if the cluster version is high enough. Unblocks cockroachdb#107394. Epic: none Release note: None
itsbilal · Aug 17, 2023 · ee5cce5 · ee5cce5
1 parent 85f5583
commit ee5cce5
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 4 deletions.
diff --git a/pkg/kv/kvnemesis/testdata/TestApplier/addsstable b/pkg/kv/kvnemesis/testdata/TestApplier/addsstable
@@ -1,6 +1,6 @@
 echo
 ----
-db1.AddSSTable(ctx, tk(1), tk(4), ... /* @s1 */) // 1115 bytes (as writes)
+db1.AddSSTable(ctx, tk(1), tk(4), ... /* @s1 */) // 1144 bytes (as writes)
 // ^-- tk(1) -> sv(s1): /Table/100/"0000000000000001"/<ts> -> /BYTES/v1
 // ^-- tk(2) -> sv(s1): /Table/100/"0000000000000002"/<ts> -> /<empty>
 // ^-- [tk(3), tk(4)) -> sv(s1): /Table/100/"000000000000000{3"-4"} -> /<empty>
diff --git a/pkg/kv/kvnemesis/testdata/TestValidate/addsstable_ingestion b/pkg/kv/kvnemesis/testdata/TestValidate/addsstable_ingestion
@@ -1,6 +1,6 @@
 echo
 ----
-db0.AddSSTable(ctx, tk(1), tk(4), ... /* @s1 */) // 1115 bytes
+db0.AddSSTable(ctx, tk(1), tk(4), ... /* @s1 */) // 1144 bytes
 // ^-- tk(1) -> sv(s1): /Table/100/"0000000000000001"/0.000000001,0 -> /BYTES/v1
 // ^-- tk(2) -> sv(s1): /Table/100/"0000000000000002"/0.000000001,0 -> /<empty>
 // ^-- [tk(3), tk(4)) -> sv(s1): /Table/100/"000000000000000{3"-4"} -> /<empty>

diff --git a/pkg/kv/kvnemesis/testdata/TestValidate/addsstable_ingestion_shadowing_scan b/pkg/kv/kvnemesis/testdata/TestValidate/addsstable_ingestion_shadowing_scan
@@ -4,7 +4,7 @@ db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
 db0.Put(ctx, tk(2), sv(1)) // @0.000000001,0 <nil>
 db0.Put(ctx, tk(3), sv(1)) // @0.000000001,0 <nil>
 db0.Put(ctx, tk(4), sv(1)) // @0.000000001,0 <nil>
-db0.AddSSTable(ctx, tk(1), tk(4), ... /* @s2 */) // 1116 bytes
+db0.AddSSTable(ctx, tk(1), tk(4), ... /* @s2 */) // 1145 bytes
 // ^-- tk(1) -> sv(s2): /Table/100/"0000000000000001"/0.000000001,0 -> /BYTES/v2
 // ^-- tk(2) -> sv(s2): /Table/100/"0000000000000002"/0.000000001,0 -> /<empty>
 // ^-- [tk(3), tk(4)) -> sv(s2): /Table/100/"000000000000000{3"-4"} -> /<empty>

diff --git a/pkg/storage/sst_writer.go b/pkg/storage/sst_writer.go
@@ -74,6 +74,9 @@ func MakeIngestionWriterOptions(ctx context.Context, cs *cluster.Settings) sstab
 		ValueBlocksEnabled.Get(&cs.SV) {
 		format = sstable.TableFormatPebblev3
 	}
+	if cs.Version.IsActive(ctx, clusterversion.V23_2_PebbleFormatVirtualSSTables) && ValueBlocksEnabled.Get(&cs.SV) {
+		format = sstable.TableFormatPebblev4
+	}
 	opts := DefaultPebbleOptions().MakeWriterOptions(0, format)
 	opts.MergerName = "nullptr"
 	return opts

diff --git a/pkg/storage/sst_writer_test.go b/pkg/storage/sst_writer_test.go
@@ -95,12 +95,25 @@ func TestMakeIngestionWriterOptions(t *testing.T) {
 		{
 			name: "with value blocks",
 			st: func() *cluster.Settings {
-				st := cluster.MakeTestingClusterSettings()
+				st := cluster.MakeTestingClusterSettingsWithVersions(
+					clusterversion.ByKey(clusterversion.V23_1EnablePebbleFormatSSTableValueBlocks),
+					clusterversion.TestingBinaryMinSupportedVersion,
+					true,
+				)
 				ValueBlocksEnabled.Override(context.Background(), &st.SV, true)
 				return st
 			}(),
 			want: sstable.TableFormatPebblev3,
 		},
+		{
+			name: "with virtual sstables",
+			st: func() *cluster.Settings {
+				st := cluster.MakeTestingClusterSettings()
+				ValueBlocksEnabled.Override(context.Background(), &st.SV, true)
+				return st
+			}(),
+			want: sstable.TableFormatPebblev4,
+		},
 	}
 
 	for _, tc := range testCases {