Merge #115447 #115462 #115593 #115595

115447: roachtest: indicate when version is too old in `UploadWorkload` r=DarrylWong,srosenberg a=renatolabs

In #115061, a `UploadWorkload` API was added to allow tests to upload "arbitrary" versions of the `workload` binary, especially useful in upgrade testing.

However, we only have `workload` binaries for v22.2+. In this commit, we add a `bool` return value to `UplodWorkload` indicating whether a binary was actually uploaded. Callers are expected to check this value before attempting to use a workload binary.

The `acceptance/version-upgrade` test is updated in this commit in the way described.

Epic: none

Release note: None

115462: storage: add cluster setting for ingest splits r=RaduBerinde a=itsbilal

Previously, the ingest-time splitting feature was only tunable through a code change. This change adds a cluster setting to allow this feature to be turned on dynamically.

Fixes #115430.

Epic: none

Release note: None

115593: ui: change some references to tenant to virtual cluster r=dhartunian a=stevendanna

This changes the "Tenant" selector on the metrics pages to use the new "Virtual Cluster" terminology.

Fixes #114170

Release note: None

<img width="1037" alt="Screenshot 2023-12-05 at 11 05 10" src="https://github.com/cockroachdb/cockroach/assets/852371/54f9ce9c-fc0b-4457-8214-de22c45b32bc">
<img width="1086" alt="Screenshot 2023-12-05 at 11 05 05" src="https://github.com/cockroachdb/cockroach/assets/852371/d0cb2f32-20a6-4522-8727-3c19ce0a91ab">


115595: server: support strict MemFS in StickyVFSRegistry r=jbowens a=pavelkalinnikov

The strict `MemFS` can be used for emulating crashes, and testing for data durability loss that they may cause if fsync is not used correctly.

Touches #113135, #114411
Epic: none
Release note: none

Co-authored-by: Renato Costa <[email protected]>
Co-authored-by: Bilal Akhtar <[email protected]>
Co-authored-by: Steven Danna <[email protected]>
Co-authored-by: Pavel Kalinnikov <[email protected]>

docs/generated/settings/settings.html

@@ -264,6 +264,7 @@
 <tr><td><div id="setting-sql-txn-read-committed-isolation-enabled" class="anchored"><code>sql.txn.read_committed_isolation.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>set to true to allow transactions to use the READ COMMITTED isolation level if specified by BEGIN/SET commands</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-txn-fingerprint-id-cache-capacity" class="anchored"><code>sql.txn_fingerprint_id_cache.capacity</code></div></td><td>integer</td><td><code>100</code></td><td>the maximum number of txn fingerprint IDs stored</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-storage-experimental-eventually-file-only-snapshots-enabled" class="anchored"><code>storage.experimental.eventually_file_only_snapshots.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>set to true to use eventually-file-only-snapshots even when kv.snapshot_receiver.excise.enabled is false</td><td>Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-storage-ingest-split-enabled" class="anchored"><code>storage.ingest_split.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>set to true to use ingest-time splitting to lower write-amplification (experimental)</td><td>Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-storage-max-sync-duration" class="anchored"><code>storage.max_sync_duration</code></div></td><td>duration</td><td><code>20s</code></td><td>maximum duration for disk operations; any operations that take longer than this setting trigger a warning log entry or process crash</td><td>Serverless/Dedicated/Self-Hosted (read-only)</td></tr>
 <tr><td><div id="setting-storage-max-sync-duration-fatal-enabled" class="anchored"><code>storage.max_sync_duration.fatal.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>if true, fatal the process when a disk operation exceeds storage.max_sync_duration</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-storage-value-blocks-enabled" class="anchored"><code>storage.value_blocks.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to true to enable writing of value blocks in sstables</td><td>Serverless/Dedicated/Self-Hosted</td></tr>

pkg/cmd/roachtest/roachtestutil/clusterupgrade/clusterupgrade.go

@@ -68,6 +68,12 @@ func (v *Version) IsCurrent() bool {
 	return v.Version.Compare(&CurrentVersion().Version) == 0
 }
 
+// AtLeast is a thin wrapper around `(*version.Version).AtLeast`,
+// allowing two `Version` objects to be compared directly.
+func (v *Version) AtLeast(other *Version) bool {
+	return v.Version.AtLeast(&other.Version)
+}
+
 // CurrentVersion returns the version associated with the current
 // build.
 func CurrentVersion() *Version {
@@ -124,7 +130,7 @@ func ClusterVersion(ctx context.Context, db *gosql.DB) (roachpb.Version, error)
 }
 
 // UploadCockroach stages the cockroach binary in the nodes.
-// Convenience function, see `UploadBinaryVersion` for more details.
+// Convenience function, see `uploadBinaryVersion` for more details.
 func UploadCockroach(
 	ctx context.Context,
 	t test.Test,
@@ -133,26 +139,38 @@ func UploadCockroach(
 	nodes option.NodeListOption,
 	v *Version,
 ) (string, error) {
-	return UploadBinaryVersion(ctx, t, l, "cockroach", c, nodes, v)
+	return uploadBinaryVersion(ctx, t, l, "cockroach", c, nodes, v)
 }
 
+// minWorkloadBinaryVersion is the minimum version for which we have
+// `workload` binaries available.
+var minWorkloadBinaryVersion = MustParseVersion("v22.2.0")
+
 // UploadWorkload stages the workload binary in the nodes.
-// Convenience function, see `UploadBinaryVersion` for more details.
+// Convenience function, see `uploadBinaryVersion` for more details.
+// The boolean return value indicates whether a workload binary was
+// uploaded to the nodes; a `false` value indicates that the version
+// passed is too old and no binary is available.
 func UploadWorkload(
 	ctx context.Context,
 	t test.Test,
 	l *logger.Logger,
 	c cluster.Cluster,
 	nodes option.NodeListOption,
 	v *Version,
-) (string, error) {
-	return UploadBinaryVersion(ctx, t, l, "workload", c, nodes, v)
+) (string, bool, error) {
+	if !v.AtLeast(minWorkloadBinaryVersion) {
+		return "", false, nil
+	}
+
+	path, err := uploadBinaryVersion(ctx, t, l, "workload", c, nodes, v)
+	return path, err == nil, err
 }
 
-// UploadBinaryVersion uploads the specified binary associated with
+// uploadBinaryVersion uploads the specified binary associated with
 // the given version to the given nodes. It returns the path of the
 // uploaded binaries on the nodes.
-func UploadBinaryVersion(
+func uploadBinaryVersion(
 	ctx context.Context,
 	t test.Test,
 	l *logger.Logger,

pkg/cmd/roachtest/roachtestutil/mixedversion/BUILD.bazel

@@ -27,7 +27,6 @@ go_library(
         "//pkg/util/randutil",
         "//pkg/util/syncutil",
         "//pkg/util/timeutil",
-        "//pkg/util/version",
         "@com_github_pkg_errors//:errors",
     ],
 )

pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go

@@ -89,7 +89,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/roachprod/vm"
 	"github.com/cockroachdb/cockroach/pkg/testutils/release"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
-	"github.com/cockroachdb/cockroach/pkg/util/version"
 	"github.com/pkg/errors"
 )
 
@@ -144,7 +143,7 @@ var (
 	// is a published ARM64 build. If we are running a mixedversion test
 	// on ARM64, older versions will be skipped even if the test
 	// requested a certain number of upgrades.
-	minSupportedARM64Version = version.MustParse("v22.2.0")
+	minSupportedARM64Version = clusterupgrade.MustParseVersion("v22.2.0")
 
 	defaultTestOptions = testOptions{
 		// We use fixtures more often than not as they are more likely to

pkg/cmd/roachtest/tests/versionupgrade.go

@@ -108,7 +108,7 @@ func runVersionUpgrade(ctx context.Context, t test.Test, c cluster.Cluster) {
 		"setup schema changer workload",
 		func(ctx context.Context, l *logger.Logger, rng *rand.Rand, h *mixedversion.Helper) error {
 			node := h.RandomNode(rng, c.All())
-			workloadPath, err := clusterupgrade.UploadWorkload(
+			workloadPath, _, err := clusterupgrade.UploadWorkload(
 				ctx, t, l, c, c.Node(node), h.Context.ToVersion,
 			)
 			if err != nil {
@@ -149,13 +149,18 @@ func runVersionUpgrade(ctx context.Context, t test.Test, c cluster.Cluster) {
 			// The schemachange workload is designed to work up to one
 			// version back. Therefore, we upload a compatible `workload`
 			// binary to `randomNode`, where the workload will run.
-			workloadPath, err := clusterupgrade.UploadWorkload(
+			workloadPath, uploaded, err := clusterupgrade.UploadWorkload(
 				ctx, t, l, c, c.Node(randomNode), h.Context.ToVersion,
 			)
 			if err != nil {
 				return errors.Wrap(err, "uploading workload binary")
 			}
 
+			if !uploaded {
+				l.Printf("Version being upgraded is too old, no workload binary available. Skipping")
+				return nil
+			}
+
 			l.Printf("running schemachange workload")
 			runCmd := roachtestutil.
 				NewCommand("%s run schemachange", workloadPath).

pkg/server/sticky_vfs.go

@@ -20,17 +20,22 @@ import (
 type StickyVFSOption func(cfg *stickyConfig)
 
 type stickyConfig struct {
-	// Preserved for anticipated future options, such as the ability to use
-	// Pebble's "strict" MemFS.
+	newFS func() *vfs.MemFS // by default vfs.NewMem
 }
 
+// UseStrictMemFS option instructs StickyVFSRegistry to produce strict in-memory
+// filesystems, i.e. to use vfs.NewStrictMem instead of vfs.NewMem.
+var UseStrictMemFS = StickyVFSOption(func(cfg *stickyConfig) {
+	cfg.newFS = vfs.NewStrictMem
+})
+
 // StickyVFSRegistry manages the lifecycle of sticky in-memory filesystems. It
 // is intended for use in demos and/or tests, where we want in-memory storage
 // nodes to persist between killed nodes.
 type StickyVFSRegistry interface {
 	// Get returns the named in-memory FS, constructing a new one if this is the
 	// first time a FS with the provided ID has been requested.
-	Get(stickyVFSID string) vfs.FS
+	Get(stickyVFSID string) *vfs.MemFS
 }
 
 // stickyVFSRegistryImpl is the bookkeeper for all active sticky filesystems,
@@ -49,18 +54,22 @@ func NewStickyVFSRegistry(opts ...StickyVFSOption) StickyVFSRegistry {
 	for _, opt := range opts {
 		opt(&registry.cfg)
 	}
+	// Use the regular in-memory filesystem, unless specified otherwise.
+	if registry.cfg.newFS == nil {
+		registry.cfg.newFS = vfs.NewMem
+	}
 	return registry
 }
 
 // Get implements the StickyVFSRegistry interface.
-func (registry *stickyVFSRegistryImpl) Get(stickyVFSID string) vfs.FS {
+func (registry *stickyVFSRegistryImpl) Get(stickyVFSID string) *vfs.MemFS {
 	registry.mu.Lock()
 	defer registry.mu.Unlock()
 
 	if fs, ok := registry.entries[stickyVFSID]; ok {
 		return fs
 	}
-	fs := vfs.NewMem()
+	fs := registry.cfg.newFS()
 	registry.entries[stickyVFSID] = fs
 	return fs
 }

pkg/storage/pebble.go

@@ -135,6 +135,21 @@ var UseExciseForSnapshots = settings.RegisterBoolSetting(
 	settings.WithPublic,
 )
 
+// IngestSplitEnabled controls whether ingest-time splitting is enabled in
+// Pebble. This feature allows for existing sstables to be split into multiple
+// virtual sstables at ingest time if that allows for an ingestion sstable to go
+// into a lower level than it would otherwise be in. No keys are masked with
+// this split; it only happens if there are no keys in that existing sstable
+// in the span of the incoming sstable.
+var IngestSplitEnabled = settings.RegisterBoolSetting(
+	settings.SystemOnly,
+	"storage.ingest_split.enabled",
+	"set to true to use ingest-time splitting to lower write-amplification (experimental)",
+	util.ConstantWithMetamorphicTestBool(
+		"storage.ingest_split.enabled", false), /* defaultValue */
+	settings.WithPublic,
+)
+
 // IngestAsFlushable controls whether ingested sstables that overlap the
 // memtable may be lazily ingested: written to the WAL and enqueued in the list
 // of flushables (eg, memtables, large batches and now lazily-ingested
@@ -1094,6 +1109,9 @@ func NewPebble(ctx context.Context, cfg PebbleConfig) (p *Pebble, err error) {
 	// See https://github.com/cockroachdb/pebble/issues/3120
 	// TODO(travers): Re-enable, once the issues are resolved.
 	opts.Experimental.MultiLevelCompactionHeuristic = pebble.NoMultiLevel{}
+	opts.Experimental.IngestSplit = func() bool {
+		return IngestSplitEnabled.Get(&cfg.Settings.SV)
+	}
 
 	auxDir := opts.FS.PathJoin(cfg.Dir, base.AuxiliaryDir)
 	if err := opts.FS.MkdirAll(auxDir, 0755); err != nil {

pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/index.tsx

@@ -413,7 +413,7 @@ export class NodeGraphs extends React.Component<
           {isSystemTenant(currentTenant) && tenantOptions.length > 1 && (
             <PageConfigItem>
               <Dropdown
-                title="Tenant"
+                title="Virtual Cluster"
                 options={tenantOptions}
                 selected={selectedTenant}
                 onChange={selection => this.setClusterPath("tenant", selection)}

pkg/ui/workspaces/db-console/src/views/reports/containers/customChart/customMetric.tsx

@@ -342,10 +342,10 @@ export class CustomChartTable extends React.Component<CustomChartTableProps> {
               <td className="metric-table__header">Source</td>
               <td className="metric-table__header">Per Node</td>
               {canViewTenantOptions && (
-                <td className="metric-table__header">Tenant</td>
+                <td className="metric-table__header">Virtual Cluster</td>
               )}
               {canViewTenantOptions && (
-                <td className="metric-table__header">Per Tenant</td>
+                <td className="metric-table__header">Per Virtual Cluster</td>
               )}
               <td className="metric-table__header"></td>
             </tr>