db: add ViaBackingFileDownload option to DownloadSpan

compaction.go

@@ -1515,6 +1515,8 @@ type downloadSpan struct {
 	// downloadSpan. Must be equal to len(doneChans)-1, i.e. there's one spare
 	// doneChan created each time a compaction starts up, for the next compaction.
 	compactionsStarted int
+
+	kind compactionKind
 }
 
 func (d *DB) addInProgressCompaction(c *compaction) {

compaction_picker.go

@@ -1795,6 +1795,9 @@ func pickManualCompaction(
 	return pc, false
 }
 
+// pickDownloadCompaction picks a download compaction for the downloadSpan,
+// which could be specified as being performed either by a copy compaction of
+// the backing file or a rewrite compaction.
 func pickDownloadCompaction(
 	vers *version,
 	opts *Options,
@@ -1809,8 +1812,11 @@ func pickDownloadCompaction(
 	if file.CompactionState == manifest.CompactionStateCompacting {
 		return nil
 	}
+	if download.kind != compactionKindCopy && download.kind != compactionKindRewrite {
+		panic("invalid download/rewrite compaction kind")
+	}
 	pc = newPickedCompaction(opts, vers, level, level, baseLevel)
-	pc.kind = compactionKindRewrite
+	pc.kind = download.kind
 	pc.startLevel.files = manifest.NewLevelSliceKeySorted(opts.Comparer.Compare, []*fileMetadata{file})
 	if !pc.setupInputs(opts, env.diskAvailBytes, pc.startLevel) {
 		// setupInputs returned false indicating there's a conflicting

db.go

@@ -1888,6 +1888,20 @@ type DownloadSpan struct {
 	StartKey []byte
 	// EndKey is exclusive.
 	EndKey []byte
+	// ViaBackingFileDownload, if true, indicates the span should be downloaded by
+	// downloading any remote backing files byte-for-byte and replacing them with
+	// the downloaded local files, while otherwise leaving the virtual SSTables
+	// as-is. If false, a "normal" rewriting compaction of the span, that iterates
+	// the keys and produces a new SSTable, is used instead. Downloading raw files
+	// can be faster when the whole file is being downloaded, as it avoids some
+	// cpu-intensive steps involved in iteration and new file construction such as
+	// compression, however it can also be wasteful when only a small portion of a
+	// larger backing file is being used by a virtual file. Additionally, if the
+	// virtual file has expensive read-time transformations, such as prefix
+	// replacement, rewriting once can persist the result of these for future use
+	// while copying only the backing file will obligate future reads to continue
+	// to compute such transforms.
+	ViaBackingFileDownload bool
 }
 
 func (d *DB) downloadSpan(ctx context.Context, span DownloadSpan) error {
@@ -1896,6 +1910,10 @@ func (d *DB) downloadSpan(ctx context.Context, span DownloadSpan) error {
 		end:   span.EndKey,
 		// Protected by d.mu.
 		doneChans: make([]chan error, 1),
+		kind:      compactionKindRewrite,
+	}
+	if span.ViaBackingFileDownload {
+		dSpan.kind = compactionKindCopy
 	}
 	dSpan.doneChans[0] = make(chan error, 1)
 	doneChan := dSpan.doneChans[0]