diff --git a/pkg/cli/cliflags/flags.go b/pkg/cli/cliflags/flags.go index e92aac3db1fb..3b5b904ff72a 100644 --- a/pkg/cli/cliflags/flags.go +++ b/pkg/cli/cliflags/flags.go @@ -1295,6 +1295,12 @@ and the system tenant using the \connect command.`, If set, disable enterprise features.`, } + DemoEnableRangefeeds = FlagInfo{ + Name: "auto-enable-rangefeeds", + Description: ` +If set to false, overrides the default demo behavior of enabling rangefeeds.`, + } + UseEmptyDatabase = FlagInfo{ Name: "empty", Description: `Deprecated in favor of --no-example-database`, diff --git a/pkg/cli/context.go b/pkg/cli/context.go index 78915b506768..efee9661293f 100644 --- a/pkg/cli/context.go +++ b/pkg/cli/context.go @@ -611,6 +611,7 @@ func setDemoContextDefaults() { demoCtx.HTTPPort, _ = strconv.Atoi(base.DefaultHTTPPort) demoCtx.WorkloadMaxQPS = 25 demoCtx.Multitenant = true + demoCtx.DefaultEnableRangefeeds = true demoCtx.disableEnterpriseFeatures = false } diff --git a/pkg/cli/demo.go b/pkg/cli/demo.go index b9f6ec7eb7d7..2faf3cd224a4 100644 --- a/pkg/cli/demo.go +++ b/pkg/cli/demo.go @@ -252,6 +252,12 @@ func runDemo(cmd *cobra.Command, gen workload.Generator) (resErr error) { } sqlCtx.ShellCtx.DemoCluster = c + if demoCtx.DefaultEnableRangefeeds { + if err = c.SetClusterSetting(ctx, "kv.rangefeed.enabled", true); err != nil { + return clierrorplus.CheckAndMaybeShout(err) + } + } + if cliCtx.IsInteractive { cliCtx.PrintfUnlessEmbedded(`# # Welcome to the CockroachDB demo database! diff --git a/pkg/cli/democluster/api.go b/pkg/cli/democluster/api.go index bcc866923977..5c01da0facbe 100644 --- a/pkg/cli/democluster/api.go +++ b/pkg/cli/democluster/api.go @@ -51,6 +51,10 @@ type DemoCluster interface { // SetupWorkload initializes the workload generator if defined. SetupWorkload(ctx context.Context) error + + // SetClusterSetting overrides a default cluster setting at system level + // and for all tenants. + SetClusterSetting(ctx context.Context, setting string, value interface{}) error } // EnableEnterprise is not implemented here in order to keep OSS/BSL builds successful. diff --git a/pkg/cli/democluster/context.go b/pkg/cli/democluster/context.go index 5ae71566c0ef..3314c9cf8906 100644 --- a/pkg/cli/democluster/context.go +++ b/pkg/cli/democluster/context.go @@ -96,6 +96,10 @@ type Context struct { // Multitenant is true if we're starting the demo cluster in // multi-tenant mode. Multitenant bool + + // DefaultEnableRangefeeds is true if rangefeeds should start + // out enabled. + DefaultEnableRangefeeds bool } // IsInteractive returns true if the demo cluster configuration diff --git a/pkg/cli/democluster/demo_cluster.go b/pkg/cli/democluster/demo_cluster.go index f7bf611fe925..8089fe45d1a5 100644 --- a/pkg/cli/democluster/demo_cluster.go +++ b/pkg/cli/democluster/demo_cluster.go @@ -1217,6 +1217,28 @@ func (c *transientCluster) maybeEnableMultiTenantMultiRegion(ctx context.Context return nil } +func (c *transientCluster) SetClusterSetting( + ctx context.Context, setting string, value interface{}, +) error { + storageURL, err := c.getNetworkURLForServer(ctx, 0, false /* includeAppName */, false /* isTenant */) + if err != nil { + return err + } + db, err := gosql.Open("postgres", storageURL.ToPQ().String()) + if err != nil { + return err + } + defer db.Close() + _, err = db.Exec(fmt.Sprintf("SET CLUSTER SETTING %s = '%v'", setting, value)) + if err != nil { + return err + } + if c.demoCtx.Multitenant { + _, err = db.Exec(fmt.Sprintf("ALTER TENANT ALL SET CLUSTER SETTING %s = '%v'", setting, value)) + } + return err +} + func (c *transientCluster) SetupWorkload(ctx context.Context) error { if err := c.maybeEnableMultiTenantMultiRegion(ctx); err != nil { return err diff --git a/pkg/cli/flags.go b/pkg/cli/flags.go index 34c7df290f65..e2b9c59dfde9 100644 --- a/pkg/cli/flags.go +++ b/pkg/cli/flags.go @@ -758,6 +758,7 @@ func init() { "For details, see: "+build.MakeIssueURL(53404)) cliflagcfg.BoolFlag(f, &demoCtx.disableEnterpriseFeatures, cliflags.DemoNoLicense) + cliflagcfg.BoolFlag(f, &demoCtx.DefaultEnableRangefeeds, cliflags.DemoEnableRangefeeds) cliflagcfg.BoolFlag(f, &demoCtx.Multitenant, cliflags.DemoMultitenant) // TODO(knz): Currently the multitenant UX for 'demo' is not diff --git a/pkg/cli/interactive_tests/test_demo_changefeeds.tcl b/pkg/cli/interactive_tests/test_demo_changefeeds.tcl new file mode 100644 index 000000000000..0ebccf35ee9b --- /dev/null +++ b/pkg/cli/interactive_tests/test_demo_changefeeds.tcl @@ -0,0 +1,42 @@ +#! /usr/bin/env expect -f + +source [file join [file dirname $argv0] common.tcl] + +start_test "Demo core changefeed using format=csv" +spawn $argv demo --format=csv + +# We should start in a populated database. +eexpect "movr>" + +# initial_scan=only prevents the changefeed from hanging waiting for more changes. +send "CREATE CHANGEFEED FOR users WITH initial_scan='only';\r" + +# header for the results of a successful changefeed +eexpect "table,key,value" + +# Statement execution time after the initial scan completes +eexpect "Time:" + +eexpect "movr>" +send_eof +eexpect eof + +end_test + +start_test "Demo with rangefeeds disabled as they are in real life" +spawn $argv demo --format=csv --auto-enable-rangefeeds=false + +# We should start in a populated database. +eexpect "movr>" + +# initial_scan=only prevents the changefeed from hanging waiting for more changes. +send "CREATE CHANGEFEED FOR users WITH initial_scan='only';\r" + +# changefeed should fail fast with an informative error. +eexpect "ERROR: rangefeeds require the kv.rangefeed.enabled setting." + +eexpect "movr>" +send_eof +eexpect eof + +end_test diff --git a/pkg/roachpb/api.proto b/pkg/roachpb/api.proto index 56458248d085..10e2768da420 100644 --- a/pkg/roachpb/api.proto +++ b/pkg/roachpb/api.proto @@ -1529,6 +1529,15 @@ message ExportRequest { // large history being broken up into target_file_size chunks and prevent // blowing up on memory usage. This option is only allowed together with // return_sst since caller should reconstruct full tables. + // + // NB: If the result contains MVCC range tombstones, this can cause MVCC range + // tombstones in two subsequent SSTs to overlap. For example, given the range + // tombstone [a-f)@5, if we return a resume key at c@3 then the response will + // contain a truncated MVCC range tombstone [a-c\0)@5 which covers the point + // keys at c, but resuming from c@3 will contain the MVCC range tombstone + // [c-f)@5 which overlaps with the MVCC range tombstone in the previous + // response for the interval [c-c\0)@5. AddSSTable will allow this overlap + // during ingestion. bool split_mid_key = 13; // Return the exported SST data in the response. @@ -1583,7 +1592,8 @@ message BulkOpSummary { reserved 4; // EntryCounts contains the number of keys processed for each tableID/indexID // pair, stored under the key (tableID << 32) | indexID. This EntryCount key - // generation logic is also available in the BulkOpSummaryID helper. + // generation logic is also available in the BulkOpSummaryID helper. It does + // not take MVCC range tombstones into account. map entry_counts = 5; } diff --git a/pkg/storage/mvcc.go b/pkg/storage/mvcc.go index 141d0cc406be..5998bdcf9146 100644 --- a/pkg/storage/mvcc.go +++ b/pkg/storage/mvcc.go @@ -4210,17 +4210,17 @@ func ComputeStatsForRange( } // MVCCExportToSST exports changes to the keyrange [StartKey, EndKey) over the -// interval (StartTS, EndTS] as a Pebble SST. Deletions are included if all -// revisions are requested or if the StartTS is non-zero. This function looks at -// MVCC versions and intents, and returns an error if an intent is found. -// MVCCExportOptions determine ranges as well as additional export options, see -// struct definition for details. +// interval (StartTS, EndTS] as a Pebble SST. See MVCCExportOptions for options. // -// Data is written to dest as it is collected. If an error is returned then -// dest contents are undefined. +// Tombstones are included if all revisions are requested (all tombstones) or if +// the StartTS is non-zero (latest tombstone), including both MVCC point +// tombstones and MVCC range tombstones. Intents within the time interval will +// return a WriteIntentError, while intents outside the time interval are +// ignored. // // Returns an export summary and a resume key that allows resuming the export if -// it reached a limit. +// it reached a limit. Data is written to dest as it is collected. If an error +// is returned then dest contents are undefined. func MVCCExportToSST( ctx context.Context, cs *cluster.Settings, reader Reader, opts MVCCExportOptions, dest io.Writer, ) (roachpb.BulkOpSummary, MVCCKey, error) { @@ -4230,39 +4230,79 @@ func MVCCExportToSST( sstWriter := MakeBackupSSTWriter(ctx, cs, dest) defer sstWriter.Close() - var rows RowCounter - iter := NewMVCCIncrementalIterator( - reader, - MVCCIncrementalIterOptions{ - EndKey: opts.EndKey, - StartTime: opts.StartTS, - EndTime: opts.EndTS, - IntentPolicy: MVCCIncrementalIterIntentPolicyAggregate, - }) + // If we're not exporting all revisions then we can mask point keys below any + // MVCC range tombstones, since we don't care about them. + var rangeKeyMasking hlc.Timestamp + if !opts.ExportAllRevisions { + rangeKeyMasking = opts.EndTS + } + + iter := NewMVCCIncrementalIterator(reader, MVCCIncrementalIterOptions{ + KeyTypes: IterKeyTypePointsAndRanges, + StartKey: opts.StartKey.Key, + EndKey: opts.EndKey, + StartTime: opts.StartTS, + EndTime: opts.EndTS, + RangeKeyMaskingBelow: rangeKeyMasking, + IntentPolicy: MVCCIncrementalIterIntentPolicyAggregate, + }) defer iter.Close() - var curKey roachpb.Key // only used if exportAllRevisions - var resumeKey roachpb.Key - var resumeTS hlc.Timestamp + paginated := opts.TargetSize > 0 trackKeyBoundary := paginated || opts.ResourceLimiter != nil firstIteration := true - for iter.SeekGE(opts.StartKey); ; { - ok, err := iter.Valid() - if err != nil { - return roachpb.BulkOpSummary{}, MVCCKey{}, err - } - if !ok { - break + skipTombstones := !opts.ExportAllRevisions && opts.StartTS.IsEmpty() + + var rows RowCounter + var curKey roachpb.Key // only used if exportAllRevisions + var resumeKey MVCCKey + var rangeKeys []MVCCRangeKeyValue + var rangeKeysEnd roachpb.Key + var rangeKeysSize int64 + + // maxRangeKeysSizeIfTruncated calculates the worst-case size of the currently + // buffered rangeKeys if we were to stop iteration after the current resumeKey + // and flush the pending range keys with the new truncation bound given by + // resumeKey. + // + // For example, if we've buffered the range keys [a-c)@2 and [a-c)@1 with + // total size 4 bytes, and then hit a byte limit between the point keys bbb@3 + // and bbb@1, we have to truncate the two range keys to [a-bbb\0)@1 and + // [a-bbb\0)@2. The size of the flushed range keys is now 10 bytes, not 4 + // bytes. Since we're never allowed to exceed MaxSize, we have to check before + // adding bbb@3 that we have room for both bbb@3 and the pending range keys if + // they were to be truncated and flushed after it. + // + // This could either truncate the range keys at resumeKey, at resumeKey.Next() + // if StopMidKey is enabled, or at the range keys' actual end key if there + // doesn't end up being any further point keys covered by it and we go on to + // flush them as-is at their normal end key. We need to make sure we have + // enough MaxSize budget to flush them in all of these cases. + maxRangeKeysSizeIfTruncated := func(resumeKey roachpb.Key) int64 { + if rangeKeysSize == 0 { + return 0 } - unsafeKey := iter.UnsafeKey() - if unsafeKey.Key.Compare(opts.EndKey) >= 0 { - break + // We could be truncated in the middle of a point key version series, which + // would require adding on a \0 byte via Key.Next(), so let's assume that. + maxSize := rangeKeysSize + if s := maxSize + int64(len(rangeKeys)*(len(resumeKey)-len(rangeKeysEnd)+1)); s > maxSize { + maxSize = s } + return maxSize + } - if iter.NumCollectedIntents() > 0 { + iter.SeekGE(opts.StartKey) + for { + if ok, err := iter.Valid(); err != nil { + return roachpb.BulkOpSummary{}, MVCCKey{}, err + } else if !ok { + break + } else if iter.NumCollectedIntents() > 0 { break } + unsafeKey := iter.UnsafeKey() + isNewKey := !opts.ExportAllRevisions || !unsafeKey.Key.Equal(curKey) if trackKeyBoundary && opts.ExportAllRevisions && isNewKey { curKey = append(curKey[:0], unsafeKey.Key...) @@ -4286,15 +4326,93 @@ func MVCCExportToSST( // split is allowed. if limit >= ResourceLimitReachedSoft && isNewKey || limit == ResourceLimitReachedHard && opts.StopMidKey { // Reached iteration limit, stop with resume span - resumeKey = append(make(roachpb.Key, 0, len(unsafeKey.Key)), unsafeKey.Key...) - if !isNewKey { - resumeTS = unsafeKey.Timestamp + resumeKey = unsafeKey.Clone() + if isNewKey { + resumeKey.Timestamp = hlc.Timestamp{} } break } } } + // When we encounter an MVCC range tombstone stack, we buffer it in + // rangeKeys until we've moved past it or iteration ends (e.g. due to a + // limit). If we return a resume key then we need to truncate the final + // range key stack (and thus the SST) to the resume key, so we can't flush + // them until we've moved past. + hasPoint, hasRange := iter.HasPointAndRange() + + // First, flush any pending range tombstones when we reach their end key. + // + // TODO(erikgrinaker): Optimize the range key case here, to avoid these + // comparisons. Pebble should expose an API to cheaply detect range key + // changes. + if len(rangeKeysEnd) > 0 && bytes.Compare(unsafeKey.Key, rangeKeysEnd) >= 0 { + for _, rkv := range rangeKeys { + mvccValue, ok, err := tryDecodeSimpleMVCCValue(rkv.Value) + if !ok && err == nil { + mvccValue, err = decodeExtendedMVCCValue(rkv.Value) + } + if err != nil { + return roachpb.BulkOpSummary{}, MVCCKey{}, errors.Wrapf(err, + "decoding mvcc value %s", rkv.Value) + } + // Export only the inner roachpb.Value, not the MVCCValue header. + mvccValue = MVCCValue{Value: mvccValue.Value} + if err := sstWriter.ExperimentalPutMVCCRangeKey(rkv.RangeKey, mvccValue); err != nil { + return roachpb.BulkOpSummary{}, MVCCKey{}, err + } + } + rows.BulkOpSummary.DataSize += rangeKeysSize + rangeKeys, rangeKeysEnd, rangeKeysSize = rangeKeys[:0], nil, 0 + } + + // If we find any new range keys and we haven't buffered any range keys yet, + // buffer them. + if hasRange && !skipTombstones && len(rangeKeys) == 0 { + rangeBounds := iter.RangeBounds() + rangeKeysEnd = append(rangeKeysEnd[:0], rangeBounds.EndKey...) + + for _, rkv := range iter.RangeKeys() { + rangeKeys = append(rangeKeys, rkv.Clone()) + rangeKeysSize += int64(len(rkv.RangeKey.StartKey) + len(rkv.RangeKey.EndKey) + len(rkv.Value)) + if !opts.ExportAllRevisions { + break + } + } + + // Check if the range keys exceed a limit, using similar logic as point + // keys. We have to check both the size of the range keys as they are (in + // case we emit them as-is), and the size of the range keys if they were + // to be truncated at the start key due to a resume span (which could + // happen if the next point key exceeds the max size). + // + // TODO(erikgrinaker): The limit logic here is a bit of a mess, but we're + // complying with the existing point key logic for now. We should get rid + // of some of the options and clean this up. + curSize := rows.BulkOpSummary.DataSize + reachedTargetSize := opts.TargetSize > 0 && uint64(curSize) >= opts.TargetSize + newSize := curSize + maxRangeKeysSizeIfTruncated(rangeBounds.Key) + reachedMaxSize := opts.MaxSize > 0 && newSize > int64(opts.MaxSize) + if paginated && (reachedTargetSize || reachedMaxSize) { + rangeKeys, rangeKeysEnd, rangeKeysSize = rangeKeys[:0], nil, 0 + resumeKey = unsafeKey.Clone() + break + } + if reachedMaxSize { + return roachpb.BulkOpSummary{}, MVCCKey{}, &ExceedMaxSizeError{ + reached: newSize, maxSize: opts.MaxSize} + } + } + + // If we're on a bare range key, step forward. We can't use NextKey() + // because there may be a point key version at the range key's start bound. + if !hasPoint { + iter.Next() + continue + } + + // Process point keys. unsafeValue := iter.UnsafeValue() skip := false if unsafeKey.IsValue() { @@ -4311,7 +4429,6 @@ func MVCCExportToSST( // Skip tombstone records when start time is zero (non-incremental) // and we are not exporting all versions. - skipTombstones := !opts.ExportAllRevisions && opts.StartTS.IsEmpty() skip = skipTombstones && mvccValue.IsTombstone() } @@ -4320,22 +4437,26 @@ func MVCCExportToSST( return roachpb.BulkOpSummary{}, MVCCKey{}, errors.Wrapf(err, "decoding %s", unsafeKey) } curSize := rows.BulkOpSummary.DataSize - reachedTargetSize := curSize > 0 && uint64(curSize) >= opts.TargetSize - newSize := curSize + int64(len(unsafeKey.Key)+len(unsafeValue)) - reachedMaxSize := opts.MaxSize > 0 && newSize > int64(opts.MaxSize) + curSizeWithRangeKeys := curSize + maxRangeKeysSizeIfTruncated(unsafeKey.Key) + reachedTargetSize := curSizeWithRangeKeys > 0 && + uint64(curSizeWithRangeKeys) >= opts.TargetSize + kvSize := int64(len(unsafeKey.Key) + len(unsafeValue)) + newSize := curSize + kvSize + newSizeWithRangeKeys := curSizeWithRangeKeys + kvSize + reachedMaxSize := opts.MaxSize > 0 && newSizeWithRangeKeys > int64(opts.MaxSize) // When paginating we stop writing in two cases: // - target size is reached and we wrote all versions of a key // - maximum size reached and we are allowed to stop mid key if paginated && (isNewKey && reachedTargetSize || opts.StopMidKey && reachedMaxSize) { - // Allocate the right size for resumeKey rather than using curKey. - resumeKey = append(make(roachpb.Key, 0, len(unsafeKey.Key)), unsafeKey.Key...) - if opts.StopMidKey && !isNewKey { - resumeTS = unsafeKey.Timestamp + resumeKey = unsafeKey.Clone() + if isNewKey || !opts.StopMidKey { + resumeKey.Timestamp = hlc.Timestamp{} } break } if reachedMaxSize { - return roachpb.BulkOpSummary{}, MVCCKey{}, &ExceedMaxSizeError{reached: newSize, maxSize: opts.MaxSize} + return roachpb.BulkOpSummary{}, MVCCKey{}, &ExceedMaxSizeError{ + reached: newSizeWithRangeKeys, maxSize: opts.MaxSize} } if unsafeKey.Timestamp.IsEmpty() { // This should never be an intent since the incremental iterator returns @@ -4375,6 +4496,41 @@ func MVCCExportToSST( return roachpb.BulkOpSummary{}, MVCCKey{}, err } + // Flush any pending buffered range keys, truncated to the resume key (if + // any). If there is a resume timestamp, i.e. when resuming in between two + // versions, the range keys must cover the resume key too. This will cause the + // next export's range keys to overlap with this one, e.g.: [a-f) with resume + // key c@7 will export range keys [a-c\0) first, and then [c-f) when resuming, + // which overlaps at [c-c\0). + if len(rangeKeys) > 0 { + // Calculate the new rangeKeysSize due to the new resume bounds. + if len(resumeKey.Key) > 0 && rangeKeysEnd.Compare(resumeKey.Key) > 0 { + oldEndLen := len(rangeKeysEnd) + rangeKeysEnd = resumeKey.Key + if resumeKey.Timestamp.IsSet() { + rangeKeysEnd = rangeKeysEnd.Next() + } + rangeKeysSize += int64(len(rangeKeys) * (len(rangeKeysEnd) - oldEndLen)) + } + for _, rkv := range rangeKeys { + rkv.RangeKey.EndKey = rangeKeysEnd + mvccValue, ok, err := tryDecodeSimpleMVCCValue(rkv.Value) + if !ok && err == nil { + mvccValue, err = decodeExtendedMVCCValue(rkv.Value) + } + if err != nil { + return roachpb.BulkOpSummary{}, MVCCKey{}, errors.Wrapf(err, + "decoding mvcc value %s", rkv.Value) + } + // Export only the inner roachpb.Value, not the MVCCValue header. + mvccValue = MVCCValue{Value: mvccValue.Value} + if err := sstWriter.ExperimentalPutMVCCRangeKey(rkv.RangeKey, mvccValue); err != nil { + return roachpb.BulkOpSummary{}, MVCCKey{}, err + } + } + rows.BulkOpSummary.DataSize += rangeKeysSize + } + if rows.BulkOpSummary.DataSize == 0 { // If no records were added to the sstable, skip completing it and return a // nil slice – the export code will discard it anyway (based on 0 DataSize). @@ -4385,7 +4541,7 @@ func MVCCExportToSST( return roachpb.BulkOpSummary{}, MVCCKey{}, err } - return rows.BulkOpSummary, MVCCKey{Key: resumeKey, Timestamp: resumeTS}, nil + return rows.BulkOpSummary, resumeKey, nil } // MVCCExportOptions contains options for MVCCExportToSST. @@ -4423,6 +4579,15 @@ type MVCCExportOptions struct { // adding all versions until it reaches next key or end of range. If true, it // would stop immediately when targetSize is reached and return the next versions // timestamp in resumeTs so that subsequent operation can pass it to firstKeyTs. + // + // NB: If the result contains MVCC range tombstones, this can cause MVCC range + // tombstones in two subsequent SSTs to overlap. For example, given the range + // tombstone [a-f)@5, if we return a resume key at c@3 then the response will + // contain a truncated MVCC range tombstone [a-c\0)@5 which covers the point + // keys at c, but resuming from c@3 will contain the MVCC range tombstone + // [c-f)@5 which overlaps with the MVCC range tombstone in the previous + // response for the interval [c-c\0)@5. AddSSTable will allow this overlap + // during ingestion. StopMidKey bool // ResourceLimiter limits how long iterator could run until it exhausts allocated // resources. Export queries limiter in its iteration loop to break out once diff --git a/pkg/storage/mvcc_history_test.go b/pkg/storage/mvcc_history_test.go index acb35d2cb351..edcae280f1e9 100644 --- a/pkg/storage/mvcc_history_test.go +++ b/pkg/storage/mvcc_history_test.go @@ -77,6 +77,7 @@ var sstIterVerify = util.ConstantWithMetamorphicTestBool("mvcc-histories-sst-ite // put_rangekey ts=[,] [localTs=[,]] k= end= // get [t=] [ts=[,]] [resolve [status=]] k= [inconsistent] [tombstones] [failOnMoreRecent] [localUncertaintyLimit=[,]] [globalUncertaintyLimit=[,]] // scan [t=] [ts=[,]] [resolve [status=]] k= [end=] [inconsistent] [tombstones] [reverse] [failOnMoreRecent] [localUncertaintyLimit=[,]] [globalUncertaintyLimit=[,]] [max=] [targetbytes=] [avoidExcess] [allowEmpty] +// export [k=] [end=] [ts=[,]] [kTs=[,]] [startTs=[,]] [maxIntents=] [allRevisions] [targetSize=] [maxSize=] [stopMidKey] // // iter_new [k=] [end=] [prefix] [kind=key|keyAndIntents] [types=pointsOnly|pointsWithRanges|pointsAndRanges|rangesOnly] [pointSynthesis [emitOnSeekGE]] [maskBelow=[,]] // iter_new_incremental [k=] [end=] [startTs=[,]] [endTs=[,]] [types=pointsOnly|pointsWithRanges|pointsAndRanges|rangesOnly] [maskBelow=[,]] [intents=error|aggregate|emit] @@ -624,6 +625,7 @@ var commands = map[string]cmd{ "del": {typDataUpdate, cmdDelete}, "del_range": {typDataUpdate, cmdDeleteRange}, "del_range_ts": {typDataUpdate, cmdDeleteRangeTombstone}, + "export": {typReadOnly, cmdExport}, "get": {typReadOnly, cmdGet}, "increment": {typDataUpdate, cmdIncrement}, "initput": {typDataUpdate, cmdInitPut}, @@ -1062,6 +1064,83 @@ func cmdPut(e *evalCtx) error { }) } +func cmdExport(e *evalCtx) error { + key, endKey := e.getKeyRange() + opts := MVCCExportOptions{ + StartKey: MVCCKey{Key: key, Timestamp: e.getTsWithName("kTs")}, + EndKey: endKey, + StartTS: e.getTsWithName("startTs"), + EndTS: e.getTs(nil), + ExportAllRevisions: e.hasArg("allRevisions"), + StopMidKey: e.hasArg("stopMidKey"), + } + if e.hasArg("maxIntents") { + e.scanArg("maxIntents", &opts.MaxIntents) + } + if e.hasArg("targetSize") { + e.scanArg("targetSize", &opts.TargetSize) + } + if e.hasArg("maxSize") { + e.scanArg("maxSize", &opts.MaxSize) + } + + sstFile := &MemFile{} + summary, resume, err := MVCCExportToSST(e.ctx, e.st, e.engine, opts, sstFile) + if err != nil { + return err + } + + e.results.buf.Printf("export: %s", &summary) + if resume.Key != nil { + e.results.buf.Printf(" resume=%s", resume) + } + e.results.buf.Printf("\n") + + iter, err := NewPebbleMemSSTIterator(sstFile.Bytes(), false /* verify */) + if err != nil { + return err + } + defer iter.Close() + + var rangeStart roachpb.Key + for iter.SeekGE(NilKey); ; iter.Next() { + if ok, err := iter.Valid(); err != nil { + return err + } else if !ok { + break + } + hasPoint, hasRange := iter.HasPointAndRange() + if hasRange { + if rangeBounds := iter.RangeBounds(); !rangeBounds.Key.Equal(rangeStart) { + rangeStart = append(rangeStart[:0], rangeBounds.Key...) + e.results.buf.Printf("export: %s/[", rangeBounds) + for i, rangeKV := range iter.RangeKeys() { + val, err := DecodeMVCCValue(rangeKV.Value) + if err != nil { + return err + } + if i > 0 { + e.results.buf.Printf(" ") + } + e.results.buf.Printf("%s=%s", rangeKV.RangeKey.Timestamp, val) + } + e.results.buf.Printf("]\n") + } + } + if hasPoint { + key := iter.UnsafeKey() + value := iter.UnsafeValue() + mvccValue, err := DecodeMVCCValue(value) + if err != nil { + return err + } + e.results.buf.Printf("export: %v -> %s\n", key, mvccValue) + } + } + + return nil +} + func cmdScan(e *evalCtx) error { txn := e.getTxn(optional) key, endKey := e.getKeyRange() diff --git a/pkg/storage/testdata/mvcc_histories/export b/pkg/storage/testdata/mvcc_histories/export new file mode 100644 index 000000000000..13a98cf77157 --- /dev/null +++ b/pkg/storage/testdata/mvcc_histories/export @@ -0,0 +1,737 @@ +# Tests MVCC export. +# +# Sets up the following dataset, where x is MVCC point tombstone, o-o is MVCC +# range tombstone, [] is intent. We include some local timestamps, which should +# not be exported. +# +# 7 [a7] [d7] [j7] [l7] [o7] +# 6 f6 +# 5 o---------------o k5 +# 4 x x d4 f4 g4 x +# 3 o-------o e3 o-------oh3 o---o +# 2 a2 f2 g2 +# 1 o---------------------------------------o +# a b c d e f g h i j k l m n o + +run ok +put_rangekey k=a end=k ts=1 +put k=a ts=2 v=a2 +del k=a ts=4 +put_rangekey k=b end=d ts=3 +del k=b ts=4 +put k=d ts=4 v=d4 +put k=e ts=3 v=e3 localTs=2 +put k=f ts=2 v=f2 +put k=g ts=2 v=g2 +put_rangekey k=f end=h ts=3 +put k=f ts=4 v=f4 +put k=g ts=4 v=g4 +put_rangekey k=c end=g ts=5 localTs=4 +put k=f ts=6 v=f6 +put k=h ts=3 v=h3 +del k=h ts=4 +put k=k ts=5 v=k5 localTs=4 +put_rangekey k=m end=n ts=3 localTs=2 +with t=A + txn_begin ts=7 + put k=a v=a7 + put k=d v=d7 + put k=j v=j7 + put k=l v=l7 + put k=o v=n7 +---- +>> at end: +txn: "A" meta={id=00000000 key=/Min pri=0.00000000 epo=0 ts=7.000000000,0 min=0,0 seq=0} lock=true stat=PENDING rts=7.000000000,0 wto=false gul=0,0 +rangekey: {a-b}/[1.000000000,0=/] +rangekey: {b-c}/[3.000000000,0=/ 1.000000000,0=/] +rangekey: {c-d}/[5.000000000,0={localTs=4.000000000,0}/ 3.000000000,0=/ 1.000000000,0=/] +rangekey: {d-f}/[5.000000000,0={localTs=4.000000000,0}/ 1.000000000,0=/] +rangekey: {f-g}/[5.000000000,0={localTs=4.000000000,0}/ 3.000000000,0=/ 1.000000000,0=/] +rangekey: {g-h}/[3.000000000,0=/ 1.000000000,0=/] +rangekey: {h-k}/[1.000000000,0=/] +rangekey: {m-n}/[3.000000000,0={localTs=2.000000000,0}/] +meta: "a"/0,0 -> txn={id=00000000 key=/Min pri=0.00000000 epo=0 ts=7.000000000,0 min=0,0 seq=0} ts=7.000000000,0 del=false klen=12 vlen=7 mergeTs= txnDidNotUpdateMeta=true +data: "a"/7.000000000,0 -> /BYTES/a7 +data: "a"/4.000000000,0 -> / +data: "a"/2.000000000,0 -> /BYTES/a2 +data: "b"/4.000000000,0 -> / +meta: "d"/0,0 -> txn={id=00000000 key=/Min pri=0.00000000 epo=0 ts=7.000000000,0 min=0,0 seq=0} ts=7.000000000,0 del=false klen=12 vlen=7 mergeTs= txnDidNotUpdateMeta=true +data: "d"/7.000000000,0 -> /BYTES/d7 +data: "d"/4.000000000,0 -> /BYTES/d4 +data: "e"/3.000000000,0 -> {localTs=2.000000000,0}/BYTES/e3 +data: "f"/6.000000000,0 -> /BYTES/f6 +data: "f"/4.000000000,0 -> /BYTES/f4 +data: "f"/2.000000000,0 -> /BYTES/f2 +data: "g"/4.000000000,0 -> /BYTES/g4 +data: "g"/2.000000000,0 -> /BYTES/g2 +data: "h"/4.000000000,0 -> / +data: "h"/3.000000000,0 -> /BYTES/h3 +meta: "j"/0,0 -> txn={id=00000000 key=/Min pri=0.00000000 epo=0 ts=7.000000000,0 min=0,0 seq=0} ts=7.000000000,0 del=false klen=12 vlen=7 mergeTs= txnDidNotUpdateMeta=true +data: "j"/7.000000000,0 -> /BYTES/j7 +data: "k"/5.000000000,0 -> {localTs=4.000000000,0}/BYTES/k5 +meta: "l"/0,0 -> txn={id=00000000 key=/Min pri=0.00000000 epo=0 ts=7.000000000,0 min=0,0 seq=0} ts=7.000000000,0 del=false klen=12 vlen=7 mergeTs= txnDidNotUpdateMeta=true +data: "l"/7.000000000,0 -> /BYTES/l7 +meta: "o"/0,0 -> txn={id=00000000 key=/Min pri=0.00000000 epo=0 ts=7.000000000,0 min=0,0 seq=0} ts=7.000000000,0 del=false klen=12 vlen=7 mergeTs= txnDidNotUpdateMeta=true +data: "o"/7.000000000,0 -> /BYTES/n7 + +# Exporting across intents will error. +run error +export k=a end=z +---- +error: (*roachpb.WriteIntentError:) conflicting intents on "a" + +run error +export k=a end=z maxIntents=100 +---- +error: (*roachpb.WriteIntentError:) conflicting intents on "a", "d", "j", "l", "o" + +run error +export k=a end=z maxIntents=3 +---- +error: (*roachpb.WriteIntentError:) conflicting intents on "a", "d", "j" + +# Export the entire dataset below the intents, with full revision history. +run ok +export k=a end=z ts=6 allRevisions +---- +export: data_size:165 +export: {a-b}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / +export: "a"/2.000000000,0 -> /BYTES/a2 +export: {b-c}/[3.000000000,0=/ 1.000000000,0=/] +export: "b"/4.000000000,0 -> / +export: {c-d}/[5.000000000,0=/ 3.000000000,0=/ 1.000000000,0=/] +export: {d-f}/[5.000000000,0=/ 1.000000000,0=/] +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "e"/3.000000000,0 -> /BYTES/e3 +export: {f-g}/[5.000000000,0=/ 3.000000000,0=/ 1.000000000,0=/] +export: "f"/6.000000000,0 -> /BYTES/f6 +export: "f"/4.000000000,0 -> /BYTES/f4 +export: "f"/2.000000000,0 -> /BYTES/f2 +export: {g-h}/[3.000000000,0=/ 1.000000000,0=/] +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "g"/2.000000000,0 -> /BYTES/g2 +export: {h-k}/[1.000000000,0=/] +export: "h"/4.000000000,0 -> / +export: "h"/3.000000000,0 -> /BYTES/h3 +export: "k"/5.000000000,0 -> /BYTES/k5 +export: {m-n}/[3.000000000,0=/] + +# Export the full revision history, at increasing end time and then at +# increasing start time. +run ok +export k=a end=z ts=1 allRevisions +---- +export: data_size:14 +export: {a-k}/[1.000000000,0=/] + +run ok +export k=a end=z ts=2 allRevisions +---- +export: data_size:38 +export: {a-k}/[1.000000000,0=/] +export: "a"/2.000000000,0 -> /BYTES/a2 +export: "f"/2.000000000,0 -> /BYTES/f2 +export: "g"/2.000000000,0 -> /BYTES/g2 + +run ok +export k=a end=z ts=3 allRevisions +---- +export: data_size:77 +export: {a-b}/[1.000000000,0=/] +export: "a"/2.000000000,0 -> /BYTES/a2 +export: {b-d}/[3.000000000,0=/ 1.000000000,0=/] +export: {d-f}/[1.000000000,0=/] +export: "e"/3.000000000,0 -> /BYTES/e3 +export: {f-h}/[3.000000000,0=/ 1.000000000,0=/] +export: "f"/2.000000000,0 -> /BYTES/f2 +export: "g"/2.000000000,0 -> /BYTES/g2 +export: {h-k}/[1.000000000,0=/] +export: "h"/3.000000000,0 -> /BYTES/h3 +export: {m-n}/[3.000000000,0=/] + +run ok +export k=a end=z ts=4 allRevisions +---- +export: data_size:104 +export: {a-b}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / +export: "a"/2.000000000,0 -> /BYTES/a2 +export: {b-d}/[3.000000000,0=/ 1.000000000,0=/] +export: "b"/4.000000000,0 -> / +export: {d-f}/[1.000000000,0=/] +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "e"/3.000000000,0 -> /BYTES/e3 +export: {f-h}/[3.000000000,0=/ 1.000000000,0=/] +export: "f"/4.000000000,0 -> /BYTES/f4 +export: "f"/2.000000000,0 -> /BYTES/f2 +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "g"/2.000000000,0 -> /BYTES/g2 +export: {h-k}/[1.000000000,0=/] +export: "h"/4.000000000,0 -> / +export: "h"/3.000000000,0 -> /BYTES/h3 +export: {m-n}/[3.000000000,0=/] + +run ok +export k=a end=z ts=5 allRevisions +---- +export: data_size:157 +export: {a-b}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / +export: "a"/2.000000000,0 -> /BYTES/a2 +export: {b-c}/[3.000000000,0=/ 1.000000000,0=/] +export: "b"/4.000000000,0 -> / +export: {c-d}/[5.000000000,0=/ 3.000000000,0=/ 1.000000000,0=/] +export: {d-f}/[5.000000000,0=/ 1.000000000,0=/] +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "e"/3.000000000,0 -> /BYTES/e3 +export: {f-g}/[5.000000000,0=/ 3.000000000,0=/ 1.000000000,0=/] +export: "f"/4.000000000,0 -> /BYTES/f4 +export: "f"/2.000000000,0 -> /BYTES/f2 +export: {g-h}/[3.000000000,0=/ 1.000000000,0=/] +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "g"/2.000000000,0 -> /BYTES/g2 +export: {h-k}/[1.000000000,0=/] +export: "h"/4.000000000,0 -> / +export: "h"/3.000000000,0 -> /BYTES/h3 +export: "k"/5.000000000,0 -> /BYTES/k5 +export: {m-n}/[3.000000000,0=/] + +run ok +export k=a end=z ts=6 allRevisions +---- +export: data_size:165 +export: {a-b}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / +export: "a"/2.000000000,0 -> /BYTES/a2 +export: {b-c}/[3.000000000,0=/ 1.000000000,0=/] +export: "b"/4.000000000,0 -> / +export: {c-d}/[5.000000000,0=/ 3.000000000,0=/ 1.000000000,0=/] +export: {d-f}/[5.000000000,0=/ 1.000000000,0=/] +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "e"/3.000000000,0 -> /BYTES/e3 +export: {f-g}/[5.000000000,0=/ 3.000000000,0=/ 1.000000000,0=/] +export: "f"/6.000000000,0 -> /BYTES/f6 +export: "f"/4.000000000,0 -> /BYTES/f4 +export: "f"/2.000000000,0 -> /BYTES/f2 +export: {g-h}/[3.000000000,0=/ 1.000000000,0=/] +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "g"/2.000000000,0 -> /BYTES/g2 +export: {h-k}/[1.000000000,0=/] +export: "h"/4.000000000,0 -> / +export: "h"/3.000000000,0 -> /BYTES/h3 +export: "k"/5.000000000,0 -> /BYTES/k5 +export: {m-n}/[3.000000000,0=/] + +run ok +export k=a end=z startTs=1 ts=6 allRevisions +---- +export: data_size:151 +export: "a"/4.000000000,0 -> / +export: "a"/2.000000000,0 -> /BYTES/a2 +export: {b-c}/[3.000000000,0=/] +export: "b"/4.000000000,0 -> / +export: {c-d}/[5.000000000,0=/ 3.000000000,0=/] +export: {d-f}/[5.000000000,0=/] +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "e"/3.000000000,0 -> /BYTES/e3 +export: {f-g}/[5.000000000,0=/ 3.000000000,0=/] +export: "f"/6.000000000,0 -> /BYTES/f6 +export: "f"/4.000000000,0 -> /BYTES/f4 +export: "f"/2.000000000,0 -> /BYTES/f2 +export: {g-h}/[3.000000000,0=/] +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "g"/2.000000000,0 -> /BYTES/g2 +export: "h"/4.000000000,0 -> / +export: "h"/3.000000000,0 -> /BYTES/h3 +export: "k"/5.000000000,0 -> /BYTES/k5 +export: {m-n}/[3.000000000,0=/] + +run ok +export k=a end=z startTs=2 ts=6 allRevisions +---- +export: data_size:127 +export: "a"/4.000000000,0 -> / +export: {b-c}/[3.000000000,0=/] +export: "b"/4.000000000,0 -> / +export: {c-d}/[5.000000000,0=/ 3.000000000,0=/] +export: {d-f}/[5.000000000,0=/] +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "e"/3.000000000,0 -> /BYTES/e3 +export: {f-g}/[5.000000000,0=/ 3.000000000,0=/] +export: "f"/6.000000000,0 -> /BYTES/f6 +export: "f"/4.000000000,0 -> /BYTES/f4 +export: {g-h}/[3.000000000,0=/] +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "h"/4.000000000,0 -> / +export: "h"/3.000000000,0 -> /BYTES/h3 +export: "k"/5.000000000,0 -> /BYTES/k5 +export: {m-n}/[3.000000000,0=/] + +run ok +export k=a end=z startTs=3 ts=6 allRevisions +---- +export: data_size:88 +export: "a"/4.000000000,0 -> / +export: "b"/4.000000000,0 -> / +export: {c-g}/[5.000000000,0=/] +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "f"/6.000000000,0 -> /BYTES/f6 +export: "f"/4.000000000,0 -> /BYTES/f4 +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "h"/4.000000000,0 -> / +export: "k"/5.000000000,0 -> /BYTES/k5 + +run ok +export k=a end=z startTs=4 ts=6 allRevisions +---- +export: data_size:61 +export: {c-g}/[5.000000000,0=/] +export: "f"/6.000000000,0 -> /BYTES/f6 +export: "k"/5.000000000,0 -> /BYTES/k5 + +run ok +export k=a end=z startTs=5 ts=6 allRevisions +---- +export: data_size:8 +export: "f"/6.000000000,0 -> /BYTES/f6 + +run ok +export k=a end=z startTs=6 ts=6 allRevisions +---- +export: + +# Export without revision history at increasing end time, then at increasing +# start time. +run ok +export k=a end=z ts=1 +---- +export: + +run ok +export k=a end=z ts=2 +---- +export: data_size:24 +export: "a"/2.000000000,0 -> /BYTES/a2 +export: "f"/2.000000000,0 -> /BYTES/f2 +export: "g"/2.000000000,0 -> /BYTES/g2 + +run ok +export k=a end=z ts=3 +---- +export: data_size:24 +export: "a"/2.000000000,0 -> /BYTES/a2 +export: "e"/3.000000000,0 -> /BYTES/e3 +export: "h"/3.000000000,0 -> /BYTES/h3 + +run ok +export k=a end=z ts=4 +---- +export: data_size:32 +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "e"/3.000000000,0 -> /BYTES/e3 +export: "f"/4.000000000,0 -> /BYTES/f4 +export: "g"/4.000000000,0 -> /BYTES/g4 + +run ok +export k=a end=z ts=5 +---- +export: data_size:16 +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "k"/5.000000000,0 -> /BYTES/k5 + +run ok +export k=a end=z ts=6 +---- +export: data_size:24 +export: "f"/6.000000000,0 -> /BYTES/f6 +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "k"/5.000000000,0 -> /BYTES/k5 + +run ok +export k=a end=z startTs=1 ts=6 +---- +export: data_size:91 +export: "a"/4.000000000,0 -> / +export: {b-c}/[3.000000000,0=/] +export: "b"/4.000000000,0 -> / +export: {c-g}/[5.000000000,0=/] +export: "f"/6.000000000,0 -> /BYTES/f6 +export: {g-h}/[3.000000000,0=/] +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "h"/4.000000000,0 -> / +export: "k"/5.000000000,0 -> /BYTES/k5 +export: {m-n}/[3.000000000,0=/] + +run ok +export k=a end=z startTs=2 ts=6 +---- +export: data_size:91 +export: "a"/4.000000000,0 -> / +export: {b-c}/[3.000000000,0=/] +export: "b"/4.000000000,0 -> / +export: {c-g}/[5.000000000,0=/] +export: "f"/6.000000000,0 -> /BYTES/f6 +export: {g-h}/[3.000000000,0=/] +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "h"/4.000000000,0 -> / +export: "k"/5.000000000,0 -> /BYTES/k5 +export: {m-n}/[3.000000000,0=/] + +run ok +export k=a end=z startTs=3 ts=6 +---- +export: data_size:72 +export: "a"/4.000000000,0 -> / +export: "b"/4.000000000,0 -> / +export: {c-g}/[5.000000000,0=/] +export: "f"/6.000000000,0 -> /BYTES/f6 +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "h"/4.000000000,0 -> / +export: "k"/5.000000000,0 -> /BYTES/k5 + +run ok +export k=a end=z startTs=4 ts=6 +---- +export: data_size:61 +export: {c-g}/[5.000000000,0=/] +export: "f"/6.000000000,0 -> /BYTES/f6 +export: "k"/5.000000000,0 -> /BYTES/k5 + +run ok +export k=a end=z startTs=5 ts=6 +---- +export: data_size:8 +export: "f"/6.000000000,0 -> /BYTES/f6 + +run ok +export k=a end=z startTs=6 ts=6 +---- +export: + +# Incremental export one timestamp at a time, with and without full revision +# history. +run ok +export k=a end=z startTs=0 ts=1 allRevisions +---- +export: data_size:14 +export: {a-k}/[1.000000000,0=/] + +run ok +export k=a end=z startTs=1 ts=2 allRevisions +---- +export: data_size:24 +export: "a"/2.000000000,0 -> /BYTES/a2 +export: "f"/2.000000000,0 -> /BYTES/f2 +export: "g"/2.000000000,0 -> /BYTES/g2 + +run ok +export k=a end=z startTs=2 ts=3 allRevisions +---- +export: data_size:39 +export: {b-d}/[3.000000000,0=/] +export: "e"/3.000000000,0 -> /BYTES/e3 +export: {f-h}/[3.000000000,0=/] +export: "h"/3.000000000,0 -> /BYTES/h3 +export: {m-n}/[3.000000000,0=/] + +run ok +export k=a end=z startTs=3 ts=4 allRevisions +---- +export: data_size:27 +export: "a"/4.000000000,0 -> / +export: "b"/4.000000000,0 -> / +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "f"/4.000000000,0 -> /BYTES/f4 +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "h"/4.000000000,0 -> / + +run ok +export k=a end=z startTs=4 ts=5 allRevisions +---- +export: data_size:53 +export: {c-g}/[5.000000000,0=/] +export: "k"/5.000000000,0 -> /BYTES/k5 + +run ok +export k=a end=z startTs=5 ts=6 allRevisions +---- +export: data_size:8 +export: "f"/6.000000000,0 -> /BYTES/f6 + +run ok +export k=a end=z startTs=0 ts=1 +---- +export: + +run ok +export k=a end=z startTs=1 ts=2 +---- +export: data_size:24 +export: "a"/2.000000000,0 -> /BYTES/a2 +export: "f"/2.000000000,0 -> /BYTES/f2 +export: "g"/2.000000000,0 -> /BYTES/g2 + +run ok +export k=a end=z startTs=2 ts=3 +---- +export: data_size:39 +export: {b-d}/[3.000000000,0=/] +export: "e"/3.000000000,0 -> /BYTES/e3 +export: {f-h}/[3.000000000,0=/] +export: "h"/3.000000000,0 -> /BYTES/h3 +export: {m-n}/[3.000000000,0=/] + +run ok +export k=a end=z startTs=3 ts=4 +---- +export: data_size:27 +export: "a"/4.000000000,0 -> / +export: "b"/4.000000000,0 -> / +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "f"/4.000000000,0 -> /BYTES/f4 +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "h"/4.000000000,0 -> / + +run ok +export k=a end=z startTs=4 ts=5 +---- +export: data_size:53 +export: {c-g}/[5.000000000,0=/] +export: "k"/5.000000000,0 -> /BYTES/k5 + +run ok +export k=a end=z startTs=5 ts=6 +---- +export: data_size:8 +export: "f"/6.000000000,0 -> /BYTES/f6 + +# TargetSize returns a resume span, and allows overflow, both when exporting the +# whole revision history and the latest version. It is not affected by +# stopMidKey. +run ok +export k=a end=z ts=6 allRevisions targetSize=1 +---- +export: data_size:11 resume="b"/0,0 +export: {a-b}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / +export: "a"/2.000000000,0 -> /BYTES/a2 + +run ok +export k=a end=z ts=6 allRevisions targetSize=1 stopMidKey +---- +export: data_size:11 resume="b"/0,0 +export: {a-b}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / +export: "a"/2.000000000,0 -> /BYTES/a2 + +run ok +export k=a end=z ts=6 targetSize=1 +---- +export: data_size:8 resume="g"/0,0 +export: "f"/6.000000000,0 -> /BYTES/f6 + +run ok +export k=a end=z startTs=1 ts=6 targetSize=1 +---- +export: data_size:1 resume="b"/0,0 +export: "a"/4.000000000,0 -> / + +# MaxSize returns an error if exceeded without TargetSize. +# +# TODO(erikgrinaker): It probably doesn't make sense for this behavior to change +# based on whether TargetSize is set or not, but keeping the existing logic for +# now. +run error +export k=a end=z ts=6 allRevisions maxSize=1 +---- +error: (*storage.ExceedMaxSizeError:) export size (3 bytes) exceeds max size (1 bytes) + +run error +export k=a end=z ts=6 allRevisions maxSize=10 +---- +error: (*storage.ExceedMaxSizeError:) export size (12 bytes) exceeds max size (10 bytes) + +# MaxSize with TargetSize will bail out before exceeding MaxSize, but it +# depends on StopMidKey. +run ok +export k=a end=z ts=6 allRevisions targetSize=1 maxSize=1 +---- +export: + +run error +export k=a end=z ts=6 allRevisions targetSize=10 maxSize=10 +---- +error: (*storage.ExceedMaxSizeError:) export size (12 bytes) exceeds max size (10 bytes) + +run ok +export k=a end=z ts=6 allRevisions targetSize=10 maxSize=10 stopMidKey +---- +export: data_size:4 resume="a"/2.000000000,0 +export: a{-\x00}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / + +run ok +export k=a end=z ts=6 allRevisions targetSize=12 maxSize=12 +---- +export: data_size:11 resume="b"/0,0 +export: {a-b}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / +export: "a"/2.000000000,0 -> /BYTES/a2 + +run error +export k=a end=z ts=6 allRevisions targetSize=17 maxSize=17 +---- +error: (*storage.ExceedMaxSizeError:) export size (18 bytes) exceeds max size (17 bytes) + +# TargetSize and MaxSize without stopMidKey will keep going to the +# end of the key as long as MaxSize isn't exceeded. +run ok +export k=a end=z ts=6 allRevisions targetSize=4 maxSize=12 +---- +export: data_size:11 resume="b"/0,0 +export: {a-b}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / +export: "a"/2.000000000,0 -> /BYTES/a2 + +# Hitting MaxSize right after including a range key with the same start key as +# the exceeding point key will emit a point-sized range key, unfortunately. This +# is also the case when we emit a covered point. However, it won't emit that +# range key if StopMidKey is disabled. +run ok +export k=a end=z ts=6 allRevisions targetSize=3 maxSize=3 stopMidKey +---- +export: data_size:3 resume="a"/4.000000000,0 +export: a{-\x00}/[1.000000000,0=/] + +run ok +export k=a end=z ts=6 allRevisions targetSize=4 maxSize=4 stopMidKey +---- +export: data_size:4 resume="a"/2.000000000,0 +export: a{-\x00}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / + +run ok +export k=a end=z ts=6 allRevisions targetSize=17 maxSize=17 stopMidKey +---- +export: data_size:17 resume="b"/4.000000000,0 +export: {a-b}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / +export: "a"/2.000000000,0 -> /BYTES/a2 +export: b{-\x00}/[3.000000000,0=/ 1.000000000,0=/] + +run error +export k=a end=z ts=6 allRevisions targetSize=17 maxSize=17 +---- +error: (*storage.ExceedMaxSizeError:) export size (18 bytes) exceeds max size (17 bytes) + +# Resuming from various bounds, with and without other options. +run ok +export k=b end=k ts=6 allRevisions +---- +export: data_size:131 +export: {b-c}/[3.000000000,0=/ 1.000000000,0=/] +export: "b"/4.000000000,0 -> / +export: {c-d}/[5.000000000,0=/ 3.000000000,0=/ 1.000000000,0=/] +export: {d-f}/[5.000000000,0=/ 1.000000000,0=/] +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "e"/3.000000000,0 -> /BYTES/e3 +export: {f-g}/[5.000000000,0=/ 3.000000000,0=/ 1.000000000,0=/] +export: "f"/6.000000000,0 -> /BYTES/f6 +export: "f"/4.000000000,0 -> /BYTES/f4 +export: "f"/2.000000000,0 -> /BYTES/f2 +export: {g-h}/[3.000000000,0=/ 1.000000000,0=/] +export: "g"/4.000000000,0 -> /BYTES/g4 +export: "g"/2.000000000,0 -> /BYTES/g2 +export: {h-k}/[1.000000000,0=/] +export: "h"/4.000000000,0 -> / +export: "h"/3.000000000,0 -> /BYTES/h3 + +run ok +export k=bbb end=ggg startTs=2 ts=5 allRevisions +---- +export: data_size:89 +export: {bbb-c}/[3.000000000,0=/] +export: {c-d}/[5.000000000,0=/ 3.000000000,0=/] +export: {d-f}/[5.000000000,0=/] +export: "d"/4.000000000,0 -> /BYTES/d4 +export: "e"/3.000000000,0 -> /BYTES/e3 +export: {f-g}/[5.000000000,0=/ 3.000000000,0=/] +export: "f"/4.000000000,0 -> /BYTES/f4 +export: g{-gg}/[3.000000000,0=/] +export: "g"/4.000000000,0 -> /BYTES/g4 + +run ok +export k=bbb end=ggg startTs=2 ts=5 +---- +export: data_size:61 +export: {bbb-c}/[3.000000000,0=/] +export: {c-g}/[5.000000000,0=/] +export: g{-gg}/[3.000000000,0=/] +export: "g"/4.000000000,0 -> /BYTES/g4 + +# Resuming from a specific key version. +run ok +export k=a kTs=4 end=c ts=6 allRevisions +---- +export: data_size:16 +export: {a-b}/[1.000000000,0=/] +export: "a"/4.000000000,0 -> / +export: "a"/2.000000000,0 -> /BYTES/a2 +export: {b-c}/[3.000000000,0=/ 1.000000000,0=/] +export: "b"/4.000000000,0 -> / + +run ok +export k=a kTs=3 end=c ts=6 allRevisions +---- +export: data_size:15 +export: {a-b}/[1.000000000,0=/] +export: "a"/2.000000000,0 -> /BYTES/a2 +export: {b-c}/[3.000000000,0=/ 1.000000000,0=/] +export: "b"/4.000000000,0 -> / + +run ok +export k=a kTs=2 end=c ts=6 allRevisions +---- +export: data_size:15 +export: {a-b}/[1.000000000,0=/] +export: "a"/2.000000000,0 -> /BYTES/a2 +export: {b-c}/[3.000000000,0=/ 1.000000000,0=/] +export: "b"/4.000000000,0 -> / + +run ok +export k=a kTs=1 end=c ts=6 allRevisions +---- +export: data_size:7 +export: {a-b}/[1.000000000,0=/] +export: {b-c}/[3.000000000,0=/ 1.000000000,0=/] +export: "b"/4.000000000,0 -> / + +run ok +export k=f kTs=4 end=g ts=6 allRevisions +---- +export: data_size:35 +export: {f-g}/[5.000000000,0=/ 3.000000000,0=/ 1.000000000,0=/] +export: "f"/4.000000000,0 -> /BYTES/f4 +export: "f"/2.000000000,0 -> /BYTES/f2 + +run ok +export k=f kTs=4 end=g startTs=2 ts=4 allRevisions +---- +export: data_size:10 +export: {f-g}/[3.000000000,0=/] +export: "f"/4.000000000,0 -> /BYTES/f4 + +run ok +export k=f kTs=3 end=g startTs=2 ts=4 allRevisions +---- +export: data_size:2 +export: {f-g}/[3.000000000,0=/] + +# Resuming from a specific key version at or below startTS. +run ok +export k=a kTs=2 end=c startTs=2 ts=6 +---- +export: data_size:3 +export: {b-c}/[3.000000000,0=/] +export: "b"/4.000000000,0 -> /