diff --git a/docs/generated/sql/bnf/create_func_stmt.bnf b/docs/generated/sql/bnf/create_func_stmt.bnf index d79b6b7bb70f..2b432c3d13f0 100644 --- a/docs/generated/sql/bnf/create_func_stmt.bnf +++ b/docs/generated/sql/bnf/create_func_stmt.bnf @@ -1,2 +1,2 @@ create_func_stmt ::= - 'CREATE' ( 'OR' 'REPLACE' | ) 'FUNCTION' func_create_name '(' ( ( ( ( func_arg | func_arg | func_arg ) ) ( ( ',' ( func_arg | func_arg | func_arg ) ) )* ) | ) ')' 'RETURNS' ( | ) ( func_arg_type ) ( ( ( ( 'AS' func_as | 'LANGUAGE' 'SQL' | ( 'CALLED' 'ON' 'NULL' 'INPUT' | 'RETURNS' 'NULL' 'ON' 'NULL' 'INPUT' | 'STRICT' | 'IMMUTABLE' | 'STABLE' | 'VOLATILE' | 'LEAKPROOF' | 'NOT' 'LEAKPROOF' ) ) ) ( ( ( 'AS' func_as | 'LANGUAGE' 'SQL' | ( 'CALLED' 'ON' 'NULL' 'INPUT' | 'RETURNS' 'NULL' 'ON' 'NULL' 'INPUT' | 'STRICT' | 'IMMUTABLE' | 'STABLE' | 'VOLATILE' | 'LEAKPROOF' | 'NOT' 'LEAKPROOF' ) ) ) )* ) | ) opt_routine_body + 'CREATE' ( 'OR' 'REPLACE' | ) 'FUNCTION' func_create_name '(' ( ( ( ( func_arg | func_arg | func_arg ) ) ( ( ',' ( func_arg | func_arg | func_arg ) ) )* ) | ) ')' 'RETURNS' ( | ) ( func_arg_type ) ( ( ( ( 'AS' ( 'SCONST' ) | 'LANGUAGE' 'SQL' | ( 'CALLED' 'ON' 'NULL' 'INPUT' | 'RETURNS' 'NULL' 'ON' 'NULL' 'INPUT' | 'STRICT' | 'IMMUTABLE' | 'STABLE' | 'VOLATILE' | 'LEAKPROOF' | 'NOT' 'LEAKPROOF' ) ) ) ( ( ( 'AS' ( 'SCONST' ) | 'LANGUAGE' 'SQL' | ( 'CALLED' 'ON' 'NULL' 'INPUT' | 'RETURNS' 'NULL' 'ON' 'NULL' 'INPUT' | 'STRICT' | 'IMMUTABLE' | 'STABLE' | 'VOLATILE' | 'LEAKPROOF' | 'NOT' 'LEAKPROOF' ) ) ) )* ) | ) opt_routine_body diff --git a/pkg/cmd/docgen/diagrams.go b/pkg/cmd/docgen/diagrams.go index d10f4a116dda..4c91cefd862b 100644 --- a/pkg/cmd/docgen/diagrams.go +++ b/pkg/cmd/docgen/diagrams.go @@ -765,9 +765,10 @@ var specs = []stmtSpec{ }, { name: "create_func_stmt", - inline: []string{"opt_or_replace", "opt_func_arg_with_default_list", "opt_return_set", "func_return_type", "opt_create_func_opt_list", "create_func_opt_list", "common_func_opt_item", "create_func_opt_item", "routine_return_stmt", "func_arg_with_default_list", "func_arg_with_default"}, - unlink: []string{"opt_or_replace", "opt_func_arg_with_default_list", "opt_return_set", "func_return_type", "opt_create_func_opt_list", "create_func_opt_list", "create_func_opt_item", "common_func_opt_item", "routine_return_stmt", "non_reserved_word_or_sconst", "func_arg_with_default_list", "func_arg_with_default", "a_expr"}, + inline: []string{"opt_or_replace", "opt_func_arg_with_default_list", "opt_return_set", "func_return_type", "opt_create_func_opt_list", "create_func_opt_list", "common_func_opt_item", "create_func_opt_item", "routine_return_stmt", "func_arg_with_default_list", "func_arg_with_default", "func_as"}, + unlink: []string{"opt_or_replace", "opt_func_arg_with_default_list", "opt_return_set", "func_return_type", "opt_create_func_opt_list", "create_func_opt_list", "create_func_opt_item", "common_func_opt_item", "routine_return_stmt", "non_reserved_word_or_sconst", "func_arg_with_default_list", "func_arg_with_default", "a_expr", "func_as"}, replace: map[string]string{ + "func_as": "'SCONST'", "non_reserved_word_or_sconst": "'SQL'", "'DEFAULT'": "", "'SETOF'": "", diff --git a/pkg/kv/kvserver/batcheval/cmd_add_sstable_test.go b/pkg/kv/kvserver/batcheval/cmd_add_sstable_test.go index 6eecd2d96ca3..d5af51a304d8 100644 --- a/pkg/kv/kvserver/batcheval/cmd_add_sstable_test.go +++ b/pkg/kv/kvserver/batcheval/cmd_add_sstable_test.go @@ -53,19 +53,18 @@ func TestEvalAddSSTable(t *testing.T) { // These are run with IngestAsWrites both disabled and enabled, and // kv.bulk_io_write.sst_rewrite_concurrency.per_call of 0 and 8. testcases := map[string]struct { - data kvs - sst kvs - reqTS int64 - toReqTS int64 // SSTTimestampToRequestTimestamp with given SST timestamp - noConflict bool // DisallowConflicts - noShadow bool // DisallowShadowing - noShadowBelow int64 // DisallowShadowingBelow - requireReqTS bool // AddSSTableRequireAtRequestTimestamp - expect kvs - expectErr interface{} // error type, substring, substring slice, or true (any) - expectErrRace interface{} - expectStatsEst bool // expect MVCCStats.ContainsEstimates, don't check stats - expectGCBytesEst bool // expect MVCCStats.GCBytesAge to be inaccurate. + data kvs + sst kvs + reqTS int64 + toReqTS int64 // SSTTimestampToRequestTimestamp with given SST timestamp + noConflict bool // DisallowConflicts + noShadow bool // DisallowShadowing + noShadowBelow int64 // DisallowShadowingBelow + requireReqTS bool // AddSSTableRequireAtRequestTimestamp + expect kvs + expectErr interface{} // error type, substring, substring slice, or true (any) + expectErrRace interface{} + expectStatsEst bool // expect MVCCStats.ContainsEstimates, don't check stats }{ // Blind writes. "blind writes below existing": { @@ -771,46 +770,52 @@ func TestEvalAddSSTable(t *testing.T) { expectErr: &roachpb.WriteTooOldError{}, }, "DisallowConflicts allows sst range keys above engine range keys": { - noConflict: true, - data: kvs{pointKV("a", 6, "d"), rangeKV("a", "b", 5, "")}, - sst: kvs{pointKV("a", 7, "a8"), rangeKV("a", "b", 8, "")}, - expect: kvs{rangeKV("a", "b", 8, ""), rangeKV("a", "b", 5, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d")}, - expectGCBytesEst: true, + noConflict: true, + data: kvs{pointKV("a", 6, "d"), rangeKV("a", "b", 5, "")}, + sst: kvs{pointKV("a", 7, "a8"), rangeKV("a", "b", 8, "")}, + expect: kvs{rangeKV("a", "b", 8, ""), rangeKV("a", "b", 5, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d")}, }, "DisallowConflicts allows fragmented sst range keys above engine range keys": { - noConflict: true, - data: kvs{pointKV("a", 6, "d"), rangeKV("a", "b", 5, "")}, - sst: kvs{pointKV("a", 7, "a8"), rangeKV("a", "b", 8, ""), rangeKV("c", "d", 8, "")}, - expect: kvs{rangeKV("a", "b", 8, ""), rangeKV("a", "b", 5, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("c", "d", 8, "")}, - expectGCBytesEst: true, + noConflict: true, + data: kvs{pointKV("a", 6, "d"), rangeKV("a", "b", 5, "")}, + sst: kvs{pointKV("a", 7, "a8"), rangeKV("a", "b", 8, ""), rangeKV("c", "d", 8, "")}, + expect: kvs{rangeKV("a", "b", 8, ""), rangeKV("a", "b", 5, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("c", "d", 8, "")}, }, "DisallowConflicts allows fragmented straddling sst range keys": { - noConflict: true, - data: kvs{pointKV("a", 6, "d"), rangeKV("b", "d", 5, "")}, - sst: kvs{pointKV("a", 7, "a8"), rangeKV("a", "c", 8, ""), rangeKV("c", "d", 7, "")}, - expect: kvs{rangeKV("a", "b", 8, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "c", 8, ""), rangeKV("b", "c", 5, ""), rangeKV("c", "d", 7, ""), rangeKV("c", "d", 5, "")}, - expectGCBytesEst: true, + noConflict: true, + data: kvs{pointKV("a", 6, "d"), rangeKV("b", "d", 5, "")}, + sst: kvs{pointKV("a", 7, "a8"), rangeKV("a", "c", 8, ""), rangeKV("c", "d", 7, "")}, + expect: kvs{rangeKV("a", "b", 8, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "c", 8, ""), rangeKV("b", "c", 5, ""), rangeKV("c", "d", 7, ""), rangeKV("c", "d", 5, "")}, }, "DisallowConflicts allows fragmented straddling sst range keys with no points": { - noConflict: true, - data: kvs{rangeKV("b", "d", 5, "")}, - sst: kvs{rangeKV("a", "c", 8, ""), rangeKV("c", "d", 7, "")}, - expect: kvs{rangeKV("a", "b", 8, ""), rangeKV("b", "c", 8, ""), rangeKV("b", "c", 5, ""), rangeKV("c", "d", 7, ""), rangeKV("c", "d", 5, "")}, - expectGCBytesEst: true, + noConflict: true, + data: kvs{rangeKV("b", "d", 5, "")}, + sst: kvs{rangeKV("a", "c", 8, ""), rangeKV("c", "d", 7, "")}, + expect: kvs{rangeKV("a", "b", 8, ""), rangeKV("b", "c", 8, ""), rangeKV("b", "c", 5, ""), rangeKV("c", "d", 7, ""), rangeKV("c", "d", 5, "")}, }, "DisallowConflicts allows engine range keys contained within sst range keys": { - noConflict: true, - data: kvs{pointKV("a", 6, "d"), rangeKV("b", "d", 5, "")}, - sst: kvs{pointKV("a", 7, "a8"), rangeKV("a", "e", 8, "")}, - expect: kvs{rangeKV("a", "b", 8, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "d", 8, ""), rangeKV("b", "d", 5, ""), rangeKV("d", "e", 8, "")}, - expectGCBytesEst: true, + noConflict: true, + data: kvs{pointKV("a", 6, "d"), rangeKV("b", "d", 5, "")}, + sst: kvs{pointKV("a", 7, "a8"), rangeKV("a", "e", 8, "")}, + expect: kvs{rangeKV("a", "b", 8, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "d", 8, ""), rangeKV("b", "d", 5, ""), rangeKV("d", "e", 8, "")}, + }, + "DisallowConflicts allows engine range keys contained within sst range keys 2": { + noConflict: true, + data: kvs{pointKV("a", 6, "d"), rangeKV("b", "d", 5, "")}, + sst: kvs{pointKV("a", 7, "a8"), rangeKV("a", "d", 8, "")}, + expect: kvs{rangeKV("a", "b", 8, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "d", 8, ""), rangeKV("b", "d", 5, "")}, + }, + "DisallowConflicts allows engine range keys contained within sst range keys 3": { + noConflict: true, + data: kvs{pointKV("a", 6, "d"), rangeKV("a", "d", 5, "")}, + sst: kvs{pointKV("a", 7, "a8"), rangeKV("b", "e", 8, "")}, + expect: kvs{rangeKV("a", "b", 5, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "d", 8, ""), rangeKV("b", "d", 5, ""), rangeKV("d", "e", 8, "")}, }, "DisallowConflicts does not skip over engine range keys covering no sst points": { - noConflict: true, - data: kvs{pointKV("a", 6, "d"), rangeKV("b", "c", 6, ""), rangeKV("c", "d", 5, "")}, - sst: kvs{pointKV("a", 7, "a8"), rangeKV("a", "e", 8, "")}, - expect: kvs{rangeKV("a", "b", 8, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "c", 8, ""), rangeKV("b", "c", 6, ""), rangeKV("c", "d", 8, ""), rangeKV("c", "d", 5, ""), rangeKV("d", "e", 8, "")}, - expectGCBytesEst: true, + noConflict: true, + data: kvs{pointKV("a", 6, "d"), rangeKV("b", "c", 6, ""), rangeKV("c", "d", 5, "")}, + sst: kvs{pointKV("a", 7, "a8"), rangeKV("a", "e", 8, "")}, + expect: kvs{rangeKV("a", "b", 8, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "c", 8, ""), rangeKV("b", "c", 6, ""), rangeKV("c", "d", 8, ""), rangeKV("c", "d", 5, ""), rangeKV("d", "e", 8, "")}, }, "DisallowConflicts does not allow conflict with engine range key covering no sst points": { noConflict: true, @@ -819,25 +824,22 @@ func TestEvalAddSSTable(t *testing.T) { expectErr: &roachpb.WriteTooOldError{}, }, "DisallowConflicts allows sst range keys contained within engine range keys": { - noConflict: true, - data: kvs{pointKV("a", 6, "d"), rangeKV("a", "e", 5, "")}, - sst: kvs{pointKV("a", 7, "a8"), rangeKV("b", "d", 8, "")}, - expect: kvs{rangeKV("a", "b", 5, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "d", 8, ""), rangeKV("b", "d", 5, ""), rangeKV("d", "e", 5, "")}, - expectGCBytesEst: true, + noConflict: true, + data: kvs{pointKV("a", 6, "d"), rangeKV("a", "e", 5, "")}, + sst: kvs{pointKV("a", 7, "a8"), rangeKV("b", "d", 8, "")}, + expect: kvs{rangeKV("a", "b", 5, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "d", 8, ""), rangeKV("b", "d", 5, ""), rangeKV("d", "e", 5, "")}, }, "DisallowConflicts allows sst range key fragmenting engine range keys": { - noConflict: true, - data: kvs{pointKV("a", 6, "d"), rangeKV("a", "c", 5, ""), rangeKV("c", "e", 6, "")}, - sst: kvs{pointKV("a", 7, "a8"), rangeKV("b", "d", 8, "")}, - expect: kvs{rangeKV("a", "b", 5, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "c", 8, ""), rangeKV("b", "c", 5, ""), rangeKV("c", "d", 8, ""), rangeKV("c", "d", 6, ""), rangeKV("d", "e", 6, "")}, - expectGCBytesEst: true, + noConflict: true, + data: kvs{pointKV("a", 6, "d"), rangeKV("a", "c", 5, ""), rangeKV("c", "e", 6, "")}, + sst: kvs{pointKV("a", 7, "a8"), rangeKV("b", "d", 8, "")}, + expect: kvs{rangeKV("a", "b", 5, ""), pointKV("a", 7, "a8"), pointKV("a", 6, "d"), rangeKV("b", "c", 8, ""), rangeKV("b", "c", 5, ""), rangeKV("c", "d", 8, ""), rangeKV("c", "d", 6, ""), rangeKV("d", "e", 6, "")}, }, "DisallowConflicts calculates stats correctly for merged range keys": { - noConflict: true, - data: kvs{rangeKV("a", "c", 8, ""), pointKV("a", 6, "d"), rangeKV("d", "e", 8, "")}, - sst: kvs{pointKV("a", 10, "de"), rangeKV("c", "d", 8, ""), pointKV("f", 10, "de")}, - expect: kvs{rangeKV("a", "e", 8, ""), pointKV("a", 10, "de"), pointKV("a", 6, "d"), pointKV("f", 10, "de")}, - expectGCBytesEst: true, + noConflict: true, + data: kvs{rangeKV("a", "c", 8, ""), pointKV("a", 6, "d"), rangeKV("d", "e", 8, "")}, + sst: kvs{pointKV("a", 10, "de"), rangeKV("c", "d", 8, ""), pointKV("f", 10, "de")}, + expect: kvs{rangeKV("a", "e", 8, ""), pointKV("a", 10, "de"), pointKV("a", 6, "d"), pointKV("f", 10, "de")}, }, "DisallowConflicts calculates stats correctly for merged range keys 2": { noConflict: true, @@ -858,25 +860,22 @@ func TestEvalAddSSTable(t *testing.T) { expectErr: "ingested range key collides with an existing one", }, "DisallowShadowing allows shadowing of keys deleted by engine range tombstones": { - noShadow: true, - data: kvs{rangeKV("a", "b", 7, ""), pointKV("a", 6, "d")}, - sst: kvs{pointKV("a", 8, "a8")}, - expect: kvs{rangeKV("a", "b", 7, ""), pointKV("a", 8, "a8"), pointKV("a", 6, "d")}, - expectGCBytesEst: true, + noShadow: true, + data: kvs{rangeKV("a", "b", 7, ""), pointKV("a", 6, "d")}, + sst: kvs{pointKV("a", 8, "a8")}, + expect: kvs{rangeKV("a", "b", 7, ""), pointKV("a", 8, "a8"), pointKV("a", 6, "d")}, }, "DisallowShadowing allows idempotent range tombstones": { - noShadow: true, - data: kvs{rangeKV("a", "b", 7, "")}, - sst: kvs{rangeKV("a", "b", 7, "")}, - expect: kvs{rangeKV("a", "b", 7, "")}, - expectGCBytesEst: true, + noShadow: true, + data: kvs{rangeKV("a", "b", 7, "")}, + sst: kvs{rangeKV("a", "b", 7, "")}, + expect: kvs{rangeKV("a", "b", 7, "")}, }, "DisallowShadowing calculates stats correctly for merged range keys with idempotence": { - noShadow: true, - data: kvs{rangeKV("b", "d", 8, ""), rangeKV("e", "f", 8, "")}, - sst: kvs{rangeKV("a", "c", 8, ""), rangeKV("d", "e", 8, "")}, - expect: kvs{rangeKV("a", "f", 8, "")}, - expectGCBytesEst: true, + noShadow: true, + data: kvs{rangeKV("b", "d", 8, ""), rangeKV("e", "f", 8, "")}, + sst: kvs{rangeKV("a", "c", 8, ""), rangeKV("d", "e", 8, "")}, + expect: kvs{rangeKV("a", "f", 8, "")}, }, "DisallowShadowingBelow disallows sst range keys shadowing live keys": { noShadowBelow: 3, @@ -885,18 +884,16 @@ func TestEvalAddSSTable(t *testing.T) { expectErr: "ingested range key collides with an existing one", }, "DisallowShadowingBelow allows shadowing of keys deleted by engine range tombstones": { - noShadowBelow: 3, - data: kvs{rangeKV("a", "b", 7, ""), pointKV("a", 6, "d")}, - sst: kvs{pointKV("a", 8, "a8")}, - expect: kvs{rangeKV("a", "b", 7, ""), pointKV("a", 8, "a8"), pointKV("a", 6, "d")}, - expectGCBytesEst: true, + noShadowBelow: 3, + data: kvs{rangeKV("a", "b", 7, ""), pointKV("a", 6, "d")}, + sst: kvs{pointKV("a", 8, "a8")}, + expect: kvs{rangeKV("a", "b", 7, ""), pointKV("a", 8, "a8"), pointKV("a", 6, "d")}, }, "DisallowShadowingBelow allows idempotent range tombstones": { - noShadowBelow: 3, - data: kvs{rangeKV("a", "b", 7, "")}, - sst: kvs{rangeKV("a", "b", 7, "")}, - expect: kvs{rangeKV("a", "b", 7, "")}, - expectGCBytesEst: true, + noShadowBelow: 3, + data: kvs{rangeKV("a", "b", 7, "")}, + sst: kvs{rangeKV("a", "b", 7, "")}, + expect: kvs{rangeKV("a", "b", 7, "")}, }, "DisallowConflict with allowed shadowing disallows idempotent range tombstones": { noConflict: true, @@ -1070,9 +1067,6 @@ func TestEvalAddSSTable(t *testing.T) { } else { require.Zero(t, stats.ContainsEstimates, "found estimated stats") expected := storageutils.EngineStats(t, engine, stats.LastUpdateNanos) - if tc.expectGCBytesEst && expected != nil { - expected.GCBytesAge = stats.GCBytesAge - } require.Equal(t, expected, stats) } }) diff --git a/pkg/storage/sst.go b/pkg/storage/sst.go index ba62947e150b..ca8997c4ee88 100644 --- a/pkg/storage/sst.go +++ b/pkg/storage/sst.go @@ -444,9 +444,12 @@ func CheckSSTConflicts( if sstRangeKeysChanged { if extHasRange && extRangeKeys.Bounds.Overlaps(sstRangeKeys.Bounds) { mergedIntoExisting := false + overlappingSection := sstRangeKeys.Bounds switch sstRangeKeys.Bounds.Key.Compare(extRangeKeys.Bounds.Key) { case -1: // sstRangeKey starts earlier than extRangeKey. Add a fragment + overlappingSection.Key = extRangeKeys.Bounds.Key + statsDiff.AgeTo(sstRangeKeys.Versions.Newest().WallTime) statsDiff.RangeKeyBytes += int64(EncodedMVCCKeyPrefixLength(extRangeKeys.Bounds.Key)) addedFragment := MVCCRangeKeyStack{ Bounds: roachpb.Span{Key: sstRangeKeys.Bounds.Key, EndKey: extRangeKeys.Bounds.Key}, @@ -460,12 +463,24 @@ func CheckSSTConflicts( } else { // Add the sst range key versions again, to account for the overlap // with extRangeKeys. - for _, v := range sstRangeKeys.Versions { - statsDiff.Add(updateStatsOnRangeKeyPutVersion(extRangeKeys, v)) + updatedStack := extRangeKeys + updatedStack.Versions = extRangeKeys.Versions.Clone() + for i, v := range sstRangeKeys.Versions { + if i == 0 { + // We do this dance to make updatedStack.Versions.Newest() == v. This + // is necessary to keep GCBytesAge calculations correct, we don't + // want updateStatsOnRangeKeyPutVersion to "lift" the GCBytesAge + // contribution of extRangeKeys' bounds. We will do that later. + // We only want it to add the version. + oldVersions := updatedStack.Versions + updatedStack.Versions = append(MVCCRangeKeyVersions{v}, oldVersions...) + } + statsDiff.Add(updateStatsOnRangeKeyPutVersion(updatedStack, v)) } } case 0: // Same start key. No need to encode the start key again. + statsDiff.AgeTo(sstRangeKeys.Versions.Newest().WallTime) statsDiff.RangeKeyCount-- statsDiff.RangeKeyBytes -= int64(EncodedMVCCKeyPrefixLength(sstRangeKeys.Bounds.Key)) case 1: @@ -476,9 +491,24 @@ func CheckSSTConflicts( } // No need to re-encode the start key, as UpdateStatsOnRangeKeySplit has already // done that for us. + statsDiff.AgeTo(sstRangeKeys.Versions.Newest().WallTime) statsDiff.RangeKeyCount-- statsDiff.RangeKeyBytes -= int64(EncodedMVCCKeyPrefixLength(sstRangeKeys.Bounds.Key)) } + if extRangeKeys.Bounds.EndKey.Compare(sstRangeKeys.Bounds.EndKey) < 0 { + overlappingSection.EndKey = extRangeKeys.Bounds.EndKey + } + // Move up the GCBytesAge contribution of the overlapping section from + // extRangeKeys.Newest up to sstRangeKeys.Newest. + { + keyBytes := int64(EncodedMVCCKeyPrefixLength(overlappingSection.Key)) + + int64(EncodedMVCCKeyPrefixLength(overlappingSection.EndKey)) + statsDiff.AgeTo(extRangeKeys.Newest().WallTime) + statsDiff.RangeKeyBytes -= keyBytes + statsDiff.AgeTo(sstRangeKeys.Newest().WallTime) + statsDiff.RangeKeyBytes += keyBytes + } + // Check if the overlapping part of sstRangeKeys and extRangeKeys has // idempotent versions. We already know this isn't a conflict, as that // check happened earlier. @@ -490,21 +520,28 @@ func CheckSSTConflicts( } // Subtract stats for this version, as it already exists in the // engine. - statsDiff.Subtract(updateStatsOnRangeKeyPutVersion(sstRangeKeys, v)) + overlappingStack := MVCCRangeKeyStack{ + Bounds: overlappingSection, + Versions: sstRangeKeys.Versions, + } + statsDiff.Subtract(updateStatsOnRangeKeyPutVersion(overlappingStack, v)) idempotentIdx++ } switch extRangeKeys.Bounds.EndKey.Compare(sstRangeKeys.Bounds.EndKey) { case +1: statsDiff.Add(UpdateStatsOnRangeKeySplit(sstRangeKeys.Bounds.EndKey, extRangeKeys.Versions)) // Remove the contribution for the end key. + statsDiff.AgeTo(sstRangeKeys.Versions.Newest().WallTime) statsDiff.RangeKeyBytes -= int64(EncodedMVCCKeyPrefixLength(sstRangeKeys.Bounds.EndKey)) case 0: // Remove the contribution for the end key. + statsDiff.AgeTo(sstRangeKeys.Versions.Newest().WallTime) statsDiff.RangeKeyBytes -= int64(EncodedMVCCKeyPrefixLength(sstRangeKeys.Bounds.EndKey)) case -1: statsDiff.Add(UpdateStatsOnRangeKeySplit(extRangeKeys.Bounds.EndKey, sstRangeKeys.Versions)) // Remove the contribution for the end key. - statsDiff.RangeKeyBytes -= int64(EncodedMVCCKeyPrefixLength(sstRangeKeys.Bounds.EndKey)) + statsDiff.AgeTo(sstRangeKeys.Versions.Newest().WallTime) + statsDiff.RangeKeyBytes -= int64(EncodedMVCCKeyPrefixLength(extRangeKeys.Bounds.EndKey)) } } } @@ -534,8 +571,16 @@ func CheckSSTConflicts( // it is possible that we missed an overlap between extRangeKeys and // sstPrevRangeKeys. Account for that here by adding the version stats // for sstPrevRangeKeys. - for _, v := range sstPrevRangeKeys.Versions { - statsDiff.Add(updateStatsOnRangeKeyPutVersion(extRangeKeys, v)) + updatedStack := extRangeKeys + updatedStack.Versions = extRangeKeys.Versions.Clone() + for i, v := range sstPrevRangeKeys.Versions { + statsDiff.Add(updateStatsOnRangeKeyPutVersion(updatedStack, v)) + if i == 0 { + // We do this dance to make updatedStack.Versions.Newest() == v. This + // is necessary to keep GCBytesAge calculations correct. + oldVersions := updatedStack.Versions + updatedStack.Versions = append(MVCCRangeKeyVersions{v}, oldVersions...) + } } } sstPrevRangeKeys = sstRangeKeys.Clone() @@ -566,29 +611,58 @@ func CheckSSTConflicts( // sstRangeKeysChanged conditional above. if sstHasRange && sstRangeKeys.Bounds.Overlaps(extRangeKeys.Bounds) && !sstRangeKeysChanged { idempotentIdx := 0 - for _, v := range sstRangeKeys.Versions { + updatedStack := extRangeKeys + if sstRangeKeys.Bounds.EndKey.Compare(extRangeKeys.Bounds.EndKey) < 0 { + updatedStack.Bounds.EndKey = sstRangeKeys.Bounds.EndKey + } + updatedStack.Versions = extRangeKeys.Versions.Clone() + for i, v := range sstRangeKeys.Versions { if len(extRangeKeys.Versions) > idempotentIdx && v.Timestamp.Equal(extRangeKeys.Versions[idempotentIdx].Timestamp) { // Skip this version, as it already exists in the engine. idempotentIdx++ continue } - statsDiff.Add(updateStatsOnRangeKeyPutVersion(extRangeKeys, v)) + statsDiff.Add(updateStatsOnRangeKeyPutVersion(updatedStack, v)) + if i == idempotentIdx { + // We do this dance to make updatedStack.Versions.Newest() == v. This + // is necessary to keep GCBytesAge calculations correct. + oldVersions := updatedStack.Versions + updatedStack.Versions = append(MVCCRangeKeyVersions{v}, oldVersions...) + } } // Check if this ext range key is going to fragment the SST range key. if sstRangeKeys.Bounds.Key.Compare(extRangeKeys.Bounds.Key) < 0 && !extRangeKeys.Versions.Equal(sstRangeKeys.Versions) && (extPrevRangeKeys.IsEmpty() || !extPrevRangeKeys.Bounds.EndKey.Equal(extRangeKeys.Bounds.Key)) { statsDiff.Add(UpdateStatsOnRangeKeySplit(extRangeKeys.Bounds.Key, sstRangeKeys.Versions)) + updatedStack := extRangeKeys + updatedStack.Versions = extRangeKeys.Versions.Clone() // Remove the contribution for versions, as that's already been added. - for _, v := range sstRangeKeys.Versions { - statsDiff.Subtract(updateStatsOnRangeKeyPutVersion(extRangeKeys, v)) + for i, v := range sstRangeKeys.Versions { + if i == 0 { + // We do this dance to make updatedStack.Versions.Newest() == v. This + // is necessary to keep GCBytesAge calculations correct. + oldVersions := updatedStack.Versions + updatedStack.Versions = append(MVCCRangeKeyVersions{v}, oldVersions...) + } + statsDiff.Subtract(updateStatsOnRangeKeyPutVersion(updatedStack, v)) } + statsDiff.AgeTo(sstRangeKeys.Versions.Newest().WallTime) statsDiff.RangeKeyBytes -= int64(EncodedMVCCKeyPrefixLength(extRangeKeys.Bounds.Key)) statsDiff.RangeKeyCount-- } else if !extPrevRangeKeys.IsEmpty() && extPrevRangeKeys.Bounds.EndKey.Equal(extRangeKeys.Bounds.Key) { + updatedStack := extRangeKeys + updatedStack.Versions = extRangeKeys.Versions.Clone() // Remove the contribution for versions, as that's already been added. - for _, v := range sstRangeKeys.Versions { - statsDiff.Subtract(updateStatsOnRangeKeyPutVersion(extRangeKeys, v)) + for i, v := range sstRangeKeys.Versions { + if i == 0 { + // We do this dance to make updatedStack.Versions.Newest() == v. This + // is necessary to keep GCBytesAge calculations correct. + oldVersions := updatedStack.Versions + updatedStack.Versions = append(MVCCRangeKeyVersions{v}, oldVersions...) + } + statsDiff.Subtract(updateStatsOnRangeKeyPutVersion(updatedStack, v)) } + statsDiff.AgeTo(sstRangeKeys.Versions.Newest().WallTime) statsDiff.RangeKeyBytes -= int64(EncodedMVCCKeyPrefixLength(extRangeKeys.Bounds.Key)) statsDiff.RangeKeyCount-- } @@ -601,14 +675,17 @@ func CheckSSTConflicts( statsDiff.Add(UpdateStatsOnRangeKeySplit(sstRangeKeys.Bounds.EndKey, extRangeKeys.Versions)) } // Remove the contribution for the end key. + statsDiff.AgeTo(sstRangeKeys.Versions.Newest().WallTime) statsDiff.RangeKeyBytes -= int64(EncodedMVCCKeyPrefixLength(sstRangeKeys.Bounds.EndKey)) case 0: // Remove the contribution for the end key. + statsDiff.AgeTo(sstRangeKeys.Versions.Newest().WallTime) statsDiff.RangeKeyBytes -= int64(EncodedMVCCKeyPrefixLength(sstRangeKeys.Bounds.EndKey)) case -1: if !extRangeKeys.Versions.Equal(sstRangeKeys.Versions) { // This ext range key's end will fragment this sst range key. statsDiff.Add(UpdateStatsOnRangeKeySplit(extRangeKeys.Bounds.EndKey, sstRangeKeys.Versions)) + statsDiff.AgeTo(sstRangeKeys.Versions.Newest().WallTime) statsDiff.RangeKeyBytes -= int64(EncodedMVCCKeyPrefixLength(extRangeKeys.Bounds.EndKey)) } } @@ -648,6 +725,8 @@ func CheckSSTConflicts( } } else if extValueDeletedByRange { // Don't double-count the current key. + version, _ := extRangeKeys.Versions.FirstAtOrAbove(extKey.Timestamp) + statsDiff.AgeTo(version.Timestamp.WallTime) statsDiff.KeyCount-- statsDiff.KeyBytes -= int64(len(extKey.Key) + 1) }