From a241910aa7bc94bf545489b43815000e46e33b56 Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Thu, 20 Jan 2022 16:41:32 -0500 Subject: [PATCH] internal/manifest: fix Overlaps L0 expansion with exclusive-end When Version.Overlaps is called for L0, the overlap window is iteratively expanded until stable. In #1432, the Overlaps function was adjusted to allow specifying that the end bound should be considered exclusive. However, #1432 failed to update the exclusivity of the end bound when the range widened. This improperly excluded files with largest keys that exactly equaled the new widened end bound. This commit also transforms the TestOverlaps test into a datadriven test, introducing a few helpers for parsing the DebugString output of a Version. Fix #1459. --- internal/base/internal.go | 13 + internal/manifest/testdata/overlaps | 504 ++++++++++++++++++++++++++++ internal/manifest/version.go | 56 +++- internal/manifest/version_test.go | 218 ++---------- 4 files changed, 602 insertions(+), 189 deletions(-) create mode 100644 internal/manifest/testdata/overlaps diff --git a/internal/base/internal.go b/internal/base/internal.go index dda48069d7..0ee3d4efbf 100644 --- a/internal/base/internal.go +++ b/internal/base/internal.go @@ -359,3 +359,16 @@ type prettyInternalKey struct { func (k prettyInternalKey) Format(s fmt.State, c rune) { fmt.Fprintf(s, "%s#%d,%s", k.formatKey(k.UserKey), k.SeqNum(), k.Kind()) } + +// ParsePrettyInternalKey parses the pretty string representation of an +// internal key. The format is #,. +func ParsePrettyInternalKey(s string) InternalKey { + x := strings.FieldsFunc(s, func(c rune) bool { return c == '#' || c == ',' }) + ukey := x[0] + kind, ok := kindsMap[x[2]] + if !ok { + panic(fmt.Sprintf("unknown kind: %q", x[2])) + } + seqNum, _ := strconv.ParseUint(x[1], 10, 64) + return MakeInternalKey([]byte(ukey), seqNum, kind) +} diff --git a/internal/manifest/testdata/overlaps b/internal/manifest/testdata/overlaps new file mode 100644 index 0000000000..c7e90cdd3d --- /dev/null +++ b/internal/manifest/testdata/overlaps @@ -0,0 +1,504 @@ +define +0: + 000700:[b#7008,SET-e#7009,SET] + 000701:[c#7018,SET-f#7019,SET] + 000702:[f#7028,SET-g#7029,SET] + 000703:[x#7038,SET-y#7039,SET] + 000704:[n#7048,SET-p#7049,SET] + 000705:[p#7058,SET-p#7059,SET] + 000706:[p#7068,SET-u#7069,SET] + 000707:[r#7078,SET-s#7079,SET] +1: + 000710:[a#7140,SET-d#72057594037927935,RANGEDEL] + 000711:[d#7108,SET-g#7109,SET] + 000712:[g#7118,SET-j#7119,SET] + 000713:[n#7128,SET-p#7129,SET] + 000714:[p#7148,SET-p#7149,SET] + 000715:[p#7138,SET-u#7139,SET] +---- +0.3: + 000704:[n#7048,SET-p#7049,SET] +0.2: + 000700:[b#7008,SET-e#7009,SET] + 000705:[p#7058,SET-p#7059,SET] +0.1: + 000701:[c#7018,SET-f#7019,SET] + 000706:[p#7068,SET-u#7069,SET] +0.0: + 000702:[f#7028,SET-g#7029,SET] + 000707:[r#7078,SET-s#7079,SET] + 000703:[x#7038,SET-y#7039,SET] +1: + 000710:[a#7140,SET-d#72057594037927935,RANGEDEL] + 000711:[d#7108,SET-g#7109,SET] + 000712:[g#7118,SET-j#7119,SET] + 000713:[n#7128,SET-p#7129,SET] + 000714:[p#7148,SET-p#7149,SET] + 000715:[p#7138,SET-u#7139,SET] + +# Level 0 + +overlaps level=0 start=a end=a exclusive-end=false +---- + +overlaps level=0 start=a end=b exclusive-end=false +---- +000700:[b#7008,SET-e#7009,SET] +000701:[c#7018,SET-f#7019,SET] +000702:[f#7028,SET-g#7029,SET] + +overlaps level=0 start=a end=d exclusive-end=false +---- +000700:[b#7008,SET-e#7009,SET] +000701:[c#7018,SET-f#7019,SET] +000702:[f#7028,SET-g#7029,SET] + +overlaps level=0 start=a end=e exclusive-end=false +---- +000700:[b#7008,SET-e#7009,SET] +000701:[c#7018,SET-f#7019,SET] +000702:[f#7028,SET-g#7029,SET] + +overlaps level=0 start=a end=g exclusive-end=false +---- +000700:[b#7008,SET-e#7009,SET] +000701:[c#7018,SET-f#7019,SET] +000702:[f#7028,SET-g#7029,SET] + +overlaps level=0 start=a end=z exclusive-end=false +---- +000700:[b#7008,SET-e#7009,SET] +000701:[c#7018,SET-f#7019,SET] +000702:[f#7028,SET-g#7029,SET] +000703:[x#7038,SET-y#7039,SET] +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=c end=e exclusive-end=false +---- +000700:[b#7008,SET-e#7009,SET] +000701:[c#7018,SET-f#7019,SET] +000702:[f#7028,SET-g#7029,SET] + +overlaps level=0 start=d end=d exclusive-end=false +---- +000700:[b#7008,SET-e#7009,SET] +000701:[c#7018,SET-f#7019,SET] +000702:[f#7028,SET-g#7029,SET] + +# The below case relies on exclusive-end changing to false after picking some file. + +overlaps level=0 start=b end=f exclusive-end=true +---- +000700:[b#7008,SET-e#7009,SET] +000701:[c#7018,SET-f#7019,SET] +000702:[f#7028,SET-g#7029,SET] + +overlaps level=0 start=g end=n exclusive-end=false +---- +000700:[b#7008,SET-e#7009,SET] +000701:[c#7018,SET-f#7019,SET] +000702:[f#7028,SET-g#7029,SET] +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=h end=i exclusive-end=false +---- + +overlaps level=0 start=h end=o exclusive-end=false +---- +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=h end=u exclusive-end=false +---- +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=k end=l exclusive-end=false +---- + +overlaps level=0 start=k end=o exclusive-end=false +---- +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=k end=p exclusive-end=false +---- +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=n end=o exclusive-end=false +---- +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=n end=z exclusive-end=false +---- +000703:[x#7038,SET-y#7039,SET] +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=o end=z exclusive-end=false +---- +000703:[x#7038,SET-y#7039,SET] +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=p end=z exclusive-end=false +---- +000703:[x#7038,SET-y#7039,SET] +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=q end=z exclusive-end=false +---- +000703:[x#7038,SET-y#7039,SET] +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=r end=s exclusive-end=false +---- +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=r end=z exclusive-end=false +---- +000703:[x#7038,SET-y#7039,SET] +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=s end=z exclusive-end=false +---- +000703:[x#7038,SET-y#7039,SET] +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=u end=z exclusive-end=false +---- +000703:[x#7038,SET-y#7039,SET] +000704:[n#7048,SET-p#7049,SET] +000705:[p#7058,SET-p#7059,SET] +000706:[p#7068,SET-u#7069,SET] +000707:[r#7078,SET-s#7079,SET] + +overlaps level=0 start=y end=z exclusive-end=false +---- +000703:[x#7038,SET-y#7039,SET] + +overlaps level=0 start=z end=z exclusive-end=false +---- + +# Level 1 + +overlaps level=1 start=a end=a exclusive-end=false +---- +000710:[a#7140,SET-d#72057594037927935,RANGEDEL] + +overlaps level=1 start=a end=b exclusive-end=false +---- +000710:[a#7140,SET-d#72057594037927935,RANGEDEL] + +overlaps level=1 start=a end=d exclusive-end=false +---- +000710:[a#7140,SET-d#72057594037927935,RANGEDEL] +000711:[d#7108,SET-g#7109,SET] + +overlaps level=1 start=a end=e exclusive-end=false +---- +000710:[a#7140,SET-d#72057594037927935,RANGEDEL] +000711:[d#7108,SET-g#7109,SET] + +overlaps level=1 start=a end=g exclusive-end=false +---- +000710:[a#7140,SET-d#72057594037927935,RANGEDEL] +000711:[d#7108,SET-g#7109,SET] +000712:[g#7118,SET-j#7119,SET] + +overlaps level=1 start=a end=g exclusive-end=true +---- +000710:[a#7140,SET-d#72057594037927935,RANGEDEL] +000711:[d#7108,SET-g#7109,SET] + +overlaps level=1 start=a end=z exclusive-end=false +---- +000710:[a#7140,SET-d#72057594037927935,RANGEDEL] +000711:[d#7108,SET-g#7109,SET] +000712:[g#7118,SET-j#7119,SET] +000713:[n#7128,SET-p#7129,SET] +000714:[p#7148,SET-p#7149,SET] +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=a end=z exclusive-end=true +---- +000710:[a#7140,SET-d#72057594037927935,RANGEDEL] +000711:[d#7108,SET-g#7109,SET] +000712:[g#7118,SET-j#7119,SET] +000713:[n#7128,SET-p#7129,SET] +000714:[p#7148,SET-p#7149,SET] +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=c end=e exclusive-end=false +---- +000710:[a#7140,SET-d#72057594037927935,RANGEDEL] +000711:[d#7108,SET-g#7109,SET] + +overlaps level=1 start=d end=d exclusive-end=false +---- +000711:[d#7108,SET-g#7109,SET] + +overlaps level=1 start=g end=n exclusive-end=false +---- +000711:[d#7108,SET-g#7109,SET] +000712:[g#7118,SET-j#7119,SET] +000713:[n#7128,SET-p#7129,SET] + +overlaps level=1 start=h end=i exclusive-end=false +---- +000712:[g#7118,SET-j#7119,SET] + +overlaps level=1 start=h end=n exclusive-end=true +---- +000712:[g#7118,SET-j#7119,SET] + +overlaps level=1 start=h end=n exclusive-end=false +---- +000712:[g#7118,SET-j#7119,SET] +000713:[n#7128,SET-p#7129,SET] + +overlaps level=1 start=h end=o exclusive-end=false +---- +000712:[g#7118,SET-j#7119,SET] +000713:[n#7128,SET-p#7129,SET] + +overlaps level=1 start=h end=u exclusive-end=false +---- +000712:[g#7118,SET-j#7119,SET] +000713:[n#7128,SET-p#7129,SET] +000714:[p#7148,SET-p#7149,SET] +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=k end=l exclusive-end=false +---- + +overlaps level=1 start=k end=o exclusive-end=false +---- +000713:[n#7128,SET-p#7129,SET] + +overlaps level=1 start=k end=p exclusive-end=false +---- +000713:[n#7128,SET-p#7129,SET] +000714:[p#7148,SET-p#7149,SET] +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=k end=p exclusive-end=true +---- +000713:[n#7128,SET-p#7129,SET] + +overlaps level=1 start=n end=o exclusive-end=false +---- +000713:[n#7128,SET-p#7129,SET] + +overlaps level=1 start=n end=z exclusive-end=false +---- +000713:[n#7128,SET-p#7129,SET] +000714:[p#7148,SET-p#7149,SET] +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=o end=z exclusive-end=false +---- +000713:[n#7128,SET-p#7129,SET] +000714:[p#7148,SET-p#7149,SET] +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=p end=z exclusive-end=false +---- +000713:[n#7128,SET-p#7129,SET] +000714:[p#7148,SET-p#7149,SET] +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=q end=z exclusive-end=false +---- +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=r end=s exclusive-end=false +---- +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=r end=z exclusive-end=false +---- +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=s end=z exclusive-end=false +---- +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=u end=z exclusive-end=false +---- +000715:[p#7138,SET-u#7139,SET] + +overlaps level=1 start=y end=z exclusive-end=false +---- + +overlaps level=1 start=z end=z exclusive-end=false +---- + +# Level 2 is empty. + +overlaps level=2 start=a end=z exclusive-end=false +---- + +# Test a scenario where an originally exclusive-end must be promoted to +# inclusive during the iterative expansion of L0 overlaps. +# +# 000003 with the f largest bound must be included. + +define +0: + 000001:[a#1,SET-d#2,SET] + 000002:[c#3,SET-f#4,SET] + 000003:[f#5,SET-f#5,SET] +---- +0.2: + 000001:[a#1,SET-d#2,SET] +0.1: + 000002:[c#3,SET-f#4,SET] +0.0: + 000003:[f#5,SET-f#5,SET] + +overlaps level=0 start=a end=b exclusive-end=true +---- +000001:[a#1,SET-d#2,SET] +000002:[c#3,SET-f#4,SET] +000003:[f#5,SET-f#5,SET] + +# The below is a verbatim reproduction of the case detected by the +# metamorphic tests in pebble#1459: The above case is already a +# simplified version of the same condition. The verbatim reproduction is +# included for completeness. + +define +0.4: + 000987:[aiinjp@20#4667,SET-fcklu@5#72057594037927935,RANGEDEL] + 000988:[fcklu@5#4668,MERGE-glpw@1#72057594037927935,RANGEDEL] + 000989:[glpw@1#4662,RANGEDEL-mlgxnog@19#72057594037927935,RANGEDEL] + 000990:[mlgxnog@19#4662,RANGEDEL-nwnmqtyvjt@5#72057594037927935,RANGEDEL] + 000991:[nwnmqtyvjt@5#4662,RANGEDEL-wmkrrxp@6#72057594037927935,RANGEDEL] +0.3: + 000978:[dygfdczcax@15#4609,DEL-vtocgpw@18#4609,DEL] + 000992:[wmkrrxp@6#4657,MERGE-yyquzcd@21#4624,SET] + 000993:[zslykqao@12#4636,SINGLEDEL-zzqwavxgrec@12#4627,DEL] +0.2: + 000981:[fhcykuix@5#4601,MERGE-kiati@10#4595,MERGE] + 000977:[mgksrvk@15#4598,DEL-mgksrvk@15#4598,DEL] + 000982:[nirnrarzktp@12#4600,MERGE-zaowx@3#4602,SET] + 000828:[zzqwavxgrec@12#4092,SINGLEDEL-zzqwavxgrec@12#4092,SINGLEDEL] +0.1: + 000980:[dusu@10#4603,SET-duyeldgvnll@21#4605,SET] + 000973:[ewqqtp@15#4591,RANGEDEL-zaygjmy@1#72057594037927935,RANGEDEL] + 000605:[zzqwavxgrec@12#2894,SET-zzqwavxgrec@12#2894,SET] +0.0: + 000910:[abddymplk@20#4370,MERGE-abddymplk@20#4370,MERGE] + 000939:[abvukibeofb@13#4439,SET-abvukibeofb@13#4439,SET] + 000975:[ajoqjxr@16#4578,MERGE-zjyqka@1#4544,DEL] + 000983:[znnoar@20#4604,SINGLEDEL-znnoar@20#4604,SINGLEDEL] + 000535:[zzqwavxgrec@12#2657,SINGLEDEL-zzqwavxgrec@12#2526,SET] +5: + 000971:[acutc@6#4227,SET-zzhra@12#72057594037927935,RANGEDEL] +6: + 000806:[gourk@18#0,SET-zzhra@2#0,SET] +---- +0.4: + 000987:[aiinjp@20#4667,SET-fcklu@5#72057594037927935,RANGEDEL] + 000988:[fcklu@5#4668,MERGE-glpw@1#72057594037927935,RANGEDEL] + 000989:[glpw@1#4662,RANGEDEL-mlgxnog@19#72057594037927935,RANGEDEL] + 000990:[mlgxnog@19#4662,RANGEDEL-nwnmqtyvjt@5#72057594037927935,RANGEDEL] + 000991:[nwnmqtyvjt@5#4662,RANGEDEL-wmkrrxp@6#72057594037927935,RANGEDEL] +0.3: + 000978:[dygfdczcax@15#4609,DEL-vtocgpw@18#4609,DEL] + 000992:[wmkrrxp@6#4657,MERGE-yyquzcd@21#4624,SET] + 000993:[zslykqao@12#4636,SINGLEDEL-zzqwavxgrec@12#4627,DEL] +0.2: + 000981:[fhcykuix@5#4601,MERGE-kiati@10#4595,MERGE] + 000977:[mgksrvk@15#4598,DEL-mgksrvk@15#4598,DEL] + 000982:[nirnrarzktp@12#4600,MERGE-zaowx@3#4602,SET] + 000828:[zzqwavxgrec@12#4092,SINGLEDEL-zzqwavxgrec@12#4092,SINGLEDEL] +0.1: + 000980:[dusu@10#4603,SET-duyeldgvnll@21#4605,SET] + 000973:[ewqqtp@15#4591,RANGEDEL-zaygjmy@1#72057594037927935,RANGEDEL] + 000605:[zzqwavxgrec@12#2894,SET-zzqwavxgrec@12#2894,SET] +0.0: + 000910:[abddymplk@20#4370,MERGE-abddymplk@20#4370,MERGE] + 000939:[abvukibeofb@13#4439,SET-abvukibeofb@13#4439,SET] + 000975:[ajoqjxr@16#4578,MERGE-zjyqka@1#4544,DEL] + 000983:[znnoar@20#4604,SINGLEDEL-znnoar@20#4604,SINGLEDEL] + 000535:[zzqwavxgrec@12#2657,SINGLEDEL-zzqwavxgrec@12#2526,SET] +5: + 000971:[acutc@6#4227,SET-zzhra@12#72057594037927935,RANGEDEL] +6: + 000806:[gourk@18#0,SET-zzhra@2#0,SET] + +overlaps level=0 start=heacptnep@12 end=kiicbzwtpe@16 exclusive-end=false +---- +000973:[ewqqtp@15#4591,RANGEDEL-zaygjmy@1#72057594037927935,RANGEDEL] +000975:[ajoqjxr@16#4578,MERGE-zjyqka@1#4544,DEL] +000977:[mgksrvk@15#4598,DEL-mgksrvk@15#4598,DEL] +000978:[dygfdczcax@15#4609,DEL-vtocgpw@18#4609,DEL] +000980:[dusu@10#4603,SET-duyeldgvnll@21#4605,SET] +000981:[fhcykuix@5#4601,MERGE-kiati@10#4595,MERGE] +000982:[nirnrarzktp@12#4600,MERGE-zaowx@3#4602,SET] +000987:[aiinjp@20#4667,SET-fcklu@5#72057594037927935,RANGEDEL] +000988:[fcklu@5#4668,MERGE-glpw@1#72057594037927935,RANGEDEL] +000989:[glpw@1#4662,RANGEDEL-mlgxnog@19#72057594037927935,RANGEDEL] +000990:[mlgxnog@19#4662,RANGEDEL-nwnmqtyvjt@5#72057594037927935,RANGEDEL] +000991:[nwnmqtyvjt@5#4662,RANGEDEL-wmkrrxp@6#72057594037927935,RANGEDEL] +000992:[wmkrrxp@6#4657,MERGE-yyquzcd@21#4624,SET] + +overlaps level=0 start=acutc@6 end=zzhra@12 exclusive-end=true +---- +000535:[zzqwavxgrec@12#2657,SINGLEDEL-zzqwavxgrec@12#2526,SET] +000605:[zzqwavxgrec@12#2894,SET-zzqwavxgrec@12#2894,SET] +000828:[zzqwavxgrec@12#4092,SINGLEDEL-zzqwavxgrec@12#4092,SINGLEDEL] +000973:[ewqqtp@15#4591,RANGEDEL-zaygjmy@1#72057594037927935,RANGEDEL] +000975:[ajoqjxr@16#4578,MERGE-zjyqka@1#4544,DEL] +000977:[mgksrvk@15#4598,DEL-mgksrvk@15#4598,DEL] +000978:[dygfdczcax@15#4609,DEL-vtocgpw@18#4609,DEL] +000980:[dusu@10#4603,SET-duyeldgvnll@21#4605,SET] +000981:[fhcykuix@5#4601,MERGE-kiati@10#4595,MERGE] +000982:[nirnrarzktp@12#4600,MERGE-zaowx@3#4602,SET] +000983:[znnoar@20#4604,SINGLEDEL-znnoar@20#4604,SINGLEDEL] +000987:[aiinjp@20#4667,SET-fcklu@5#72057594037927935,RANGEDEL] +000988:[fcklu@5#4668,MERGE-glpw@1#72057594037927935,RANGEDEL] +000989:[glpw@1#4662,RANGEDEL-mlgxnog@19#72057594037927935,RANGEDEL] +000990:[mlgxnog@19#4662,RANGEDEL-nwnmqtyvjt@5#72057594037927935,RANGEDEL] +000991:[nwnmqtyvjt@5#4662,RANGEDEL-wmkrrxp@6#72057594037927935,RANGEDEL] +000992:[wmkrrxp@6#4657,MERGE-yyquzcd@21#4624,SET] +000993:[zslykqao@12#4636,SINGLEDEL-zzqwavxgrec@12#4627,DEL] diff --git a/internal/manifest/version.go b/internal/manifest/version.go index b0cc6114d5..609849b107 100644 --- a/internal/manifest/version.go +++ b/internal/manifest/version.go @@ -8,6 +8,8 @@ import ( "bytes" "fmt" "sort" + "strconv" + "strings" "sync" "sync/atomic" @@ -305,6 +307,7 @@ func NewVersion( // they appear within `files`. Some tests depend on this behavior in // order to test consistency checking, etc. Once we've constructed the // initial B-Tree, we swap out the btreeCmp for the correct one. + // TODO(jackson): Adjust or remove the tests and remove this. v.Levels[l].tree, _ = makeBTree(btreeCmpSpecificOrder(files[l]), files[l]) if l == 0 { @@ -438,6 +441,47 @@ func (v *Version) DebugString(format base.FormatKey) string { return buf.String() } +// ParseVersionDebug parses a Version from its DebugString output. +func ParseVersionDebug( + cmp Compare, formatKey base.FormatKey, flushSplitBytes int64, s string, +) (*Version, error) { + var level int + var files [NumLevels][]*FileMetadata + for _, l := range strings.Split(s, "\n") { + l = strings.TrimSpace(l) + + switch l[:2] { + case "0.", "0:", "1:", "2:", "3:", "4:", "5:", "6:": + var err error + level, err = strconv.Atoi(l[:1]) + if err != nil { + return nil, err + } + default: + // Example format: + // 000971:[acutc@6#4227,SET-zzhra@12#72057594037927935,RANGEDEL] + delim := map[rune]bool{':': true, '[': true, '-': true, ']': true} + fields := strings.FieldsFunc(l, func(c rune) bool { return delim[c] }) + fileNum, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil { + return nil, err + } + files[level] = append(files[level], &FileMetadata{ + FileNum: base.FileNum(fileNum), + Smallest: base.ParsePrettyInternalKey(fields[1]), + Largest: base.ParsePrettyInternalKey(fields[2]), + }) + } + } + // Reverse the order of L0 files. This ensures we construct the same + // sublevels. (They're printed from higher sublevel to lower, which means in + // a partial order that represents newest to oldest). + for i := 0; i < len(files[0])/2; i++ { + files[0][i], files[0][len(files[0])-i-1] = files[0][len(files[0])-i-1], files[0][i] + } + return NewVersion(cmp, formatKey, flushSplitBytes, files), nil +} + // Refs returns the number of references to the version. func (v *Version) Refs() int32 { return atomic.LoadInt32(&v.refs) @@ -526,7 +570,9 @@ func (v *Version) Contains(level int, cmp Compare, m *FileMetadata) bool { // and the computation is repeated until [start, end] stabilizes. // The returned files are a subsequence of the input files, i.e., the ordering // is not changed. -func (v *Version) Overlaps(level int, cmp Compare, start, end []byte, exclusiveEnd bool) LevelSlice { +func (v *Version) Overlaps( + level int, cmp Compare, start, end []byte, exclusiveEnd bool, +) LevelSlice { if level == 0 { // Indices that have been selected as overlapping. l0 := v.Levels[level] @@ -564,8 +610,14 @@ func (v *Version) Overlaps(level int, cmp Compare, start, end []byte, exclusiveE start = smallest restart = true } - if cmp(largest, end) > 0 { + if v := cmp(largest, end); v > 0 { end = largest + exclusiveEnd = meta.Largest.IsExclusiveSentinel() + restart = true + } else if v == 0 && exclusiveEnd && !meta.Largest.IsExclusiveSentinel() { + // Only update the exclusivity of our existing `end` + // bound. + exclusiveEnd = false restart = true } } diff --git a/internal/manifest/version_test.go b/internal/manifest/version_test.go index 381de2070f..09a65af712 100644 --- a/internal/manifest/version_test.go +++ b/internal/manifest/version_test.go @@ -5,6 +5,7 @@ package manifest import ( + "bytes" "fmt" "strconv" "strings" @@ -14,6 +15,7 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/datadriven" + "github.com/cockroachdb/pebble/internal/testkeys" "github.com/cockroachdb/pebble/vfs" "github.com/cockroachdb/redact" ) @@ -89,194 +91,36 @@ func TestIkeyRange(t *testing.T) { } func TestOverlaps(t *testing.T) { - m00 := &FileMetadata{ - FileNum: 700, - Size: 1, - Smallest: base.ParseInternalKey("b.SET.7008"), - Largest: base.ParseInternalKey("e.SET.7009"), - } - m01 := &FileMetadata{ - FileNum: 701, - Size: 1, - Smallest: base.ParseInternalKey("c.SET.7018"), - Largest: base.ParseInternalKey("f.SET.7019"), - } - m02 := &FileMetadata{ - FileNum: 702, - Size: 1, - Smallest: base.ParseInternalKey("f.SET.7028"), - Largest: base.ParseInternalKey("g.SET.7029"), - } - m03 := &FileMetadata{ - FileNum: 703, - Size: 1, - Smallest: base.ParseInternalKey("x.SET.7038"), - Largest: base.ParseInternalKey("y.SET.7039"), - } - m04 := &FileMetadata{ - FileNum: 704, - Size: 1, - Smallest: base.ParseInternalKey("n.SET.7048"), - Largest: base.ParseInternalKey("p.SET.7049"), - } - m05 := &FileMetadata{ - FileNum: 705, - Size: 1, - Smallest: base.ParseInternalKey("p.SET.7058"), - Largest: base.ParseInternalKey("p.SET.7059"), - } - m06 := &FileMetadata{ - FileNum: 706, - Size: 1, - Smallest: base.ParseInternalKey("p.SET.7068"), - Largest: base.ParseInternalKey("u.SET.7069"), - } - m07 := &FileMetadata{ - FileNum: 707, - Size: 1, - Smallest: base.ParseInternalKey("r.SET.7078"), - Largest: base.ParseInternalKey("s.SET.7079"), - } - - m10 := &FileMetadata{ - FileNum: 710, - Size: 1, - Smallest: base.ParseInternalKey("a.SET.7140"), - Largest: base.InternalKey{ - UserKey: []byte("d"), - Trailer: base.InternalKeyRangeDeleteSentinel, - }, - } - m11 := &FileMetadata{ - FileNum: 711, - Size: 1, - Smallest: base.ParseInternalKey("d.SET.7108"), - Largest: base.ParseInternalKey("g.SET.7109"), - } - m12 := &FileMetadata{ - FileNum: 712, - Size: 1, - Smallest: base.ParseInternalKey("g.SET.7118"), - Largest: base.ParseInternalKey("j.SET.7119"), - } - m13 := &FileMetadata{ - FileNum: 713, - Size: 1, - Smallest: base.ParseInternalKey("n.SET.7128"), - Largest: base.ParseInternalKey("p.SET.7129"), - } - m14 := &FileMetadata{ - FileNum: 714, - Size: 1, - Smallest: base.ParseInternalKey("p.SET.7148"), - Largest: base.ParseInternalKey("p.SET.7149"), - } - m15 := &FileMetadata{ - FileNum: 715, - Size: 1, - Smallest: base.ParseInternalKey("p.SET.7138"), - Largest: base.ParseInternalKey("u.SET.7139"), - } - - v := Version{ - Levels: [NumLevels]LevelMetadata{ - 0: levelMetadata(0, m00, m01, m02, m03, m04, m05, m06, m07), - 1: levelMetadata(1, m10, m11, m12, m13, m14, m15), - }, - } - - testCases := []struct { - level int - ukey0, ukey1 string - exclusiveEnd bool - want string - }{ - // Level 0: m00=b-e, m01=c-f, m02=f-g, m03=x-y, m04=n-p, m05=p-p, m06=p-u, m07=r-s. - // Note that: - // - the slice isn't sorted (e.g. m02=f-g, m03=x-y, m04=n-p), - // - m00 and m01 overlap (not just touch), - // - m06 contains m07, - // - m00, m01 and m02 transitively overlap/touch each other, and - // - m04, m05, m06 and m07 transitively overlap/touch each other. - {0, "a", "a", false, ""}, - {0, "a", "b", false, "m00 m01 m02"}, - {0, "a", "d", false, "m00 m01 m02"}, - {0, "a", "e", false, "m00 m01 m02"}, - {0, "a", "g", false, "m00 m01 m02"}, - {0, "a", "z", false, "m00 m01 m02 m03 m04 m05 m06 m07"}, - {0, "c", "e", false, "m00 m01 m02"}, - {0, "d", "d", false, "m00 m01 m02"}, - {0, "b", "f", true, "m00 m01"}, - {0, "g", "n", false, "m00 m01 m02 m04 m05 m06 m07"}, - {0, "h", "i", false, ""}, - {0, "h", "o", false, "m04 m05 m06 m07"}, - {0, "h", "u", false, "m04 m05 m06 m07"}, - {0, "k", "l", false, ""}, - {0, "k", "o", false, "m04 m05 m06 m07"}, - {0, "k", "p", false, "m04 m05 m06 m07"}, - {0, "n", "o", false, "m04 m05 m06 m07"}, - {0, "n", "z", false, "m03 m04 m05 m06 m07"}, - {0, "o", "z", false, "m03 m04 m05 m06 m07"}, - {0, "p", "z", false, "m03 m04 m05 m06 m07"}, - {0, "q", "z", false, "m03 m04 m05 m06 m07"}, - {0, "r", "s", false, "m04 m05 m06 m07"}, - {0, "r", "z", false, "m03 m04 m05 m06 m07"}, - {0, "s", "z", false, "m03 m04 m05 m06 m07"}, - {0, "u", "z", false, "m03 m04 m05 m06 m07"}, - {0, "y", "z", false, "m03"}, - {0, "z", "z", false, ""}, - - // Level 1: m10=a-d* m11=d-g, m12=g-j, m13=n-p, m14=p-p, m15=p-u. - // d* - exclusive, rangedel sentinel - {1, "a", "a", false, "m10"}, - {1, "a", "b", false, "m10"}, - {1, "a", "d", false, "m10 m11"}, - {1, "a", "e", false, "m10 m11"}, - {1, "a", "g", false, "m10 m11 m12"}, - {1, "a", "g", true, "m10 m11"}, - {1, "a", "z", false, "m10 m11 m12 m13 m14 m15"}, - {1, "c", "e", false, "m10 m11"}, - {1, "d", "d", false, "m11"}, - {1, "g", "n", false, "m11 m12 m13"}, - {1, "h", "i", false, "m12"}, - {1, "h", "n", true, "m12"}, - {1, "h", "n", false, "m12 m13"}, - {1, "h", "o", false, "m12 m13"}, - {1, "h", "u", false, "m12 m13 m14 m15"}, - {1, "k", "l", false, ""}, - {1, "k", "o", false, "m13"}, - {1, "k", "p", false, "m13 m14 m15"}, - {1, "k", "p", true, "m13"}, - {1, "n", "o", false, "m13"}, - {1, "n", "z", false, "m13 m14 m15"}, - {1, "o", "z", false, "m13 m14 m15"}, - {1, "p", "z", false, "m13 m14 m15"}, - {1, "q", "z", false, "m15"}, - {1, "r", "s", false, "m15"}, - {1, "r", "z", false, "m15"}, - {1, "s", "z", false, "m15"}, - {1, "u", "z", false, "m15"}, - {1, "y", "z", false, ""}, - {1, "z", "z", false, ""}, - - // Level 2: empty. - {2, "a", "z", false, ""}, - } - - cmp := base.DefaultComparer.Compare - for _, tc := range testCases { - overlaps := v.Overlaps(tc.level, cmp, []byte(tc.ukey0), []byte(tc.ukey1), tc.exclusiveEnd) - iter := overlaps.Iter() - var s []string - for meta := iter.First(); meta != nil; meta = iter.Next() { - s = append(s, fmt.Sprintf("m%02d", meta.FileNum%100)) - } - got := strings.Join(s, " ") - if got != tc.want { - t.Errorf("level=%d, range=%s-%s (exclusiveEnd = %t)\ngot %v\nwant %v", - tc.level, tc.ukey0, tc.ukey1, tc.exclusiveEnd, got, tc.want) + var v *Version + cmp := testkeys.Comparer.Compare + fmtKey := testkeys.Comparer.FormatKey + datadriven.RunTest(t, "testdata/overlaps", func(d *datadriven.TestData) string { + switch d.Cmd { + case "define": + var err error + v, err = ParseVersionDebug(cmp, fmtKey, 64>>10 /* flush split bytes */, d.Input) + if err != nil { + return err.Error() + } + return v.DebugString(fmtKey) + case "overlaps": + var level int + var start, end string + var exclusiveEnd bool + d.ScanArgs(t, "level", &level) + d.ScanArgs(t, "start", &start) + d.ScanArgs(t, "end", &end) + d.ScanArgs(t, "exclusive-end", &exclusiveEnd) + var buf bytes.Buffer + v.Overlaps(level, testkeys.Comparer.Compare, []byte(start), []byte(end), exclusiveEnd).Each(func(f *FileMetadata) { + fmt.Fprintf(&buf, "%06d:[%s-%s]\n", f.FileNum, + f.Smallest.Pretty(fmtKey), f.Largest.Pretty(fmtKey)) + }) + return buf.String() + default: + return fmt.Sprintf("unknown command: %s", d.Cmd) } - } + }) } func TestContains(t *testing.T) {