Skip to content

Commit

Permalink
pebble: improve syntax for external ingestion test
Browse files Browse the repository at this point in the history
Use a better syntax for the external ingestion test, which allows
specifying prefix and suffix replacement per object.
  • Loading branch information
RaduBerinde committed Feb 27, 2024
1 parent aed0b13 commit 86c76bf
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 70 deletions.
89 changes: 53 additions & 36 deletions data_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1284,46 +1284,63 @@ func runIngestCmd(td *datadriven.TestData, d *DB, fs vfs.FS) error {
return nil
}

func runIngestExternalCmd(td *datadriven.TestData, d *DB, locator string) error {
external := make([]ExternalFile, 0)
usePrefixChange := false
var fromPrefix, toPrefix []byte
var syntheticSuffix []byte
for i := range td.CmdArgs {
switch td.CmdArgs[i].Key {
case "prefix-replace":
vals := td.CmdArgs[i].Vals
if len(vals) != 2 {
return errors.New("usage: prefix-replace=(from,to)")
}
fromPrefix = []byte(vals[0])
toPrefix = []byte(vals[1])
usePrefixChange = true
case "suffix-replace":
syntheticSuffix = []byte(td.CmdArgs[i].Vals[0])
}
}
for _, arg := range strings.Split(td.Input, "\n") {
fields := strings.Split(arg, ",")
if len(fields) != 4 {
return errors.New("usage: path,size,smallest,largest")
func runIngestExternalCmd(t testing.TB, td *datadriven.TestData, d *DB, locator string) error {
var external []ExternalFile
for _, line := range strings.Split(td.Input, "\n") {
usageErr := func(info interface{}) {
t.Helper()
td.Fatalf(t, "error parsing %q: %v; "+
"usage: obj bounds=(smallest,largest) [size=x] [prefix-replace=(from,to)] [synthetic-prefix=prefix] [synthetic-suffix=suffix]",
line, info,
)
}
ef := ExternalFile{}
ef.Locator = remote.Locator(locator)
ef.ObjName = fields[0]
sizeInt, err := strconv.Atoi(fields[1])
objName, args, err := datadriven.ParseLine(line)
if err != nil {
return err
usageErr(err)
}
ef.Size = uint64(sizeInt)
ef.SmallestUserKey = []byte(fields[2])
ef.LargestUserKey = []byte(fields[3])
ef.HasPointKey = true
if usePrefixChange {
ef.ContentPrefix = fromPrefix
ef.SyntheticPrefix = toPrefix
ef := ExternalFile{
Locator: remote.Locator(locator),
ObjName: objName,
HasPointKey: true,
Size: 10,
}
ef.SyntheticSuffix = syntheticSuffix
for _, arg := range args {
nArgs := func(n int) {
if len(arg.Vals) != n {
usageErr(fmt.Sprintf("%s must have %d arguments", arg.Key, n))
}
}
switch arg.Key {
case "bounds":
nArgs(2)
ef.SmallestUserKey = []byte(arg.Vals[0])
ef.LargestUserKey = []byte(arg.Vals[1])

case "size":
nArgs(1)
arg.Scan(t, 0, &ef.Size)

case "prefix-replace":
nArgs(2)
ef.ContentPrefix = []byte(arg.Vals[0])
ef.SyntheticPrefix = []byte(arg.Vals[1])

case "synthetic-prefix":
nArgs(1)
ef.SyntheticPrefix = []byte(arg.Vals[0])

case "synthetic-suffix":
nArgs(1)
ef.SyntheticSuffix = []byte(arg.Vals[0])

default:
usageErr(fmt.Sprintf("unknown argument %v", arg.Key))
}
}
if ef.SmallestUserKey == nil {
usageErr("no bounds specified")
}

external = append(external, ef)
}

Expand Down
2 changes: 1 addition & 1 deletion ingest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1816,7 +1816,7 @@ func TestIngestExternal(t *testing.T) {

case "ingest-external":
flushed = false
if err := runIngestExternalCmd(td, d, "external-locator"); err != nil {
if err := runIngestExternalCmd(t, td, d, "external-locator"); err != nil {
return err.Error()
}
// Wait for a possible flush.
Expand Down
66 changes: 33 additions & 33 deletions testdata/ingest_external
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ set c foobar
----

ingest-external
f1,5,a,cc
f1 bounds=(a,cc)
----

lsm
Expand Down Expand Up @@ -39,7 +39,7 @@ set c foobar
----

ingest-external
f2,5,a,c
f2 bounds=(a,c)
----

lsm
Expand Down Expand Up @@ -73,14 +73,14 @@ set h foo
# This ingestion should error out due to the overlap between file spans.

ingest-external
f3,10,c,f
f4,10,e,h
f3 bounds=(c,f)
f4 bounds=(e,h)
----
pebble: external sstables have overlapping ranges

ingest-external
f3,10,c,f
f4,10,f,hh
f3 bounds=(c,f)
f4 bounds=(f,hh)
----

lsm
Expand Down Expand Up @@ -159,8 +159,8 @@ set eg foo
set eh foo
----

ingest-external prefix-replace=(e,f)
f5,10,ff,fi
ingest-external
f5 bounds=(ff,fi) prefix-replace=(e,f)
----

iter
Expand Down Expand Up @@ -202,14 +202,14 @@ set eg foo
set eh foo
----

ingest-external prefix-replace=(e,f)
f5,10,ff,fi
ingest-external
f5 bounds=(ff,fi) prefix-replace=(e,f)
----
pebble: format major version too old for synthetic prefix ingestion


ingest-external suffix-replace=@5
f5,10,ff,fi
ingest-external
f5 bounds=(ff,fi) synthetic-suffix=@5
----
pebble: format major version too old for synthetic suffix ingestion

Expand All @@ -224,8 +224,8 @@ set b@2 foo
set c@1 foo
----

ingest-external suffix-replace=@5
f1,10,a,d
ingest-external
f1 bounds=(a,d) synthetic-suffix=@5
----

iter
Expand All @@ -238,13 +238,13 @@ b@5: (foo, .)
c@5: (foo, .)

# Verify that we require bounds without suffix if we use suffix replacement.
ingest-external suffix-replace=@5
f6,10,a@1,z@10
ingest-external
f6 bounds=(a@1,z@10) synthetic-suffix=@5
----
pebble: synthetic suffix is set but smallest key has suffix

ingest-external suffix-replace=@5
f6,10,a,z@10
ingest-external
f6 bounds=(a,z@10) synthetic-suffix=@5
----
pebble: synthetic suffix is set but largest key has suffix

Expand All @@ -262,8 +262,8 @@ del-range f u
----

ingest-external
f6,10,a,c
f6,10,g,v
f6 bounds=(a,c)
f6 bounds=(g,v)
----

# The previous element cannot be i, because it is inside the [g, v) portion of
Expand All @@ -286,8 +286,8 @@ set x bar
----

ingest-external
f8,10,x,y
f7,10,a,b
f8 bounds=(x,y)
f7 bounds=(a,b)
----

iter
Expand All @@ -314,9 +314,9 @@ set fh foo
set fi foo
----

ingest-external prefix-replace=(f,g)
f7,10,gc,gf
f8,10,gg,gj
ingest-external
f7 bounds=(gc,gf) prefix-replace=(f,g)
f8 bounds=(gg,gj) prefix-replace=(f,g)
----

iter
Expand Down Expand Up @@ -373,8 +373,8 @@ build-remote f9
set i foo
----

ingest-external prefix-replace=(,c)
f9,10,cg,ck
ingest-external
f9 bounds=(cg,ck) synthetic-prefix=c
----

iter
Expand Down Expand Up @@ -405,8 +405,8 @@ build-remote f10
del-range i j
----

ingest-external prefix-replace=(,c)
f10,10,cg,ck
ingest-external
f10 bounds=(cg,ck) synthetic-prefix=c
----

iter
Expand All @@ -433,13 +433,13 @@ set ux ux
----

# Replace prefix with one greater than the original one.
ingest-external prefix-replace=(d,e)
ext,10,ea,ed
ingest-external
ext bounds=(ea,ed) prefix-replace=(d,e)
----

# Replace prefix with one smaller than the original one.
ingest-external prefix-replace=(d,b)
ext,10,ba,bd
ingest-external
ext bounds=(ba,bd) prefix-replace=(d,b)
----

# Write some keys so we actually perform a compaction.
Expand Down

0 comments on commit 86c76bf

Please sign in to comment.