diff --git a/objstorage/objstorageprovider/provider_test.go b/objstorage/objstorageprovider/provider_test.go index ce58be2881..f11279648c 100644 --- a/objstorage/objstorageprovider/provider_test.go +++ b/objstorage/objstorageprovider/provider_test.go @@ -98,13 +98,14 @@ func TestProvider(t *testing.T) { opts := objstorage.CreateOptions{ SharedCleanupMethod: objstorage.SharedRefTracking, } - if len(d.CmdArgs) == 3 && d.CmdArgs[2].Key == "no-ref-tracking" { - d.CmdArgs = d.CmdArgs[:2] + if len(d.CmdArgs) == 5 && d.CmdArgs[4].Key == "no-ref-tracking" { + d.CmdArgs = d.CmdArgs[:4] opts.SharedCleanupMethod = objstorage.SharedNoCleanup } var fileNum base.FileNum var typ string - scanArgs(" [no-ref-tracking]", &fileNum, &typ) + var salt, size int + scanArgs(" [no-ref-tracking]", &fileNum, &typ, &salt, &size) switch typ { case "local": case "shared": @@ -116,7 +117,10 @@ func TestProvider(t *testing.T) { if err != nil { return err.Error() } - require.NoError(t, w.Write([]byte(d.Input))) + data := make([]byte, size) + // TODO(radu): write in chunks? + genData(byte(salt), 0, data) + require.NoError(t, w.Write(data)) require.NoError(t, w.Finish()) return log.String() @@ -125,13 +129,14 @@ func TestProvider(t *testing.T) { opts := objstorage.CreateOptions{ SharedCleanupMethod: objstorage.SharedRefTracking, } - if len(d.CmdArgs) == 3 && d.CmdArgs[2].Key == "no-ref-tracking" { - d.CmdArgs = d.CmdArgs[:2] + if len(d.CmdArgs) == 5 && d.CmdArgs[4].Key == "no-ref-tracking" { + d.CmdArgs = d.CmdArgs[:4] opts.SharedCleanupMethod = objstorage.SharedNoCleanup } var fileNum base.FileNum var typ string - scanArgs(" [no-ref-tracking]", &fileNum, &typ) + var salt, size int + scanArgs(" [no-ref-tracking]", &fileNum, &typ, &salt, &size) switch typ { case "local": case "shared": @@ -144,8 +149,10 @@ func TestProvider(t *testing.T) { tmpFilename := fmt.Sprintf("temp-file-%d", tmpFileCounter) f, err := fs.Create(tmpFilename) require.NoError(t, err) - n, err := f.Write([]byte(d.Input)) - require.Equal(t, len(d.Input), n) + data := make([]byte, size) + genData(byte(salt), 0, data) + n, err := f.Write(data) + require.Equal(t, len(data), n) require.NoError(t, err) require.NoError(t, f.Close()) @@ -162,10 +169,23 @@ func TestProvider(t *testing.T) { if err != nil { return err.Error() } - data := make([]byte, int(r.Size())) - err = r.ReadAt(ctx, data, 0) - require.NoError(t, err) - return log.String() + fmt.Sprintf("data: %s\n", string(data)) + rh := r.NewReadHandle(ctx) + log.Infof("size: %d", r.Size()) + for _, l := range strings.Split(d.Input, "\n") { + var offset, size int + fmt.Sscanf(l, "%d %d", &offset, &size) + data := make([]byte, size) + err := rh.ReadAt(ctx, data, int64(offset)) + if err != nil { + log.Infof("%d %d: %v", offset, size, err) + } else { + salt := checkData(d, t, offset, data) + log.Infof("%d %d: ok (salt %d)", offset, size, salt) + } + } + require.NoError(t, rh.Close()) + require.NoError(t, r.Close()) + return log.String() case "remove": var fileNum base.FileNum @@ -289,3 +309,30 @@ func TestNotExistError(t *testing.T) { }) } } + +// genData generates object data that can be checked later with checkData. +func genData(salt byte, offset int, p []byte) { + for i := range p { + p[i] = salt ^ xor(offset+i) + } +} + +func checkData(d *datadriven.TestData, t *testing.T, offset int, p []byte) (salt byte) { + t.Helper() + salt = p[0] ^ xor(offset) + for i := range p { + if p[i]^xor(offset+i) != salt { + d.Fatalf(t, "invalid data") + } + } + return salt +} + +// xor returns the XOR of all bytes representing the integer. +func xor(n int) byte { + v := uint64(n) + v ^= v >> 32 + v ^= v >> 16 + v ^= v >> 8 + return byte(v) +} diff --git a/objstorage/objstorageprovider/testdata/provider/local b/objstorage/objstorageprovider/testdata/provider/local index c8cd4171c0..a2f4ce375b 100644 --- a/objstorage/objstorageprovider/testdata/provider/local +++ b/objstorage/objstorageprovider/testdata/provider/local @@ -5,7 +5,7 @@ open p0 0 mkdir-all: p0 0755 open-dir: p0 -create 1 local +create 1 local 1 1024 foo ---- create: p0/000001.sst @@ -13,21 +13,44 @@ foo close: p0/000001.sst read 1 +0 512 +0 1024 +512 1024 ---- -data: foo + open: p0/000001.sst +size: 1024 + read-at(0, 512): p0/000001.sst +0 512: ok (salt 1) + read-at(0, 1024): p0/000001.sst +0 1024: ok (salt 1) + prefetch(512, 65536): p0/000001.sst + read-at(512, 1024): p0/000001.sst +512 1024: EOF + close: p0/000001.sst # A provider without shared storage creates object with shared preference # locally. -create 2 shared -bar +create 2 shared 2 1024 ---- create: p0/000002.sst sync-data: p0/000002.sst close: p0/000002.sst read 2 +0 512 +0 1024 +512 1024 ---- -data: bar + open: p0/000002.sst +size: 1024 + read-at(0, 512): p0/000002.sst +0 512: ok (salt 2) + read-at(0, 1024): p0/000002.sst +0 1024: ok (salt 2) + prefetch(512, 65536): p0/000002.sst + read-at(512, 1024): p0/000002.sst +512 1024: EOF + close: p0/000002.sst remove 1 ---- @@ -41,27 +64,35 @@ read 1 ---- file 000001 (type 2) unknown to the objstorage provider: file does not exist -link-or-copy 3 local -three +link-or-copy 3 local 3 100 ---- create: temp-file-1 close: temp-file-1 link: temp-file-1 -> p0/000003.sst read 3 +0 100 ---- -data: three + open: p0/000003.sst +size: 100 + read-at(0, 100): p0/000003.sst +0 100: ok (salt 3) + close: p0/000003.sst -link-or-copy 4 shared -four +link-or-copy 4 shared 4 1234 ---- create: temp-file-2 close: temp-file-2 link: temp-file-2 -> p0/000004.sst read 4 +0 1234 ---- -data: four + open: p0/000004.sst +size: 1234 + read-at(0, 1234): p0/000004.sst +0 1234: ok (salt 4) + close: p0/000004.sst close ---- diff --git a/objstorage/objstorageprovider/testdata/provider/local_readahead b/objstorage/objstorageprovider/testdata/provider/local_readahead new file mode 100644 index 0000000000..0f9873f0ad --- /dev/null +++ b/objstorage/objstorageprovider/testdata/provider/local_readahead @@ -0,0 +1,51 @@ +open p1 1 +---- + mkdir-all: p1 0755 + open-dir: p1 + open-dir: p1 + create: p1/SHARED-CATALOG-000001 + sync: p1/SHARED-CATALOG-000001 + create: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 + close: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 + sync: p1 + sync: p1/SHARED-CATALOG-000001 + +create 1 local 1 2000000 +---- + create: p1/000001.sst + sync-data: p1/000001.sst + sync-data: p1/000001.sst + close: p1/000001.sst + +# We should see prefetch calls, and eventually a reopen +# (with sequential reads option). +read 1 +0 1000 +1000 15000 +16000 30000 +46000 10000 +56000 50000 +106000 30000 +140000 80000 +---- + open: p1/000001.sst +size: 2000000 + read-at(0, 1000): p1/000001.sst +0 1000: ok (salt 1) + read-at(1000, 15000): p1/000001.sst +1000 15000: ok (salt 1) + prefetch(16000, 65536): p1/000001.sst + read-at(16000, 30000): p1/000001.sst +16000 30000: ok (salt 1) + read-at(46000, 10000): p1/000001.sst +46000 10000: ok (salt 1) + prefetch(56000, 131072): p1/000001.sst + read-at(56000, 50000): p1/000001.sst +56000 50000: ok (salt 1) + read-at(106000, 30000): p1/000001.sst +106000 30000: ok (salt 1) + open: p1/000001.sst + read-at(140000, 80000): p1/000001.sst +140000 80000: ok (salt 1) + close: p1/000001.sst + close: p1/000001.sst diff --git a/objstorage/objstorageprovider/testdata/provider/shared_attach b/objstorage/objstorageprovider/testdata/provider/shared_attach index 77df361773..69cdbbcdbf 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_attach +++ b/objstorage/objstorageprovider/testdata/provider/shared_attach @@ -13,32 +13,28 @@ open p1 1 sync: p1 sync: p1/SHARED-CATALOG-000001 -create 1 shared -obj-one +create 1 shared 1 100 ---- create object "61a6-1-000001.sst" - close writer for "61a6-1-000001.sst" after 7 bytes + close writer for "61a6-1-000001.sst" after 100 bytes create object "61a6-1-000001.sst.ref.1.000001" close writer for "61a6-1-000001.sst.ref.1.000001" after 0 bytes -create 2 shared -obj-two +create 2 shared 2 200 ---- create object "a629-1-000002.sst" - close writer for "a629-1-000002.sst" after 7 bytes + close writer for "a629-1-000002.sst" after 200 bytes create object "a629-1-000002.sst.ref.1.000002" close writer for "a629-1-000002.sst.ref.1.000002" after 0 bytes -create 3 shared -obj-three +create 3 shared 3 300 ---- create object "eaac-1-000003.sst" - close writer for "eaac-1-000003.sst" after 9 bytes + close writer for "eaac-1-000003.sst" after 300 bytes create object "eaac-1-000003.sst.ref.1.000003" close writer for "eaac-1-000003.sst.ref.1.000003" after 0 bytes -create 100 local -obj-one +create 100 local 100 15 ---- create: p1/000100.sst sync-data: p1/000100.sst @@ -83,8 +79,7 @@ open p2 2 sync: p2 sync: p2/SHARED-CATALOG-000001 -create 100 shared -obj-one-hundred +create 100 shared 100 15 ---- create object "fd72-2-000100.sst" close writer for "fd72-2-000100.sst" after 15 bytes @@ -118,25 +113,39 @@ list 000103 -> shared://eaac-1-000003.sst read 101 +0 100 +15 10 ---- size of object "61a6-1-000001.sst.ref.2.000101": 0 - size of object "61a6-1-000001.sst": 7 - read object "61a6-1-000001.sst" at 0: 7 bytes + size of object "61a6-1-000001.sst": 100 +size: 100 + read object "61a6-1-000001.sst" at 0: 100 bytes +0 100: ok (salt 1) + close reader for "61a6-1-000001.sst" + read object "61a6-1-000001.sst" at 15: 85 bytes +15 10: ok (salt 1) close reader for "61a6-1-000001.sst" -data: obj-one read 102 +0 200 +90 100 ---- size of object "a629-1-000002.sst.ref.2.000102": 0 - size of object "a629-1-000002.sst": 7 - read object "a629-1-000002.sst" at 0: 7 bytes + size of object "a629-1-000002.sst": 200 +size: 200 + read object "a629-1-000002.sst" at 0: 200 bytes +0 200: ok (salt 2) + close reader for "a629-1-000002.sst" + read object "a629-1-000002.sst" at 90: 110 bytes +90 100: ok (salt 2) close reader for "a629-1-000002.sst" -data: obj-two read 103 +0 300 ---- size of object "eaac-1-000003.sst.ref.2.000103": 0 - size of object "eaac-1-000003.sst": 9 - read object "eaac-1-000003.sst" at 0: 9 bytes + size of object "eaac-1-000003.sst": 300 +size: 300 + read object "eaac-1-000003.sst" at 0: 300 bytes +0 300: ok (salt 3) close reader for "eaac-1-000003.sst" -data: obj-three diff --git a/objstorage/objstorageprovider/testdata/provider/shared_attach_after_unref b/objstorage/objstorageprovider/testdata/provider/shared_attach_after_unref index 00d883f10a..d13f692fd4 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_attach_after_unref +++ b/objstorage/objstorageprovider/testdata/provider/shared_attach_after_unref @@ -12,11 +12,10 @@ open p5 5 sync: p5 sync: p5/SHARED-CATALOG-000001 -create 1 shared -obj-one +create 1 shared 1 100 ---- create object "d632-5-000001.sst" - close writer for "d632-5-000001.sst" after 7 bytes + close writer for "d632-5-000001.sst" after 100 bytes create object "d632-5-000001.sst.ref.5.000001" close writer for "d632-5-000001.sst.ref.5.000001" after 0 bytes @@ -56,11 +55,10 @@ switch p5 close-backing p5b1 ---- -create 2 shared -obj-two +create 2 shared 2 100 ---- create object "1ab5-5-000002.sst" - close writer for "1ab5-5-000002.sst" after 7 bytes + close writer for "1ab5-5-000002.sst" after 100 bytes create object "1ab5-5-000002.sst.ref.5.000002" close writer for "1ab5-5-000002.sst.ref.5.000002" after 0 bytes diff --git a/objstorage/objstorageprovider/testdata/provider/shared_attach_multi b/objstorage/objstorageprovider/testdata/provider/shared_attach_multi index 013fe746a0..343cb4b8ab 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_attach_multi +++ b/objstorage/objstorageprovider/testdata/provider/shared_attach_multi @@ -12,11 +12,10 @@ open p1 1 sync: p1 sync: p1/SHARED-CATALOG-000001 -create 1 shared -obj-one +create 1 shared 1 100 ---- create object "61a6-1-000001.sst" - close writer for "61a6-1-000001.sst" after 7 bytes + close writer for "61a6-1-000001.sst" after 100 bytes create object "61a6-1-000001.sst.ref.1.000001" close writer for "61a6-1-000001.sst.ref.1.000001" after 0 bytes diff --git a/objstorage/objstorageprovider/testdata/provider/shared_basic b/objstorage/objstorageprovider/testdata/provider/shared_basic index a73efcf45d..e560eef520 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_basic +++ b/objstorage/objstorageprovider/testdata/provider/shared_basic @@ -13,32 +13,37 @@ open p1 1 sync: p1 sync: p1/SHARED-CATALOG-000001 -create 1 local -obj-one +create 1 local 1 100 ---- create: p1/000001.sst sync-data: p1/000001.sst close: p1/000001.sst read 1 +0 100 ---- -data: obj-one + open: p1/000001.sst +size: 100 + read-at(0, 100): p1/000001.sst +0 100: ok (salt 1) + close: p1/000001.sst -create 2 shared -obj-two +create 2 shared 2 100 ---- create object "a629-1-000002.sst" - close writer for "a629-1-000002.sst" after 7 bytes + close writer for "a629-1-000002.sst" after 100 bytes create object "a629-1-000002.sst.ref.1.000002" close writer for "a629-1-000002.sst.ref.1.000002" after 0 bytes read 2 +0 100 ---- size of object "a629-1-000002.sst.ref.1.000002": 0 - size of object "a629-1-000002.sst": 7 - read object "a629-1-000002.sst" at 0: 7 bytes + size of object "a629-1-000002.sst": 100 +size: 100 + read object "a629-1-000002.sst" at 0: 100 bytes +0 100: ok (salt 2) close reader for "a629-1-000002.sst" -data: obj-two list ---- @@ -57,6 +62,8 @@ open p1 1 mkdir-all: p1 0755 open-dir: p1 open-dir: p1 + open: p1/SHARED-CATALOG-000001 + close: p1/SHARED-CATALOG-000001 list ---- @@ -73,34 +80,41 @@ remove 2 list (prefix="a629-1-000002.sst.ref.", delimiter="") delete object "a629-1-000002.sst" -link-or-copy 3 local -three +link-or-copy 3 local 3 100 ---- create: temp-file-1 close: temp-file-1 link: temp-file-1 -> p1/000003.sst read 3 +0 100 ---- -data: three + open: p1/000003.sst +size: 100 + read-at(0, 100): p1/000003.sst +0 100: ok (salt 3) + close: p1/000003.sst -link-or-copy 4 shared -four +link-or-copy 4 shared 4 100 ---- create: temp-file-2 close: temp-file-2 create object "2f2f-1-000004.sst" - close writer for "2f2f-1-000004.sst" after 4 bytes + open: temp-file-2 + close writer for "2f2f-1-000004.sst" after 100 bytes create object "2f2f-1-000004.sst.ref.1.000004" close writer for "2f2f-1-000004.sst.ref.1.000004" after 0 bytes + close: temp-file-2 read 4 +0 100 ---- size of object "2f2f-1-000004.sst.ref.1.000004": 0 - size of object "2f2f-1-000004.sst": 4 - read object "2f2f-1-000004.sst" at 0: 4 bytes + size of object "2f2f-1-000004.sst": 100 +size: 100 + read object "2f2f-1-000004.sst" at 0: 100 bytes +0 100: ok (salt 4) close reader for "2f2f-1-000004.sst" -data: four close ---- diff --git a/objstorage/objstorageprovider/testdata/provider/shared_no_ref b/objstorage/objstorageprovider/testdata/provider/shared_no_ref index fa9d1e2b71..2f67a50ab9 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_no_ref +++ b/objstorage/objstorageprovider/testdata/provider/shared_no_ref @@ -13,51 +13,56 @@ open p1 1 sync: p1 sync: p1/SHARED-CATALOG-000001 -create 1 shared no-ref-tracking -obj-one +create 1 shared 1 100 no-ref-tracking ---- create object "61a6-1-000001.sst" - close writer for "61a6-1-000001.sst" after 7 bytes + close writer for "61a6-1-000001.sst" after 100 bytes read 1 +0 100 ---- - size of object "61a6-1-000001.sst": 7 - read object "61a6-1-000001.sst" at 0: 7 bytes + size of object "61a6-1-000001.sst": 100 +size: 100 + read object "61a6-1-000001.sst" at 0: 100 bytes +0 100: ok (salt 1) close reader for "61a6-1-000001.sst" -data: obj-one -create 2 shared no-ref-tracking -obj-two +create 2 shared 2 100 no-ref-tracking ---- create object "a629-1-000002.sst" - close writer for "a629-1-000002.sst" after 7 bytes + close writer for "a629-1-000002.sst" after 100 bytes read 2 +0 100 ---- - size of object "a629-1-000002.sst": 7 - read object "a629-1-000002.sst" at 0: 7 bytes + size of object "a629-1-000002.sst": 100 +size: 100 + read object "a629-1-000002.sst" at 0: 100 bytes +0 100: ok (salt 2) close reader for "a629-1-000002.sst" -data: obj-two list ---- 000001 -> shared://61a6-1-000001.sst 000002 -> shared://a629-1-000002.sst -link-or-copy 3 shared no-ref-tracking -obj-three +link-or-copy 3 shared 3 100 no-ref-tracking ---- create: temp-file-1 close: temp-file-1 create object "eaac-1-000003.sst" - close writer for "eaac-1-000003.sst" after 9 bytes + open: temp-file-1 + close writer for "eaac-1-000003.sst" after 100 bytes + close: temp-file-1 read 3 +0 100 ---- - size of object "eaac-1-000003.sst": 9 - read object "eaac-1-000003.sst" at 0: 9 bytes + size of object "eaac-1-000003.sst": 100 +size: 100 + read object "eaac-1-000003.sst" at 0: 100 bytes +0 100: ok (salt 3) close reader for "eaac-1-000003.sst" -data: obj-three close ---- @@ -70,6 +75,8 @@ open p1 1 mkdir-all: p1 0755 open-dir: p1 open-dir: p1 + open: p1/SHARED-CATALOG-000001 + close: p1/SHARED-CATALOG-000001 list ---- @@ -78,25 +85,31 @@ list 000003 -> shared://eaac-1-000003.sst read 1 +0 100 ---- - size of object "61a6-1-000001.sst": 7 - read object "61a6-1-000001.sst" at 0: 7 bytes + size of object "61a6-1-000001.sst": 100 +size: 100 + read object "61a6-1-000001.sst" at 0: 100 bytes +0 100: ok (salt 1) close reader for "61a6-1-000001.sst" -data: obj-one read 2 +0 100 ---- - size of object "a629-1-000002.sst": 7 - read object "a629-1-000002.sst" at 0: 7 bytes + size of object "a629-1-000002.sst": 100 +size: 100 + read object "a629-1-000002.sst" at 0: 100 bytes +0 100: ok (salt 2) close reader for "a629-1-000002.sst" -data: obj-two read 3 +0 100 ---- - size of object "eaac-1-000003.sst": 9 - read object "eaac-1-000003.sst" at 0: 9 bytes + size of object "eaac-1-000003.sst": 100 +size: 100 + read object "eaac-1-000003.sst" at 0: 100 bytes +0 100: ok (salt 3) close reader for "eaac-1-000003.sst" -data: obj-three save-backing b1 1 ---- @@ -130,18 +143,22 @@ list 000102 -> shared://61a6-1-000001.sst read 101 +0 100 ---- - size of object "61a6-1-000001.sst": 7 - read object "61a6-1-000001.sst" at 0: 7 bytes + size of object "61a6-1-000001.sst": 100 +size: 100 + read object "61a6-1-000001.sst" at 0: 100 bytes +0 100: ok (salt 1) close reader for "61a6-1-000001.sst" -data: obj-one read 102 +0 100 ---- - size of object "61a6-1-000001.sst": 7 - read object "61a6-1-000001.sst" at 0: 7 bytes + size of object "61a6-1-000001.sst": 100 +size: 100 + read object "61a6-1-000001.sst" at 0: 100 bytes +0 100: ok (salt 1) close reader for "61a6-1-000001.sst" -data: obj-one # In this mode, all removes should be no-ops on the shared backend. remove 101 diff --git a/objstorage/objstorageprovider/testdata/provider/shared_remove b/objstorage/objstorageprovider/testdata/provider/shared_remove index 82c12946af..353cc04c8d 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_remove +++ b/objstorage/objstorageprovider/testdata/provider/shared_remove @@ -10,27 +10,24 @@ open p1 1 sync: p1 sync: p1/SHARED-CATALOG-000001 -create 1 shared -obj-one +create 1 shared 1 100 ---- create object "61a6-1-000001.sst" - close writer for "61a6-1-000001.sst" after 7 bytes + close writer for "61a6-1-000001.sst" after 100 bytes create object "61a6-1-000001.sst.ref.1.000001" close writer for "61a6-1-000001.sst.ref.1.000001" after 0 bytes -create 2 shared -obj-two +create 2 shared 2 100 ---- create object "a629-1-000002.sst" - close writer for "a629-1-000002.sst" after 7 bytes + close writer for "a629-1-000002.sst" after 100 bytes create object "a629-1-000002.sst.ref.1.000002" close writer for "a629-1-000002.sst.ref.1.000002" after 0 bytes -create 3 shared -obj-three +create 3 shared 3 100 ---- create object "eaac-1-000003.sst" - close writer for "eaac-1-000003.sst" after 9 bytes + close writer for "eaac-1-000003.sst" after 100 bytes create object "eaac-1-000003.sst.ref.1.000003" close writer for "eaac-1-000003.sst.ref.1.000003" after 0 bytes @@ -52,11 +49,10 @@ open p2 2 sync: p2 sync: p2/SHARED-CATALOG-000001 -create 4 shared -obj-four +create 4 shared 4 100 ---- create object "4c52-2-000004.sst" - close writer for "4c52-2-000004.sst" after 8 bytes + close writer for "4c52-2-000004.sst" after 100 bytes create object "4c52-2-000004.sst.ref.2.000004" close writer for "4c52-2-000004.sst.ref.2.000004" after 0 bytes diff --git a/objstorage/objstorageprovider/testdata/readahead b/objstorage/objstorageprovider/testdata/readahead index 0bb7383b08..d2bb217314 100644 --- a/objstorage/objstorageprovider/testdata/readahead +++ b/objstorage/objstorageprovider/testdata/readahead @@ -1,3 +1,6 @@ +reset +---- + read 2048, 16 ---- @@ -136,7 +139,7 @@ limit: 73216 read 7780, 16 ----- +--- readahead: 0 numReads: 4 size: 131072 @@ -146,17 +149,13 @@ limit: 73216 read 7880, 16 ---- -readahead: 0 -numReads: 5 -size: 131072 -prevSize: 65536 -limit: 73216 +expected 2 args: offset, size read 7980, 16 ---- readahead: 0 -numReads: 6 +numReads: 4 size: 131072 prevSize: 65536 limit: 73216 @@ -165,7 +164,7 @@ read 73416, 16 ---- readahead: 131072 -numReads: 7 +numReads: 5 size: 262144 prevSize: 131072 limit: 204488 @@ -174,7 +173,7 @@ read 204488, 16 ---- readahead: 262144 -numReads: 8 +numReads: 6 size: 262144 prevSize: 262144 limit: 466632 @@ -185,7 +184,7 @@ read 466632, 16 ---- readahead: 262144 -numReads: 9 +numReads: 7 size: 262144 prevSize: 262144 limit: 728776 @@ -196,7 +195,7 @@ cache-read 728770, 16 ---- readahead: 0 -numReads: 9 +numReads: 7 size: 262144 prevSize: 262144 limit: 728786 @@ -205,7 +204,7 @@ read 728780, 16 ---- readahead: 262144 -numReads: 10 +numReads: 8 size: 262144 prevSize: 262144 limit: 990924 diff --git a/testdata/checkpoint b/testdata/checkpoint index ead1c79df8..131e6ebacc 100644 --- a/testdata/checkpoint +++ b/testdata/checkpoint @@ -5,6 +5,7 @@ open-dir: db lock: db/LOCK open-dir: db open-dir: db +open: db/CURRENT create: db/MANIFEST-000001 sync: db/MANIFEST-000001 remove: db/temporary.000001.dbtmp @@ -145,18 +146,22 @@ sync: checkpoints/checkpoint1 close: checkpoints/checkpoint1 link: db/000005.sst -> checkpoints/checkpoint1/000005.sst link: db/000007.sst -> checkpoints/checkpoint1/000007.sst +open: db/MANIFEST-000001 create: checkpoints/checkpoint1/MANIFEST-000001 sync-data: checkpoints/checkpoint1/MANIFEST-000001 close: checkpoints/checkpoint1/MANIFEST-000001 +close: db/MANIFEST-000001 open-dir: checkpoints/checkpoint1 create: checkpoints/checkpoint1/marker.manifest.000001.MANIFEST-000001 sync-data: checkpoints/checkpoint1/marker.manifest.000001.MANIFEST-000001 close: checkpoints/checkpoint1/marker.manifest.000001.MANIFEST-000001 sync: checkpoints/checkpoint1 close: checkpoints/checkpoint1 +open: db/000006.log create: checkpoints/checkpoint1/000006.log sync-data: checkpoints/checkpoint1/000006.log close: checkpoints/checkpoint1/000006.log +close: db/000006.log sync: checkpoints/checkpoint1 close: checkpoints/checkpoint1 @@ -180,18 +185,22 @@ close: checkpoints/checkpoint2/marker.format-version.000001.014 sync: checkpoints/checkpoint2 close: checkpoints/checkpoint2 link: db/000007.sst -> checkpoints/checkpoint2/000007.sst +open: db/MANIFEST-000001 create: checkpoints/checkpoint2/MANIFEST-000001 sync-data: checkpoints/checkpoint2/MANIFEST-000001 close: checkpoints/checkpoint2/MANIFEST-000001 +close: db/MANIFEST-000001 open-dir: checkpoints/checkpoint2 create: checkpoints/checkpoint2/marker.manifest.000001.MANIFEST-000001 sync-data: checkpoints/checkpoint2/marker.manifest.000001.MANIFEST-000001 close: checkpoints/checkpoint2/marker.manifest.000001.MANIFEST-000001 sync: checkpoints/checkpoint2 close: checkpoints/checkpoint2 +open: db/000006.log create: checkpoints/checkpoint2/000006.log sync-data: checkpoints/checkpoint2/000006.log close: checkpoints/checkpoint2/000006.log +close: db/000006.log sync: checkpoints/checkpoint2 close: checkpoints/checkpoint2 @@ -212,18 +221,22 @@ sync: checkpoints/checkpoint3 close: checkpoints/checkpoint3 link: db/000005.sst -> checkpoints/checkpoint3/000005.sst link: db/000007.sst -> checkpoints/checkpoint3/000007.sst +open: db/MANIFEST-000001 create: checkpoints/checkpoint3/MANIFEST-000001 sync-data: checkpoints/checkpoint3/MANIFEST-000001 close: checkpoints/checkpoint3/MANIFEST-000001 +close: db/MANIFEST-000001 open-dir: checkpoints/checkpoint3 create: checkpoints/checkpoint3/marker.manifest.000001.MANIFEST-000001 sync-data: checkpoints/checkpoint3/marker.manifest.000001.MANIFEST-000001 close: checkpoints/checkpoint3/marker.manifest.000001.MANIFEST-000001 sync: checkpoints/checkpoint3 close: checkpoints/checkpoint3 +open: db/000006.log create: checkpoints/checkpoint3/000006.log sync-data: checkpoints/checkpoint3/000006.log close: checkpoints/checkpoint3/000006.log +close: db/000006.log sync: checkpoints/checkpoint3 close: checkpoints/checkpoint3 @@ -238,11 +251,44 @@ sync-data: db/000009.sst close: db/000009.sst sync: db sync: db/MANIFEST-000001 +open: db/000005.sst +read-at(744, 53): db/000005.sst +read-at(707, 37): db/000005.sst +read-at(79, 628): db/000005.sst +read-at(52, 27): db/000005.sst +open: db/000005.sst +close: db/000005.sst +open: db/000009.sst +read-at(732, 53): db/000009.sst +read-at(695, 37): db/000009.sst +read-at(67, 628): db/000009.sst +read-at(40, 27): db/000009.sst +open: db/000009.sst +close: db/000009.sst +open: db/000007.sst +read-at(744, 53): db/000007.sst +read-at(707, 37): db/000007.sst +read-at(79, 628): db/000007.sst +read-at(52, 27): db/000007.sst +open: db/000007.sst +close: db/000007.sst +open: db/000005.sst +read-at(0, 52): db/000005.sst +open: db/000007.sst +read-at(0, 52): db/000007.sst create: db/000010.sst +close: db/000005.sst +open: db/000009.sst +read-at(0, 40): db/000009.sst +close: db/000007.sst +close: db/000009.sst sync-data: db/000010.sst close: db/000010.sst sync: db sync: db/MANIFEST-000001 +close: db/000005.sst +close: db/000007.sst +close: db/000009.sst remove: db/000005.sst remove: db/000007.sst remove: db/000009.sst @@ -280,10 +326,28 @@ open-dir: checkpoints/checkpoint1 lock: checkpoints/checkpoint1/LOCK open-dir: checkpoints/checkpoint1 open-dir: checkpoints/checkpoint1 +open: checkpoints/checkpoint1/MANIFEST-000001 +close: checkpoints/checkpoint1/MANIFEST-000001 open-dir: checkpoints/checkpoint1 +open: checkpoints/checkpoint1/OPTIONS-000003 +close: checkpoints/checkpoint1/OPTIONS-000003 +open: checkpoints/checkpoint1/000006.log +close: checkpoints/checkpoint1/000006.log scan checkpoints/checkpoint1 ---- +open: checkpoints/checkpoint1/000007.sst +read-at(744, 53): checkpoints/checkpoint1/000007.sst +read-at(707, 37): checkpoints/checkpoint1/000007.sst +read-at(79, 628): checkpoints/checkpoint1/000007.sst +read-at(52, 27): checkpoints/checkpoint1/000007.sst +read-at(0, 52): checkpoints/checkpoint1/000007.sst +open: checkpoints/checkpoint1/000005.sst +read-at(744, 53): checkpoints/checkpoint1/000005.sst +read-at(707, 37): checkpoints/checkpoint1/000005.sst +read-at(79, 628): checkpoints/checkpoint1/000005.sst +read-at(52, 27): checkpoints/checkpoint1/000005.sst +read-at(0, 52): checkpoints/checkpoint1/000005.sst a 1 b 5 c 3 @@ -295,6 +359,12 @@ g 10 scan db ---- +open: db/000010.sst +read-at(766, 53): db/000010.sst +read-at(729, 37): db/000010.sst +read-at(101, 628): db/000010.sst +read-at(74, 27): db/000010.sst +read-at(0, 74): db/000010.sst a 1 b 5 c 3 @@ -321,10 +391,22 @@ open-dir: checkpoints/checkpoint2 lock: checkpoints/checkpoint2/LOCK open-dir: checkpoints/checkpoint2 open-dir: checkpoints/checkpoint2 +open: checkpoints/checkpoint2/MANIFEST-000001 +close: checkpoints/checkpoint2/MANIFEST-000001 open-dir: checkpoints/checkpoint2 +open: checkpoints/checkpoint2/OPTIONS-000003 +close: checkpoints/checkpoint2/OPTIONS-000003 +open: checkpoints/checkpoint2/000006.log +close: checkpoints/checkpoint2/000006.log scan checkpoints/checkpoint2 ---- +open: checkpoints/checkpoint2/000007.sst +read-at(744, 53): checkpoints/checkpoint2/000007.sst +read-at(707, 37): checkpoints/checkpoint2/000007.sst +read-at(79, 628): checkpoints/checkpoint2/000007.sst +read-at(52, 27): checkpoints/checkpoint2/000007.sst +read-at(0, 52): checkpoints/checkpoint2/000007.sst b 5 d 7 e 8 @@ -349,10 +431,28 @@ open-dir: checkpoints/checkpoint3 lock: checkpoints/checkpoint3/LOCK open-dir: checkpoints/checkpoint3 open-dir: checkpoints/checkpoint3 +open: checkpoints/checkpoint3/MANIFEST-000001 +close: checkpoints/checkpoint3/MANIFEST-000001 open-dir: checkpoints/checkpoint3 +open: checkpoints/checkpoint3/OPTIONS-000003 +close: checkpoints/checkpoint3/OPTIONS-000003 +open: checkpoints/checkpoint3/000006.log +close: checkpoints/checkpoint3/000006.log scan checkpoints/checkpoint3 ---- +open: checkpoints/checkpoint3/000007.sst +read-at(744, 53): checkpoints/checkpoint3/000007.sst +read-at(707, 37): checkpoints/checkpoint3/000007.sst +read-at(79, 628): checkpoints/checkpoint3/000007.sst +read-at(52, 27): checkpoints/checkpoint3/000007.sst +read-at(0, 52): checkpoints/checkpoint3/000007.sst +open: checkpoints/checkpoint3/000005.sst +read-at(744, 53): checkpoints/checkpoint3/000005.sst +read-at(707, 37): checkpoints/checkpoint3/000005.sst +read-at(79, 628): checkpoints/checkpoint3/000005.sst +read-at(52, 27): checkpoints/checkpoint3/000005.sst +read-at(0, 52): checkpoints/checkpoint3/000005.sst a 1 b 5 c 3 diff --git a/testdata/cleaner b/testdata/cleaner index e5b4059cbc..cd96e7d2fb 100644 --- a/testdata/cleaner +++ b/testdata/cleaner @@ -8,6 +8,7 @@ open-dir: db_wal lock: db/LOCK open-dir: db open-dir: db +open: db/CURRENT create: db/MANIFEST-000001 sync: db/MANIFEST-000001 remove: db/temporary.000001.dbtmp @@ -65,11 +66,33 @@ sync: db sync: db/MANIFEST-000001 mkdir-all: db_wal/archive 0755 rename: db_wal/000004.log -> db_wal/archive/000004.log +open: db/000005.sst +read-at(744, 53): db/000005.sst +read-at(707, 37): db/000005.sst +read-at(79, 628): db/000005.sst +read-at(52, 27): db/000005.sst +open: db/000005.sst +close: db/000005.sst +open: db/000007.sst +read-at(718, 53): db/000007.sst +read-at(681, 37): db/000007.sst +read-at(53, 628): db/000007.sst +read-at(26, 27): db/000007.sst +open: db/000007.sst +close: db/000007.sst +open: db/000005.sst +read-at(0, 52): db/000005.sst create: db/000008.sst +close: db/000005.sst +open: db/000007.sst +read-at(0, 26): db/000007.sst +close: db/000007.sst sync-data: db/000008.sst close: db/000008.sst sync: db sync: db/MANIFEST-000001 +close: db/000005.sst +close: db/000007.sst mkdir-all: db/archive 0755 rename: db/000005.sst -> db/archive/000005.sst mkdir-all: db/archive 0755 @@ -109,6 +132,7 @@ open-dir: db1_wal lock: db1/LOCK open-dir: db1 open-dir: db1 +open: db1/CURRENT create: db1/MANIFEST-000001 sync: db1/MANIFEST-000001 remove: db1/temporary.000001.dbtmp @@ -178,7 +202,16 @@ open-dir: db1_wal lock: db1/LOCK open-dir: db1 open-dir: db1 +open: db1/CURRENT +read-at(0, 16): db1/CURRENT +close: db1/CURRENT +open: db1/MANIFEST-000001 +close: db1/MANIFEST-000001 open-dir: db1 +open: db1/OPTIONS-000003 +close: db1/OPTIONS-000003 +open: db1_wal/000004.log +close: db1_wal/000004.log create: db1/MANIFEST-000458 sync: db1/MANIFEST-000458 remove: db1/temporary.000458.dbtmp diff --git a/testdata/event_listener b/testdata/event_listener index 9785c3e820..afdc789b40 100644 --- a/testdata/event_listener +++ b/testdata/event_listener @@ -7,6 +7,7 @@ open-dir: wal lock: db/LOCK open-dir: db open-dir: db +open: db/CURRENT create: db/MANIFEST-000001 sync: db/MANIFEST-000001 remove: db/temporary.000001.dbtmp @@ -149,6 +150,26 @@ sync: db remove: db/MANIFEST-000001 [JOB 7] MANIFEST deleted 000001 [JOB 8] compacting(default) L0 [000005 000008] (1.5 K) + L6 [] (0 B) +open: db/000005.sst +read-at(717, 53): db/000005.sst +read-at(680, 37): db/000005.sst +read-at(52, 628): db/000005.sst +read-at(25, 27): db/000005.sst +open: db/000005.sst +close: db/000005.sst +open: db/000008.sst +read-at(717, 53): db/000008.sst +read-at(680, 37): db/000008.sst +read-at(52, 628): db/000008.sst +read-at(25, 27): db/000008.sst +open: db/000008.sst +close: db/000008.sst +open: db/000005.sst +read-at(0, 25): db/000005.sst +open: db/000008.sst +read-at(0, 25): db/000008.sst +close: db/000008.sst +close: db/000005.sst create: db/000010.sst [JOB 8] compacting: sstable created 000010 sync-data: db/000010.sst @@ -163,6 +184,8 @@ remove: db/marker.manifest.000003.MANIFEST-000009 sync: db [JOB 8] MANIFEST created 000011 [JOB 8] compacted(default) L0 [000005 000008] (1.5 K) + L6 [] (0 B) -> L6 [000010] (770 B), in 1.0s (3.0s total), output rate 770 B/s +close: db/000005.sst +close: db/000008.sst remove: db/000005.sst [JOB 8] sstable deleted 000005 remove: db/000008.sst @@ -204,9 +227,22 @@ remove: db/MANIFEST-000009 ingest ---- +open: ext/0 +read-at(773, 53): ext/0 +read-at(736, 37): ext/0 +read-at(53, 683): ext/0 +read-at(26, 27): ext/0 +read-at(0, 26): ext/0 +close: ext/0 link: ext/0 -> db/000015.sst [JOB 12] ingesting: sstable created 000015 sync: db +open: db/000013.sst +read-at(717, 53): db/000013.sst +read-at(680, 37): db/000013.sst +read-at(52, 628): db/000013.sst +read-at(25, 27): db/000013.sst +read-at(0, 25): db/000013.sst create: db/MANIFEST-000016 close: db/MANIFEST-000014 sync: db/MANIFEST-000016 @@ -249,6 +285,20 @@ zmemtbl 0 0 B ingest-flushable ---- sync-data: wal/000012.log +open: ext/a +read-at(773, 53): ext/a +read-at(736, 37): ext/a +read-at(53, 683): ext/a +read-at(26, 27): ext/a +read-at(0, 26): ext/a +close: ext/a +open: ext/b +read-at(773, 53): ext/b +read-at(736, 37): ext/b +read-at(53, 683): ext/b +read-at(26, 27): ext/b +read-at(0, 26): ext/b +close: ext/b link: ext/a -> db/000017.sst [JOB 13] ingesting: sstable created 000017 link: ext/b -> db/000018.sst @@ -354,18 +404,22 @@ link: db/000022.sst -> checkpoint/000022.sst link: db/000017.sst -> checkpoint/000017.sst link: db/000010.sst -> checkpoint/000010.sst link: db/000018.sst -> checkpoint/000018.sst +open: db/MANIFEST-000023 create: checkpoint/MANIFEST-000023 sync-data: checkpoint/MANIFEST-000023 close: checkpoint/MANIFEST-000023 +close: db/MANIFEST-000023 open-dir: checkpoint create: checkpoint/marker.manifest.000001.MANIFEST-000023 sync-data: checkpoint/marker.manifest.000001.MANIFEST-000023 close: checkpoint/marker.manifest.000001.MANIFEST-000023 sync: checkpoint close: checkpoint +open: wal/000021.log create: checkpoint/000021.log sync-data: checkpoint/000021.log close: checkpoint/000021.log +close: wal/000021.log sync: checkpoint close: checkpoint @@ -376,6 +430,7 @@ pebble: file deletion disablement invariant violated close ---- close: db +close: db/000013.sst sync-data: wal/000021.log close: wal/000021.log close: db/MANIFEST-000023 diff --git a/vfs/logging_fs.go b/vfs/logging_fs.go index ee0c3e281c..3ba558458f 100644 --- a/vfs/logging_fs.go +++ b/vfs/logging_fs.go @@ -37,6 +37,15 @@ func (fs *loggingFS) Create(name string) (File, error) { return newLoggingFile(f, name, fs.logFn), nil } +func (fs *loggingFS) Open(name string, opts ...OpenOption) (File, error) { + fs.logFn("open: %s", name) + f, err := fs.FS.Open(name, opts...) + if err != nil { + return nil, err + } + return newLoggingFile(f, name, fs.logFn), nil +} + func (fs *loggingFS) Link(oldname, newname string) error { fs.logFn("link: %s -> %s", oldname, newname) return fs.FS.Link(oldname, newname) @@ -122,3 +131,13 @@ func (f *loggingFile) SyncTo(length int64) (fullSync bool, err error) { f.logFn("sync-to(%d): %s", length, f.name) return f.File.SyncTo(length) } + +func (f *loggingFile) ReadAt(p []byte, offset int64) (int, error) { + f.logFn("read-at(%d, %d): %s", offset, len(p), f.name) + return f.File.ReadAt(p, offset) +} + +func (f *loggingFile) Prefetch(offset int64, length int64) error { + f.logFn("prefetch(%d, %d): %s", offset, length, f.name) + return f.File.Prefetch(offset, length) +}