From 91041935852ab094e0202472d4f18af57a8c9cc9 Mon Sep 17 00:00:00 2001 From: Fabio Barone Date: Sun, 3 Feb 2019 19:19:22 -0500 Subject: [PATCH 1/5] swarm/storage: GetAllReferences returns all chunk references --- swarm/storage/filestore.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index 2d8d82d95a50..4894a846cf76 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -96,3 +96,41 @@ func (f *FileStore) Store(ctx context.Context, data io.Reader, size int64, toEnc func (f *FileStore) HashSize() int { return f.hashFunc().Size() } + +// Public API. This endpoint returns all chunk hashes (only) for a given file +func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader) (addrs []Address, err error) { + var addrs = make([]Address, 0) + // create a special kind of putter, which only will store the references + putter := &HashExplorer{ + hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, false), + References: make([]Reference, 0), + } + // do the actual splitting anyway, no way around it + _, _, err := PyramidSplit(ctx, data, putter, putter) + if err != nil { + return nil, err + } + // collect all references + for _, ref := range putter.References { + addrs = append(addrs, ref) + } + return addrs, nil +} + +// HashExplorer is a special kind of putter which will only store chunk references +type HashExplorer struct { + *hasherStore + References []Reference +} + +// HashExplorer's Put will add just the chunk hashes to its `References` +func (he *HashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) { + // Need to do the actual Put, which returns the references + ref, err := he.hasherStore.Put(ctx, chunkData) + if err != nil { + return nil, err + } + // internally store the reference + he.References = append(he.References, ref) + return ref, nil +} From eb2d20ff17c3275b9f010d45dc4c0870cf6f499f Mon Sep 17 00:00:00 2001 From: Fabio Barone Date: Sun, 3 Feb 2019 20:07:24 -0500 Subject: [PATCH 2/5] swarm/storage: GetAllReferences test and fixes --- swarm/storage/filestore.go | 6 +++--- swarm/storage/filestore_test.go | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index 4894a846cf76..410b4407b1c2 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -99,20 +99,20 @@ func (f *FileStore) HashSize() int { // Public API. This endpoint returns all chunk hashes (only) for a given file func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader) (addrs []Address, err error) { - var addrs = make([]Address, 0) + addrs = make([]Address, 0) // create a special kind of putter, which only will store the references putter := &HashExplorer{ hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, false), References: make([]Reference, 0), } // do the actual splitting anyway, no way around it - _, _, err := PyramidSplit(ctx, data, putter, putter) + _, _, err = PyramidSplit(ctx, data, putter, putter) if err != nil { return nil, err } // collect all references for _, ref := range putter.References { - addrs = append(addrs, ref) + addrs = append(addrs, Address(ref)) } return addrs, nil } diff --git a/swarm/storage/filestore_test.go b/swarm/storage/filestore_test.go index fb0f761a4a6d..f46ce86b767b 100644 --- a/swarm/storage/filestore_test.go +++ b/swarm/storage/filestore_test.go @@ -173,3 +173,39 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) { t.Fatalf("Comparison error after clearing memStore.") } } + +// TestGetAllReferences only tests that GetAllReferences returns an expected +// number of references for a given file +func TestGetAllReferences(t *testing.T) { + tdb, cleanup, err := newTestDbStore(false, false) + defer cleanup() + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + db := tdb.LDBStore + memStore := NewMemStore(NewDefaultStoreParams(), db) + localStore := &LocalStore{ + memStore: memStore, + DbStore: db, + } + fileStore := NewFileStore(localStore, NewFileStoreParams()) + + checkRefs := func(dataSize int, expectedLen int) { + slice := testutil.RandomBytes(1, dataSize) + + addrs, err := fileStore.GetAllReferences(context.Background(), bytes.NewReader(slice)) + if err != nil { + t.Fatal(err) + } + if len(addrs) != expectedLen { + t.Fatalf("Expected reference array length to be %d, but is %d", expectedLen, len(addrs)) + } + } + + // testRuns[i] and expectedLen[i] are dataSize and expected length respectively + testRuns := []int{1024, 8192, 16000, 30000, 1000000} + expectedLens := []int{1, 3, 5, 9, 248} + for i, r := range testRuns { + checkRefs(r, expectedLens[i]) + } +} From 3dc906568084985529515dc2e89c7dd25a2c1a51 Mon Sep 17 00:00:00 2001 From: Fabio Barone Date: Sun, 3 Feb 2019 20:31:11 -0500 Subject: [PATCH 3/5] swarm/storage: with toEncrypt flag --- swarm/storage/filestore.go | 4 ++-- swarm/storage/filestore_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index 410b4407b1c2..9ed57a5b3ac0 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -98,11 +98,11 @@ func (f *FileStore) HashSize() int { } // Public API. This endpoint returns all chunk hashes (only) for a given file -func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader) (addrs []Address, err error) { +func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs []Address, err error) { addrs = make([]Address, 0) // create a special kind of putter, which only will store the references putter := &HashExplorer{ - hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, false), + hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt), References: make([]Reference, 0), } // do the actual splitting anyway, no way around it diff --git a/swarm/storage/filestore_test.go b/swarm/storage/filestore_test.go index f46ce86b767b..2dbccdf114e2 100644 --- a/swarm/storage/filestore_test.go +++ b/swarm/storage/filestore_test.go @@ -193,7 +193,7 @@ func TestGetAllReferences(t *testing.T) { checkRefs := func(dataSize int, expectedLen int) { slice := testutil.RandomBytes(1, dataSize) - addrs, err := fileStore.GetAllReferences(context.Background(), bytes.NewReader(slice)) + addrs, err := fileStore.GetAllReferences(context.Background(), bytes.NewReader(slice), false) if err != nil { t.Fatal(err) } From 7b56fd29cf4b19c4b4a7928a1fd2545d7f0a5a36 Mon Sep 17 00:00:00 2001 From: Fabio Barone Date: Mon, 4 Feb 2019 11:45:10 -0500 Subject: [PATCH 4/5] swarm/storage: sorted references for GetAllReferences --- swarm/storage/filestore.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index 9ed57a5b3ac0..d9a7a9374feb 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -19,6 +19,7 @@ package storage import ( "context" "io" + "sort" ) /* @@ -98,8 +99,7 @@ func (f *FileStore) HashSize() int { } // Public API. This endpoint returns all chunk hashes (only) for a given file -func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs []Address, err error) { - addrs = make([]Address, 0) +func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs AddressCollection, err error) { // create a special kind of putter, which only will store the references putter := &HashExplorer{ hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt), @@ -111,9 +111,12 @@ func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncr return nil, err } // collect all references + arr := make([]Address, 0) for _, ref := range putter.References { - addrs = append(addrs, Address(ref)) + arr = append(arr, Address(ref)) } + addrs = NewAddressCollection(len(arr)) + sort.Sort(addrs) return addrs, nil } From ce973dc31a440791616f80fe3e32badd3b988c0e Mon Sep 17 00:00:00 2001 From: Fabio Barone Date: Mon, 4 Feb 2019 13:06:56 -0500 Subject: [PATCH 5/5] swarm/storage: bug fix in GetAllReferences --- swarm/storage/filestore.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index d9a7a9374feb..aebe03c1ef6b 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -111,11 +111,10 @@ func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncr return nil, err } // collect all references - arr := make([]Address, 0) + addrs = NewAddressCollection(0) for _, ref := range putter.References { - arr = append(arr, Address(ref)) + addrs = append(addrs, Address(ref)) } - addrs = NewAddressCollection(len(arr)) sort.Sort(addrs) return addrs, nil }