From f9271177107ff4ab767b43102ecebf41f2daf922 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Thu, 15 Sep 2022 14:48:31 +0400 Subject: [PATCH 1/5] dont shadow var --- carserver/station_api_impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carserver/station_api_impl.go b/carserver/station_api_impl.go index 46a088b..f98aa67 100644 --- a/carserver/station_api_impl.go +++ b/carserver/station_api_impl.go @@ -76,7 +76,7 @@ func (s *StationAPIImpl) createOrUpdateReqStatsUnlocked(ctx context.Context, cre if err == datastore.ErrNotFound { stats := station.ReqStats{} createFn(&stats) - bz, err = json.Marshal(stats) + bz, err := json.Marshal(stats) if err != nil { return fmt.Errorf("failed to marshal retrieval stats to json: %w", err) } From 572791abef3b620ce56b3669e86a6e5f2a5651f2 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Thu, 15 Sep 2022 15:19:32 +0400 Subject: [PATCH 2/5] run tsts --- .github/workflows/go.yml | 2 +- carstore/carstore_test.go | 5 ++--- carstore/gateway_api.go | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b970cc6..51dd4b8 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -21,4 +21,4 @@ jobs: run: go build -v ./... - name: Test - run: go test -v -race ./... \ No newline at end of file + run: go test -v -race ./carstore \ No newline at end of file diff --git a/carstore/carstore_test.go b/carstore/carstore_test.go index c1ce699..f0418cc 100644 --- a/carstore/carstore_test.go +++ b/carstore/carstore_test.go @@ -14,8 +14,6 @@ import ( cid "github.com/ipfs/go-cid" - "golang.org/x/sync/errgroup" - "github.com/filecoin-project/dagstore" "github.com/filecoin-project/saturn-l2/logs" @@ -93,6 +91,7 @@ func TestPersistentCache(t *testing.T) { csh.assertStorageStats(t, station.StorageStats{BytesCurrentlyStored: uint64(len(bz))}) } +/* func TestPersistentCacheConcurrent(t *testing.T) { ctx := context.Background() carv1File := "../testdata/files/sample-v1.car" @@ -226,7 +225,7 @@ func TestDownloadTimeout(t *testing.T) { csh.fetchAndAssertNotFound(reqID, rootcid) require.EqualValues(t, 0, csh.ms.nDownloaded()) csh.assertStorageStats(t, station.StorageStats{BytesCurrentlyStored: 0}) -} +}*/ func (csh *carstoreHarness) assertAvailable(t *testing.T, ctx context.Context, c cid.Cid) { require.Eventually(t, func() bool { diff --git a/carstore/gateway_api.go b/carstore/gateway_api.go index 7f3ee27..a47e522 100644 --- a/carstore/gateway_api.go +++ b/carstore/gateway_api.go @@ -94,6 +94,7 @@ func (gw *GatewayReader) Read(p []byte) (int, error) { } func (gw *GatewayReader) Close() error { + fmt.Println("\n Recoding downloaded data", gw.n) if err := gw.sapi.RecordDataDownloaded(gw.ctx, gw.n); err != nil { log.Errorw("failed to record download stats", "err", err) return err From 859cd4d0bd541247f6c12eabfacc852d5c99c28c Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Thu, 15 Sep 2022 15:24:03 +0400 Subject: [PATCH 3/5] run all tests --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 51dd4b8..b970cc6 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -21,4 +21,4 @@ jobs: run: go build -v ./... - name: Test - run: go test -v -race ./carstore \ No newline at end of file + run: go test -v -race ./... \ No newline at end of file From f840ef73df748f3f321c3ead1ef71f00b6501492 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Thu, 15 Sep 2022 15:28:55 +0400 Subject: [PATCH 4/5] run all tests --- carstore/carstore_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/carstore/carstore_test.go b/carstore/carstore_test.go index f0418cc..e1ca245 100644 --- a/carstore/carstore_test.go +++ b/carstore/carstore_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "golang.org/x/sync/errgroup" + "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/multicodec" @@ -91,7 +93,6 @@ func TestPersistentCache(t *testing.T) { csh.assertStorageStats(t, station.StorageStats{BytesCurrentlyStored: uint64(len(bz))}) } -/* func TestPersistentCacheConcurrent(t *testing.T) { ctx := context.Background() carv1File := "../testdata/files/sample-v1.car" @@ -225,7 +226,7 @@ func TestDownloadTimeout(t *testing.T) { csh.fetchAndAssertNotFound(reqID, rootcid) require.EqualValues(t, 0, csh.ms.nDownloaded()) csh.assertStorageStats(t, station.StorageStats{BytesCurrentlyStored: 0}) -}*/ +} func (csh *carstoreHarness) assertAvailable(t *testing.T, ctx context.Context, c cid.Cid) { require.Eventually(t, func() bool { From 3d7cc2dfef73cdc8712748bd91df59a01afa4110 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Thu, 15 Sep 2022 15:36:31 +0400 Subject: [PATCH 5/5] logging for tests --- carstore/carstore_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/carstore/carstore_test.go b/carstore/carstore_test.go index e1ca245..2a11222 100644 --- a/carstore/carstore_test.go +++ b/carstore/carstore_test.go @@ -3,6 +3,7 @@ package carstore import ( "context" "errors" + "fmt" "sync" "testing" "time" @@ -47,20 +48,25 @@ func TestPersistentCache(t *testing.T) { csh.assertStorageStats(t, station.StorageStats{BytesCurrentlyStored: 0}) // first hit -> not found csh.fetchAndAssertNotFound(reqID, rootcid) + fmt.Println("\n data downloaded after first hit is", csh.ms.nDownloaded()) // second hit -> not found csh.fetchAndAssertNotFound(reqID, rootcid) + fmt.Println("\n data downloaded after second hit is", csh.ms.nDownloaded()) // wait till l2 has fetched and cached it csh.assertAvailable(t, ctx, rootcid) + fmt.Println("\n data downloaded after caching is", csh.ms.nDownloaded()) // third hit -> found csh.fetchAndAssertFound(ctx, reqID, rootcid) require.EqualValues(t, len(bz), csh.ms.nDownloaded()) + fmt.Println("\n data downloaded is after third hit", csh.ms.nDownloaded()) // fourth hit -> found csh.fetchAndAssertFound(ctx, reqID, rootcid) require.EqualValues(t, len(bz), csh.ms.nDownloaded()) + fmt.Println("\n data downloaded after fetching success is", csh.ms.nDownloaded()) // wait for shard to become reclaimable again require.Eventually(t, func() bool { @@ -76,14 +82,18 @@ func TestPersistentCache(t *testing.T) { require.Len(t, res.Shards, 1) csh.assertStorageStats(t, station.StorageStats{BytesCurrentlyStored: 0}) + fmt.Println("\n data downloaded after dagstore GC is", csh.ms.nDownloaded()) + // fetch car -> fails as we do not have it but will trigger a fetch again csh.fetchAndAssertNotFound(reqID, rootcid) + fmt.Println("\n data downloaded after fetch trigger fails is", csh.ms.nDownloaded()) // fetch car -> works now as car file was downloaded in the previous fetch require.Eventually(t, func() bool { err = csh.cs.FetchAndWriteCAR(reqID, rootcid, func(_ bstore.Blockstore) error { return nil }) + fmt.Println("\n data downloaded after a cycle is", csh.ms.nDownloaded()) return err == nil }, 50*time.Second, 200*time.Millisecond)