From 6f0b5c23f509da33ad95b127d84a9d73d6585a2e Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 1 Jul 2021 22:14:42 +1000 Subject: [PATCH 01/10] feat: report block count on `dag import` --- core/commands/dag/dag.go | 13 ++++++++++--- core/commands/dag/import.go | 10 ++++++++-- test/sharness/t0054-dag-car-import-export.sh | 20 +++++++++++++------- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 8cd9864a6b6..aee028876bc 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -55,7 +55,8 @@ type ResolveOutput struct { // CarImportOutput is the output type of the 'dag import' commands type CarImportOutput struct { - Root RootMeta + BlockCount uint64 + Root *RootMeta } // RootMeta is the metadata for a root pinning response @@ -160,8 +161,9 @@ var DagResolveCmd = &cmds.Command{ } type importResult struct { - roots map[cid.Cid]struct{} - err error + blockCount uint64 + roots map[cid.Cid]struct{} + err error } // DagImportCmd is a command for importing a car to ipfs @@ -206,6 +208,11 @@ Maximum supported CAR version: 1 return nil } + if event.Root == nil { + fmt.Fprintf(w, "Imported %d blocks\n", event.BlockCount) + return nil + } + enc, err := cmdenv.GetLowLevelCidEncoder(req) if err != nil { return err diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index b83af8b911d..cf929de3500 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -55,6 +55,10 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return done.err } + if err := res.Emit(&CarImportOutput{BlockCount: done.blockCount}); err != nil { + return err + } + // It is not guaranteed that a root in a header is actually present in the same ( or any ) // .car file. This is the case in version 1, and ideally in further versions too // Accumulate any root CID seen in a header, and supplement its actual node if/when encountered @@ -101,7 +105,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment failedPins++ } - if err := res.Emit(&CarImportOutput{Root: ret}); err != nil { + if err := res.Emit(&CarImportOutput{Root: &ret}); err != nil { return err } } @@ -126,6 +130,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, batch := ipld.NewBatch(req.Context, api.Dag()) roots := make(map[cid.Cid]struct{}) + var blockCount uint64 it := req.Files.Entries() for it.Next() { @@ -176,6 +181,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, if err := batch.Add(req.Context, nd); err != nil { return err } + blockCount++ } return nil @@ -197,5 +203,5 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, return } - ret <- importResult{roots: roots} + ret <- importResult{blockCount: blockCount, roots: roots} } diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index 450591ff792..e7f8a32d4f7 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -56,14 +56,16 @@ run_online_imp_exp_tests() { reset_blockstore 1 cat > basic_import_expected <naked_root_import_json_expected <multiroot_import_json_expected < multiroot_import_json_actual @@ -182,14 +185,17 @@ test_expect_success "multiroot import expected output" ' ' +cat >pin_import_expected << EOE +{"BlockCount":1198,"Root":null} +EOE test_expect_success "pin-less import works" ' ipfs dag import --enc=json --pin-roots=false \ ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \ ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \ > no-pin_import_actual ' -test_expect_success "expected silence on --pin-roots=false" ' - test_cmp /dev/null no-pin_import_actual +test_expect_success "expected no pins on --pin-roots=false" ' + test_cmp pin_import_expected no-pin_import_actual ' From daab8574a18a1ba37fd1be648dc058ebce8a2287 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 2 Jul 2021 10:42:29 +1000 Subject: [PATCH 02/10] fix: clean-up dag import message format --- core/commands/dag/dag.go | 14 +++++++++++--- core/commands/dag/import.go | 2 +- test/sharness/t0054-dag-car-import-export.sh | 16 ++++++++-------- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index aee028876bc..a7cfce94dd5 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -55,8 +55,8 @@ type ResolveOutput struct { // CarImportOutput is the output type of the 'dag import' commands type CarImportOutput struct { - BlockCount uint64 - Root *RootMeta + BlockCount *uint64 `json:",omitempty"` + Root *RootMeta `json:",omitempty"` } // RootMeta is the metadata for a root pinning response @@ -208,11 +208,19 @@ Maximum supported CAR version: 1 return nil } + // event should have only one of `Root` or `BlockCount` set, not both if event.Root == nil { - fmt.Fprintf(w, "Imported %d blocks\n", event.BlockCount) + if event.BlockCount == nil { + return fmt.Errorf("Unexpected message from DAG import") + } + fmt.Fprintf(w, "Imported %d blocks\n", *event.BlockCount) return nil } + if event.BlockCount != nil { + return fmt.Errorf("Unexpected message from DAG import") + } + enc, err := cmdenv.GetLowLevelCidEncoder(req) if err != nil { return err diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index cf929de3500..edbbae01288 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -55,7 +55,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return done.err } - if err := res.Emit(&CarImportOutput{BlockCount: done.blockCount}); err != nil { + if err := res.Emit(&CarImportOutput{BlockCount: &done.blockCount}); err != nil { return err } diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index e7f8a32d4f7..aba826c126c 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -63,9 +63,9 @@ Pinned root${tab}bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy$ EOE cat >naked_root_import_json_expected <multiroot_import_json_expected < multiroot_import_json_actual @@ -186,7 +186,7 @@ test_expect_success "multiroot import expected output" ' cat >pin_import_expected << EOE -{"BlockCount":1198,"Root":null} +{"BlockCount":1198} EOE test_expect_success "pin-less import works" ' ipfs dag import --enc=json --pin-roots=false \ From 875c3f06e7a36096448d25686aad490949401378 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 7 Sep 2021 14:35:40 -0700 Subject: [PATCH 03/10] Only print stats when --stats flag is passed This applies to both text and json output encoings. - Stats data is now contained within a Stats datastructure - Stats are printed after root so that first line of output is the same as previously, even when stats are output using --stats --- core/commands/dag/dag.go | 25 +++++++++++++++++-------- core/commands/dag/import.go | 15 +++++++++++---- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index a7cfce94dd5..b64dfdc00c0 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -16,9 +16,10 @@ import ( ) const ( + pinRootsOptionName = "pin-roots" progressOptionName = "progress" silentOptionName = "silent" - pinRootsOptionName = "pin-roots" + statsOptionName = "stats" ) // DagCmd provides a subset of commands for interacting with ipld dag objects @@ -53,10 +54,14 @@ type ResolveOutput struct { RemPath string } +type CarImportStats struct { + BlockCount uint64 +} + // CarImportOutput is the output type of the 'dag import' commands type CarImportOutput struct { - BlockCount *uint64 `json:",omitempty"` - Root *RootMeta `json:",omitempty"` + Root *RootMeta `json:",omitempty"` + Stats *CarImportStats `json:",omitempty"` } // RootMeta is the metadata for a root pinning response @@ -195,8 +200,9 @@ Maximum supported CAR version: 1 cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(), }, Options: []cmds.Option{ - cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), + cmds.BoolOption(silentOptionName, "No output."), + cmds.BoolOption(statsOptionName, "Output stats."), }, Type: CarImportOutput{}, Run: dagImport, @@ -208,16 +214,19 @@ Maximum supported CAR version: 1 return nil } - // event should have only one of `Root` or `BlockCount` set, not both + // event should have only one of `Root` or `Stats` set, not both if event.Root == nil { - if event.BlockCount == nil { + if event.Stats == nil { return fmt.Errorf("Unexpected message from DAG import") } - fmt.Fprintf(w, "Imported %d blocks\n", *event.BlockCount) + stats, _ := req.Options[statsOptionName].(bool) + if stats { + fmt.Fprintf(w, "Imported %d blocks\n", event.Stats.BlockCount) + } return nil } - if event.BlockCount != nil { + if event.Stats != nil { return fmt.Errorf("Unexpected message from DAG import") } diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index edbbae01288..8cfc38894d1 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -55,10 +55,6 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return done.err } - if err := res.Emit(&CarImportOutput{BlockCount: &done.blockCount}); err != nil { - return err - } - // It is not guaranteed that a root in a header is actually present in the same ( or any ) // .car file. This is the case in version 1, and ideally in further versions too // Accumulate any root CID seen in a header, and supplement its actual node if/when encountered @@ -119,6 +115,17 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment } } + stats, _ := req.Options[statsOptionName].(bool) + if stats { + if err := res.Emit(&CarImportOutput{ + Stats: &CarImportStats{ + BlockCount: done.blockCount, + }, + }); err != nil { + return err + } + } + return nil } From 52c970f4e61bebf10042b5645350e0d1dafea16e Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 7 Sep 2021 16:25:10 -0700 Subject: [PATCH 04/10] fix sharness test --- test/sharness/t0054-dag-car-import-export.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index aba826c126c..39aa77e4ab6 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -41,7 +41,7 @@ do_import() { while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done & while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done & - ipfsi "$node" dag import "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null + ipfsi "$node" dag import --stats "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null result=$? rm -f spin.gc &>/dev/null @@ -63,9 +63,9 @@ Pinned root${tab}bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy$ EOE cat >naked_root_import_json_expected < naked_import_result_json_actual ' @@ -172,13 +172,13 @@ test_expect_success "correct error" ' cat >multiroot_import_json_expected < multiroot_import_json_actual + ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual ' test_expect_success "multiroot import expected output" ' test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual @@ -186,10 +186,10 @@ test_expect_success "multiroot import expected output" ' cat >pin_import_expected << EOE -{"BlockCount":1198} +{"Stats":{"BlockCount":1198}} EOE test_expect_success "pin-less import works" ' - ipfs dag import --enc=json --pin-roots=false \ + ipfs dag import --stats --enc=json --pin-roots=false \ ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \ ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \ > no-pin_import_actual @@ -200,7 +200,7 @@ test_expect_success "expected no pins on --pin-roots=false" ' test_expect_success "naked root import works" ' - ipfs dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ + ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ > naked_root_import_json_actual ' test_expect_success "naked root import expected output" ' From c14d1bc9d87fbe499829d9d05bff543dcbb389c9 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 7 Sep 2021 23:50:47 -0700 Subject: [PATCH 05/10] Add PayloadBytesCount to stats --- core/commands/dag/dag.go | 12 +++++++----- core/commands/dag/import.go | 17 ++++++++++++----- test/sharness/t0054-dag-car-import-export.sh | 8 ++++---- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index b64dfdc00c0..2bcbe8dede4 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -55,7 +55,8 @@ type ResolveOutput struct { } type CarImportStats struct { - BlockCount uint64 + BlockCount uint64 + PayloadBytesCount uint64 } // CarImportOutput is the output type of the 'dag import' commands @@ -166,9 +167,10 @@ var DagResolveCmd = &cmds.Command{ } type importResult struct { - blockCount uint64 - roots map[cid.Cid]struct{} - err error + blockCount uint64 + payloadBytesCount uint64 + roots map[cid.Cid]struct{} + err error } // DagImportCmd is a command for importing a car to ipfs @@ -221,7 +223,7 @@ Maximum supported CAR version: 1 } stats, _ := req.Options[statsOptionName].(bool) if stats { - fmt.Fprintf(w, "Imported %d blocks\n", event.Stats.BlockCount) + fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.PayloadBytesCount) } return nil } diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index 8cfc38894d1..46207007016 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -117,11 +117,13 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment stats, _ := req.Options[statsOptionName].(bool) if stats { - if err := res.Emit(&CarImportOutput{ + err = res.Emit(&CarImportOutput{ Stats: &CarImportStats{ - BlockCount: done.blockCount, + BlockCount: done.blockCount, + PayloadBytesCount: done.payloadBytesCount, }, - }); err != nil { + }) + if err != nil { return err } } @@ -137,7 +139,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, batch := ipld.NewBatch(req.Context, api.Dag()) roots := make(map[cid.Cid]struct{}) - var blockCount uint64 + var blockCount, payloadBytesCount uint64 it := req.Files.Entries() for it.Next() { @@ -189,6 +191,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, return err } blockCount++ + ndSize, _ := nd.Size() + payloadBytesCount += ndSize } return nil @@ -210,5 +214,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, return } - ret <- importResult{blockCount: blockCount, roots: roots} + ret <- importResult{ + blockCount: blockCount, + payloadBytesCount: payloadBytesCount, + roots: roots} } diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index 39aa77e4ab6..064fe7bbf27 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -56,7 +56,7 @@ run_online_imp_exp_tests() { reset_blockstore 1 cat > basic_import_expected <naked_root_import_json_expected <multiroot_import_json_expected < multiroot_import_json_actual @@ -186,7 +186,7 @@ test_expect_success "multiroot import expected output" ' cat >pin_import_expected << EOE -{"Stats":{"BlockCount":1198}} +{"Stats":{"BlockCount":1198,"PayloadBytesCount":468513}} EOE test_expect_success "pin-less import works" ' ipfs dag import --stats --enc=json --pin-roots=false \ From 56b87df8cfa0b6d35138afd6fe1098a6a964edbf Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 8 Sep 2021 02:34:17 -0700 Subject: [PATCH 06/10] Attempt to stabilize flaky tests --- test/sharness/lib/test-lib.sh | 2 +- test/sharness/t0041-ping.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index f4b3c68db16..a20668f50e1 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -269,7 +269,7 @@ test_launch_ipfs_daemon() { # wait for api file to show up test_expect_success "api file shows up" ' - test_wait_for_file 50 100ms "$IPFS_PATH/api" + test_wait_for_file 50 200ms "$IPFS_PATH/api" ' test_set_address_vars actual_daemon diff --git a/test/sharness/t0041-ping.sh b/test/sharness/t0041-ping.sh index 276cd4802d1..c4665b9ba14 100755 --- a/test/sharness/t0041-ping.sh +++ b/test/sharness/t0041-ping.sh @@ -43,7 +43,7 @@ test_expect_success "test ping 0" ' ' test_expect_success "test ping offline" ' - iptb stop 1 && + iptb stop 1 && sleep 2 && ! ipfsi 0 ping -n2 -- "$PEERID_1" ' From 2636d56e0956826908ebfdaad82262ae0b7c5f42 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 9 Sep 2021 08:51:32 -0700 Subject: [PATCH 07/10] Rename PayloadBytesCount to BlockBytesCount --- core/commands/dag/dag.go | 14 +++++++------- core/commands/dag/import.go | 14 +++++++------- test/sharness/t0054-dag-car-import-export.sh | 6 +++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 2bcbe8dede4..8fabe628948 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -55,8 +55,8 @@ type ResolveOutput struct { } type CarImportStats struct { - BlockCount uint64 - PayloadBytesCount uint64 + BlockCount uint64 + BlockBytesCount uint64 } // CarImportOutput is the output type of the 'dag import' commands @@ -167,10 +167,10 @@ var DagResolveCmd = &cmds.Command{ } type importResult struct { - blockCount uint64 - payloadBytesCount uint64 - roots map[cid.Cid]struct{} - err error + blockCount uint64 + blockBytesCount uint64 + roots map[cid.Cid]struct{} + err error } // DagImportCmd is a command for importing a car to ipfs @@ -223,7 +223,7 @@ Maximum supported CAR version: 1 } stats, _ := req.Options[statsOptionName].(bool) if stats { - fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.PayloadBytesCount) + fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.BlockBytesCount) } return nil } diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index 46207007016..a75954e6394 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -119,8 +119,8 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment if stats { err = res.Emit(&CarImportOutput{ Stats: &CarImportStats{ - BlockCount: done.blockCount, - PayloadBytesCount: done.payloadBytesCount, + BlockCount: done.blockCount, + BlockBytesCount: done.blockBytesCount, }, }) if err != nil { @@ -139,7 +139,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, batch := ipld.NewBatch(req.Context, api.Dag()) roots := make(map[cid.Cid]struct{}) - var blockCount, payloadBytesCount uint64 + var blockCount, blockBytesCount uint64 it := req.Files.Entries() for it.Next() { @@ -192,7 +192,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, } blockCount++ ndSize, _ := nd.Size() - payloadBytesCount += ndSize + blockBytesCount += ndSize } return nil @@ -215,7 +215,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, } ret <- importResult{ - blockCount: blockCount, - payloadBytesCount: payloadBytesCount, - roots: roots} + blockCount: blockCount, + blockBytesCount: blockBytesCount, + roots: roots} } diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index 064fe7bbf27..ba56b8fff1f 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -65,7 +65,7 @@ EOE cat >naked_root_import_json_expected <multiroot_import_json_expected < multiroot_import_json_actual @@ -186,7 +186,7 @@ test_expect_success "multiroot import expected output" ' cat >pin_import_expected << EOE -{"Stats":{"BlockCount":1198,"PayloadBytesCount":468513}} +{"Stats":{"BlockCount":1198,"BlockBytesCount":468513}} EOE test_expect_success "pin-less import works" ' ipfs dag import --stats --enc=json --pin-roots=false \ From 160b9a6e26cb63a5d6c35a5e347b381d0d7ca41d Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 9 Sep 2021 17:06:15 -0700 Subject: [PATCH 08/10] Correctly calculate size or imported dag --- core/commands/dag/import.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index a75954e6394..a013c6d6364 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -191,8 +191,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, return err } blockCount++ - ndSize, _ := nd.Size() - blockBytesCount += ndSize + blockBytesCount += uint64(len(nd.RawData())) } return nil From ed9b606c5ddf071981c6ff1890b2caafd7635ac6 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 13 Sep 2021 15:01:13 +1000 Subject: [PATCH 09/10] Use RawSize of original block for import bytes calc --- core/commands/dag/import.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index a013c6d6364..244260601bf 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -191,7 +191,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, return err } blockCount++ - blockBytesCount += uint64(len(nd.RawData())) + blockBytesCount += uint64(len(block.RawData())) } return nil From 02fe73573fdd3dca9afb62587bec12bf0add393e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 21 Sep 2021 22:41:04 +0200 Subject: [PATCH 10/10] test: dag import without --stats basic regression tests for the default output (text and json) --- test/sharness/t0054-dag-car-import-export.sh | 44 ++++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index ba56b8fff1f..311833748f0 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -41,7 +41,7 @@ do_import() { while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done & while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done & - ipfsi "$node" dag import --stats "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null + ipfsi "$node" dag import "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null result=$? rm -f spin.gc &>/dev/null @@ -55,20 +55,25 @@ run_online_imp_exp_tests() { reset_blockstore 0 reset_blockstore 1 - cat > basic_import_expected < basic_import_stats_expected < basic_import_expected + # Explainer: + # naked_root_import_json_expected output is produced by dag import of combined_naked_roots_genesis_and_128.car + # executed when roots are already present in the repo - thus the BlockCount=0 + # (if blocks were not present in the repo, blockstore: block not found would be returned) cat >naked_root_import_json_expected < basic_import_actual + ' + + test_expect_success "basic import output with --stats as expected" ' + test_cmp_sorted basic_import_stats_expected basic_import_actual + ' + test_expect_success "basic fetch+export 1" ' ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car ' @@ -119,7 +136,7 @@ EOE cat ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car > pipe_testnet & cat ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car > pipe_devnet & - do_import 0 \ + do_import 0 --stats \ pipe_testnet \ pipe_devnet \ ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ @@ -136,7 +153,7 @@ EOE ' test_expect_success "fifo-import output as expected" ' - test_cmp_sorted basic_import_expected basic_fifo_import_actual + test_cmp_sorted basic_import_stats_expected basic_fifo_import_actual ' } @@ -170,20 +187,29 @@ test_expect_success "correct error" ' test_cmp_sorted offline_fetch_error_expected offline_fetch_error_actual ' - -cat >multiroot_import_json_expected <multiroot_import_json_stats_expected < multiroot_import_json_actual +# output without --stats line +head -3 multiroot_import_json_stats_expected > multiroot_import_json_expected + +test_expect_success "multiroot import works (--enc=json)" ' + ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual ' test_expect_success "multiroot import expected output" ' test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual ' +test_expect_success "multiroot import works with --stats" ' + ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual +' +test_expect_success "multiroot import expected output" ' + test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual +' + cat >pin_import_expected << EOE {"Stats":{"BlockCount":1198,"BlockBytesCount":468513}}