From a715fb3063117263ccfde577b7ae798708105d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Mar 2022 23:25:29 +0100 Subject: [PATCH 1/2] feat: Make ClientCalcCommP multithreaded --- node/impl/client/client.go | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 1730b7a7a1b..7ecf6a776b4 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -47,7 +47,6 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-commp-utils/ffiwrapper" "github.com/filecoin-project/go-commp-utils/writer" datatransfer "github.com/filecoin-project/go-data-transfer" @@ -1263,28 +1262,12 @@ func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Addre } func (a *API) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet, error) { - - // Hard-code the sector type to 32GiBV1_1, because: - // - ffiwrapper.GeneratePieceCIDFromFile requires a RegisteredSealProof - // - commP itself is sector-size independent, with rather low probability of that changing - // ( note how the final rust call is identical for every RegSP type ) - // https://github.com/filecoin-project/rust-filecoin-proofs-api/blob/v5.0.0/src/seal.rs#L1040-L1050 - // - // IF/WHEN this changes in the future we will have to be able to calculate - // "old style" commP, and thus will need to introduce a version switch or similar - arbitraryProofType := abi.RegisteredSealProof_StackedDrg64GiBV1_1 - rdr, err := os.Open(inpath) if err != nil { return nil, err } defer rdr.Close() //nolint:errcheck - stat, err := rdr.Stat() - if err != nil { - return nil, err - } - // check that the data is a car file; if it's not, retrieval won't work _, err = car.ReadHeader(bufio.NewReader(rdr)) if err != nil { @@ -1295,16 +1278,20 @@ func (a *API) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet return nil, xerrors.Errorf("seek to start: %w", err) } - pieceReader, pieceSize := padreader.New(rdr, uint64(stat.Size())) - commP, err := ffiwrapper.GeneratePieceCIDFromFile(arbitraryProofType, pieceReader, pieceSize) + w := &writer.Writer{} + _, err = io.CopyBuffer(w, rdr, make([]byte, writer.CommPBuf)) + if err != nil { + return nil, xerrors.Errorf("copy inte commp writer: %w", err) + } + commp, err := w.Sum() if err != nil { return nil, xerrors.Errorf("computing commP failed: %w", err) } return &api.CommPRet{ - Root: commP, - Size: pieceSize, + Root: commp.PieceCID, + Size: commp.PieceSize.Unpadded(), }, nil } From 0d9e34d6516bd97a668b1a873fe5d53a47161f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 10 Mar 2022 22:41:45 +0100 Subject: [PATCH 2/2] client impl: Fix typo in ClientCalcCommP CopyBuffer error --- node/impl/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 7ecf6a776b4..f105d315277 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -1281,7 +1281,7 @@ func (a *API) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet w := &writer.Writer{} _, err = io.CopyBuffer(w, rdr, make([]byte, writer.CommPBuf)) if err != nil { - return nil, xerrors.Errorf("copy inte commp writer: %w", err) + return nil, xerrors.Errorf("copy into commp writer: %w", err) } commp, err := w.Sum()