From dd7c1e4e2c0cb4d371fecaa361de8065dd3f8307 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 7 Apr 2021 09:51:41 -0700 Subject: [PATCH 1/6] support padding out smaller files --- go.mod | 1 + storagemarket/impl/provider_environments.go | 59 +++++++++++++++++-- .../impl/providerstates/provider_states.go | 4 +- .../providerstates/provider_states_test.go | 2 +- .../impl/providerutils/providerutils.go | 10 ++-- .../impl/providerutils/providerutils_test.go | 2 +- 6 files changed, 65 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index d6cc1778..36907b45 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/filecoin-project/go-fil-markets go 1.13 require ( + github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f github.com/filecoin-project/go-address v0.0.3 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434 diff --git a/storagemarket/impl/provider_environments.go b/storagemarket/impl/provider_environments.go index 8e30a0cf..430aace4 100644 --- a/storagemarket/impl/provider_environments.go +++ b/storagemarket/impl/provider_environments.go @@ -10,8 +10,11 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" + ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-commp-utils/zerocomm" "github.com/filecoin-project/go-multistore" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/piecestore" @@ -49,16 +52,64 @@ func (p *providerDealEnvironment) DeleteStore(storeID multistore.StoreID) error return p.p.multiStore.Delete(storeID) } -func (p *providerDealEnvironment) GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, filestore.Path, error) { +func (p *providerDealEnvironment) GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node, pieceSize abi.PaddedPieceSize) (cid.Cid, filestore.Path, error) { proofType, err := p.p.spn.GetProofType(context.TODO(), p.p.actor, nil) if err != nil { return cid.Undef, "", err } + + var pieceCid cid.Cid + var path filestore.Path + var psize abi.UnpaddedPieceSize + if p.p.universalRetrievalEnabled { - return providerutils.GeneratePieceCommitmentWithMetadata(p.p.fs, p.p.pio.GeneratePieceCommitment, proofType, payloadCid, selector, storeID) + pieceCid, psize, path, err = providerutils.GeneratePieceCommitmentWithMetadata(p.p.fs, p.p.pio.GeneratePieceCommitment, proofType, payloadCid, selector, storeID) + } else { + pieceCid, psize, err = p.p.pio.GeneratePieceCommitment(proofType, payloadCid, selector, storeID) + } + if err != nil { + return cid.Undef, "", err + } + + if psize.Padded() < pieceSize { + // need to pad up! + paddedCid, err := ZeroPadPieceCommitment(pieceCid, psize, pieceSize.Unpadded()) + if err != nil { + return cid.Undef, "", err + } + + pieceCid = paddedCid } - pieceCid, _, err := p.p.pio.GeneratePieceCommitment(proofType, payloadCid, selector, storeID) - return pieceCid, filestore.Path(""), err + + return pieceCid, path, nil +} + +// TODO: move this to a proper utility location +func ZeroPadPieceCommitment(c cid.Cid, curSize abi.UnpaddedPieceSize, toSize abi.UnpaddedPieceSize) (cid.Cid, error) { + cur := c + for curSize < toSize { + + zc := zerocomm.ZeroPieceCommitment(curSize) + + p, err := ffi.GenerateUnsealedCID(abi.RegisteredSealProof_StackedDrg32GiBV1, []abi.PieceInfo{ + abi.PieceInfo{ + Size: curSize.Padded(), + PieceCID: cur, + }, + abi.PieceInfo{ + Size: curSize.Padded(), + PieceCID: zc, + }, + }) + if err != nil { + return cid.Undef, err + } + + cur = p + curSize = curSize * 2 + } + + return cur, nil } func (p *providerDealEnvironment) GeneratePieceReader(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node) (io.ReadCloser, uint64, error, <-chan error) { diff --git a/storagemarket/impl/providerstates/provider_states.go b/storagemarket/impl/providerstates/provider_states.go index e3de73e6..60794ce6 100644 --- a/storagemarket/impl/providerstates/provider_states.go +++ b/storagemarket/impl/providerstates/provider_states.go @@ -40,7 +40,7 @@ type ProviderDealEnvironment interface { Node() storagemarket.StorageProviderNode Ask() storagemarket.StorageAsk DeleteStore(storeID multistore.StoreID) error - GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, filestore.Path, error) + GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node, psize abi.PaddedPieceSize) (cid.Cid, filestore.Path, error) GeneratePieceReader(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node) (io.ReadCloser, uint64, error, <-chan error) SendSignedResponse(ctx context.Context, response *network.Response) error Disconnect(proposalCid cid.Cid) error @@ -197,7 +197,7 @@ func DecideOnProposal(ctx fsm.Context, environment ProviderDealEnvironment, deal // VerifyData verifies that data received for a deal matches the pieceCID // in the proposal func VerifyData(ctx fsm.Context, environment ProviderDealEnvironment, deal storagemarket.MinerDeal) error { - pieceCid, metadataPath, err := environment.GeneratePieceCommitment(deal.StoreID, deal.Ref.Root, shared.AllSelector()) + pieceCid, metadataPath, err := environment.GeneratePieceCommitment(deal.StoreID, deal.Ref.Root, shared.AllSelector(), deal.Proposal.PieceSize) if err != nil { return ctx.Trigger(storagemarket.ProviderEventDataVerificationFailed, xerrors.Errorf("error generating CommP: %w", err), filestore.Path(""), filestore.Path("")) } diff --git a/storagemarket/impl/providerstates/provider_states_test.go b/storagemarket/impl/providerstates/provider_states_test.go index 6bec0033..1e8bafe4 100644 --- a/storagemarket/impl/providerstates/provider_states_test.go +++ b/storagemarket/impl/providerstates/provider_states_test.go @@ -1471,7 +1471,7 @@ func (fe *fakeEnvironment) GeneratePieceReader(storeID *multistore.StoreID, payl return fe.pieceReader, fe.pieceSize, fe.generatePieceReaderErr, errChan } -func (fe *fakeEnvironment) GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, filestore.Path, error) { +func (fe *fakeEnvironment) GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node, psize abi.PaddedPieceSize) (cid.Cid, filestore.Path, error) { return fe.pieceCid, fe.metadataPath, fe.generateCommPError } diff --git a/storagemarket/impl/providerutils/providerutils.go b/storagemarket/impl/providerutils/providerutils.go index 433666e3..902f419b 100644 --- a/storagemarket/impl/providerutils/providerutils.go +++ b/storagemarket/impl/providerutils/providerutils.go @@ -85,19 +85,19 @@ func GeneratePieceCommitmentWithMetadata( proofType abi.RegisteredSealProof, payloadCid cid.Cid, selector ipld.Node, - storeID *multistore.StoreID) (cid.Cid, filestore.Path, error) { + storeID *multistore.StoreID) (cid.Cid, abi.UnpaddedPieceSize, filestore.Path, error) { metadataFile, err := fileStore.CreateTemp() if err != nil { - return cid.Cid{}, "", err + return cid.Cid{}, 0, "", err } blockRecorder := blockrecorder.RecordEachBlockTo(metadataFile) - pieceCid, _, err := commPGenerator(proofType, payloadCid, selector, storeID, blockRecorder) + pieceCid, psize, err := commPGenerator(proofType, payloadCid, selector, storeID, blockRecorder) _ = metadataFile.Close() if err != nil { _ = fileStore.Delete(metadataFile.Path()) - return cid.Cid{}, "", err + return cid.Cid{}, 0, "", err } - return pieceCid, metadataFile.Path(), err + return pieceCid, psize, metadataFile.Path(), err } // LoadBlockLocations loads a metadata file then converts it to a map of cid -> blockLocation diff --git a/storagemarket/impl/providerutils/providerutils_test.go b/storagemarket/impl/providerutils/providerutils_test.go index bcfadf4d..1f30516f 100644 --- a/storagemarket/impl/providerutils/providerutils_test.go +++ b/storagemarket/impl/providerutils/providerutils_test.go @@ -161,7 +161,7 @@ func TestCommPGenerationWithMetadata(t *testing.T) { t.Run(testName, func(t *testing.T) { fcp := &fakeCommPGenerator{pieceCid, pieceSize, testCase.commPErr} fs := shared_testutil.NewTestFileStore(testCase.fileStoreParams) - resultPieceCid, resultMetadataPath, resultErr := providerutils.GeneratePieceCommitmentWithMetadata( + resultPieceCid, _, resultMetadataPath, resultErr := providerutils.GeneratePieceCommitmentWithMetadata( fs, fcp.GenerateCommPToFile, proofType, payloadCid, selector, &storeID) require.Equal(t, resultPieceCid, testCase.expectedPieceCid) require.Equal(t, resultMetadataPath, testCase.expectedMetadataPath) From 2c677fb42a190d600d48b5e4cae3f33973ba3e3f Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 7 Apr 2021 12:54:21 -0700 Subject: [PATCH 2/6] wire through proper piece size ( forward-port from https://github.com/filecoin-project/go-fil-markets/pull/537 ) --- storagemarket/impl/providerstates/provider_states.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/storagemarket/impl/providerstates/provider_states.go b/storagemarket/impl/providerstates/provider_states.go index 60794ce6..35cc52ab 100644 --- a/storagemarket/impl/providerstates/provider_states.go +++ b/storagemarket/impl/providerstates/provider_states.go @@ -309,7 +309,7 @@ func HandoffDeal(ctx fsm.Context, environment ProviderDealEnvironment, deal stor } // Hand the deal off to the process that adds it to a sector - packingInfo, err = handoffDeal(ctx.Context(), environment, deal, file, uint64(file.Size())) + packingInfo, err = handoffDeal(ctx.Context(), environment, deal, file, uint64(file.Size()), deal.Proposal.PieceSize) if err != nil { err = xerrors.Errorf("packing piece at path %s: %w", deal.PiecePath, err) return ctx.Trigger(storagemarket.ProviderEventDealHandoffFailed, err) @@ -324,7 +324,7 @@ func HandoffDeal(ctx fsm.Context, environment ProviderDealEnvironment, deal stor // Hand the deal off to the process that adds it to a sector var packingErr error - packingInfo, packingErr = handoffDeal(ctx.Context(), environment, deal, pieceReader, pieceSize) + packingInfo, packingErr = handoffDeal(ctx.Context(), environment, deal, pieceReader, pieceSize, deal.Proposal.PieceSize) // Close the read side of the pipe err = pieceReader.Close() @@ -360,8 +360,8 @@ func HandoffDeal(ctx fsm.Context, environment ProviderDealEnvironment, deal stor return ctx.Trigger(storagemarket.ProviderEventDealHandedOff) } -func handoffDeal(ctx context.Context, environment ProviderDealEnvironment, deal storagemarket.MinerDeal, reader io.Reader, size uint64) (*storagemarket.PackingResult, error) { - paddedReader, paddedSize := padreader.New(reader, size) +func handoffDeal(ctx context.Context, environment ProviderDealEnvironment, deal storagemarket.MinerDeal, reader io.Reader, payloadSize uint64, pieceSize abi.PaddedPieceSize) (*storagemarket.PackingResult, error) { + paddedReader, _ := padreader.New(reader, payloadSize) return environment.Node().OnDealComplete( ctx, storagemarket.MinerDeal{ @@ -374,7 +374,7 @@ func handoffDeal(ctx context.Context, environment ProviderDealEnvironment, deal DealID: deal.DealID, FastRetrieval: deal.FastRetrieval, }, - paddedSize, + pieceSize.Unpadded(), paddedReader, ) } From 253218d11e13b970a7d9834f4c565f4c991d72dc Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Mon, 7 Jun 2021 21:13:45 +0200 Subject: [PATCH 3/6] Do not pull the entire ffi just for padding alone --- go.mod | 5 ++- go.sum | 14 +++++-- storagemarket/impl/provider_environments.go | 42 +++++---------------- 3 files changed, 22 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 04e30122..377fb9e7 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,13 @@ module github.com/filecoin-project/go-fil-markets go 1.13 require ( - github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f github.com/filecoin-project/go-address v0.0.3 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 - github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434 + github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7 github.com/filecoin-project/go-data-transfer v1.7.0 github.com/filecoin-project/go-ds-versioning v0.1.0 + github.com/filecoin-project/go-fil-commcid v0.1.0 + github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc diff --git a/go.sum b/go.sum index fd27605e..0187ea48 100644 --- a/go.sum +++ b/go.sum @@ -104,8 +104,8 @@ github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7 github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= -github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434 h1:0kHszkYP3hgApcjl5x4rpwONhN9+j7XDobf6at5XfHs= -github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= +github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7 h1:U9Z+76pHCKBmtdxFV7JFZJj7OVm12I6dEKwtMVbq5p0= +github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo= @@ -114,8 +114,11 @@ github.com/filecoin-project/go-data-transfer v1.7.0/go.mod h1:GLRr5BmLEqsLwXfiRD github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= +github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88OqLYEo6roi+GiIeOh8= +github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= +github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= +github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -407,6 +410,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= @@ -645,8 +650,9 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/storagemarket/impl/provider_environments.go b/storagemarket/impl/provider_environments.go index 430aace4..39f00aca 100644 --- a/storagemarket/impl/provider_environments.go +++ b/storagemarket/impl/provider_environments.go @@ -10,9 +10,9 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" - ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-commp-utils/zerocomm" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" "github.com/filecoin-project/go-multistore" "github.com/filecoin-project/go-state-types/abi" @@ -73,45 +73,21 @@ func (p *providerDealEnvironment) GeneratePieceCommitment(storeID *multistore.St if psize.Padded() < pieceSize { // need to pad up! - paddedCid, err := ZeroPadPieceCommitment(pieceCid, psize, pieceSize.Unpadded()) + rawPaddedCommp, err := commp.PadCommP( + // we know how long a pieceCid "hash" is, just blindly extract the trailing 32 bytes + pieceCid.Hash()[len(pieceCid.Hash())-32:], + uint64(psize.Padded()), + uint64(pieceSize), + ) if err != nil { return cid.Undef, "", err } - - pieceCid = paddedCid + pieceCid, _ = commcid.DataCommitmentV1ToCID(rawPaddedCommp) } return pieceCid, path, nil } -// TODO: move this to a proper utility location -func ZeroPadPieceCommitment(c cid.Cid, curSize abi.UnpaddedPieceSize, toSize abi.UnpaddedPieceSize) (cid.Cid, error) { - cur := c - for curSize < toSize { - - zc := zerocomm.ZeroPieceCommitment(curSize) - - p, err := ffi.GenerateUnsealedCID(abi.RegisteredSealProof_StackedDrg32GiBV1, []abi.PieceInfo{ - abi.PieceInfo{ - Size: curSize.Padded(), - PieceCID: cur, - }, - abi.PieceInfo{ - Size: curSize.Padded(), - PieceCID: zc, - }, - }) - if err != nil { - return cid.Undef, err - } - - cur = p - curSize = curSize * 2 - } - - return cur, nil -} - func (p *providerDealEnvironment) GeneratePieceReader(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node) (io.ReadCloser, uint64, error, <-chan error) { return p.p.pio.GeneratePieceReader(payloadCid, selector, storeID) } From 009133372db965c9ad4b6d53558803ea277e3d00 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Mon, 5 Jul 2021 20:05:23 +0200 Subject: [PATCH 4/6] Actually write out the padding - AP depends on it --- go.mod | 5 ++++- go.sum | 4 ++-- storagemarket/impl/providerstates/provider_states.go | 9 ++++++++- .../impl/providerstates/provider_states_test.go | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 377fb9e7..b71e7f43 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 github.com/filecoin-project/go-multistore v0.0.3 - github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 + github.com/filecoin-project/go-padreader v0.0.0-20210705163151-ca3f7051a78a github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.0 @@ -49,4 +49,7 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 ) +// FIXME - temporary until https://github.com/filecoin-project/go-padreader/pull/6 merges +replace github.com/filecoin-project/go-padreader => github.com/ribasushi/go-padreader v0.0.0-20210705163151-ca3f7051a78a + replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi diff --git a/go.sum b/go.sum index 0187ea48..ad334a2a 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,6 @@ github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxl github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+eEvrDCGJoPLxFpDynFjYfBjI= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ= -github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 h1:+/4aUeUoKr6AKfPE3mBhXA5spIV6UcKdTYDPNU2Tdmg= -github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak= github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= @@ -754,6 +752,8 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/ribasushi/go-padreader v0.0.0-20210705163151-ca3f7051a78a h1:Qlok8N/JeVs6jlDT+fuxbskSCiIk5m1Kiz2N28YHKEs= +github.com/ribasushi/go-padreader v0.0.0-20210705163151-ca3f7051a78a/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= diff --git a/storagemarket/impl/providerstates/provider_states.go b/storagemarket/impl/providerstates/provider_states.go index 35cc52ab..48f09250 100644 --- a/storagemarket/impl/providerstates/provider_states.go +++ b/storagemarket/impl/providerstates/provider_states.go @@ -361,7 +361,14 @@ func HandoffDeal(ctx fsm.Context, environment ProviderDealEnvironment, deal stor } func handoffDeal(ctx context.Context, environment ProviderDealEnvironment, deal storagemarket.MinerDeal, reader io.Reader, payloadSize uint64, pieceSize abi.PaddedPieceSize) (*storagemarket.PackingResult, error) { - paddedReader, _ := padreader.New(reader, payloadSize) + // because we use the PadReader directly during AP we need to produce the + // correct amount of zeroes + // (alternative would be to keep precise track of sector offsets for each + // piece which is just too much work for a seldom used feature) + paddedReader, err := padreader.NewInflator(reader, payloadSize, pieceSize.Unpadded()) + if err != nil { + return nil, err + } return environment.Node().OnDealComplete( ctx, storagemarket.MinerDeal{ diff --git a/storagemarket/impl/providerstates/provider_states_test.go b/storagemarket/impl/providerstates/provider_states_test.go index 1e8bafe4..fe5bbbce 100644 --- a/storagemarket/impl/providerstates/provider_states_test.go +++ b/storagemarket/impl/providerstates/provider_states_test.go @@ -1393,7 +1393,7 @@ func makeExecutor(ctx context.Context, environment.ask = defaultAsk } if environment.pieceSize == 0 { - environment.pieceSize = uint64(defaultPieceSize) + environment.pieceSize = uint64(defaultPieceSize.Unpadded()) } if environment.pieceReader == nil { environment.pieceReader = newStubbedReadCloser(nil) From c1cf1dca582c6331b7538d2f345802548fd341fa Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Mon, 5 Jul 2021 21:54:22 +0200 Subject: [PATCH 5/6] Also properly pad up car files on import --- storagemarket/impl/provider.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/storagemarket/impl/provider.go b/storagemarket/impl/provider.go index bb8835d8..388b91bd 100644 --- a/storagemarket/impl/provider.go +++ b/storagemarket/impl/provider.go @@ -18,6 +18,8 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer" versioning "github.com/filecoin-project/go-ds-versioning/pkg" versionedfsm "github.com/filecoin-project/go-ds-versioning/pkg/fsm" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" "github.com/filecoin-project/go-multistore" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" @@ -308,7 +310,7 @@ func (p *Provider) ImportDataForDeal(ctx context.Context, propCid cid.Cid, data _ = n // TODO: verify n? - pieceSize := uint64(tempfi.Size()) + carSize := uint64(tempfi.Size()) _, err = tempfi.Seek(0, io.SeekStart) if err != nil { @@ -322,16 +324,30 @@ func (p *Provider) ImportDataForDeal(ctx context.Context, propCid cid.Cid, data return xerrors.Errorf("failed to determine proof type: %w", err) } - pieceCid, err := generatePieceCommitment(proofType, tempfi, pieceSize) + pieceCid, err := generatePieceCommitment(proofType, tempfi, carSize) if err != nil { cleanup() return xerrors.Errorf("failed to generate commP: %w", err) } + if carSizePadded := padreader.PaddedSize(carSize).Padded(); carSizePadded < d.Proposal.PieceSize { + // need to pad up! + rawPaddedCommp, err := commp.PadCommP( + // we know how long a pieceCid "hash" is, just blindly extract the trailing 32 bytes + pieceCid.Hash()[len(pieceCid.Hash())-32:], + uint64(carSizePadded), + uint64(d.Proposal.PieceSize), + ) + if err != nil { + return err + } + pieceCid, _ = commcid.DataCommitmentV1ToCID(rawPaddedCommp) + } + // Verify CommP matches if !pieceCid.Equals(d.Proposal.PieceCID) { cleanup() - return xerrors.Errorf("given data does not match expected commP (got: %x, expected %x)", pieceCid, d.Proposal.PieceCID) + return xerrors.Errorf("given data does not match expected commP (got: %s, expected %s)", pieceCid, d.Proposal.PieceCID) } return p.deals.Send(propCid, storagemarket.ProviderEventVerifiedData, tempfi.Path(), filestore.Path("")) From ccfb7d1c54057b3fd3bce41c3ee8b085f492dca5 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Fri, 23 Jul 2021 21:24:50 +0200 Subject: [PATCH 6/6] Update to now-merged filecoin-project/go-padreader --- go.mod | 5 +---- go.sum | 5 +++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b71e7f43..090e4741 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 github.com/filecoin-project/go-multistore v0.0.3 - github.com/filecoin-project/go-padreader v0.0.0-20210705163151-ca3f7051a78a + github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.0 @@ -49,7 +49,4 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 ) -// FIXME - temporary until https://github.com/filecoin-project/go-padreader/pull/6 merges -replace github.com/filecoin-project/go-padreader => github.com/ribasushi/go-padreader v0.0.0-20210705163151-ca3f7051a78a - replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi diff --git a/go.sum b/go.sum index ad334a2a..4c603ad2 100644 --- a/go.sum +++ b/go.sum @@ -126,6 +126,9 @@ github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxl github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+eEvrDCGJoPLxFpDynFjYfBjI= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ= +github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak= +github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 h1:0BogtftbcgyBx4lP2JWM00ZK7/pXmgnrDqKp9aLTgVs= +github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ= github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= @@ -752,8 +755,6 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/ribasushi/go-padreader v0.0.0-20210705163151-ca3f7051a78a h1:Qlok8N/JeVs6jlDT+fuxbskSCiIk5m1Kiz2N28YHKEs= -github.com/ribasushi/go-padreader v0.0.0-20210705163151-ca3f7051a78a/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=