support padding out smaller files

go.mod

@@ -3,6 +3,7 @@ module github.com/filecoin-project/go-fil-markets
 go 1.13
 
 require (
+	github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f
 	github.com/filecoin-project/go-address v0.0.3
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434

storagemarket/impl/provider_environments.go

@@ -10,8 +10,11 @@ import (
 	"github.com/libp2p/go-libp2p-core/peer"
 	"golang.org/x/xerrors"
 
+	ffi "github.com/filecoin-project/filecoin-ffi"
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-commp-utils/zerocomm"
 	"github.com/filecoin-project/go-multistore"
+	"github.com/filecoin-project/go-state-types/abi"
 
 	"github.com/filecoin-project/go-fil-markets/filestore"
 	"github.com/filecoin-project/go-fil-markets/piecestore"
@@ -49,16 +52,64 @@ func (p *providerDealEnvironment) DeleteStore(storeID multistore.StoreID) error
 	return p.p.multiStore.Delete(storeID)
 }
 
-func (p *providerDealEnvironment) GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, filestore.Path, error) {
+func (p *providerDealEnvironment) GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node, pieceSize abi.PaddedPieceSize) (cid.Cid, filestore.Path, error) {
 	proofType, err := p.p.spn.GetProofType(context.TODO(), p.p.actor, nil)
 	if err != nil {
 		return cid.Undef, "", err
 	}
+
+	var pieceCid cid.Cid
+	var path filestore.Path
+	var psize abi.UnpaddedPieceSize
+
 	if p.p.universalRetrievalEnabled {
-		return providerutils.GeneratePieceCommitmentWithMetadata(p.p.fs, p.p.pio.GeneratePieceCommitment, proofType, payloadCid, selector, storeID)
+		pieceCid, psize, path, err = providerutils.GeneratePieceCommitmentWithMetadata(p.p.fs, p.p.pio.GeneratePieceCommitment, proofType, payloadCid, selector, storeID)
+	} else {
+		pieceCid, psize, err = p.p.pio.GeneratePieceCommitment(proofType, payloadCid, selector, storeID)
+	}
+	if err != nil {
+		return cid.Undef, "", err
+	}
+
+	if psize.Padded() < pieceSize {
+		// need to pad up!
+		paddedCid, err := ZeroPadPieceCommitment(pieceCid, psize, pieceSize.Unpadded())
+		if err != nil {
+			return cid.Undef, "", err
+		}
+
+		pieceCid = paddedCid
 	}
-	pieceCid, _, err := p.p.pio.GeneratePieceCommitment(proofType, payloadCid, selector, storeID)
-	return pieceCid, filestore.Path(""), err
+
+	return pieceCid, path, nil
+}
+
+// TODO: move this to a proper utility location
+func ZeroPadPieceCommitment(c cid.Cid, curSize abi.UnpaddedPieceSize, toSize abi.UnpaddedPieceSize) (cid.Cid, error) {
+	cur := c
+	for curSize < toSize {
+
+		zc := zerocomm.ZeroPieceCommitment(curSize)
+
+		p, err := ffi.GenerateUnsealedCID(abi.RegisteredSealProof_StackedDrg32GiBV1, []abi.PieceInfo{
+			abi.PieceInfo{
+				Size:     curSize.Padded(),
+				PieceCID: cur,
+			},
+			abi.PieceInfo{
+				Size:     curSize.Padded(),
+				PieceCID: zc,
+			},
+		})
+		if err != nil {
+			return cid.Undef, err
+		}
+
+		cur = p
+		curSize = curSize * 2
+	}
+
+	return cur, nil
 }
 
 func (p *providerDealEnvironment) GeneratePieceReader(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node) (io.ReadCloser, uint64, error, <-chan error) {

storagemarket/impl/providerstates/provider_states.go

@@ -40,7 +40,7 @@ type ProviderDealEnvironment interface {
 	Node() storagemarket.StorageProviderNode
 	Ask() storagemarket.StorageAsk
 	DeleteStore(storeID multistore.StoreID) error
-	GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, filestore.Path, error)
+	GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node, psize abi.PaddedPieceSize) (cid.Cid, filestore.Path, error)
 	GeneratePieceReader(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node) (io.ReadCloser, uint64, error, <-chan error)
 	SendSignedResponse(ctx context.Context, response *network.Response) error
 	Disconnect(proposalCid cid.Cid) error
@@ -197,7 +197,7 @@ func DecideOnProposal(ctx fsm.Context, environment ProviderDealEnvironment, deal
 // VerifyData verifies that data received for a deal matches the pieceCID
 // in the proposal
 func VerifyData(ctx fsm.Context, environment ProviderDealEnvironment, deal storagemarket.MinerDeal) error {
-	pieceCid, metadataPath, err := environment.GeneratePieceCommitment(deal.StoreID, deal.Ref.Root, shared.AllSelector())
+	pieceCid, metadataPath, err := environment.GeneratePieceCommitment(deal.StoreID, deal.Ref.Root, shared.AllSelector(), deal.Proposal.PieceSize)
 	if err != nil {
 		return ctx.Trigger(storagemarket.ProviderEventDataVerificationFailed, xerrors.Errorf("error generating CommP: %w", err), filestore.Path(""), filestore.Path(""))
 	}

storagemarket/impl/providerstates/provider_states_test.go

@@ -1471,7 +1471,7 @@ func (fe *fakeEnvironment) GeneratePieceReader(storeID *multistore.StoreID, payl
 	return fe.pieceReader, fe.pieceSize, fe.generatePieceReaderErr, errChan
 }
 
-func (fe *fakeEnvironment) GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, filestore.Path, error) {
+func (fe *fakeEnvironment) GeneratePieceCommitment(storeID *multistore.StoreID, payloadCid cid.Cid, selector ipld.Node, psize abi.PaddedPieceSize) (cid.Cid, filestore.Path, error) {
 	return fe.pieceCid, fe.metadataPath, fe.generateCommPError
 }
 

storagemarket/impl/providerutils/providerutils.go

@@ -85,19 +85,19 @@ func GeneratePieceCommitmentWithMetadata(
 	proofType abi.RegisteredSealProof,
 	payloadCid cid.Cid,
 	selector ipld.Node,
-	storeID *multistore.StoreID) (cid.Cid, filestore.Path, error) {
+	storeID *multistore.StoreID) (cid.Cid, abi.UnpaddedPieceSize, filestore.Path, error) {
 	metadataFile, err := fileStore.CreateTemp()
 	if err != nil {
-		return cid.Cid{}, "", err
+		return cid.Cid{}, 0, "", err
 	}
 	blockRecorder := blockrecorder.RecordEachBlockTo(metadataFile)
-	pieceCid, _, err := commPGenerator(proofType, payloadCid, selector, storeID, blockRecorder)
+	pieceCid, psize, err := commPGenerator(proofType, payloadCid, selector, storeID, blockRecorder)
 	_ = metadataFile.Close()
 	if err != nil {
 		_ = fileStore.Delete(metadataFile.Path())
-		return cid.Cid{}, "", err
+		return cid.Cid{}, 0, "", err
 	}
-	return pieceCid, metadataFile.Path(), err
+	return pieceCid, psize, metadataFile.Path(), err
 }
 
 // LoadBlockLocations loads a metadata file then converts it to a map of cid -> blockLocation

storagemarket/impl/providerutils/providerutils_test.go

@@ -161,7 +161,7 @@ func TestCommPGenerationWithMetadata(t *testing.T) {
 		t.Run(testName, func(t *testing.T) {
 			fcp := &fakeCommPGenerator{pieceCid, pieceSize, testCase.commPErr}
 			fs := shared_testutil.NewTestFileStore(testCase.fileStoreParams)
-			resultPieceCid, resultMetadataPath, resultErr := providerutils.GeneratePieceCommitmentWithMetadata(
+			resultPieceCid, _, resultMetadataPath, resultErr := providerutils.GeneratePieceCommitmentWithMetadata(
 				fs, fcp.GenerateCommPToFile, proofType, payloadCid, selector, &storeID)
 			require.Equal(t, resultPieceCid, testCase.expectedPieceCid)
 			require.Equal(t, resultMetadataPath, testCase.expectedMetadataPath)