From 71b8eea5e3c8cf335cb8db2396e83311f9cdd7f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 13 Jan 2020 17:40:17 +0100 Subject: [PATCH 1/2] Update statestore (#38) --- go.mod | 2 +- go.sum | 4 ++-- storagemarket/impl/client.go | 4 ++-- storagemarket/impl/client_utils.go | 2 +- storagemarket/impl/provider.go | 2 +- storagemarket/impl/provider_utils.go | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 388f2c2b..2b9be579 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce - github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15 + github.com/filecoin-project/go-statestore v0.1.0 github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c diff --git a/go.sum b/go.sum index 8f5c4156..0b80bd36 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= -github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15 h1:6uEnOV0KO1jzART47HpCrnpAiMfDXHv7RoT/qCfcN9Q= -github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= +github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= +github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= diff --git a/storagemarket/impl/client.go b/storagemarket/impl/client.go index 82dae974..ace43dea 100644 --- a/storagemarket/impl/client.go +++ b/storagemarket/impl/client.go @@ -131,7 +131,7 @@ func (c *Client) onIncoming(deal *ClientDeal) { func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { log.Infof("Client deal %s updated state to %s", update.id, storagemarket.DealStates[update.newState]) var deal ClientDeal - err := c.deals.Mutate(update.id, func(d *ClientDeal) error { + err := c.deals.Get(update.id).Mutate(func(d *ClientDeal) error { d.State = update.newState if update.mut != nil { update.mut(d) @@ -285,7 +285,7 @@ func (c *Client) List() ([]ClientDeal, error) { func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) { var out ClientDeal - if err := c.deals.Get(d, &out); err != nil { + if err := c.deals.Get(d).Get(&out); err != nil { return nil, err } return &out, nil diff --git a/storagemarket/impl/client_utils.go b/storagemarket/impl/client_utils.go index 3c3b6dc0..b5d90e53 100644 --- a/storagemarket/impl/client_utils.go +++ b/storagemarket/impl/client_utils.go @@ -158,7 +158,7 @@ func (c *ClientRequestValidator) ValidatePull( } var deal ClientDeal - err := c.deals.Get(dealVoucher.Proposal, &deal) + err := c.deals.Get(dealVoucher.Proposal).Get(&deal) if err != nil { return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal) } diff --git a/storagemarket/impl/provider.go b/storagemarket/impl/provider.go index 7d3f9bf9..738f291a 100644 --- a/storagemarket/impl/provider.go +++ b/storagemarket/impl/provider.go @@ -174,7 +174,7 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { return } var deal MinerDeal - err := p.deals.Mutate(update.id, func(d *MinerDeal) error { + err := p.deals.Get(update.id).Mutate(func(d *MinerDeal) error { d.State = update.newState if update.mut != nil { update.mut(d) diff --git a/storagemarket/impl/provider_utils.go b/storagemarket/impl/provider_utils.go index acabcc57..c3109b32 100644 --- a/storagemarket/impl/provider_utils.go +++ b/storagemarket/impl/provider_utils.go @@ -20,7 +20,7 @@ import ( ) func (p *Provider) failDeal(ctx context.Context, id cid.Cid, cerr error) { - if err := p.deals.End(id); err != nil { + if err := p.deals.Get(id).End(); err != nil { log.Warnf("deals.End: %s", err) } @@ -150,7 +150,7 @@ func (m *ProviderRequestValidator) ValidatePush( } var deal MinerDeal - err := m.deals.Get(dealVoucher.Proposal, &deal) + err := m.deals.Get(dealVoucher.Proposal).Get(&deal) if err != nil { return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal) } From 74d100f305f83cb95a5c09e93e18213a952d52a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 14 Jan 2020 02:54:28 +0100 Subject: [PATCH 2/2] Use sectorbuilder GenerateCommitment directly (#41) --- go.mod | 5 +- go.sum | 41 ++++++++++++++++ pieceio/pieceio.go | 21 +++----- pieceio/pieceio_test.go | 51 ++++++-------------- pieceio/sectorcalculator/sectorcalculator.go | 45 ----------------- storagemarket/impl/client_utils.go | 13 +---- 6 files changed, 68 insertions(+), 108 deletions(-) delete mode 100644 pieceio/sectorcalculator/sectorcalculator.go diff --git a/go.mod b/go.mod index 2b9be579..0f56a9c9 100644 --- a/go.mod +++ b/go.mod @@ -3,17 +3,18 @@ module github.com/filecoin-project/go-fil-markets go 1.13 require ( - github.com/filecoin-project/filecoin-ffi v0.0.0-20191213130254-f261762ff8ed + github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590 github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce + github.com/filecoin-project/go-sectorbuilder v0.0.1 github.com/filecoin-project/go-statestore v0.1.0 github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 - github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10 + github.com/ipfs/go-cid v0.0.4 github.com/ipfs/go-datastore v0.1.1 github.com/ipfs/go-graphsync v0.0.4 github.com/ipfs/go-ipfs-blockstore v0.1.0 diff --git a/go.sum b/go.sum index 0b80bd36..2ff73bf0 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,14 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= +github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= +github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -35,6 +39,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSY github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= +github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY= +github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -48,6 +54,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f h1:L2jaVU8TvWTx7iZPhlYvUE8vkoOnj778XuKavz8W36g= github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f/go.mod h1:rCbpXPva2NKF9/J4X6sr7hbKBgQCxyFtRj7KOZqoIms= @@ -57,6 +65,10 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= +github.com/filecoin-project/go-sectorbuilder v0.0.1 h1:yiLSEprWA1E43DFTSCXLSuCstYuDKiI6RCXiYz4GaRs= +github.com/filecoin-project/go-sectorbuilder v0.0.1/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -132,6 +144,8 @@ github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUP github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10 h1:5mRf2p8Bv2iKiuPsGrQUrx38rdBm2T/03JCM6VWzoMc= github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds= +github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI= @@ -358,10 +372,15 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= @@ -379,6 +398,8 @@ github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVq github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= @@ -407,11 +428,14 @@ github.com/multiformats/go-multihash v0.0.6/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX6wmozr0Hp4= github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-varint v0.0.2 h1:6sUvyh2YHpJCb8RZ6eYzj6iJQ4+chWYmyIHxszqlPTA= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= @@ -426,6 +450,12 @@ github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc= +github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -481,6 +511,8 @@ github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= @@ -510,6 +542,8 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= +go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= @@ -537,6 +571,8 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -581,8 +617,11 @@ golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -616,6 +655,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/pieceio/pieceio.go b/pieceio/pieceio.go index 5c64021d..9120435d 100644 --- a/pieceio/pieceio.go +++ b/pieceio/pieceio.go @@ -4,16 +4,12 @@ import ( "context" "fmt" "github.com/filecoin-project/go-fil-markets/filestore" + "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "io" ) -type SectorCalculator interface { - // GeneratePieceCommitment takes a PADDED io stream and a total size and generates a commP - GeneratePieceCommitment(piece io.Reader, pieceSize uint64) ([]byte, error) -} - type PadReader interface { // PaddedSize returns the expected size of a piece after it's been padded PaddedSize(size uint64) uint64 @@ -27,14 +23,13 @@ type CarIO interface { } type pieceIO struct { - padReader PadReader - carIO CarIO - sectorCalculator SectorCalculator - store filestore.FileStore + padReader PadReader + carIO CarIO + store filestore.FileStore } -func NewPieceIO(padReader PadReader, carIO CarIO, sectorCalculator SectorCalculator, store filestore.FileStore) PieceIO { - return &pieceIO{padReader, carIO, sectorCalculator, store} +func NewPieceIO(padReader PadReader, carIO CarIO, store filestore.FileStore) PieceIO { + return &pieceIO{padReader, carIO, store} } func (pio *pieceIO) GeneratePieceCommitment(bs ReadStore, payloadCid cid.Cid, selector ipld.Node) ([]byte, filestore.Path, error) { @@ -70,12 +65,12 @@ func (pio *pieceIO) GeneratePieceCommitment(bs ReadStore, payloadCid cid.Cid, se cleanup() return nil, "", err } - commitment, err := pio.sectorCalculator.GeneratePieceCommitment(f, paddedSize) + commitment, err := sectorbuilder.GeneratePieceCommitment(f, paddedSize) if err != nil { cleanup() return nil, "", err } - return commitment, f.Path(), nil + return commitment[:], f.Path(), nil } func (pio *pieceIO) ReadPiece(r io.Reader, bs WriteStore) (cid.Cid, error) { diff --git a/pieceio/pieceio_test.go b/pieceio/pieceio_test.go index 345e7ea5..908ed28d 100644 --- a/pieceio/pieceio_test.go +++ b/pieceio/pieceio_test.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/go-fil-markets/pieceio/cario" pmocks "github.com/filecoin-project/go-fil-markets/pieceio/mocks" "github.com/filecoin-project/go-fil-markets/pieceio/padreader" - "github.com/filecoin-project/go-fil-markets/pieceio/sectorcalculator" + "github.com/filecoin-project/go-sectorbuilder" dag "github.com/ipfs/go-merkledag" dstest "github.com/ipfs/go-merkledag/test" ipldfree "github.com/ipld/go-ipld-prime/impl/free" @@ -25,13 +25,12 @@ import ( func Test_ThereAndBackAgain(t *testing.T) { tempDir := filestore.Path("./tempDir") - sc := sectorcalculator.NewSectorCalculator(tempDir) pr := padreader.NewPadReader() cio := cario.NewCarIO() store, err := filestore.NewLocalFileStore(tempDir) require.NoError(t, err) - pio := pieceio.NewPieceIO(pr, cio, sc, store) + pio := pieceio.NewPieceIO(pr, cio, store) require.NoError(t, err) sourceBserv := dstest.Bserv() @@ -114,13 +113,12 @@ func Test_ThereAndBackAgain(t *testing.T) { func Test_StoreRestoreMemoryBuffer(t *testing.T) { tempDir := filestore.Path("./tempDir") - sc := sectorcalculator.NewSectorCalculator(tempDir) pr := padreader.NewPadReader() cio := cario.NewCarIO() store, err := filestore.NewLocalFileStore(tempDir) require.NoError(t, err) - pio := pieceio.NewPieceIO(pr, cio, sc, store) + pio := pieceio.NewPieceIO(pr, cio, store) sourceBserv := dstest.Bserv() sourceBs := sourceBserv.Blockstore() @@ -171,9 +169,9 @@ func Test_StoreRestoreMemoryBuffer(t *testing.T) { _, err = f.Read(buf) require.NoError(t, err) buffer := bytes.NewBuffer(buf) - secondCommitment, err := sc.GeneratePieceCommitment(buffer, uint64(info.Size())) + secondCommitment, err := sectorbuilder.GeneratePieceCommitment(buffer, uint64(info.Size())) require.NoError(t, err) - require.Equal(t, commitment, secondCommitment) + require.Equal(t, commitment, secondCommitment[:]) } func Test_Failures(t *testing.T) { @@ -212,13 +210,12 @@ func Test_Failures(t *testing.T) { t.Run("create temp file fails", func(t *testing.T) { fsmock := fsmocks.FileStore{} fsmock.On("CreateTemp").Return(nil, fmt.Errorf("Failed")) - pio := pieceio.NewPieceIO(nil, nil, nil, &fsmock) + pio := pieceio.NewPieceIO(nil, nil, &fsmock) _, _, err := pio.GeneratePieceCommitment(sourceBs, nd3.Cid(), node) require.Error(t, err) }) t.Run("write CAR fails", func(t *testing.T) { tempDir := filestore.Path("./tempDir") - sc := sectorcalculator.NewSectorCalculator(tempDir) pr := padreader.NewPadReader() store, err := filestore.NewLocalFileStore(tempDir) require.NoError(t, err) @@ -226,13 +223,11 @@ func Test_Failures(t *testing.T) { ciomock := pmocks.CarIO{} any := mock.Anything ciomock.On("WriteCar", any, any, any, any, any).Return(fmt.Errorf("failed to write car")) - pio := pieceio.NewPieceIO(pr, &ciomock, sc, store) + pio := pieceio.NewPieceIO(pr, &ciomock, store) _, _, err = pio.GeneratePieceCommitment(sourceBs, nd3.Cid(), node) require.Error(t, err) }) t.Run("padding fails", func(t *testing.T) { - tempDir := filestore.Path("./tempDir") - sc := sectorcalculator.NewSectorCalculator(tempDir) pr := padreader.NewPadReader() cio := cario.NewCarIO() @@ -244,7 +239,7 @@ func Test_Failures(t *testing.T) { counter := 0 size := 0 - mockfile.On("Write", mock.Anything).Run(func (args mock.Arguments) { + mockfile.On("Write", mock.Anything).Run(func(args mock.Arguments) { arg := args[0] buf := arg.([]byte) size := len(buf) @@ -255,13 +250,11 @@ func Test_Failures(t *testing.T) { mockfile.On("Close").Return(nil).Once() mockfile.On("Path").Return(filestore.Path("mock")).Once() - pio := pieceio.NewPieceIO(pr, cio, sc, &fsmock) + pio := pieceio.NewPieceIO(pr, cio, &fsmock) _, _, err := pio.GeneratePieceCommitment(sourceBs, nd3.Cid(), node) require.Error(t, err) }) t.Run("incorrect padding", func(t *testing.T) { - tempDir := filestore.Path("./tempDir") - sc := sectorcalculator.NewSectorCalculator(tempDir) pr := padreader.NewPadReader() cio := cario.NewCarIO() @@ -273,7 +266,7 @@ func Test_Failures(t *testing.T) { counter := 0 size := 0 - mockfile.On("Write", mock.Anything).Run(func (args mock.Arguments) { + mockfile.On("Write", mock.Anything).Run(func(args mock.Arguments) { arg := args[0] buf := arg.([]byte) size := len(buf) @@ -284,13 +277,11 @@ func Test_Failures(t *testing.T) { mockfile.On("Close").Return(nil).Once() mockfile.On("Path").Return(filestore.Path("mock")).Once() - pio := pieceio.NewPieceIO(pr, cio, sc, &fsmock) + pio := pieceio.NewPieceIO(pr, cio, &fsmock) _, _, err := pio.GeneratePieceCommitment(sourceBs, nd3.Cid(), node) require.Error(t, err) }) t.Run("seek fails", func(t *testing.T) { - tempDir := filestore.Path("./tempDir") - sc := sectorcalculator.NewSectorCalculator(tempDir) pr := padreader.NewPadReader() cio := cario.NewCarIO() @@ -302,7 +293,7 @@ func Test_Failures(t *testing.T) { counter := 0 size := 0 - mockfile.On("Write", mock.Anything).Run(func (args mock.Arguments) { + mockfile.On("Write", mock.Anything).Run(func(args mock.Arguments) { arg := args[0] buf := arg.([]byte) size := len(buf) @@ -314,22 +305,8 @@ func Test_Failures(t *testing.T) { mockfile.On("Path").Return(filestore.Path("mock")).Once() mockfile.On("Seek", mock.Anything, mock.Anything).Return(int64(0), fmt.Errorf("seek failed")) - pio := pieceio.NewPieceIO(pr, cio, sc, &fsmock) + pio := pieceio.NewPieceIO(pr, cio, &fsmock) _, _, err := pio.GeneratePieceCommitment(sourceBs, nd3.Cid(), node) require.Error(t, err) }) - t.Run("generate piece commitment fails", func(t *testing.T) { - tempDir := filestore.Path("./tempDir") - sc := pmocks.SectorCalculator{} - pr := padreader.NewPadReader() - cio := cario.NewCarIO() - - sc.On("GeneratePieceCommitment", mock.Anything, mock.Anything, mock.Anything).Return([]byte{}, fmt.Errorf("commitment failed")) - - store, err := filestore.NewLocalFileStore(tempDir) - require.NoError(t, err) - pio := pieceio.NewPieceIO(pr, cio, &sc, store) - _, _, err = pio.GeneratePieceCommitment(sourceBs, nd3.Cid(), node) - require.Error(t, err) - }) -} \ No newline at end of file +} diff --git a/pieceio/sectorcalculator/sectorcalculator.go b/pieceio/sectorcalculator/sectorcalculator.go deleted file mode 100644 index 78a1333d..00000000 --- a/pieceio/sectorcalculator/sectorcalculator.go +++ /dev/null @@ -1,45 +0,0 @@ -package sectorcalculator - -import ( - ffi "github.com/filecoin-project/filecoin-ffi" - "github.com/filecoin-project/go-fil-markets/filestore" - "github.com/filecoin-project/go-fil-markets/pieceio" - "io" - "io/ioutil" - "os" -) - -type sectorCalculator struct { - tempDir filestore.Path -} - -func NewSectorCalculator(tempDir filestore.Path) pieceio.SectorCalculator { - return §orCalculator{tempDir} -} - -func (s sectorCalculator) GeneratePieceCommitment(piece io.Reader, pieceSize uint64) ([]byte, error) { - f, ok := piece.(*os.File) // try to avoid yet another temp file - var err error - if !ok { - f, err = ioutil.TempFile(string(s.tempDir), "") - if err != nil { - return nil, err - } - defer func() { // cleanup - f.Close() - os.Remove(f.Name()) - }() - _, err = io.Copy(f, io.LimitReader(piece, int64(pieceSize))) - if err != nil { - return nil, err - } - _, err = f.Seek(0, io.SeekStart) - if err != nil { - return nil, err - } - } else { - defer f.Close() - } - commP, err := ffi.GeneratePieceCommitmentFromFile(f, pieceSize) - return commP[:], err -} diff --git a/storagemarket/impl/client_utils.go b/storagemarket/impl/client_utils.go index b5d90e53..5cbe1462 100644 --- a/storagemarket/impl/client_utils.go +++ b/storagemarket/impl/client_utils.go @@ -2,8 +2,6 @@ package storageimpl import ( "context" - "io/ioutil" - "os" "runtime" "github.com/ipfs/go-cid" @@ -15,9 +13,8 @@ import ( "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/pieceio/padreader" - "github.com/filecoin-project/go-fil-markets/pieceio/sectorcalculator" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" ) @@ -63,13 +60,7 @@ func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, uint64, error pr, psize := padreader.NewPaddedReader(uf, uint64(s)) - dir, err := ioutil.TempDir("", "sector") - if err != nil { - return nil, 0, err - } - defer os.RemoveAll(dir) - calculator := sectorcalculator.NewSectorCalculator(filestore.Path(dir)) - commp, err := calculator.GeneratePieceCommitment(pr, psize) + commp, err := sectorbuilder.GeneratePieceCommitment(pr, psize) if err != nil { return nil, 0, xerrors.Errorf("generating CommP: %w", err) }