diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 862825ce83..cd083cf1d4 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -2,17 +2,11 @@ package commands import ( "fmt" - "os" - "path/filepath" - "sync" - ipfscfg "github.com/ipfs/go-ipfs-config" - "github.com/ipfs/go-ipfs/plugin/loader" - "github.com/ipfs/go-ipfs/repo/fsrepo" - "github.com/ipfs/interface-go-ipfs-core/options" "github.com/spf13/cobra" cfg "github.com/lazyledger/lazyledger-core/config" + "github.com/lazyledger/lazyledger-core/ipfs" tmos "github.com/lazyledger/lazyledger-core/libs/os" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" "github.com/lazyledger/lazyledger-core/p2p" @@ -79,7 +73,6 @@ func initFilesWithConfig(config *cfg.Config) error { if tmos.FileExists(genFile) { logger.Info("Found genesis file", "path", genFile) } else { - genDoc := types.GenesisDoc{ ChainID: fmt.Sprintf("test-chain-%v", tmrand.Str(6)), GenesisTime: tmtime.Now(), @@ -106,90 +99,6 @@ func initFilesWithConfig(config *cfg.Config) error { logger.Info("Generated genesis file", "path", genFile) } - if err := InitIpfs(config); err != nil { - return err - } - - return nil -} - -// InitIpfs takes a few config flags from the tendermint config.IPFS -// and applies them to the freshly created IPFS repo. -// The IPFS config will stored under config.IPFS.ConfigRootPath. -// TODO(ismail) move into separate file, and consider making IPFS initialization -// independent from the `tendermint init` subcommand. -// TODO(ismail): add counter part in ResetAllCmd -func InitIpfs(config *cfg.Config) error { - repoRoot := config.IPFSRepoRoot() - if fsrepo.IsInitialized(repoRoot) { - logger.Info("IPFS was already initialized", "ipfs-path", repoRoot) - return nil - } - var conf *ipfscfg.Config - - identity, err := ipfscfg.CreateIdentity(os.Stdout, []options.KeyGenerateOption{ - options.Key.Type(options.Ed25519Key), - }) - if err != nil { - return err - } - - logger.Info("initializing IPFS node", "ipfs-path", repoRoot) - - if err := tmos.EnsureDir(repoRoot, 0700); err != nil { - return err - } - - conf, err = ipfscfg.InitWithIdentity(identity) - if err != nil { - return err - } - - applyFromTmConfig(conf, config.IPFS) - if err := setupPlugins(repoRoot); err != nil { - return err - } - - if err := fsrepo.Init(repoRoot, conf); err != nil { - return err - } - return nil -} - -// Inject replies on several global vars internally. -// For instance fsrepo.AddDatastoreConfigHandler will error -// if called multiple times with the same datastore. -// But for CI and integration tests, we want to setup the plugins -// for each repo but only inject once s.t. we can init multiple -// repos from the same runtime. -// TODO(ismail): find a more elegant way to achieve the same. -var injectPluginsOnce sync.Once - -func setupPlugins(path string) error { - // Load plugins. This will skip the repo if not available. - plugins, err := loader.NewPluginLoader(filepath.Join(path, "plugins")) - if err != nil { - return fmt.Errorf("error loading plugins: %s", err) - } - - if err := plugins.Initialize(); err != nil { - return fmt.Errorf("error initializing plugins: %s", err) - } - - injectPluginsOnce.Do(func() { - err = plugins.Inject() - }) - if err != nil { - return fmt.Errorf("error injecting plugins once: %w", err) - } - - return nil -} - -func applyFromTmConfig(ipfsConf *ipfscfg.Config, tmConf *cfg.IPFSConfig) { - ipfsConf.Addresses.API = ipfscfg.Strings{tmConf.API} - ipfsConf.Addresses.Gateway = ipfscfg.Strings{tmConf.Gateway} - ipfsConf.Addresses.Swarm = tmConf.Swarm - ipfsConf.Addresses.Announce = tmConf.Announce - ipfsConf.Addresses.NoAnnounce = tmConf.NoAnnounce + // TODO(ismail): add counter part in ResetAllCmd + return ipfs.InitRepo(config.IPFS.Path(), logger) } diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 4e42ff3767..88fff8c6a9 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -10,12 +10,14 @@ import ( "github.com/spf13/cobra" cfg "github.com/lazyledger/lazyledger-core/config" + "github.com/lazyledger/lazyledger-core/ipfs" tmos "github.com/lazyledger/lazyledger-core/libs/os" nm "github.com/lazyledger/lazyledger-core/node" ) var ( genesisHash []byte + initIPFS bool ) // AddNodeFlags exposes some common configuration options on the command-line @@ -89,6 +91,23 @@ func AddNodeFlags(cmd *cobra.Command) { "db-dir", config.DBPath, "database directory") + + cmd.Flags().String( + "ipfs.repo-path", + config.IPFS.RepoPath, + "custom IPFS repository path. Defaults to `.{RootDir}/ipfs`", + ) + cmd.Flags().Bool( + "ipfs.serve-api", + config.IPFS.ServeAPI, + "set this to expose IPFS API(useful for debugging)", + ) + cmd.Flags().BoolVar( + &initIPFS, + "ipfs.init", + false, + "set this to initialize repository for embedded IPFS node. Flag is ignored if repo is already initialized", + ) } // NewRunNodeCmd returns the command that allows the CLI to start a node. @@ -103,7 +122,11 @@ func NewRunNodeCmd(nodeProvider nm.Provider) *cobra.Command { return err } - n, err := nodeProvider(config, logger) + n, err := nodeProvider( + config, + ipfs.Embedded(initIPFS, config.IPFS, logger), + logger, + ) if err != nil { return fmt.Errorf("failed to create node: %w", err) } diff --git a/config/config.go b/config/config.go index 5f8551b436..8996dfeb9d 100644 --- a/config/config.go +++ b/config/config.go @@ -8,6 +8,8 @@ import ( "os" "path/filepath" "time" + + "github.com/lazyledger/lazyledger-core/ipfs" ) const ( @@ -66,7 +68,7 @@ type Config struct { TxIndex *TxIndexConfig `mapstructure:"tx-index"` Instrumentation *InstrumentationConfig `mapstructure:"instrumentation"` // Options for IPFS service - IPFS *IPFSConfig `mapstructure:"ipfs"` + IPFS *ipfs.Config `mapstructure:"ipfs"` } // DefaultConfig returns a default configuration for a Tendermint node @@ -81,7 +83,7 @@ func DefaultConfig() *Config { Consensus: DefaultConsensusConfig(), TxIndex: DefaultTxIndexConfig(), Instrumentation: DefaultInstrumentationConfig(), - IPFS: DefaultIPFSConfig(), + IPFS: ipfs.DefaultConfig(), } } @@ -108,6 +110,7 @@ func (cfg *Config) SetRoot(root string) *Config { cfg.P2P.RootDir = root cfg.Mempool.RootDir = root cfg.Consensus.RootDir = root + cfg.IPFS.RootDir = root return cfg } @@ -841,16 +844,16 @@ func DefaultConsensusConfig() *ConsensusConfig { // TestConsensusConfig returns a configuration for testing the consensus service func TestConsensusConfig() *ConsensusConfig { cfg := DefaultConsensusConfig() - cfg.TimeoutPropose = 40 * time.Millisecond - cfg.TimeoutProposeDelta = 1 * time.Millisecond - cfg.TimeoutPrevote = 10 * time.Millisecond - cfg.TimeoutPrevoteDelta = 1 * time.Millisecond - cfg.TimeoutPrecommit = 10 * time.Millisecond - cfg.TimeoutPrecommitDelta = 1 * time.Millisecond + cfg.TimeoutPropose = 100 * time.Millisecond + cfg.TimeoutProposeDelta = 10 * time.Millisecond + cfg.TimeoutPrevote = 40 * time.Millisecond + cfg.TimeoutPrevoteDelta = 10 * time.Millisecond + cfg.TimeoutPrecommit = 40 * time.Millisecond + cfg.TimeoutPrecommitDelta = 10 * time.Millisecond // NOTE: when modifying, make sure to update time_iota_ms (testGenesisFmt) in toml.go - cfg.TimeoutCommit = 10 * time.Millisecond + cfg.TimeoutCommit = 40 * time.Millisecond cfg.SkipTimeoutCommit = true - cfg.PeerGossipSleepDuration = 5 * time.Millisecond + cfg.PeerGossipSleepDuration = 10 * time.Millisecond cfg.PeerQueryMaj23SleepDuration = 250 * time.Millisecond cfg.DoubleSignCheckHeight = int64(0) return cfg @@ -1007,8 +1010,8 @@ func DefaultInstrumentationConfig() *InstrumentationConfig { } } -func TetsIpfsConfig() *IPFSConfig { - return DefaultIPFSConfig() +func TetsIpfsConfig() *ipfs.Config { + return ipfs.DefaultConfig() } // TestInstrumentationConfig returns a default configuration for metrics diff --git a/config/ipfs_config.go b/config/ipfs_config.go deleted file mode 100644 index 1475a4b1c4..0000000000 --- a/config/ipfs_config.go +++ /dev/null @@ -1,34 +0,0 @@ -package config - -// IPFSConfig defines a subset of the IPFS config that will be passed to the IPFS init and IPFS node (as a service) -// spun up by the tendermint node. -// It is mostly concerned about port configuration (Addresses). -type IPFSConfig struct { - // is where the generated IPFS config and files will be stored. - // The default is ~/.tendermint/ipfs. - ConfigRootPath string - // TODO: can we avoid copying the fields from ipfs' config.Addresses here? - // TODO: also, these are only used on init. Maybe ConfigRootPath is sufficient? - API string // address for the local API (RPC) - Gateway string // address to listen on for IPFS HTTP object gateway - // swarm related options: - Swarm []string // addresses for the swarm to listen on - Announce []string // swarm addresses to announce to the network - NoAnnounce []string // swarm addresses not to announce to the network -} - -// DefaultIPFSConfig returns a default config different from the default IPFS config. -// This avoids conflicts with existing installations when running LazyLedger-core node -// locally for testing purposes. -func DefaultIPFSConfig() *IPFSConfig { - return &IPFSConfig{ - ConfigRootPath: ".ipfs/", - API: "/ip4/127.0.0.1/tcp/5002", - Gateway: "/ip4/127.0.0.1/tcp/5002", - Swarm: []string{"/ip4/0.0.0.0/tcp/4002", "/ip6/::/tcp/4002"}, - } -} - -func (cfg *Config) IPFSRepoRoot() string { - return rootify(cfg.IPFS.ConfigRootPath, cfg.RootDir) -} diff --git a/config/toml.go b/config/toml.go index 13a338f295..0076343ffe 100644 --- a/config/toml.go +++ b/config/toml.go @@ -439,23 +439,9 @@ namespace = "{{ .Instrumentation.Namespace }}" ####################################################### [ipfs] -# IPFS repo root. -repo-root = "{{ .IPFS.ConfigRootPath}}" - -## Below options will be passed to ipfs when initializing an ipfs repo. -## They will be written into the repo-root/config on init. -## To modify the generated config, edit repo-root/config accordingly. - -# Address for the local API (RPC). -api = "{{ .IPFS.API }}" -# Address to listen on for IPFS HTTP object gateway. -gateway = "{{ .IPFS.Gateway }}" -# Addresses for the swarm to listen on -swarm = [{{ range .IPFS.Swarm }}{{ printf "%q, " . }}{{end}}] -# Swarm addresses to announce to the network -announce = [{{ range .IPFS.Announce }}{{ printf "%q, " . }}{{end}}] -# Swarm addresses not to announce to the network -no-announce = [{{ range .IPFS.NoAnnounce }}{{ printf "%q, " . }}{{end}}] +# IPFS related configuration +repo-path = "{{ .IPFS.RepoPath}}" +serve-api = "{{ .IPFS.ServeAPI}}" ` /****** these are for test settings ***********/ diff --git a/consensus/common_test.go b/consensus/common_test.go index b70dcf7fd3..ca2c0cfd5a 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -52,7 +52,7 @@ type cleanupFunc func() var ( config *cfg.Config // NOTE: must be reset for each _test.go file consensusReplayConfig *cfg.Config - ensureTimeout = 600 * time.Millisecond + ensureTimeout = 1000 * time.Millisecond ) func ensureDir(dir string, mode os.FileMode) { diff --git a/consensus/state.go b/consensus/state.go index f491887e4e..28c7c11fa4 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -12,7 +12,7 @@ import ( "time" "github.com/gogo/protobuf/proto" - ipfsapi "github.com/ipfs/interface-go-ipfs-core" + ipface "github.com/ipfs/interface-go-ipfs-core" cfg "github.com/lazyledger/lazyledger-core/config" cstypes "github.com/lazyledger/lazyledger-core/consensus/types" "github.com/lazyledger/lazyledger-core/crypto" @@ -25,6 +25,7 @@ import ( "github.com/lazyledger/lazyledger-core/libs/service" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" "github.com/lazyledger/lazyledger-core/p2p" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" sm "github.com/lazyledger/lazyledger-core/state" "github.com/lazyledger/lazyledger-core/types" @@ -93,7 +94,7 @@ type State struct { // store blocks and commits blockStore sm.BlockStore - IpfsAPI ipfsapi.CoreAPI + ipfs ipface.CoreAPI // create and execute blocks blockExec *sm.BlockExecutor @@ -206,6 +207,13 @@ func NewState( //---------------------------------------- // Public interface +// SetIPFSApi sets the IPFSAPI +func (cs *State) SetIPFSApi(api ipface.CoreAPI) { + cs.mtx.Lock() + defer cs.mtx.Unlock() + cs.ipfs = api +} + // SetLogger implements Service. func (cs *State) SetLogger(l log.Logger) { cs.BaseService.Logger = l @@ -1111,12 +1119,12 @@ func (cs *State) defaultDecideProposal(height int64, round int32) { // post data to ipfs // TODO(evan): don't hard code context and timeout - if cs.IpfsAPI != nil { + if cs.ipfs != nil { // longer timeouts result in block proposers failing to propose blocks in time. ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*1500) defer cancel() // TODO: post data to IPFS in a goroutine - err := block.PutBlock(ctx, cs.IpfsAPI.Dag()) + err := ipld.PutBlock(ctx, cs.ipfs.Dag(), block) if err != nil { cs.Logger.Error(fmt.Sprintf("failure to post block data to IPFS: %s", err.Error())) } diff --git a/go.mod b/go.mod index 1f410fba0b..8344c0358a 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-ipfs v0.8.0 github.com/ipfs/go-ipfs-api v0.2.0 - github.com/ipfs/go-ipfs-config v0.12.0 + github.com/ipfs/go-ipfs-config v0.11.0 github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-path v0.0.9 // indirect github.com/ipfs/go-verifcid v0.0.1 @@ -30,6 +30,7 @@ require ( github.com/lazyledger/rsmt2d v0.2.0 github.com/libp2p/go-buffer-pool v0.0.2 github.com/minio/highwayhash v1.0.1 + github.com/multiformats/go-multiaddr v0.3.1 github.com/multiformats/go-multihash v0.0.14 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index f5b7be7fbd..7b845a1959 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,7 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +contrib.go.opencensus.io/exporter/prometheus v0.2.0 h1:9PUk0/8V0LGoPqVCrf8fQZJkFGBxudu8jOjQSMwoD6w= contrib.go.opencensus.io/exporter/prometheus v0.2.0/go.mod h1:TYmVAyE8Tn1lyPcltF5IYYfWp2KHu7lQGIZnj8iZMys= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -52,9 +53,11 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= @@ -78,6 +81,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d h1:QgeLLoPD3kRVmeu/1al9iIpIANMi9O1zXFm8BnYGCJg= @@ -115,6 +119,7 @@ github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitf github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -182,6 +187,7 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302/go.mod h1:qBlWZqWeVx9BjvqBsnC/8RUlAYpIFmPvgROcw0n1scE= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -192,6 +198,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= @@ -205,9 +212,11 @@ github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gabriel-vasile/mimetype v1.1.2 h1:gaPnPcNor5aZSVCJVSGipcpbgMWiAAj9z182ocSGbHU= github.com/gabriel-vasile/mimetype v1.1.2/go.mod h1:6CDPel/o/3/s4+bp6kIbsWATq8pmgOisOPG40CJa6To= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-bindata/go-bindata/v3 v3.1.3 h1:F0nVttLC3ws0ojc7p60veTurcOm//D4QBODNM7EGrCI= github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-delve/delve v1.5.0/go.mod h1:c6b3a1Gry6x8a4LGCe/CWzrocrfaHvkUxCj3k4bvSUQ= @@ -446,9 +455,8 @@ github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7Na github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-cmds v0.5.0 h1:T1ZT6Qu3IUCp6FgU2IzVtvGLaexEWo9q13+S5ic+Q5Y= github.com/ipfs/go-ipfs-cmds v0.5.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= +github.com/ipfs/go-ipfs-config v0.11.0 h1:w4t2pz415Gtg6MTUKAq06C7ezC59/Us+k3+n1Tje+wg= github.com/ipfs/go-ipfs-config v0.11.0/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0= -github.com/ipfs/go-ipfs-config v0.12.0 h1:wxqN3ohBlis1EkhkzIKuF+XLx4YNn9rNpiSOYw3DFZc= -github.com/ipfs/go-ipfs-config v0.12.0/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -523,12 +531,14 @@ github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3 github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN1EJOjc= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= +github.com/ipfs/go-pinning-service-http-client v0.1.0 h1:Au0P4NglL5JfzhNSZHlZ1qra+IcJyO3RWMd9EYCwqSY= github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= github.com/ipfs/interface-go-ipfs-core v0.4.0 h1:+mUiamyHIwedqP8ZgbCIwpy40oX7QcXUbo4CZOeJVJg= github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o= +github.com/ipld/go-car v0.1.1-0.20201015032735-ff6ccdc46acc h1:BdI33Q56hLWG9Ef0WbQ7z+dwmbRYhTb45SMjw0RudbQ= github.com/ipld/go-car v0.1.1-0.20201015032735-ff6ccdc46acc/go.mod h1:WdIgzcEjFqydQ7jH+BXzGYxVCmLeAs5nP8Vu3Rege2Y= github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018 h1:RbRHv8epkmvBYA5cGfz68GUSbOgx5j/7ObLIl4Rsif0= @@ -575,10 +585,12 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= @@ -689,9 +701,11 @@ github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfx github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0 h1:Qfl+e5+lfDgwdrXdu4YNCWyEo3fWuP+WgN9mN0iWviQ= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= +github.com/libp2p/go-libp2p-gostream v0.3.0 h1:rnas//vRdHYCr7bjraZJISPwZV8OGMjeX5k5fN5Ax44= github.com/libp2p/go-libp2p-gostream v0.3.0/go.mod h1:pLBQu8db7vBMNINGsAwLL/ZCE8wng5V1FThoaE5rNjc= github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go= github.com/libp2p/go-libp2p-host v0.0.3/go.mod h1:Y/qPyA6C8j2coYyos1dfRm0I8+nvd4TGrDGt4tA7JR8= +github.com/libp2p/go-libp2p-http v0.2.0 h1:GYeVd+RZzkRa8XFLITqOpcrIQG6KbFLPJqII6HHBHzY= github.com/libp2p/go-libp2p-http v0.2.0/go.mod h1:GlNKFqDZHe25LVy2CvnZKx75/jLtMaD3VxZV6N39X7E= github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= github.com/libp2p/go-libp2p-interface-connmgr v0.0.4/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= @@ -887,17 +901,21 @@ github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561/go.mod h1:9vuHe github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -1098,6 +1116,7 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/statsd_exporter v0.15.0 h1:UiwC1L5HkxEPeapXdm2Ye0u1vUJfTj7uwT5yydYpa1E= github.com/prometheus/statsd_exporter v0.15.0/go.mod h1:Dv8HnkoLQkeEjkIE4/2ndAA7WL1zHKK7WMqFQqu72rw= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -1141,6 +1160,7 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -1233,6 +1253,7 @@ github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= +github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1 h1:ctS9Anw/KozviCCtK6VWMz5kPL9nbQzbQY4yfqlIV4M= github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1/go.mod h1:tKH72zYNt/exx6/5IQO6L9LoQ0rEjd5SbbWaDTs9Zso= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= @@ -1407,6 +1428,7 @@ golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1555,6 +1577,7 @@ google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1607,6 +1630,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/ipfs/config.go b/ipfs/config.go new file mode 100644 index 0000000000..8ce42dc8f6 --- /dev/null +++ b/ipfs/config.go @@ -0,0 +1,31 @@ +package ipfs + +import "path/filepath" + +// Config defines a subset of the IPFS config that will be passed to the IPFS init and IPFS node (as a service) +// spun up by the tendermint node. +// It is mostly concerned about port configuration (Addresses). +type Config struct { + RootDir string + // is where the generated IPFS config and files will be stored. + // The default is ~/.tendermint/ipfs. + RepoPath string `mapstructure:"repo-path"` + ServeAPI bool `mapstructure:"serve-api"` +} + +// DefaultConfig returns a default config different from the default IPFS config. +// This avoids conflicts with existing installations when running LazyLedger-core node +// locally for testing purposes. +func DefaultConfig() *Config { + return &Config{ + RepoPath: "ipfs", + ServeAPI: false, + } +} + +func (cfg *Config) Path() string { + if filepath.IsAbs(cfg.RepoPath) { + return cfg.RepoPath + } + return filepath.Join(cfg.RootDir, cfg.RepoPath) +} diff --git a/ipfs/defaults.go b/ipfs/defaults.go new file mode 100644 index 0000000000..7e3cd53555 --- /dev/null +++ b/ipfs/defaults.go @@ -0,0 +1,201 @@ +package ipfs + +import ( + "os" + + ipfscfg "github.com/ipfs/go-ipfs-config" + "github.com/ipfs/interface-go-ipfs-core/options" +) + +// BootstrapPeers is a list of default bootstrap peers for IPFS network +// TODO(Wondertan): Change this to LL defaults at some point +var BootstrapPeers, _ = ipfscfg.DefaultBootstrapPeers() + +// DefaultFullNodeConfig provides default embedded IPFS configuration for FullNode +func DefaultFullNodeConfig() (*ipfscfg.Config, error) { + identity, err := ipfscfg.CreateIdentity(os.Stdout, []options.KeyGenerateOption{ + options.Key.Type(options.Ed25519Key), + }) + if err != nil { + return nil, err + } + + var conf = &ipfscfg.Config{ + // ed25519 PKI identity for p2p communication + // TODO(Wondertan): Any reason not reuse ed25519 key from consensus identity? + Identity: identity, + // List of libp2p peer addresses the node automatically connects on start + Bootstrap: ipfscfg.BootstrapPeerStrings(BootstrapPeers), + // Configure all the addresses the node listens to + Addresses: ipfscfg.Addresses{ + // List of libp2p addresses to listen to + Swarm: []string{ + // listen on all network interfaces. + // NOTE: We don't use IPv6 for now, but we might add support for it at some point. + "/ip4/0.0.0.0/tcp/16001", + }, + API: []string{ + "/ip4/127.0.0.1/tcp/17001", + }, + // List of public libp2p WAN address the Node can be connected with + // NOTE: Should be configured by Node administrator manually + Announce: nil, + // List of libp2p address the Node listens to but not accessible from the WAN. + // It's a good manner to fil this, as that preserves connected peers from trying connections to those + // NOTE: Should be configured by Node administrator manually + NoAnnounce: nil, + // List of addresses IPFS API server listens to + // we do not run IPFS Gateway + Gateway: nil, + }, + // Networking configuration + Swarm: ipfscfg.SwarmConfig{ + // auto port forwarding is nice + DisableNatPortMap: false, + // metrics are useful + DisableBandwidthMetrics: false, + // relays are cool from engineering standpoint, but don't use them + EnableRelayHop: false, EnableAutoRelay: false, + // ConnMgr manager controls the amount of conns a Node holds. If the amount of conns hits HighWater, ConnMgr + // trims amount of connections down to LowWater + // NOTE: Connections can by prioritized using Tagging in ConnMgr in order to keep important conns alive + ConnMgr: ipfscfg.ConnMgr{ + // we use defaults from IPFS for HighWater + HighWater: 900, + // we use defaults from IPFS for HighWater + LowWater: 600, + // defines how often ConnMgr should check conns for trimming, we use default from IPFS + GracePeriod: "20s", + // just a basic connection manager + Type: "basic", + }, + // Configuration for all networking protocols + Transports: ipfscfg.Transports{ + Network: transport{ + // we default to TCP as it's most common, battle tested transport, but we may migrate to the one below + TCP: ipfscfg.True, + // we might default to QUIC at some point, but for now TCP is more common and reliable + QUIC: ipfscfg.False, + // we don't use Websocket as we don't target for browser nodes + Websocket: ipfscfg.False, + // we don't use relays as it's expensive in terms of traffic for relay nodes + Relay: ipfscfg.False, + }, + // Security protocol wraps IO conns in a private encrypted tunnel + Security: security{ + // we default to Noise, as it's designed for p2p interactions in the first place and is 0-RTT + Noise: ipfscfg.DefaultPriority, + // we don't use SECIO as it's now deprecated and has know issues + SECIO: ipfscfg.Disabled, + // we don't allow TLS option as we use noise only + TLS: ipfscfg.Disabled, + }, + // Multiplexers provide ability to run multiple logical streams over one transport stream, e.g TCP conn + Multiplexers: multiplexers{ + // we default to Yamux due to production readiness + Yamux: ipfscfg.DefaultPriority, + // we don't use Mplex as it's more like a toy multiplexer for testings and etc + Mplex: ipfscfg.Disabled, + }, + }, + // this ia manual peer blacklist + AddrFilters: nil, + }, + // Persistent KV store configuration + Datastore: ipfscfg.Datastore{ + // Configuration for badger kv we default to + Spec: map[string]interface{}{ + "type": "measure", + "prefix": "badger.datastore", + "child": map[string]interface{}{ + "type": "badgerds", + "path": "badgerds", + "syncWrites": false, + "truncate": true, + }, + }, + // we don't use bloom filtered blockstore + BloomFilterSize: 0, + // we don't use GC so values below does not have any affect + StorageGCWatermark: 0, + GCPeriod: "", + StorageMax: "", + }, + // Configuration for CID reproviding + // In kDHT all records have TTL, thus we have to regularly(Interval) reprovide/reannounce stored CID to the + // network. Otherwise information that the node stores something will be lost. Should be in tact with kDHT + // record cleaning configuration + Reprovider: ipfscfg.Reprovider{ + Interval: "12h", + Strategy: "all", + }, + Routing: ipfscfg.Routing{ + // Full node must be available from the WAN thus 'dhtserver' + // Depending on the node type/use-case different modes should be used. + Type: "dhtserver", + }, + // Configuration for plugins + Plugins: ipfscfg.Plugins{ + // Disable preloaded but unused plugins + Plugins: map[string]ipfscfg.Plugin{ + "ds-flatfs": { + Disabled: true, + }, + "ds-level": { + Disabled: true, + }, + "ipld-git": { + Disabled: true, + }, + }, + }, + // Unused fields: + // we currently don't use PubSub + Pubsub: ipfscfg.PubsubConfig{}, + // List of all experimental IPFS features + // We currently don't use any of them + Experimental: ipfscfg.Experiments{}, + // bi-directional agreement between peers to hold connections with + Peering: ipfscfg.Peering{}, + // local network discovery is useful, but there is no practical reason to have two FullNode in one LAN + Discovery: ipfscfg.Discovery{}, + // we don't use AutoNAT service for now + AutoNAT: ipfscfg.AutoNATConfig{}, + // we use API in some cases, but we do not need additional headers + API: ipfscfg.API{}, + // we do not use FUSE and thus mounts + Mounts: ipfscfg.Mounts{}, + // we do not use IPFS + Ipns: ipfscfg.Ipns{}, + // we do not run Gateway + Gateway: ipfscfg.Gateway{}, + // we don't use Pinning as we don't ues GC + Pinning: ipfscfg.Pinning{}, + // unused in IPFS and/or deprecated + Provider: ipfscfg.Provider{}, + } + + return conf, err +} + +// anonymous type aliases to keep default configuration succinct +type ( + //nolint + transport = struct { + QUIC ipfscfg.Flag `json:",omitempty"` + TCP ipfscfg.Flag `json:",omitempty"` + Websocket ipfscfg.Flag `json:",omitempty"` + Relay ipfscfg.Flag `json:",omitempty"` + } + //nolint + security = struct { + TLS ipfscfg.Priority `json:",omitempty"` + SECIO ipfscfg.Priority `json:",omitempty"` + Noise ipfscfg.Priority `json:",omitempty"` + } + //nolint + multiplexers = struct { + Yamux ipfscfg.Priority `json:",omitempty"` + Mplex ipfscfg.Priority `json:",omitempty"` + } +) diff --git a/ipfs/embedded.go b/ipfs/embedded.go new file mode 100644 index 0000000000..27f0fc14ec --- /dev/null +++ b/ipfs/embedded.go @@ -0,0 +1,155 @@ +package ipfs + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "sync" + + ipfscfg "github.com/ipfs/go-ipfs-config" + "github.com/ipfs/go-ipfs/commands" + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/core/coreapi" + "github.com/ipfs/go-ipfs/core/corehttp" + "github.com/ipfs/go-ipfs/core/node/libp2p" + "github.com/ipfs/go-ipfs/repo" + "github.com/ipfs/go-ipfs/repo/fsrepo" + coreiface "github.com/ipfs/interface-go-ipfs-core" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" + + "github.com/lazyledger/lazyledger-core/libs/log" +) + +// Embedded is the provider that embeds IPFS node within the same process. +// It also returns closable for graceful node shutdown. +func Embedded(init bool, cfg *Config, logger log.Logger) APIProvider { + return func() (coreiface.CoreAPI, io.Closer, error) { + path := cfg.Path() + defer os.Setenv(ipfscfg.EnvDir, path) + + // NOTE: no need to validate the path before + if err := plugins(path); err != nil { + return nil, nil, err + } + // Init Repo if requested + if init { + if err := InitRepo(path, logger); err != nil { + return nil, nil, err + } + } + // Open the repo + repo, err := fsrepo.Open(path) + if err != nil { + var nrerr fsrepo.NoRepoError + if errors.As(err, &nrerr) { + return nil, nil, fmt.Errorf("no IPFS repo found in %s.\nplease use flag: --ipfs-init", nrerr.Path) + } + return nil, nil, err + } + // Construct the node + nodeOptions := &core.BuildCfg{ + Online: true, + // This option sets the node to be a full DHT node (both fetching and storing DHT Records) + Routing: libp2p.DHTOption, + // This option sets the node to be a client DHT node (only fetching records) + // Routing: libp2p.DHTClientOption, + Repo: repo, + } + // Internally, ipfs decorates the context with a + // context.WithCancel. Which is then used for lifecycle management. + // We do not make use of this context and rely on calling + // Close() on the node instead + ctx := context.Background() + // It is essential that we create a fresh instance of ipfs node on + // each start as internally the node gets only stopped once per instance. + // At least in ipfs 0.7.0; see: + // https://github.com/lazyledger/go-ipfs/blob/dd295e45608560d2ada7d7c8a30f1eef3f4019bb/core/builder.go#L48-L57 + node, err := core.NewNode(ctx, nodeOptions) + if err != nil { + _ = repo.Close() + return nil, nil, err + } + // Serve API if requested + if cfg.ServeAPI { + if err := serveAPI(path, repo, node); err != nil { + _ = node.Close() + return nil, nil, err + } + } + // Wrap Node and create CoreAPI + api, err := coreapi.NewCoreAPI(node) + if err != nil { + _ = node.Close() + return nil, nil, fmt.Errorf("failed to create an instance of the IPFS core API: %w", err) + } + + logger.Info("Successfully created embedded IPFS node", "ipfs-repo", path) + return api, node, nil + } +} + +// serveAPI creates and HTTP server for IPFS API. +func serveAPI(path string, repo repo.Repo, node *core.IpfsNode) error { + cfg, err := repo.Config() + if err != nil { + return err + } + // go through every configured API address and start to listen to them + listeners := make([]manet.Listener, len(cfg.Addresses.API)) + for i, addr := range cfg.Addresses.API { + apiMaddr, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return fmt.Errorf("invalid IPFS API address: %q (err: %w)", addr, err) + } + + listeners[i], err = manet.Listen(apiMaddr) + if err != nil { + return fmt.Errorf("listen(%s) for IPFS API failed: %w", apiMaddr, err) + } + } + + for _, listener := range listeners { + fmt.Printf("IPFS API server listening on %s\n", listener.Multiaddr()) + } + // notify IPFS Repo about running api, it will create a file with an address to simplify access through CLI + if len(listeners) > 0 { + if err := node.Repo.SetAPIAddr(listeners[0].Multiaddr()); err != nil { + return fmt.Errorf("repo.SetAPIAddr() for IPFS API failed: %w", err) + } + } + // configure HTTP server with options + var opts = []corehttp.ServeOption{ + corehttp.CommandsOption(commands.Context{ + ConfigRoot: path, + LoadConfig: func(string) (*ipfscfg.Config, error) { + return cfg, nil + }, + ConstructNode: func() (*core.IpfsNode, error) { + return node, nil + }, + ReqLog: &commands.ReqLog{}, + }), + corehttp.CheckVersionOption(), + corehttp.HostnameOption(), + } + + errc := make(chan error) + var wg sync.WaitGroup + for _, apiLis := range listeners { + wg.Add(1) + go func(lis manet.Listener) { + defer wg.Done() + errc <- corehttp.Serve(node, manet.NetListener(lis), opts...) + }(apiLis) + } + + go func() { + wg.Wait() + close(errc) + }() + + return err +} diff --git a/ipfs/init.go b/ipfs/init.go new file mode 100644 index 0000000000..b979495536 --- /dev/null +++ b/ipfs/init.go @@ -0,0 +1,38 @@ +package ipfs + +import ( + "github.com/ipfs/go-ipfs/repo/fsrepo" + + "github.com/lazyledger/lazyledger-core/libs/log" + tmos "github.com/lazyledger/lazyledger-core/libs/os" +) + +// InitRepo initialize IPFS repository under the given path. +// It does nothing if repo is already created. +func InitRepo(path string, logger log.Logger) error { + if fsrepo.IsInitialized(path) { + logger.Info("IPFS is already initialized", "ipfs-path", path) + return nil + } + + if err := plugins(path); err != nil { + return err + } + + if err := tmos.EnsureDir(path, 0700); err != nil { + return err + } + + // TODO: Define node types, pass a node type as param and get relative config instead + cfg, err := DefaultFullNodeConfig() + if err != nil { + return err + } + + if err := fsrepo.Init(path, cfg); err != nil { + return err + } + + logger.Info("Successfully initialized IPFS repository", "ipfs-path", path) + return nil +} diff --git a/ipfs/mock.go b/ipfs/mock.go new file mode 100644 index 0000000000..1be81c7517 --- /dev/null +++ b/ipfs/mock.go @@ -0,0 +1,30 @@ +package ipfs + +import ( + "io" + + "github.com/ipfs/go-ipfs/core/coreapi" + coremock "github.com/ipfs/go-ipfs/core/mock" + coreiface "github.com/ipfs/interface-go-ipfs-core" + + "github.com/lazyledger/lazyledger-core/ipfs/plugin" +) + +// Mock provides simple mock IPFS API useful for testing +func Mock() APIProvider { + return func() (coreiface.CoreAPI, io.Closer, error) { + plugin.EnableNMT() + + nd, err := coremock.NewMockNode() + if err != nil { + return nil, nil, err + } + + api, err := coreapi.NewCoreAPI(nd) + if err != nil { + return nil, nil, err + } + + return api, nd, nil + } +} diff --git a/ipfs/plugin/.gitignore b/ipfs/plugin/.gitignore new file mode 100644 index 0000000000..eac6ba81e6 --- /dev/null +++ b/ipfs/plugin/.gitignore @@ -0,0 +1 @@ +nmt-plugin.so diff --git a/p2p/ipld/plugin/Makefile b/ipfs/plugin/Makefile similarity index 86% rename from p2p/ipld/plugin/Makefile rename to ipfs/plugin/Makefile index a46da5343e..bddb9341a9 100644 --- a/p2p/ipld/plugin/Makefile +++ b/ipfs/plugin/Makefile @@ -22,13 +22,13 @@ endif .PHONY: install build -lazyledger-plugin.so: plugin.go +nmt-plugin.so: main/main.go $(GOCC) build $(GOFLAGS) -buildmode=plugin -o "$@" "$<" chmod +x "$@" -build: lazyledger-plugin.so +build: nmt-plugin.so @echo "Built against" $(IPFS_VERSION) install: build mkdir -p "$(IPFS_PATH)/plugins/" - cp -f lazyledger-plugin.so "$(IPFS_PATH)/plugins/lazyledger-plugin.so" + cp -f nmt-plugin.so "$(IPFS_PATH)/plugins/nmt-plugin.so" diff --git a/ipfs/plugin/main/main.go b/ipfs/plugin/main/main.go new file mode 100644 index 0000000000..cc1b69e7b0 --- /dev/null +++ b/ipfs/plugin/main/main.go @@ -0,0 +1,13 @@ +package main + +import ( + ipfsplugin "github.com/ipfs/go-ipfs/plugin" + + "github.com/lazyledger/lazyledger-core/ipfs/plugin" +) + +// Plugins is an exported list of plugins that will be loaded by go-ipfs. +//nolint:deadcode +var Plugins = []ipfsplugin.Plugin{ + plugin.Nmt{}, +} diff --git a/p2p/ipld/plugin/nodes/nodes.go b/ipfs/plugin/nmt.go similarity index 77% rename from p2p/ipld/plugin/nodes/nodes.go rename to ipfs/plugin/nmt.go index 1df8a29d1d..fb28a8f01e 100644 --- a/p2p/ipld/plugin/nodes/nodes.go +++ b/ipfs/plugin/nmt.go @@ -1,9 +1,8 @@ -package nodes +package plugin import ( "bufio" "bytes" - "context" "crypto/sha256" "errors" "fmt" @@ -11,8 +10,6 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs/core/coredag" - "github.com/ipfs/go-ipfs/plugin" ipld "github.com/ipfs/go-ipld-format" "github.com/lazyledger/nmt" mh "github.com/multiformats/go-multihash" @@ -22,10 +19,10 @@ const ( // Below used multiformats (one codec, one multihash) seem free: // https://github.com/multiformats/multicodec/blob/master/table.csv - // Nmt is the codec used for leaf and inner nodes of an Namespaced Merkle Tree. - Nmt = 0x7700 + // NmtCodec is the codec used for leaf and inner nodes of an Namespaced Merkle Tree. + NmtCodec = 0x7700 - // NmtCodecName is the name used during registry of the Nmt codec + // NmtCodecName is the name used during registry of the NmtCodec codec NmtCodecName = "nmt-node" // Sha256Namespace8Flagged is the multihash code used to hash blocks @@ -53,10 +50,10 @@ func init() { sumSha256Namespace8Flagged, ) // this should already happen when the plugin is injected but it doesn't for some CI tests - ipld.DefaultBlockDecoder.Register(Nmt, NmtNodeParser) + ipld.DefaultBlockDecoder.Register(NmtCodec, NmtNodeParser) // register the codecs in the global maps - cid.Codecs[NmtCodecName] = Nmt - cid.CodecToStr[Nmt] = NmtCodecName + cid.Codecs[NmtCodecName] = NmtCodec + cid.CodecToStr[NmtCodec] = NmtCodecName } func mustRegisterNamespacedCodec( @@ -91,34 +88,6 @@ func sumSha256Namespace8Flagged(data []byte, _length int) ([]byte, error) { return nmt.Sha256Namespace8FlaggedInner(data[1:]), nil } -var Plugins = []plugin.Plugin{&LazyLedgerPlugin{}} - -var _ plugin.PluginIPLD = &LazyLedgerPlugin{} - -type LazyLedgerPlugin struct{} - -func (l LazyLedgerPlugin) RegisterBlockDecoders(dec ipld.BlockDecoder) error { - dec.Register(Nmt, NmtNodeParser) - return nil -} - -func (l LazyLedgerPlugin) RegisterInputEncParsers(iec coredag.InputEncParsers) error { - iec.AddParser("raw", DagParserFormatName, DataSquareRowOrColumnRawInputParser) - return nil -} - -func (l LazyLedgerPlugin) Name() string { - return "LazyLedger" -} - -func (l LazyLedgerPlugin) Version() string { - return "0.0.0" -} - -func (l LazyLedgerPlugin) Init(env *plugin.Environment) error { - return nil -} - // DataSquareRowOrColumnRawInputParser reads the raw shares and extract the IPLD nodes from the NMT tree. // Note, to parse without any error the input has to be of the form: // @@ -176,7 +145,7 @@ func (n nmtNodeCollector) ipldNodes() []ipld.Node { } func (n *nmtNodeCollector) visit(hash []byte, children ...[]byte) { - cid := mustCidFromNamespacedSha256(hash) + cid := MustCidFromNamespacedSha256(hash) switch len(children) { case 1: n.nodes = prependNode(nmtLeafNode{ @@ -201,66 +170,6 @@ func prependNode(newNode ipld.Node, nodes []ipld.Node) []ipld.Node { return nodes } -// NmtNodeAdder adds ipld.Nodes to the underlying ipld.Batch if it is inserted -// into an nmt tree -type NmtNodeAdder struct { - ctx context.Context - batch *ipld.Batch - leaves *cid.Set - err error -} - -// NewNmtNodeAdder returns a new NmtNodeAdder with the provided context and -// batch. Note that the context provided should have a timeout -// It is not thread-safe. -func NewNmtNodeAdder(ctx context.Context, batch *ipld.Batch) *NmtNodeAdder { - return &NmtNodeAdder{ - batch: batch, - ctx: ctx, - leaves: cid.NewSet(), - } -} - -// Visit can be inserted into an nmt tree to create ipld.Nodes while computing the root -func (n *NmtNodeAdder) Visit(hash []byte, children ...[]byte) { - if n.err != nil { - return // protect from further visits if there is an error - } - - cid := mustCidFromNamespacedSha256(hash) - switch len(children) { - case 1: - if n.leaves.Visit(cid) { - n.err = n.batch.Add(n.ctx, nmtLeafNode{ - cid: cid, - Data: children[0], - }) - } - case 2: - n.err = n.batch.Add(n.ctx, nmtNode{ - cid: cid, - l: children[0], - r: children[1], - }) - default: - panic("expected a binary tree") - } -} - -// Batch return the ipld.Batch originally provided to the NmtNodeAdder -func (n *NmtNodeAdder) Batch() *ipld.Batch { - return n.batch -} - -// Commit checks for errors happened during Visit and if absent commits data to inner Batch. -func (n *NmtNodeAdder) Commit() error { - if n.err != nil { - return n.err - } - - return n.batch.Commit() -} - func NmtNodeParser(block blocks.Block) (ipld.Node, error) { // length of the domain separator for leaf and inner nodes: const prefixOffset = 1 @@ -306,6 +215,10 @@ type nmtNode struct { l, r []byte } +func NewNMTNode(id cid.Cid, l, r []byte) ipld.Node { + return nmtNode{id, l, r} +} + func (n nmtNode) RawData() []byte { return append([]byte{nmt.NodePrefix}, append(n.l, n.r...)...) } @@ -385,8 +298,8 @@ func (n nmtNode) Copy() ipld.Node { } func (n nmtNode) Links() []*ipld.Link { - leftCid := mustCidFromNamespacedSha256(n.l) - rightCid := mustCidFromNamespacedSha256(n.r) + leftCid := MustCidFromNamespacedSha256(n.l) + rightCid := MustCidFromNamespacedSha256(n.r) return []*ipld.Link{{Cid: leftCid}, {Cid: rightCid}} } @@ -404,6 +317,10 @@ type nmtLeafNode struct { Data []byte } +func NewNMTLeafNode(id cid.Cid, data []byte) ipld.Node { + return &nmtLeafNode{id, data} +} + func (l nmtLeafNode) RawData() []byte { return append([]byte{nmt.LeafPrefix}, l.Data...) } @@ -470,12 +387,12 @@ func CidFromNamespacedSha256(namespacedHash []byte) (cid.Cid, error) { if err != nil { return cid.Undef, err } - return cid.NewCidV1(Nmt, mh.Multihash(buf)), nil + return cid.NewCidV1(NmtCodec, mh.Multihash(buf)), nil } -// mustCidFromNamespacedSha256 is a wrapper around cidFromNamespacedSha256 that panics +// MustCidFromNamespacedSha256 is a wrapper around cidFromNamespacedSha256 that panics // in case of an error. Use with care and only in places where no error should occur. -func mustCidFromNamespacedSha256(hash []byte) cid.Cid { +func MustCidFromNamespacedSha256(hash []byte) cid.Cid { cid, err := CidFromNamespacedSha256(hash) if err != nil { panic( diff --git a/p2p/ipld/plugin/nodes/nodes_test.go b/ipfs/plugin/nmt_test.go similarity index 98% rename from p2p/ipld/plugin/nodes/nodes_test.go rename to ipfs/plugin/nmt_test.go index 4a71da1809..d2b122fdd9 100644 --- a/p2p/ipld/plugin/nodes/nodes_test.go +++ b/ipfs/plugin/nmt_test.go @@ -1,4 +1,4 @@ -package nodes +package plugin import ( "bytes" @@ -96,7 +96,7 @@ func TestNodeCollector(t *testing.T) { t.Errorf("hashes don't match\ngot: %v\nwant: %v", got, want) } - if mustCidFromNamespacedSha256(rootDigest.Bytes()).String() != rootNodeCid.String() { + if MustCidFromNamespacedSha256(rootDigest.Bytes()).String() != rootNodeCid.String() { t.Error("root cid nod and hash not identical") } @@ -118,7 +118,7 @@ func TestNodeCollector(t *testing.T) { } hasher := nmt.NewNmtHasher(sha256.New(), namespaceSize, true) for _, leaf := range tt.leafData { - leafCid := mustCidFromNamespacedSha256(hasher.HashLeaf(leaf)) + leafCid := MustCidFromNamespacedSha256(hasher.HashLeaf(leaf)) _, has := hasMap[leafCid.String()] if !has { t.Errorf("leaf CID not found in collected nodes. missing: %s", leafCid.String()) diff --git a/ipfs/plugin/plugin.go b/ipfs/plugin/plugin.go new file mode 100644 index 0000000000..4770583962 --- /dev/null +++ b/ipfs/plugin/plugin.go @@ -0,0 +1,47 @@ +package plugin + +import ( + "github.com/ipfs/go-ipfs/core/coredag" + "github.com/ipfs/go-ipfs/plugin" + ipld "github.com/ipfs/go-ipld-format" +) + +func EnableNMT() { + _ = RegisterBlockDecoders(ipld.DefaultBlockDecoder) + _ = RegisterInputEncParsers(coredag.DefaultInputEncParsers) +} + +// Nmt is the IPLD plugin for NMT data structure. +type Nmt struct{} + +func (l Nmt) RegisterBlockDecoders(dec ipld.BlockDecoder) error { + return RegisterBlockDecoders(dec) +} + +func RegisterBlockDecoders(dec ipld.BlockDecoder) error { + dec.Register(NmtCodec, NmtNodeParser) + return nil +} + +func (l Nmt) RegisterInputEncParsers(iec coredag.InputEncParsers) error { + return RegisterInputEncParsers(iec) +} + +func RegisterInputEncParsers(iec coredag.InputEncParsers) error { + iec.AddParser("raw", DagParserFormatName, DataSquareRowOrColumnRawInputParser) + return nil +} + +func (l Nmt) Name() string { + return "NMT" +} + +func (l Nmt) Version() string { + return "0.0.0" +} + +func (l Nmt) Init(_ *plugin.Environment) error { + return nil +} + +var _ plugin.Plugin = &Nmt{} diff --git a/ipfs/plugins.go b/ipfs/plugins.go new file mode 100644 index 0000000000..32466e0f5a --- /dev/null +++ b/ipfs/plugins.go @@ -0,0 +1,54 @@ +package ipfs + +import ( + "fmt" + "path/filepath" + "sync" + + "github.com/ipfs/go-ipfs/plugin/loader" + + "github.com/lazyledger/lazyledger-core/ipfs/plugin" +) + +// pluginsOnce ensures that plugins are loaded/injected only once in a runtime. +// Otherwise repeatable loading of preloaded IPFS plugins errors. +// This is specifically needed for tests in 'node' package which spin up multiple embedded nodes. +// TODO(liamsi): consider using Mock() in those tests. +var pluginsOnce sync.Once + +// one and only loader instance within the application +var ldr *loader.PluginLoader + +// plugins loads default and custom IPFS plugins. +// Even though the NMT plugin can be loaded in much simpler way(plugin.EnableNMT), +// we still need to ensure default plugins likes "badger" are loaded. +func plugins(path string) (err error) { + pluginsOnce.Do(func() { + ldr, err = loader.NewPluginLoader(filepath.Join(path, "plugins")) + if err != nil { + err = fmt.Errorf("error loading plugins: %w", err) + return + } + + if err = ldr.Load(&plugin.Nmt{}); err != nil { + err = fmt.Errorf("error loading NMT plugin: %w", err) + return + } + + if err = ldr.Initialize(); err != nil { + err = fmt.Errorf("error initializing plugins:%w", err) + return + } + + if err = ldr.Inject(); err != nil { + err = fmt.Errorf("error injecting plugins: %w", err) + return + } + }) + + // TODO: Ideally we should handle ldr.Start()/ldr.Close() as well, however IPFS plugins system struggles from side + // effects. Those methods force us to create PluginLoader for every IPFS Node/Repo, but we + // can't create multiple PluginLoader instances in a runtime as each preloads default IPFS plugins producing errors + // with no possible workaround due to strong encapsulation. + return +} diff --git a/ipfs/provider.go b/ipfs/provider.go new file mode 100644 index 0000000000..9266350c19 --- /dev/null +++ b/ipfs/provider.go @@ -0,0 +1,10 @@ +package ipfs + +import ( + "io" + + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +// APIProvider allows customizable IPFS core APIs. +type APIProvider func() (coreiface.CoreAPI, io.Closer, error) diff --git a/libs/db/badgerdb/db.go b/libs/db/badgerdb/db.go index 94a727dc46..122ec117e8 100644 --- a/libs/db/badgerdb/db.go +++ b/libs/db/badgerdb/db.go @@ -6,6 +6,7 @@ import ( "path/filepath" "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v3/options" "github.com/lazyledger/lazyledger-core/libs/db" ) @@ -22,7 +23,8 @@ func NewDB(dbName, dir string) (*BadgerDB, error) { } opts := badger.DefaultOptions(path) opts.SyncWrites = false // note that we have Sync methods - opts.Logger = nil // badger is too chatty by default + // TODO(ismail): investigate if we don't want a logger here at least for errors though: + opts.Logger = nil // badger is too chatty by default return NewDBWithOptions(opts) } @@ -37,6 +39,22 @@ func NewDBWithOptions(opts badger.Options) (*BadgerDB, error) { return &BadgerDB{db: db}, nil } +// NewInMemoryDB creates a light weight in-memory BadgerDB. +// Mainly useful for unit-tests. +func NewInMemoryDB() (*BadgerDB, error) { + opts := badger.DefaultOptions("") + opts.InMemory = true + opts.NumCompactors = 2 // minimize number of go-routines + opts.Compression = options.None // this is supposed to be short-lived + opts.ZSTDCompressionLevel = 0 // this is supposed to be short-lived + opts.Logger = nil // there is not much that can go wrong in-memory + db, err := badger.Open(opts) + if err != nil { + return nil, err + } + return &BadgerDB{db: db}, nil +} + type BadgerDB struct { db *badger.DB } diff --git a/node/node.go b/node/node.go index 8af8c37e0e..1235c65b2d 100644 --- a/node/node.go +++ b/node/node.go @@ -5,19 +5,14 @@ import ( "context" "errors" "fmt" + "io" "net" "net/http" _ "net/http/pprof" // nolint: gosec // securely exposed on separate, optional port - "path/filepath" "strings" "time" - ipfscore "github.com/ipfs/go-ipfs/core" - coreapi "github.com/ipfs/go-ipfs/core/coreapi" - "github.com/ipfs/go-ipfs/core/node/libp2p" - "github.com/ipfs/go-ipfs/plugin/loader" - "github.com/ipfs/go-ipfs/repo/fsrepo" - "github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes" + ipface "github.com/ipfs/interface-go-ipfs-core" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/cors" @@ -28,6 +23,7 @@ import ( cs "github.com/lazyledger/lazyledger-core/consensus" "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/evidence" + "github.com/lazyledger/lazyledger-core/ipfs" dbm "github.com/lazyledger/lazyledger-core/libs/db" "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" tmjson "github.com/lazyledger/lazyledger-core/libs/json" @@ -71,6 +67,11 @@ func DefaultDBProvider(ctx *DBContext) (dbm.DB, error) { return badgerdb.NewDB(ctx.ID, ctx.Config.DBDir()) } +// InMemDBProvider provides an in-memory DB. +func InMemDBProvider(ctx *DBContext) (dbm.DB, error) { + return badgerdb.NewInMemoryDB() +} + // GenesisDocProvider returns a GenesisDoc. // It allows the GenesisDoc to be pulled from sources other than the // filesystem, for instance from a distributed key-value store cluster. @@ -85,12 +86,12 @@ func DefaultGenesisDocProviderFunc(config *cfg.Config) GenesisDocProvider { } // Provider takes a config and a logger and returns a ready to go Node. -type Provider func(*cfg.Config, log.Logger) (*Node, error) +type Provider func(*cfg.Config, ipfs.APIProvider, log.Logger) (*Node, error) // DefaultNewNode returns a Tendermint node with default settings for the // PrivValidator, ClientCreator, GenesisDoc, and DBProvider. // It implements NodeProvider. -func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) { +func DefaultNewNode(config *cfg.Config, ipfs ipfs.APIProvider, logger log.Logger) (*Node, error) { nodeKey, err := p2p.LoadOrGenNodeKey(config.NodeKeyFile()) if err != nil { return nil, fmt.Errorf("failed to load or gen node key %s: %w", config.NodeKeyFile(), err) @@ -107,9 +108,9 @@ func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) { proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), DefaultGenesisDocProviderFunc(config), DefaultDBProvider, + ipfs, DefaultMetricsProvider(config.Instrumentation), logger, - EmbedIpfsNode(true), ) } @@ -173,22 +174,6 @@ func StateProvider(stateProvider statesync.StateProvider) Option { } } -// IpfsPluginsWereLoaded indicates that all IPFS plugin were already loaded. -// Setting up plugins will be skipped when creating the IPFS node. -func IpfsPluginsWereLoaded(wereAlreadyLoaded bool) Option { - return func(n *Node) { - n.areIpfsPluginsAlreadyLoaded = wereAlreadyLoaded - } -} - -// IpfsPluginsWereLoaded indicates that all IPFS plugin were already loaded. -// Setting up plugins will skipped when creating the IPFS node. -func EmbedIpfsNode(embed bool) Option { - return func(n *Node) { - n.embedIpfsNode = embed - } -} - //------------------------------------------------------------------------------ // Node is the highest level interface to a full Tendermint node. @@ -229,10 +214,9 @@ type Node struct { txIndexer txindex.TxIndexer indexerService *txindex.IndexerService prometheusSrv *http.Server - // we store a ref to the full IpfsNode (instead of ipfs' CoreAPI) so we can Close() it OnStop() - embedIpfsNode bool // whether the node should start an IPFS node on startup - ipfsNode *ipfscore.IpfsNode // ipfs node - areIpfsPluginsAlreadyLoaded bool // avoid injecting plugins twice in tests etc + + ipfsAPI ipface.CoreAPI + ipfsClose io.Closer } func initDBs(config *cfg.Config, dbProvider DBProvider) (blockStore *store.BlockStore, stateDB dbm.DB, err error) { @@ -411,6 +395,7 @@ func createConsensusReactor(config *cfg.Config, csMetrics *cs.Metrics, waitSync bool, eventBus *types.EventBus, + ipfs ipface.CoreAPI, consensusLogger log.Logger) (*cs.Reactor, *cs.State) { consensusState := cs.NewState( @@ -422,6 +407,7 @@ func createConsensusReactor(config *cfg.Config, evidencePool, cs.StateMetrics(csMetrics), ) + consensusState.SetIPFSApi(ipfs) consensusState.SetLogger(consensusLogger) if privValidator != nil { consensusState.SetPrivValidator(privValidator) @@ -647,6 +633,7 @@ func NewNode(config *cfg.Config, clientCreator proxy.ClientCreator, genesisDocProvider GenesisDocProvider, dbProvider DBProvider, + ipfsProvider ipfs.APIProvider, metricsProvider MetricsProvider, logger log.Logger, options ...Option) (*Node, error) { @@ -750,6 +737,11 @@ func NewNode(config *cfg.Config, sm.BlockExecutorWithMetrics(smMetrics), ) + ipfs, ipfsclose, err := ipfsProvider() + if err != nil { + return nil, err + } + // Make BlockchainReactor. Don't start fast sync if we're doing a state sync first. bcReactor, err := createBlockchainReactor(config, state, blockExec, blockStore, fastSync && !stateSync, logger) if err != nil { @@ -765,7 +757,7 @@ func NewNode(config *cfg.Config, } consensusReactor, consensusState := createConsensusReactor( config, state, blockExec, blockStore, mempool, evidencePool, - privValidator, csMetrics, stateSync || fastSync, eventBus, consensusLogger, + privValidator, csMetrics, stateSync || fastSync, eventBus, ipfs, consensusLogger, ) // Set up state sync reactor, and schedule a sync if requested. @@ -866,6 +858,8 @@ func NewNode(config *cfg.Config, txIndexer: txIndexer, indexerService: indexerService, eventBus: eventBus, + ipfsAPI: ipfs, + ipfsClose: ipfsclose, } node.BaseService = *service.NewBaseService(logger, "Node", node) @@ -950,23 +944,6 @@ func (n *Node) OnStart() error { return fmt.Errorf("failed to start state sync: %w", err) } } - if n.embedIpfsNode { - // It is essential that we create a fresh instance of ipfs node on - // each start as internally the node gets only stopped once per instance. - // At least in ipfs 0.7.0; see: - // https://github.com/lazyledger/go-ipfs/blob/dd295e45608560d2ada7d7c8a30f1eef3f4019bb/core/builder.go#L48-L57 - n.ipfsNode, err = createIpfsNode(n.config, n.areIpfsPluginsAlreadyLoaded, n.Logger) - if err != nil { - return fmt.Errorf("failed to create IPFS node: %w", err) - } - - ipfsAPI, err := coreapi.NewCoreAPI(n.ipfsNode) - if err != nil { - return fmt.Errorf("failed to create an instance of the IPFS core API: %w", err) - } - - n.consensusState.IpfsAPI = ipfsAPI - } return nil } @@ -1027,14 +1004,8 @@ func (n *Node) OnStop() { } } - if n.ipfsNode != nil { - // Internally, the node gets shut down by cancelling the - // context that was passed into the node. - // Calling Close() seems cleaner and we don't - // need to keep a reference to the context around. - if err := n.ipfsNode.Close(); err != nil { - n.Logger.Error("ipfsNode.Close()", err) - } + if err := n.ipfsClose.Close(); err != nil { + n.Logger.Error("ipfsClose.Close()", err) } } @@ -1446,66 +1417,6 @@ func createAndStartPrivValidatorSocketClient( return pvscWithRetries, nil } -func createIpfsNode(config *cfg.Config, arePluginsAlreadyLoaded bool, logger log.Logger) (*ipfscore.IpfsNode, error) { - repoRoot := config.IPFSRepoRoot() - logger.Info("creating node in repo", "ipfs-root", repoRoot) - if !fsrepo.IsInitialized(repoRoot) { - // TODO: sentinel err - return nil, fmt.Errorf("ipfs repo root: %v not intitialized", repoRoot) - } - if !arePluginsAlreadyLoaded { - if err := setupPlugins(repoRoot, logger); err != nil { - return nil, err - } - } - // Open the repo - repo, err := fsrepo.Open(repoRoot) - if err != nil { - return nil, err - } - - // Construct the node - nodeOptions := &ipfscore.BuildCfg{ - Online: true, - // This option sets the node to be a full DHT node (both fetching and storing DHT Records) - Routing: libp2p.DHTOption, - // This option sets the node to be a client DHT node (only fetching records) - // Routing: libp2p.DHTClientOption, - Repo: repo, - } - // Internally, ipfs decorates the context with a - // context.WithCancel. Which is then used for lifecycle management. - // We do not make use of this context and rely on calling - // Close() on the node instead - ctx := context.Background() - node, err := ipfscore.NewNode(ctx, nodeOptions) - if err != nil { - return nil, err - } - // run as daemon: - node.IsDaemon = true - return node, nil -} - -func setupPlugins(path string, logger log.Logger) error { - // Load plugins. This will skip the repo if not available. - plugins, err := loader.NewPluginLoader(filepath.Join(path, "plugins")) - if err != nil { - return fmt.Errorf("error loading plugins: %s", err) - } - if err := plugins.Load(&nodes.LazyLedgerPlugin{}); err != nil { - return fmt.Errorf("error loading lazyledger plugin: %s", err) - } - if err := plugins.Initialize(); err != nil { - return fmt.Errorf("error initializing plugins: plugins.Initialize(): %s", err) - } - if err := plugins.Inject(); err != nil { - logger.Error("error initializing plugins: could not Inject()", "err", err) - } - - return nil -} - // splitAndTrimEmpty slices s into all subslices separated by sep and returns a // slice of the string s with all leading and trailing Unicode code points // contained in cutset removed. If sep is empty, SplitAndTrim splits after each diff --git a/node/node_test.go b/node/node_test.go index e29a82aacd..cb5a351814 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -19,6 +19,7 @@ import ( "github.com/lazyledger/lazyledger-core/crypto/ed25519" "github.com/lazyledger/lazyledger-core/crypto/tmhash" "github.com/lazyledger/lazyledger-core/evidence" + "github.com/lazyledger/lazyledger-core/ipfs" dbm "github.com/lazyledger/lazyledger-core/libs/db" "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" @@ -34,13 +35,35 @@ import ( tmtime "github.com/lazyledger/lazyledger-core/types/time" ) +func defaultNewTestNode(config *cfg.Config, logger log.Logger) (*Node, error) { + nodeKey, err := p2p.LoadOrGenNodeKey(config.NodeKeyFile()) + if err != nil { + return nil, fmt.Errorf("failed to load or gen node key %s: %w", config.NodeKeyFile(), err) + } + + pval, err := privval.LoadOrGenFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) + if err != nil { + return nil, err + } + + return NewNode(config, + pval, + nodeKey, + proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), + DefaultGenesisDocProviderFunc(config), + InMemDBProvider, + ipfs.Mock(), + DefaultMetricsProvider(config.Instrumentation), + logger, + ) +} + func TestNodeStartStop(t *testing.T) { config := cfg.ResetTestRoot("node_node_test") defer os.RemoveAll(config.RootDir) // create & start node - n, err := DefaultNewNode(config, log.TestingLogger()) - n.embedIpfsNode = false // TODO: or init ipfs upfront + n, err := defaultNewTestNode(config, log.TestingLogger()) require.NoError(t, err) err = n.Start() require.NoError(t, err) @@ -103,8 +126,7 @@ func TestNodeDelayedStart(t *testing.T) { now := tmtime.Now() // create & start node - n, err := DefaultNewNode(config, log.TestingLogger()) - n.embedIpfsNode = false // TODO: or init ipfs upfront + n, err := defaultNewTestNode(config, log.TestingLogger()) n.GenesisDoc().GenesisTime = now.Add(2 * time.Second) require.NoError(t, err) @@ -121,7 +143,7 @@ func TestNodeSetAppVersion(t *testing.T) { defer os.RemoveAll(config.RootDir) // create & start node - n, err := DefaultNewNode(config, log.TestingLogger()) + n, err := defaultNewTestNode(config, log.TestingLogger()) require.NoError(t, err) // default config uses the kvstore app @@ -137,6 +159,7 @@ func TestNodeSetAppVersion(t *testing.T) { } func TestNodeSetPrivValTCP(t *testing.T) { + t.Skip("TODO(ismail): Mock these conns using net.Pipe instead") addr := "tcp://" + testFreeAddr(t) config := cfg.ResetTestRoot("node_priv_val_tcp_test") @@ -164,7 +187,8 @@ func TestNodeSetPrivValTCP(t *testing.T) { }() defer signerServer.Stop() //nolint:errcheck // ignore for tests - n, err := DefaultNewNode(config, log.TestingLogger()) + logger := log.TestingLogger() + n, err := defaultNewTestNode(config, logger) require.NoError(t, err) assert.IsType(t, &privval.RetrySignerClient{}, n.PrivValidator()) } @@ -177,7 +201,7 @@ func TestPrivValidatorListenAddrNoProtocol(t *testing.T) { defer os.RemoveAll(config.RootDir) config.BaseConfig.PrivValidatorListenAddr = addrNoPrefix - _, err := DefaultNewNode(config, log.TestingLogger()) + _, err := defaultNewTestNode(config, log.TestingLogger()) assert.Error(t, err) } @@ -208,7 +232,8 @@ func TestNodeSetPrivValIPC(t *testing.T) { }() defer pvsc.Stop() //nolint:errcheck // ignore for tests - n, err := DefaultNewNode(config, log.TestingLogger()) + logger := log.TestingLogger() + n, err := defaultNewTestNode(config, logger) require.NoError(t, err) assert.IsType(t, &privval.RetrySignerClient{}, n.PrivValidator()) } @@ -513,7 +538,8 @@ func TestNodeNewNodeCustomReactors(t *testing.T) { nodeKey, proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), DefaultGenesisDocProviderFunc(config), - DefaultDBProvider, + InMemDBProvider, + ipfs.Mock(), DefaultMetricsProvider(config.Instrumentation), log.TestingLogger(), CustomReactors(map[string]p2p.Reactor{"FOO": cr, "BLOCKCHAIN": customBlockchainReactor}), diff --git a/p2p/ipld/nmt_adder.go b/p2p/ipld/nmt_adder.go new file mode 100644 index 0000000000..b8d5b94322 --- /dev/null +++ b/p2p/ipld/nmt_adder.go @@ -0,0 +1,63 @@ +package ipld + +import ( + "context" + + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + + "github.com/lazyledger/lazyledger-core/ipfs/plugin" +) + +// NmtNodeAdder adds ipld.Nodes to the underlying ipld.Batch if it is inserted +// into an nmt tree +type NmtNodeAdder struct { + ctx context.Context + batch *ipld.Batch + leaves *cid.Set + err error +} + +// NewNmtNodeAdder returns a new NmtNodeAdder with the provided context and +// batch. Note that the context provided should have a timeout +// It is not thread-safe. +func NewNmtNodeAdder(ctx context.Context, batch *ipld.Batch) *NmtNodeAdder { + return &NmtNodeAdder{ + batch: batch, + ctx: ctx, + leaves: cid.NewSet(), + } +} + +// Visit can be inserted into an nmt tree to create ipld.Nodes while computing the root +func (n *NmtNodeAdder) Visit(hash []byte, children ...[]byte) { + if n.err != nil { + return // protect from further visits if there is an error + } + + id := plugin.MustCidFromNamespacedSha256(hash) + switch len(children) { + case 1: + if n.leaves.Visit(id) { + n.err = n.batch.Add(n.ctx, plugin.NewNMTLeafNode(id, children[0])) + } + case 2: + n.err = n.batch.Add(n.ctx, plugin.NewNMTNode(id, children[0], children[1])) + default: + panic("expected a binary tree") + } +} + +// Batch return the ipld.Batch originally provided to the NmtNodeAdder +func (n *NmtNodeAdder) Batch() *ipld.Batch { + return n.batch +} + +// Commit checks for errors happened during Visit and if absent commits data to inner Batch. +func (n *NmtNodeAdder) Commit() error { + if n.err != nil { + return n.err + } + + return n.batch.Commit() +} diff --git a/p2p/ipld/plugin/.gitignore b/p2p/ipld/plugin/.gitignore deleted file mode 100644 index d0cabade58..0000000000 --- a/p2p/ipld/plugin/.gitignore +++ /dev/null @@ -1 +0,0 @@ -lazyledger-plugin.so diff --git a/p2p/ipld/plugin/plugin.go b/p2p/ipld/plugin/plugin.go deleted file mode 100644 index 70ffbaaa92..0000000000 --- a/p2p/ipld/plugin/plugin.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - "github.com/ipfs/go-ipfs/plugin" - - "github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes" -) - -// Plugins is an exported list of plugins that will be loaded by go-ipfs. -//nolint:deadcode -var Plugins = []plugin.Plugin{ - &nodes.LazyLedgerPlugin{}, -} diff --git a/p2p/ipld/racedetector/is_active.go b/p2p/ipld/racedetector/is_active.go new file mode 100644 index 0000000000..e33a4097d0 --- /dev/null +++ b/p2p/ipld/racedetector/is_active.go @@ -0,0 +1,7 @@ +package racedetector + +var raceDetectorActive = false + +func IsActive() bool { + return raceDetectorActive +} diff --git a/p2p/ipld/race_detector_file.go b/p2p/ipld/racedetector/race_detector_file.go similarity index 74% rename from p2p/ipld/race_detector_file.go rename to p2p/ipld/racedetector/race_detector_file.go index 85da7b27bb..c45e926f3c 100644 --- a/p2p/ipld/race_detector_file.go +++ b/p2p/ipld/racedetector/race_detector_file.go @@ -1,6 +1,6 @@ // +build race -package ipld +package racedetector func init() { raceDetectorActive = true diff --git a/p2p/ipld/read.go b/p2p/ipld/read.go index d58255c76b..84ebcb04b0 100644 --- a/p2p/ipld/read.go +++ b/p2p/ipld/read.go @@ -11,7 +11,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core/path" "github.com/lazyledger/rsmt2d" - "github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes" + "github.com/lazyledger/lazyledger-core/ipfs/plugin" "github.com/lazyledger/lazyledger-core/types" ) @@ -39,7 +39,7 @@ func RetrieveBlockData( // half of the rows for _, row := range uniqueRandNumbers(edsWidth/2, edsWidth) { for _, col := range uniqueRandNumbers(edsWidth/2, edsWidth) { - rootCid, err := nodes.CidFromNamespacedSha256(rowRoots[row]) + rootCid, err := plugin.CidFromNamespacedSha256(rowRoots[row]) if err != nil { return types.Data{}, err } diff --git a/p2p/ipld/read_test.go b/p2p/ipld/read_test.go index aca4b46780..d0128c2a2e 100644 --- a/p2p/ipld/read_test.go +++ b/p2p/ipld/read_test.go @@ -23,12 +23,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes" + "github.com/lazyledger/lazyledger-core/ipfs/plugin" + "github.com/lazyledger/lazyledger-core/p2p/ipld/racedetector" "github.com/lazyledger/lazyledger-core/types" ) -var raceDetectorActive = false - func TestLeafPath(t *testing.T) { type test struct { name string @@ -122,7 +121,7 @@ func TestGetLeafData(t *testing.T) { } // compute the root and create a cid for the root hash - rootCid, err := nodes.CidFromNamespacedSha256(root.Bytes()) + rootCid, err := plugin.CidFromNamespacedSha256(root.Bytes()) if err != nil { t.Error(err) } @@ -247,20 +246,20 @@ func TestRetrieveBlockData(t *testing.T) { t.Run(fmt.Sprintf("%s size %d", tc.name, tc.squareSize), func(t *testing.T) { // if we're using the race detector, skip some large tests due to time and // concurrency constraints - if raceDetectorActive && tc.squareSize > 8 { + if racedetector.IsActive() && tc.squareSize > 8 { t.Skip("Not running large test due to time and concurrency constraints while race detector is active.") } background := context.Background() blockData := generateRandomBlockData(tc.squareSize*tc.squareSize, adjustedMsgSize) - block := types.Block{ + block := &types.Block{ Data: blockData, LastCommit: &types.Commit{}, } // if an error is exected, don't put the block if !tc.expectErr { - err := block.PutBlock(background, ipfsAPI.Dag()) + err := PutBlock(background, ipfsAPI.Dag(), block) if err != nil { t.Fatal(err) } @@ -327,7 +326,7 @@ func getNmtRoot( batch *format.Batch, namespacedData [][]byte, ) (namespace.IntervalDigest, error) { - na := nodes.NewNmtNodeAdder(ctx, batch) + na := NewNmtNodeAdder(ctx, batch) tree := nmt.New(sha256.New(), nmt.NamespaceIDSize(types.NamespaceSize), nmt.NodeVisitor(na.Visit)) for _, leaf := range namespacedData { err := tree.Push(leaf) diff --git a/p2p/ipld/sample.go b/p2p/ipld/sample.go index 96c158c20a..9777cb615d 100644 --- a/p2p/ipld/sample.go +++ b/p2p/ipld/sample.go @@ -7,7 +7,7 @@ import ( "github.com/ipfs/go-cid" "github.com/lazyledger/nmt/namespace" - "github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes" + "github.com/lazyledger/lazyledger-core/ipfs/plugin" "github.com/lazyledger/lazyledger-core/types" ) @@ -40,7 +40,7 @@ func (s Sample) Leaf(dah *types.DataAvailabilityHeader) (cid.Cid, uint32, error) leaf = s.Col } - rootCid, err := nodes.CidFromNamespacedSha256(root.Bytes()) + rootCid, err := plugin.CidFromNamespacedSha256(root.Bytes()) if err != nil { return cid.Undef, 0, err } diff --git a/p2p/ipld/validate_test.go b/p2p/ipld/validate_test.go index 8ea0a4066d..75cce6ad54 100644 --- a/p2p/ipld/validate_test.go +++ b/p2p/ipld/validate_test.go @@ -27,13 +27,13 @@ func TestValidateAvailability(t *testing.T) { ipfsAPI := mockedIpfsAPI(t) blockData := generateRandomBlockData(squareSize*squareSize, adjustedMsgSize) - block := types.Block{ + block := &types.Block{ Data: blockData, LastCommit: &types.Commit{}, } block.Hash() - err := block.PutBlock(ctx, ipfsAPI.Dag()) + err := PutBlock(ctx, ipfsAPI.Dag(), block) require.NoError(t, err) calls := 0 diff --git a/p2p/ipld/write.go b/p2p/ipld/write.go new file mode 100644 index 0000000000..97ebd6bbdf --- /dev/null +++ b/p2p/ipld/write.go @@ -0,0 +1,60 @@ +package ipld + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + + format "github.com/ipfs/go-ipld-format" + "github.com/lazyledger/nmt" + "github.com/lazyledger/rsmt2d" + + "github.com/lazyledger/lazyledger-core/types" +) + +// PutBlock posts and pins erasured block data to IPFS using the provided +// ipld.NodeAdder. Note: the erasured data is currently recomputed +func PutBlock(ctx context.Context, adder format.NodeAdder, block *types.Block) error { + if adder == nil { + return errors.New("no ipfs node adder provided") + } + + // recompute the shares + namespacedShares, _ := block.Data.ComputeShares() + shares := namespacedShares.RawShares() + + // don't do anything if there is no data to put on IPFS + if len(shares) == 0 { + return nil + } + + // recompute the eds + eds, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree) + if err != nil { + return fmt.Errorf("failure to recompute the extended data square: %w", err) + } + + // add namespaces to erasured shares and flatten the eds + leaves := types.FlattenNamespacedEDS(namespacedShares, eds) + + // create nmt adder wrapping batch adder + batchAdder := NewNmtNodeAdder(ctx, format.NewBatch(ctx, adder)) + + // iterate through each set of col and row leaves + for _, leafSet := range leaves { + tree := nmt.New(sha256.New(), nmt.NodeVisitor(batchAdder.Visit)) + for _, share := range leafSet { + err = tree.Push(share) + if err != nil { + return err + } + } + + // compute the root in order to collect the ipld.Nodes + tree.Root() + } + + // commit the batch to ipfs + return batchAdder.Commit() +} diff --git a/p2p/ipld/write_test.go b/p2p/ipld/write_test.go new file mode 100644 index 0000000000..e4f0c8a851 --- /dev/null +++ b/p2p/ipld/write_test.go @@ -0,0 +1,86 @@ +package ipld + +import ( + "context" + "testing" + "time" + + "github.com/ipfs/go-ipfs/core/coreapi" + coremock "github.com/ipfs/go-ipfs/core/mock" + "github.com/stretchr/testify/require" + + "github.com/lazyledger/lazyledger-core/ipfs/plugin" + "github.com/lazyledger/lazyledger-core/types" +) + +func TestPutBlock(t *testing.T) { + ipfsNode, err := coremock.NewMockNode() + if err != nil { + t.Error(err) + } + + ipfsAPI, err := coreapi.NewCoreAPI(ipfsNode) + if err != nil { + t.Error(err) + } + + maxOriginalSquareSize := types.MaxSquareSize / 2 + maxShareCount := maxOriginalSquareSize * maxOriginalSquareSize + + testCases := []struct { + name string + blockData types.Data + expectErr bool + errString string + }{ + {"no leaves", generateRandomMsgOnlyData(0), false, ""}, + {"single leaf", generateRandomMsgOnlyData(1), false, ""}, + {"16 leaves", generateRandomMsgOnlyData(16), false, ""}, + {"max square size", generateRandomMsgOnlyData(maxShareCount), false, ""}, + } + ctx := context.Background() + for _, tc := range testCases { + tc := tc + + block := &types.Block{Data: tc.blockData} + + t.Run(tc.name, func(t *testing.T) { + err = PutBlock(ctx, ipfsAPI.Dag(), block) + if tc.expectErr { + require.Error(t, err) + require.Contains(t, err.Error(), tc.errString) + return + } + + require.NoError(t, err) + + timeoutCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + block.Hash() + for _, rowRoot := range block.DataAvailabilityHeader.RowsRoots.Bytes() { + // recreate the cids using only the computed roots + cid, err := plugin.CidFromNamespacedSha256(rowRoot) + if err != nil { + t.Error(err) + } + + // retrieve the data from IPFS + _, err = ipfsAPI.Dag().Get(timeoutCtx, cid) + if err != nil { + t.Errorf("Root not found: %s", cid.String()) + } + } + }) + } +} + +func generateRandomMsgOnlyData(msgCount int) types.Data { + out := make([]types.Message, msgCount) + for i, msg := range generateRandNamespacedRawData(msgCount, types.NamespaceSize, types.MsgShareSize-2) { + out[i] = types.Message{NamespaceID: msg[:types.NamespaceSize], Data: msg[types.NamespaceSize:]} + } + return types.Data{ + Messages: types.Messages{MessagesList: out}, + } +} diff --git a/p2p/transport_test.go b/p2p/transport_test.go index e13c81534c..2e01462e16 100644 --- a/p2p/transport_test.go +++ b/p2p/transport_test.go @@ -615,7 +615,7 @@ func TestTransportHandshake(t *testing.T) { t.Fatal(err) } - ni, err := handshake(c, 20*time.Millisecond, emptyNodeInfo()) + ni, err := handshake(c, 100*time.Millisecond, emptyNodeInfo()) if err != nil { t.Fatal(err) } @@ -651,7 +651,7 @@ func testSetupMultiplexTransport(t *testing.T) *MultiplexTransport { } // give the listener some time to get ready - time.Sleep(20 * time.Millisecond) + time.Sleep(100 * time.Millisecond) return mt } diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index 2eed8f1eda..5a8b8e747a 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -3,22 +3,15 @@ package rpctest import ( "context" "fmt" - "io/ioutil" "os" "path/filepath" "strings" "time" - ipfscfg "github.com/ipfs/go-ipfs-config" - "github.com/ipfs/go-ipfs/plugin/loader" - "github.com/ipfs/go-ipfs/repo/fsrepo" - "github.com/ipfs/interface-go-ipfs-core/options" abci "github.com/lazyledger/lazyledger-core/abci/types" - "github.com/lazyledger/lazyledger-core/libs/log" - tmos "github.com/lazyledger/lazyledger-core/libs/os" - "github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes" - cfg "github.com/lazyledger/lazyledger-core/config" + "github.com/lazyledger/lazyledger-core/ipfs" + "github.com/lazyledger/lazyledger-core/libs/log" tmnet "github.com/lazyledger/lazyledger-core/libs/net" nm "github.com/lazyledger/lazyledger-core/node" "github.com/lazyledger/lazyledger-core/p2p" @@ -180,17 +173,12 @@ func NewTendermint(app abci.Application, opts *Options) *nm.Node { panic(err) } - config.IPFS = cfg.DefaultIPFSConfig() - err = initIpfs(config, opts.loadIpfsPlugins, logger) - if err != nil { - panic(err) - } node, err := nm.NewNode(config, pv, nodeKey, papp, nm.DefaultGenesisDocProviderFunc(config), - nm.DefaultDBProvider, + nm.InMemDBProvider, + ipfs.Mock(), nm.DefaultMetricsProvider(config.Instrumentation), logger, - nm.IpfsPluginsWereLoaded(true), ) if err != nil { panic(err) @@ -198,54 +186,6 @@ func NewTendermint(app abci.Application, opts *Options) *nm.Node { return node } -// initIpfs a slightly modified commands.InitIpfs. -// To avoid depending on the commands package here, we accept some code duplication. -// In case commands.InitIpfs gets refactored into a separate package, -// it could be used instead. -func initIpfs(config *cfg.Config, loadPlugins bool, log log.Logger) error { - repoRoot := config.IPFSRepoRoot() - if fsrepo.IsInitialized(repoRoot) { - log.Info("IPFS repo already initialized", "repo-root", repoRoot) - return nil - } - var conf *ipfscfg.Config - - identity, err := ipfscfg.CreateIdentity(ioutil.Discard, []options.KeyGenerateOption{ - options.Key.Type(options.Ed25519Key), - }) - if err != nil { - return err - } - if err := tmos.EnsureDir(repoRoot, 0700); err != nil { - return err - } - if loadPlugins { - plugins, err := loader.NewPluginLoader(filepath.Join(repoRoot, "plugins")) - if err != nil { - return fmt.Errorf("error loading plugins: %s", err) - } - if err := plugins.Load(&nodes.LazyLedgerPlugin{}); err != nil { - return err - } - if err := plugins.Initialize(); err != nil { - return fmt.Errorf("error initializing plugins: %s", err) - } - if err := plugins.Inject(); err != nil { - return fmt.Errorf("error initializing plugins: %s", err) - } - } - conf, err = ipfscfg.InitWithIdentity(identity) - if err != nil { - return fmt.Errorf("initializing config failed, InitWithIdentity(): %w", err) - } - - if err := fsrepo.Init(repoRoot, conf); err != nil { - return err - } - - return nil -} - // SuppressStdout is an option that tries to make sure the RPC test Tendermint // node doesn't log anything to stdout. func SuppressStdout(o *Options) { diff --git a/test/e2e/app/main.go b/test/e2e/app/main.go index 32a8a69f98..d2f77d0036 100644 --- a/test/e2e/app/main.go +++ b/test/e2e/app/main.go @@ -11,6 +11,7 @@ import ( "github.com/lazyledger/lazyledger-core/config" "github.com/lazyledger/lazyledger-core/crypto/ed25519" + "github.com/lazyledger/lazyledger-core/ipfs" tmflags "github.com/lazyledger/lazyledger-core/libs/cli/flags" "github.com/lazyledger/lazyledger-core/libs/log" tmnet "github.com/lazyledger/lazyledger-core/libs/net" @@ -103,6 +104,7 @@ func startNode(cfg *Config) error { proxy.NewLocalClientCreator(app), node.DefaultGenesisDocProviderFunc(tmcfg), node.DefaultDBProvider, + ipfs.Embedded(true, ipfs.DefaultConfig(), nodeLogger), node.DefaultMetricsProvider(tmcfg.Instrumentation), nodeLogger, ) diff --git a/test/e2e/runner/load.go b/test/e2e/runner/load.go index 3758fc80c6..def189c76f 100644 --- a/test/e2e/runner/load.go +++ b/test/e2e/runner/load.go @@ -25,7 +25,7 @@ func Load(ctx context.Context, testnet *e2e.Testnet) error { if concurrency == 0 { concurrency = 1 } - initialTimeout := 1 * time.Minute + initialTimeout := 5 * time.Minute stallTimeout := 30 * time.Second chTx := make(chan types.Tx) diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 76048fb4f7..059f504607 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -19,9 +19,9 @@ import ( "github.com/BurntSushi/toml" - "github.com/lazyledger/lazyledger-core/cmd/tendermint/commands" "github.com/lazyledger/lazyledger-core/config" "github.com/lazyledger/lazyledger-core/crypto/ed25519" + "github.com/lazyledger/lazyledger-core/ipfs" "github.com/lazyledger/lazyledger-core/p2p" "github.com/lazyledger/lazyledger-core/privval" e2e "github.com/lazyledger/lazyledger-core/test/e2e/pkg" @@ -87,7 +87,7 @@ func Setup(testnet *e2e.Testnet) error { return err } // todo(evan): the path should be a constant - cfg.IPFS.ConfigRootPath = filepath.Join(nodeDir, ".ipfs") + cfg.IPFS.RepoPath = filepath.Join(nodeDir, ".ipfs") config.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), cfg) // panics appCfg, err := MakeAppConfig(node) @@ -115,7 +115,7 @@ func Setup(testnet *e2e.Testnet) error { filepath.Join(nodeDir, PrivvalDummyKeyFile), filepath.Join(nodeDir, PrivvalDummyStateFile), )).Save() - err = commands.InitIpfs(cfg) + err = ipfs.InitRepo(cfg.IPFS.RepoPath, logger) if err != nil { return err } diff --git a/types/block.go b/types/block.go index 481fdc9105..fd92f507c7 100644 --- a/types/block.go +++ b/types/block.go @@ -2,8 +2,6 @@ package types import ( "bytes" - "context" - "crypto/sha256" "errors" "fmt" "math" @@ -12,8 +10,6 @@ import ( "github.com/gogo/protobuf/proto" gogotypes "github.com/gogo/protobuf/types" - format "github.com/ipfs/go-ipld-format" - "github.com/lazyledger/nmt" "github.com/lazyledger/nmt/namespace" "github.com/lazyledger/rsmt2d" @@ -26,7 +22,6 @@ import ( tmmath "github.com/lazyledger/lazyledger-core/libs/math" "github.com/lazyledger/lazyledger-core/libs/protoio" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" - "github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" tmversion "github.com/lazyledger/lazyledger-core/proto/tendermint/version" "github.com/lazyledger/lazyledger-core/version" @@ -225,6 +220,7 @@ func (b *Block) fillHeader() { } } +// TODO: Move out from 'types' package // fillDataAvailabilityHeader fills in any remaining DataAvailabilityHeader fields // that are a function of the block data. func (b *Block) fillDataAvailabilityHeader() { @@ -252,7 +248,7 @@ func (b *Block) fillDataAvailabilityHeader() { } // flatten the square and add namespaces - leaves := flattenNamespacedEDS(namespacedShares, extendedDataSquare) + leaves := FlattenNamespacedEDS(namespacedShares, extendedDataSquare) // compute the roots for each col/row for i, leafSet := range leaves { @@ -293,55 +289,10 @@ func mustPush(rowTree *nmt.NamespacedMerkleTree, nidAndData []byte) { } } -// PutBlock posts and pins erasured block data to IPFS using the provided -// ipld.NodeAdder. Note: the erasured data is currently recomputed -func (b *Block) PutBlock(ctx context.Context, nodeAdder format.NodeAdder) error { - if nodeAdder == nil { - return errors.New("no ipfs node adder provided") - } - - // recompute the shares - namespacedShares, _ := b.Data.ComputeShares() - shares := namespacedShares.RawShares() - - // don't do anything if there is no data to put on IPFS - if len(shares) == 0 { - return nil - } - - // recompute the eds - eds, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree) - if err != nil { - return fmt.Errorf("failure to recompute the extended data square: %w", err) - } - - // add namespaces to erasured shares and flatten the eds - leaves := flattenNamespacedEDS(namespacedShares, eds) - - // create nmt adder wrapping batch adder - batchAdder := nodes.NewNmtNodeAdder(ctx, format.NewBatch(ctx, nodeAdder)) - - // iterate through each set of col and row leaves - for _, leafSet := range leaves { - tree := nmt.New(sha256.New(), nmt.NodeVisitor(batchAdder.Visit)) - for _, share := range leafSet { - err = tree.Push(share) - if err != nil { - return err - } - } - - // compute the root in order to collect the ipld.Nodes - tree.Root() - } - - // commit the batch to ipfs - return batchAdder.Commit() -} - -// flattenNamespacedEDS returns a flattend extendedDataSquare with namespaces +// TODO: Move out from 'types' package +// FlattenNamespacedEDS returns a flattend extendedDataSquare with namespaces // added to each share. NOTE: output is columns first then rows -func flattenNamespacedEDS(nss NamespacedShares, eds *rsmt2d.ExtendedDataSquare) [][][]byte { +func FlattenNamespacedEDS(nss NamespacedShares, eds *rsmt2d.ExtendedDataSquare) [][][]byte { squareWidth := eds.Width() originalDataWidth := squareWidth / 2 diff --git a/types/block_test.go b/types/block_test.go index 667e1c303a..6f000ef7a9 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -4,7 +4,6 @@ import ( // it is ok to use math/rand here: we do not need a cryptographically secure random // number generator here and we can run the tests a bit faster stdbytes "bytes" - "context" "encoding/hex" "math" mrand "math/rand" @@ -15,9 +14,6 @@ import ( "time" gogotypes "github.com/gogo/protobuf/types" - coreapi "github.com/ipfs/go-ipfs/core/coreapi" - coremock "github.com/ipfs/go-ipfs/core/mock" - "github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1336,68 +1332,6 @@ func TestCommit_ValidateBasic(t *testing.T) { } } -func TestPutBlock(t *testing.T) { - ipfsNode, err := coremock.NewMockNode() - if err != nil { - t.Error(err) - } - - ipfsAPI, err := coreapi.NewCoreAPI(ipfsNode) - if err != nil { - t.Error(err) - } - - maxOriginalSquareSize := MaxSquareSize / 2 - maxShareCount := maxOriginalSquareSize * maxOriginalSquareSize - - testCases := []struct { - name string - blockData Data - expectErr bool - errString string - }{ - {"no leaves", generateRandomMsgOnlyData(0), false, ""}, - {"single leaf", generateRandomMsgOnlyData(1), false, ""}, - {"16 leaves", generateRandomMsgOnlyData(16), false, ""}, - {"max square size", generateRandomMsgOnlyData(maxShareCount), false, ""}, - } - ctx := context.Background() - for _, tc := range testCases { - tc := tc - - block := &Block{Data: tc.blockData} - - t.Run(tc.name, func(t *testing.T) { - err = block.PutBlock(ctx, ipfsAPI.Dag()) - if tc.expectErr { - require.Error(t, err) - require.Contains(t, err.Error(), tc.errString) - return - } - - require.NoError(t, err) - - timeoutCtx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - - block.fillDataAvailabilityHeader() - for _, rowRoot := range block.DataAvailabilityHeader.RowsRoots.Bytes() { - // recreate the cids using only the computed roots - cid, err := nodes.CidFromNamespacedSha256(rowRoot) - if err != nil { - t.Error(err) - } - - // retrieve the data from IPFS - _, err = ipfsAPI.Dag().Get(timeoutCtx, cid) - if err != nil { - t.Errorf("Root not found: %s", cid.String()) - } - } - }) - } -} - func TestPaddedLength(t *testing.T) { type test struct { input, expected int @@ -1462,16 +1396,6 @@ func TestNextHighestPowerOf2(t *testing.T) { } } -func generateRandomMsgOnlyData(msgCount int) Data { - out := make([]Message, msgCount) - for i, msg := range generateRandNamespacedRawData(msgCount, NamespaceSize, MsgShareSize-2) { - out[i] = Message{NamespaceID: msg[:NamespaceSize], Data: msg[NamespaceSize:]} - } - return Data{ - Messages: Messages{MessagesList: out}, - } -} - // this code is copy pasted from the plugin, and should likely be exported in the plugin instead func generateRandNamespacedRawData(total int, nidSize int, leafSize int) [][]byte { data := make([][]byte, total)