diff --git a/cmd/run_cmd.go b/cmd/run_cmd.go index ad81ff5..f7d32eb 100644 --- a/cmd/run_cmd.go +++ b/cmd/run_cmd.go @@ -61,22 +61,22 @@ var RunCmd = &cli.Command{ DefaultText: "Undefined CIDs", }, &cli.IntFlag{ - Name: "workers", - Usage: "max number of CIDs publish and ping workers", - EnvVars: []string{"IPFS_CID_HOARDER_BATCH_SIZE"}, - DefaultText: "1 worker", + Name: "publishers", + Usage: "number of concurrent CID publishers that will be spawned", + EnvVars: []string{"IPFS_CID_HOARDER_PUBLISHERS"}, + DefaultText: "default: 1", + }, + &cli.IntFlag{ + Name: "pingers", + Usage: "number of concurrent pingers that will execute the ping tasks", + EnvVars: []string{"IPFS_CID_HOARDER_PINGERS"}, + DefaultText: "default: 250", }, &cli.IntFlag{ Name: "hosts", Usage: "number of libp2p hosts that will be used by the ping workers", EnvVars: []string{"IPFS_CID_HOARDER_HOSTS"}, - DefaultText: "10 hosts", - }, - &cli.BoolFlag{ - Name: "single-publisher", - Usage: "defines whether the hoarderder uses only a single publisher worker (improves publish time accuracy)", - EnvVars: []string{"IPFS_CID_HOARDER_SINGLE_PUBLISHER"}, - DefaultText: "false", + DefaultText: "default: 1", }, &cli.StringFlag{ Name: "pub-interval", @@ -141,23 +141,23 @@ func RunHoarder(ctx *cli.Context) error { // Initialize the CidHoarder log.WithFields(log.Fields{ - "log-level": conf.LogLevel, - "port": conf.Port, - "metrics-ip": conf.MetricsIP, - "metrics-port": conf.MetricsPort, - "database": conf.Database, - "cid-size": conf.CidContentSize, - "cid-number": conf.CidNumber, - "workers": conf.Workers, - "hosts": conf.Hosts, - "single-publisher": conf.SinglePublisher, - "req-interval": conf.ReqInterval, - "pub-interval": conf.PubInterval, - "task-timeout": conf.TaskTimeout, - "cid-ping-time": conf.CidPingTime, - "k": conf.K, - "prov-op": conf.ProvideOperation, - "blacklisted-ua": conf.BlacklistedUA, + "log-level": conf.LogLevel, + "port": conf.Port, + "metrics-ip": conf.MetricsIP, + "metrics-port": conf.MetricsPort, + "database": conf.Database, + "cid-size": conf.CidContentSize, + "cid-number": conf.CidNumber, + "publishers": conf.Publishers, + "pingers": conf.Pingers, + "hosts": conf.Hosts, + "req-interval": conf.ReqInterval, + "pub-interval": conf.PubInterval, + "task-timeout": conf.TaskTimeout, + "cid-ping-time": conf.CidPingTime, + "k": conf.K, + "prov-op": conf.ProvideOperation, + "blacklisted-ua": conf.BlacklistedUA, }).Info("running cid-hoarder") cidHoarder, err := hoarder.NewCidHoarder(ctx.Context, conf) if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index f2efeb9..5de4039 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -24,9 +24,9 @@ var DefaultConfig = Config{ Database: "postgres://user:password@ip:port/db", CidContentSize: 1024, // 1MB in KBs CidNumber: 10, - Workers: 250, + Publishers: 1, + Pingers: 250, Hosts: 10, - SinglePublisher: true, PubInterval: "80s", TaskTimeout: "80s", ReqInterval: "30m", @@ -45,9 +45,9 @@ type Config struct { Database string `json:"database-endpoint"` CidContentSize int `json:"cid-content-size"` CidNumber int `json:"cid-number"` - Workers int `json:"workers"` + Publishers int `json:"publishers"` + Pingers int `json:"pingers"` Hosts int `json:"hosts"` - SinglePublisher bool `json:"single-publisher"` PubInterval string `json:"pub-interval"` TaskTimeout string `json:"task-timeout"` ReqInterval string `json:"req-interval"` @@ -97,16 +97,16 @@ func (c *Config) Apply(ctx *cli.Context) { c.CidNumber = ctx.Int("cid-number") } - if ctx.IsSet("workers") { - c.Workers = ctx.Int("workers") + if ctx.IsSet("publishers") { + c.Publishers = ctx.Int("publishers") } - if ctx.IsSet("hosts") { - c.Hosts = ctx.Int("hosts") + if ctx.IsSet("pingers") { + c.Pingers = ctx.Int("pingers") } - if ctx.IsSet("single-publisher") { - c.SinglePublisher = ctx.Bool("single-publisher") + if ctx.IsSet("hosts") { + c.Hosts = ctx.Int("hosts") } if ctx.IsSet("pub-interval") { diff --git a/pkg/hoarder/hoarder.go b/pkg/hoarder/hoarder.go index b25e01b..4ff8128 100644 --- a/pkg/hoarder/hoarder.go +++ b/pkg/hoarder/hoarder.go @@ -91,7 +91,7 @@ func NewCidHoarder(ctx context.Context, conf *config.Config) (*CidHoarder, error dbInstance, reqInterval, taskTimeout, - conf.Workers, + conf.Pingers, conf.Hosts, cidSet) if err != nil { @@ -102,10 +102,6 @@ func NewCidHoarder(ctx context.Context, conf *config.Config) (*CidHoarder, error // select the provide operation that we want to perform: publisherHostOpts := hostOpts publisherHostOpts.WithNotifier = true // the only time were want to have the notifier - pubWorkers := 1 - if !conf.SinglePublisher { - pubWorkers = conf.Workers - } cidPublisher, err := NewCidPublisher( ctx, &studyWG, @@ -118,7 +114,7 @@ func NewCidHoarder(ctx context.Context, conf *config.Config) (*CidHoarder, error ), cidSet, conf.K, - pubWorkers, + conf.Publishers, reqInterval, pubInterval, cidPingTime,