Merge pull request #23 from cortze/feat/split-pub-workers

make distinction between pingers and publishers on config
cortze · Jun 8, 2023 · 421d504 · 421d504
2 parents d721098 + 79e07c2
commit 421d504
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 44 deletions.
diff --git a/cmd/run_cmd.go b/cmd/run_cmd.go
@@ -61,22 +61,22 @@ var RunCmd = &cli.Command{
 			DefaultText: "Undefined CIDs",
 		},
 		&cli.IntFlag{
-			Name:        "workers",
-			Usage:       "max number of CIDs publish and ping workers",
-			EnvVars:     []string{"IPFS_CID_HOARDER_BATCH_SIZE"},
-			DefaultText: "1 worker",
+			Name:        "publishers",
+			Usage:       "number of concurrent CID publishers that will be spawned",
+			EnvVars:     []string{"IPFS_CID_HOARDER_PUBLISHERS"},
+			DefaultText: "default: 1",
+		},
+		&cli.IntFlag{
+			Name:        "pingers",
+			Usage:       "number of concurrent pingers that will execute the ping tasks",
+			EnvVars:     []string{"IPFS_CID_HOARDER_PINGERS"},
+			DefaultText: "default: 250",
 		},
 		&cli.IntFlag{
 			Name:        "hosts",
 			Usage:       "number of libp2p hosts that will be used by the ping workers",
 			EnvVars:     []string{"IPFS_CID_HOARDER_HOSTS"},
-			DefaultText: "10 hosts",
-		},
-		&cli.BoolFlag{
-			Name:        "single-publisher",
-			Usage:       "defines whether the hoarderder uses only a single publisher worker (improves publish time accuracy)",
-			EnvVars:     []string{"IPFS_CID_HOARDER_SINGLE_PUBLISHER"},
-			DefaultText: "false",
+			DefaultText: "default: 1",
 		},
 		&cli.StringFlag{
 			Name:        "pub-interval",
@@ -141,23 +141,23 @@ func RunHoarder(ctx *cli.Context) error {
 
 	// Initialize the CidHoarder
 	log.WithFields(log.Fields{
-		"log-level":        conf.LogLevel,
-		"port":             conf.Port,
-		"metrics-ip":       conf.MetricsIP,
-		"metrics-port":     conf.MetricsPort,
-		"database":         conf.Database,
-		"cid-size":         conf.CidContentSize,
-		"cid-number":       conf.CidNumber,
-		"workers":          conf.Workers,
-		"hosts":            conf.Hosts,
-		"single-publisher": conf.SinglePublisher,
-		"req-interval":     conf.ReqInterval,
-		"pub-interval":     conf.PubInterval,
-		"task-timeout":     conf.TaskTimeout,
-		"cid-ping-time":    conf.CidPingTime,
-		"k":                conf.K,
-		"prov-op":          conf.ProvideOperation,
-		"blacklisted-ua":   conf.BlacklistedUA,
+		"log-level":      conf.LogLevel,
+		"port":           conf.Port,
+		"metrics-ip":     conf.MetricsIP,
+		"metrics-port":   conf.MetricsPort,
+		"database":       conf.Database,
+		"cid-size":       conf.CidContentSize,
+		"cid-number":     conf.CidNumber,
+		"publishers":     conf.Publishers,
+		"pingers":        conf.Pingers,
+		"hosts":          conf.Hosts,
+		"req-interval":   conf.ReqInterval,
+		"pub-interval":   conf.PubInterval,
+		"task-timeout":   conf.TaskTimeout,
+		"cid-ping-time":  conf.CidPingTime,
+		"k":              conf.K,
+		"prov-op":        conf.ProvideOperation,
+		"blacklisted-ua": conf.BlacklistedUA,
 	}).Info("running cid-hoarder")
 	cidHoarder, err := hoarder.NewCidHoarder(ctx.Context, conf)
 	if err != nil {

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -24,9 +24,9 @@ var DefaultConfig = Config{
 	Database:         "postgres://user:password@ip:port/db",
 	CidContentSize:   1024, // 1MB in KBs
 	CidNumber:        10,
-	Workers:          250,
+	Publishers:       1,
+	Pingers:          250,
 	Hosts:            10,
-	SinglePublisher:  true,
 	PubInterval:      "80s",
 	TaskTimeout:      "80s",
 	ReqInterval:      "30m",
@@ -45,9 +45,9 @@ type Config struct {
 	Database         string `json:"database-endpoint"`
 	CidContentSize   int    `json:"cid-content-size"`
 	CidNumber        int    `json:"cid-number"`
-	Workers          int    `json:"workers"`
+	Publishers       int    `json:"publishers"`
+	Pingers          int    `json:"pingers"`
 	Hosts            int    `json:"hosts"`
-	SinglePublisher  bool   `json:"single-publisher"`
 	PubInterval      string `json:"pub-interval"`
 	TaskTimeout      string `json:"task-timeout"`
 	ReqInterval      string `json:"req-interval"`
@@ -97,16 +97,16 @@ func (c *Config) Apply(ctx *cli.Context) {
 			c.CidNumber = ctx.Int("cid-number")
 		}
 
-		if ctx.IsSet("workers") {
-			c.Workers = ctx.Int("workers")
+		if ctx.IsSet("publishers") {
+			c.Publishers = ctx.Int("publishers")
 		}
 
-		if ctx.IsSet("hosts") {
-			c.Hosts = ctx.Int("hosts")
+		if ctx.IsSet("pingers") {
+			c.Pingers = ctx.Int("pingers")
 		}
 
-		if ctx.IsSet("single-publisher") {
-			c.SinglePublisher = ctx.Bool("single-publisher")
+		if ctx.IsSet("hosts") {
+			c.Hosts = ctx.Int("hosts")
 		}
 
 		if ctx.IsSet("pub-interval") {

diff --git a/pkg/hoarder/hoarder.go b/pkg/hoarder/hoarder.go
@@ -91,7 +91,7 @@ func NewCidHoarder(ctx context.Context, conf *config.Config) (*CidHoarder, error
 		dbInstance,
 		reqInterval,
 		taskTimeout,
-		conf.Workers,
+		conf.Pingers,
 		conf.Hosts,
 		cidSet)
 	if err != nil {
@@ -102,10 +102,6 @@ func NewCidHoarder(ctx context.Context, conf *config.Config) (*CidHoarder, error
 	// select the provide operation that we want to perform:
 	publisherHostOpts := hostOpts
 	publisherHostOpts.WithNotifier = true // the only time were want to have the notifier
-	pubWorkers := 1
-	if !conf.SinglePublisher {
-		pubWorkers = conf.Workers
-	}
 	cidPublisher, err := NewCidPublisher(
 		ctx,
 		&studyWG,
@@ -118,7 +114,7 @@ func NewCidHoarder(ctx context.Context, conf *config.Config) (*CidHoarder, error
 		),
 		cidSet,
 		conf.K,
-		pubWorkers,
+		conf.Publishers,
 		reqInterval,
 		pubInterval,
 		cidPingTime,