Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

make distinction between pingers and publishers on config #23

Merged
merged 1 commit into from
Jun 8, 2023
Merged

make distinction between pingers and publishers on config #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 28 additions & 28 deletions cmd/run_cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,22 +61,22 @@ var RunCmd = &cli.Command{
DefaultText: "Undefined CIDs",
},
&cli.IntFlag{
Name: "workers",
Usage: "max number of CIDs publish and ping workers",
EnvVars: []string{"IPFS_CID_HOARDER_BATCH_SIZE"},
DefaultText: "1 worker",
Name: "publishers",
Usage: "number of concurrent CID publishers that will be spawned",
EnvVars: []string{"IPFS_CID_HOARDER_PUBLISHERS"},
DefaultText: "default: 1",
},
&cli.IntFlag{
Name: "pingers",
Usage: "number of concurrent pingers that will execute the ping tasks",
EnvVars: []string{"IPFS_CID_HOARDER_PINGERS"},
DefaultText: "default: 250",
},
&cli.IntFlag{
Name: "hosts",
Usage: "number of libp2p hosts that will be used by the ping workers",
EnvVars: []string{"IPFS_CID_HOARDER_HOSTS"},
DefaultText: "10 hosts",
},
&cli.BoolFlag{
Name: "single-publisher",
Usage: "defines whether the hoarderder uses only a single publisher worker (improves publish time accuracy)",
EnvVars: []string{"IPFS_CID_HOARDER_SINGLE_PUBLISHER"},
DefaultText: "false",
DefaultText: "default: 1",
},
&cli.StringFlag{
Name: "pub-interval",
Expand Down Expand Up @@ -141,23 +141,23 @@ func RunHoarder(ctx *cli.Context) error {

// Initialize the CidHoarder
log.WithFields(log.Fields{
"log-level": conf.LogLevel,
"port": conf.Port,
"metrics-ip": conf.MetricsIP,
"metrics-port": conf.MetricsPort,
"database": conf.Database,
"cid-size": conf.CidContentSize,
"cid-number": conf.CidNumber,
"workers": conf.Workers,
"hosts": conf.Hosts,
"single-publisher": conf.SinglePublisher,
"req-interval": conf.ReqInterval,
"pub-interval": conf.PubInterval,
"task-timeout": conf.TaskTimeout,
"cid-ping-time": conf.CidPingTime,
"k": conf.K,
"prov-op": conf.ProvideOperation,
"blacklisted-ua": conf.BlacklistedUA,
"log-level": conf.LogLevel,
"port": conf.Port,
"metrics-ip": conf.MetricsIP,
"metrics-port": conf.MetricsPort,
"database": conf.Database,
"cid-size": conf.CidContentSize,
"cid-number": conf.CidNumber,
"publishers": conf.Publishers,
"pingers": conf.Pingers,
"hosts": conf.Hosts,
"req-interval": conf.ReqInterval,
"pub-interval": conf.PubInterval,
"task-timeout": conf.TaskTimeout,
"cid-ping-time": conf.CidPingTime,
"k": conf.K,
"prov-op": conf.ProvideOperation,
"blacklisted-ua": conf.BlacklistedUA,
}).Info("running cid-hoarder")
cidHoarder, err := hoarder.NewCidHoarder(ctx.Context, conf)
if err != nil {
Expand Down
20 changes: 10 additions & 10 deletions pkg/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ var DefaultConfig = Config{
Database: "postgres://user:password@ip:port/db",
CidContentSize: 1024, // 1MB in KBs
CidNumber: 10,
Workers: 250,
Publishers: 1,
Pingers: 250,
Hosts: 10,
SinglePublisher: true,
PubInterval: "80s",
TaskTimeout: "80s",
ReqInterval: "30m",
Expand All @@ -45,9 +45,9 @@ type Config struct {
Database string `json:"database-endpoint"`
CidContentSize int `json:"cid-content-size"`
CidNumber int `json:"cid-number"`
Workers int `json:"workers"`
Publishers int `json:"publishers"`
Pingers int `json:"pingers"`
Hosts int `json:"hosts"`
SinglePublisher bool `json:"single-publisher"`
PubInterval string `json:"pub-interval"`
TaskTimeout string `json:"task-timeout"`
ReqInterval string `json:"req-interval"`
Expand Down Expand Up @@ -97,16 +97,16 @@ func (c *Config) Apply(ctx *cli.Context) {
c.CidNumber = ctx.Int("cid-number")
}

if ctx.IsSet("workers") {
c.Workers = ctx.Int("workers")
if ctx.IsSet("publishers") {
c.Publishers = ctx.Int("publishers")
}

if ctx.IsSet("hosts") {
c.Hosts = ctx.Int("hosts")
if ctx.IsSet("pingers") {
c.Pingers = ctx.Int("pingers")
}

if ctx.IsSet("single-publisher") {
c.SinglePublisher = ctx.Bool("single-publisher")
if ctx.IsSet("hosts") {
c.Hosts = ctx.Int("hosts")
}

if ctx.IsSet("pub-interval") {
Expand Down
8 changes: 2 additions & 6 deletions pkg/hoarder/hoarder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func NewCidHoarder(ctx context.Context, conf *config.Config) (*CidHoarder, error
dbInstance,
reqInterval,
taskTimeout,
conf.Workers,
conf.Pingers,
conf.Hosts,
cidSet)
if err != nil {
Expand All @@ -102,10 +102,6 @@ func NewCidHoarder(ctx context.Context, conf *config.Config) (*CidHoarder, error
// select the provide operation that we want to perform:
publisherHostOpts := hostOpts
publisherHostOpts.WithNotifier = true // the only time were want to have the notifier
pubWorkers := 1
if !conf.SinglePublisher {
pubWorkers = conf.Workers
}
cidPublisher, err := NewCidPublisher(
ctx,
&studyWG,
Expand All @@ -118,7 +114,7 @@ func NewCidHoarder(ctx context.Context, conf *config.Config) (*CidHoarder, error
),
cidSet,
conf.K,
pubWorkers,
conf.Publishers,
reqInterval,
pubInterval,
cidPingTime,
Expand Down