Add max tweets number to configuration

SpecializedGeneralist · Mar 23, 2021 · e178d3b · e178d3b
1 parent 48bf1a7
commit e178d3b
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 4 deletions.
diff --git a/pkg/configuration/configuration.go b/pkg/configuration/configuration.go
@@ -68,6 +68,7 @@ type TweetsFetchingConfiguration struct {
 	SleepingTime                     time.Duration `yaml:"sleeping_time"`
 	OmitTweetsPublishedBeforeEnabled bool          `yaml:"omit_tweets_published_before_enabled"`
 	OmitTweetsPublishedBefore        time.Time     `yaml:"omit_tweets_published_before"`
+	MaxTweetsNumber                  int           `yaml:"max_tweets_number"`
 	NewWebResourceRoutingKey         string        `yaml:"new_web_resource_routing_key"`
 	NewTweetRoutingKey               string        `yaml:"new_tweet_routing_key"`
 	NewWebArticleRoutingKey          string        `yaml:"new_web_article_routing_key"`

diff --git a/pkg/configuration/configuration_test.go b/pkg/configuration/configuration_test.go
@@ -54,6 +54,7 @@ func TestFromYAMLFile(t *testing.T) {
 				SleepingTime:                     5 * time.Minute,
 				OmitTweetsPublishedBeforeEnabled: true,
 				OmitTweetsPublishedBefore:        omitPublishedBefore,
+				MaxTweetsNumber:                  3200,
 				NewWebResourceRoutingKey:         "",
 				NewTweetRoutingKey:               "new_tweet",
 				NewWebArticleRoutingKey:          "new_web_article",

diff --git a/pkg/tasks/tweetsfetching/worker.go b/pkg/tasks/tweetsfetching/worker.go
@@ -28,8 +28,6 @@ type Worker struct {
 	scraper *twitterscraper.Scraper
 }
 
-const maxTweetsNumber = 50
-
 func NewWorker(
 	config configuration.Configuration,
 	db *gorm.DB,
@@ -72,9 +70,9 @@ func (w *Worker) processTwitterSource(logger zerolog.Logger, ts *models.TwitterS
 
 	switch ts.Type {
 	case models.UserTwitterSource:
-		ch = w.scraper.GetTweets(context.Background(), ts.Value, maxTweetsNumber)
+		ch = w.scraper.GetTweets(context.Background(), ts.Value, w.config.TweetsFetching.MaxTweetsNumber)
 	case models.SearchTwitterSource:
-		ch = w.scraper.SearchTweets(context.Background(), ts.Value, maxTweetsNumber)
+		ch = w.scraper.SearchTweets(context.Background(), ts.Value, w.config.TweetsFetching.MaxTweetsNumber)
 	default:
 		return fmt.Errorf("unexpected twitter-source type %#v", ts.Type)
 	}

diff --git a/sample-configuration.yml b/sample-configuration.yml
@@ -22,6 +22,7 @@ tweets_fetching:
   sleeping_time: 5m
   omit_tweets_published_before_enabled: true
   omit_tweets_published_before: '2020-12-01T00:00:00Z'
+  max_tweets_number: 3200
   new_web_resource_routing_key: ''
   new_tweet_routing_key: 'new_tweet'
   new_web_article_routing_key: 'new_web_article'