diff --git a/code/src/crawler.ts b/code/src/crawler.ts index 1bb385f..b59b001 100644 --- a/code/src/crawler.ts +++ b/code/src/crawler.ts @@ -15,6 +15,8 @@ export const createCrawler = async (config: Config) => { headless: true, }, }, + /** The default values scale up and down too quickly for larger runs, these values are half that */ + autoscaledPoolOptions: { scaleDownStepRatio: 0.025, scaleUpStepRatio: 0.025 }, retryOnBlocked: true, requestHandlerTimeoutSecs: 3 * 60, proxyConfiguration, diff --git a/shared/CHANGELOG.md b/shared/CHANGELOG.md index 09d846e..8aa4ae3 100644 --- a/shared/CHANGELOG.md +++ b/shared/CHANGELOG.md @@ -6,6 +6,9 @@ This changelog tracks updates to both GTP Scraper and Extended GPT Scraper actor - Set this model as the default one for the the *Pay Per Result* scraper with a set token limit. - With this, the maximum token limit for the *Pay Per Result* scraper was increased by 150%. +*Fixes* +- Fixed concurrency scaling issues that were causing the Actor to fail due to scaling too quickly. + # 2024-05-20 *Features* - Added support for GPT-4o model. (Extended GPT Scraper only)