diff --git a/readme.md b/readme.md index 8f8449e4b..ef97fe009 100644 --- a/readme.md +++ b/readme.md @@ -912,6 +912,13 @@ Default: `Infinity` The maximum amount of items that should be emitted. +###### pagination.backoff + +Type: `number`\ +Default: `0` + +Milliseconds to wait before the next request is triggered. + ###### pagination.requestLimit Type: `number`\ diff --git a/source/as-promise/types.ts b/source/as-promise/types.ts index 45255f3bc..c551ae989 100644 --- a/source/as-promise/types.ts +++ b/source/as-promise/types.ts @@ -37,6 +37,7 @@ export interface PaginationOptions { paginate?: (response: Response, allItems: T[], currentItems: T[]) => Options | false; shouldContinue?: (item: T, allItems: T[], currentItems: T[]) => boolean; countLimit?: number; + backoff?: number; requestLimit?: number; stackAllItems?: boolean; }; diff --git a/source/create.ts b/source/create.ts index a4a4b7d2e..a930dd230 100644 --- a/source/create.ts +++ b/source/create.ts @@ -54,6 +54,9 @@ const errors = { UploadError }; +// The `delay` package weighs 10KB (!) +const delay = async (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + const {normalizeArguments, mergeOptions} = PromisableRequest; const getPromiseOrStream = (options: NormalizedOptions): GotReturn => options.isStream ? new Request(options.url, options) : asPromise(options); @@ -214,6 +217,11 @@ const create = (defaults: InstanceDefaults): Got => { let numberOfRequests = 0; while (numberOfRequests < pagination.requestLimit) { + if (numberOfRequests !== 0) { + // eslint-disable-next-line no-await-in-loop + await delay(pagination.backoff); + } + // TODO: Throw when result is not an instance of Response // eslint-disable-next-line no-await-in-loop const result = (await got(normalizedOptions)) as Response; diff --git a/source/index.ts b/source/index.ts index 9f72d091a..47cdabc93 100644 --- a/source/index.ts +++ b/source/index.ts @@ -108,6 +108,7 @@ const defaults: InstanceDefaults = { filter: () => true, shouldContinue: () => true, countLimit: Infinity, + backoff: 0, requestLimit: 10000, stackAllItems: true }, diff --git a/test/pagination.ts b/test/pagination.ts index 0e208bc2c..0b2f96a16 100644 --- a/test/pagination.ts +++ b/test/pagination.ts @@ -1,5 +1,6 @@ import {URL} from 'url'; import test from 'ava'; +import delay = require('delay'); import getStream = require('get-stream'); import got, {Response} from '../source'; import withServer, {withBodyParsingServer} from './helpers/with-server'; @@ -449,6 +450,33 @@ test('`requestLimit` works', withServer, async (t, server, got) => { t.deepEqual(results, [1]); }); +test('`backoff` works', withServer, async (t, server, got) => { + attachHandler(server, 2); + + const backoff = 200; + + const asyncIterator: AsyncIterator = got.paginate('', { + pagination: { + backoff + } + }); + + t.is((await asyncIterator.next()).value, 1); + + let receivedLastOne = false; + const promise = asyncIterator.next(); + (async () => { + await promise; + receivedLastOne = true; + })(); + + await delay(backoff / 2); + t.false(receivedLastOne); + + await delay((backoff / 2) + 100); + t.true(receivedLastOne); +}); + test('`stackAllItems` set to true', withServer, async (t, server, got) => { attachHandler(server, 3);