Add backoff option to pagination

readme.md

@@ -912,6 +912,13 @@ Default: `Infinity`
 
 The maximum amount of items that should be emitted.
 
+###### pagination.backoff
+
+Type: `number`\
+Default: `0`
+
+Milliseconds to wait before the next request is triggered.
+
 ###### pagination.requestLimit
 
 Type: `number`\

source/as-promise/types.ts

@@ -37,6 +37,7 @@ export interface PaginationOptions<T, R> {
 		paginate?: (response: Response<R>, allItems: T[], currentItems: T[]) => Options | false;
 		shouldContinue?: (item: T, allItems: T[], currentItems: T[]) => boolean;
 		countLimit?: number;
+		backoff?: number;
 		requestLimit?: number;
 		stackAllItems?: boolean;
 	};

source/create.ts

@@ -54,6 +54,9 @@ const errors = {
 	UploadError
 };
 
+// The `delay` package weighs 10KB (!)
+const delay = async (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+
 const {normalizeArguments, mergeOptions} = PromisableRequest;
 
 const getPromiseOrStream = (options: NormalizedOptions): GotReturn => options.isStream ? new Request(options.url, options) : asPromise(options);
@@ -214,6 +217,11 @@ const create = (defaults: InstanceDefaults): Got => {
 
 		let numberOfRequests = 0;
 		while (numberOfRequests < pagination.requestLimit) {
+			if (numberOfRequests !== 0) {
+				// eslint-disable-next-line no-await-in-loop
+				await delay(pagination.backoff);
+			}
+
 			// TODO: Throw when result is not an instance of Response
 			// eslint-disable-next-line no-await-in-loop
 			const result = (await got(normalizedOptions)) as Response;

source/index.ts

@@ -108,6 +108,7 @@ const defaults: InstanceDefaults = {
 			filter: () => true,
 			shouldContinue: () => true,
 			countLimit: Infinity,
+			backoff: 0,
 			requestLimit: 10000,
 			stackAllItems: true
 		},

test/pagination.ts

@@ -1,5 +1,6 @@
 import {URL} from 'url';
 import test from 'ava';
+import delay = require('delay');
 import getStream = require('get-stream');
 import got, {Response} from '../source';
 import withServer, {withBodyParsingServer} from './helpers/with-server';
@@ -449,6 +450,33 @@ test('`requestLimit` works', withServer, async (t, server, got) => {
 	t.deepEqual(results, [1]);
 });
 
+test('`backoff` works', withServer, async (t, server, got) => {
+	attachHandler(server, 2);
+
+	const backoff = 200;
+
+	const asyncIterator: AsyncIterator<number> = got.paginate<number>('', {
+		pagination: {
+			backoff
+		}
+	});
+
+	t.is((await asyncIterator.next()).value, 1);
+
+	let receivedLastOne = false;
+	const promise = asyncIterator.next();
+	(async () => {
+		await promise;
+		receivedLastOne = true;
+	})();
+
+	await delay(backoff / 2);
+	t.false(receivedLastOne);
+
+	await delay((backoff / 2) + 100);
+	t.true(receivedLastOne);
+});
+
 test('`stackAllItems` set to true', withServer, async (t, server, got) => {
 	attachHandler(server, 3);