From c183ba1ac0bc878ce4c99139a810e7f45cd7e85d Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Mon, 29 Aug 2022 18:50:49 +1000 Subject: [PATCH 01/32] fix: `utils.sleep` now returns `Promise` --- src/utils/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/utils.ts b/src/utils/utils.ts index f7c904194..65d3ee6e5 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -81,8 +81,8 @@ function pathIncludes(p1: string, p2: string): boolean { ); } -async function sleep(ms: number) { - return await new Promise((r) => setTimeout(r, ms)); +async function sleep(ms: number): Promise { + return await new Promise((r) => setTimeout(r, ms)); } function isEmptyObject(o) { From dfd9eb21bf36ed46051ca73a678786133061894c Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Wed, 31 Aug 2022 20:22:30 +1000 Subject: [PATCH 02/32] fix: `globalTeardown.ts` now forces the removal of `globalDataDir` --- tests/globalTeardown.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/globalTeardown.ts b/tests/globalTeardown.ts index c199c4d5b..0e3e5d30d 100644 --- a/tests/globalTeardown.ts +++ b/tests/globalTeardown.ts @@ -10,7 +10,7 @@ async function teardown() { console.log('GLOBAL TEARDOWN'); const globalDataDir = process.env['GLOBAL_DATA_DIR']!; console.log(`Destroying Global Data Dir: ${globalDataDir}`); - await fs.promises.rm(globalDataDir, { recursive: true }); + await fs.promises.rm(globalDataDir, { recursive: true, force: true }); } export default teardown; From dd88963f33e0b4bc79ff81e4272ae8bcca25311a Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Sat, 3 Sep 2022 17:02:47 +1000 Subject: [PATCH 03/32] npm: added `ts-node-inspect` in order to start the debugging port --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 29403fed3..6815c25ae 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "postbuild": "shx cp -fR src/proto dist && shx cp -f src/notifications/*.json dist/notifications/ && shx cp -f src/claims/*.json dist/claims/ && shx cp -f src/status/*.json dist/status/", "postversion": "npm install --package-lock-only --ignore-scripts --silent", "ts-node": "ts-node", + "ts-node-inspect": "node --require ts-node/register --inspect", "test": "jest", "lint": "eslint '{src,tests,scripts,benches}/**/*.{js,ts}'", "lintfix": "eslint '{src,tests,scripts,benches}/**/*.{js,ts}' --fix", From 6a16034133381682b2d6f3c3e59659a1c26c4fe2 Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Mon, 12 Sep 2022 00:47:45 +1000 Subject: [PATCH 04/32] build: target ES2022 --- tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index 2fffd2833..a12043658 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,7 +14,7 @@ "resolveJsonModule": true, "moduleResolution": "node", "module": "CommonJS", - "target": "ES2021", + "target": "ES2022", "baseUrl": "./src", "paths": { "@": ["index"], From 2ef9a8890432b58d049973b963d0110850cadf18 Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Sun, 11 Sep 2022 16:11:18 +1000 Subject: [PATCH 05/32] feat: introducing `Timer` as an object for tracking `setTimeout`, to be used for async deadlines --- src/timer/Timer.ts | 196 ++++++++++++++++++++++++++++++++++++++ src/timer/errors.ts | 10 ++ src/timer/index.ts | 2 + tests/timer/Timer.test.ts | 109 +++++++++++++++++++++ 4 files changed, 317 insertions(+) create mode 100644 src/timer/Timer.ts create mode 100644 src/timer/errors.ts create mode 100644 src/timer/index.ts create mode 100644 tests/timer/Timer.test.ts diff --git a/src/timer/Timer.ts b/src/timer/Timer.ts new file mode 100644 index 000000000..15287bd32 --- /dev/null +++ b/src/timer/Timer.ts @@ -0,0 +1,196 @@ +import { performance } from 'perf_hooks'; +import { CreateDestroy } from '@matrixai/async-init/dist/CreateDestroy'; +import * as timerErrors from './errors'; + +/** + * Unlike `setTimeout` or `setInterval`, + * this will not keep the NodeJS event loop alive + */ +interface Timer extends CreateDestroy {} +@CreateDestroy() +class Timer implements Promise { + public static createTimer({ + handler, + delay = 0, + }: { + handler?: () => T; + delay?: number; + } = {}): Timer { + return new this({ handler, delay }); + } + + /** + * Delay in milliseconds + * This may be `Infinity` + */ + public readonly delay: number; + + /** + * Timestamp when this is constructed + * Guaranteed to be weakly monotonic within the process lifetime + * Compare this with `performance.now()` not `Date.now()` + */ + public readonly timestamp: Date; + + /** + * Timestamp when this is scheduled to finish and execute the handler + * Guaranteed to be weakly monotonic within the process lifetime + * Compare this with `performance.now()` not `Date.now()` + */ + public readonly scheduled?: Date; + + /** + * Handler to be executed + */ + protected handler?: () => T; + + /** + * Deconstructed promise + */ + protected p: Promise; + + /** + * Resolve deconstructed promise + */ + protected resolveP: (value?: T) => void; + + /** + * Reject deconstructed promise + */ + protected rejectP: (reason?: timerErrors.ErrorTimer) => void; + + /** + * Internal timeout reference + */ + protected timeoutRef?: ReturnType; + + /** + * Whether the timer has timed out + * This is only `true` when the timer resolves + * If the timer rejects, this stays `false` + */ + protected _status: 'resolved' | 'rejected' | null = null; + + constructor({ + handler, + delay = 0, + }: { + handler?: () => T; + delay?: number; + } = {}) { + // Clip to delay >= 0 + delay = Math.max(delay, 0); + // Coerce NaN to minimal delay of 0 + if (isNaN(delay)) delay = 0; + this.handler = handler; + this.delay = delay; + this.p = new Promise((resolve, reject) => { + this.resolveP = resolve.bind(this.p); + this.rejectP = reject.bind(this.p); + }); + // If the delay is Infinity, there is no `setTimeout` + // therefore this promise will never resolve + // it may still reject however + if (isFinite(delay)) { + this.timeoutRef = setTimeout(() => void this.destroy('resolve'), delay); + if (typeof this.timeoutRef.unref === 'function') { + // Do not keep the event loop alive + this.timeoutRef.unref(); + } + this.timestamp = new Date(performance.timeOrigin + performance.now()); + this.scheduled = new Date(this.timestamp.getTime() + delay); + } else { + // There is no `setTimeout` nor `setInterval` + // so the event loop will not be kept alive + this.timestamp = new Date(performance.timeOrigin + performance.now()); + } + } + + public get [Symbol.toStringTag](): string { + return this.constructor.name; + } + + public get status(): 'resolved' | 'rejected' | null { + return this._status; + } + + public async destroy(type: 'resolve' | 'reject' = 'resolve'): Promise { + clearTimeout(this.timeoutRef); + delete this.timeoutRef; + if (type === 'resolve') { + this._status = 'resolved'; + if (this.handler != null) { + this.resolveP(this.handler()); + } else { + this.resolveP(); + } + } else if (type === 'reject') { + this._status = 'rejected'; + this.rejectP(new timerErrors.ErrorTimerCancelled()); + } + } + + /** + * Gets the remaining time in milliseconds + * This will return `Infinity` if `delay` is `Infinity` + * This will return `0` if status is `resolved` or `rejected` + */ + public getTimeout(): number { + if (this._status !== null) return 0; + if (this.scheduled == null) return Infinity; + return Math.max( + Math.trunc( + this.scheduled.getTime() - (performance.timeOrigin + performance.now()), + ), + 0, + ); + } + + /** + * To remaining time as a string + * This may return `'Infinity'` if `this.delay` is `Infinity` + */ + public toString(): string { + return this.getTimeout().toString(); + } + + /** + * To remaining time as a number + * This may return `Infinity` if `this.delay` is `Infinity` + */ + public valueOf(): number { + return this.getTimeout(); + } + + public then( + onFulfilled?: + | ((value: T) => TResult1 | PromiseLike) + | undefined + | null, + onRejected?: + | ((reason: any) => TResult2 | PromiseLike) + | undefined + | null, + ): Promise { + return this.p.then(onFulfilled, onRejected); + } + + public catch( + onRejected?: + | ((reason: any) => TResult | PromiseLike) + | undefined + | null, + ): Promise { + return this.p.catch(onRejected); + } + + public finally(onFinally?: (() => void) | undefined | null): Promise { + return this.p.finally(onFinally); + } + + public cancel() { + void this.destroy('reject'); + } +} + +export default Timer; diff --git a/src/timer/errors.ts b/src/timer/errors.ts new file mode 100644 index 000000000..b9767f636 --- /dev/null +++ b/src/timer/errors.ts @@ -0,0 +1,10 @@ +import { ErrorPolykey, sysexits } from '../errors'; + +class ErrorTimer extends ErrorPolykey {} + +class ErrorTimerCancelled extends ErrorTimer { + static description = 'Timer is cancelled'; + exitCode = sysexits.USAGE; +} + +export { ErrorTimer, ErrorTimerCancelled }; diff --git a/src/timer/index.ts b/src/timer/index.ts new file mode 100644 index 000000000..641d7a25d --- /dev/null +++ b/src/timer/index.ts @@ -0,0 +1,2 @@ +export { default as Timer } from './Timer'; +export * as errors from './errors'; diff --git a/tests/timer/Timer.test.ts b/tests/timer/Timer.test.ts new file mode 100644 index 000000000..be32b16c0 --- /dev/null +++ b/tests/timer/Timer.test.ts @@ -0,0 +1,109 @@ +import { performance } from 'perf_hooks'; +import { Timer } from '@/timer'; +import * as timerErrors from '@/timer/errors'; +import { sleep } from '@/utils'; + +describe(Timer.name, () => { + test('timer is thenable and awaitable', async () => { + const t1 = new Timer(); + expect(await t1).toBeUndefined(); + expect(t1.status).toBe('resolved'); + const t2 = new Timer(); + await expect(t2).resolves.toBeUndefined(); + expect(t2.status).toBe('resolved'); + }); + test('timer delays', async () => { + const t1 = new Timer({ delay: 20, handler: () => 1 }); + const t2 = new Timer({ delay: 10, handler: () => 2 }); + const result = await Promise.any([t1, t2]); + expect(result).toBe(2); + }); + test('timer handlers', async () => { + const t1 = new Timer({ handler: () => 123 }); + expect(await t1).toBe(123); + expect(t1.status).toBe('resolved'); + const t2 = new Timer({ delay: 100, handler: () => '123' }); + expect(await t2).toBe('123'); + expect(t2.status).toBe('resolved'); + }); + test('timer cancellation', async () => { + const t1 = new Timer({ delay: 100 }); + t1.cancel(); + await expect(t1).rejects.toThrow(timerErrors.ErrorTimerCancelled); + expect(t1.status).toBe('rejected'); + const t2 = new Timer({ delay: 100 }); + const results = await Promise.all([ + (async () => { + try { + await t2; + } catch (e) { + return e; + } + })(), + (async () => { + t2.cancel(); + })(), + ]); + expect(results[0]).toBeInstanceOf(timerErrors.ErrorTimerCancelled); + expect(t2.status).toBe('rejected'); + }); + test('timer timestamps', async () => { + const start = new Date(performance.timeOrigin + performance.now()); + await sleep(10); + const t = new Timer({ delay: 100 }); + expect(t.status).toBeNull(); + expect(t.timestamp).toBeAfter(start); + expect(t.scheduled).toBeAfter(start); + expect(t.scheduled).toBeAfterOrEqualTo(t.timestamp); + const delta = t.scheduled!.getTime() - t.timestamp.getTime(); + expect(t.getTimeout()).toBeLessThanOrEqual(delta); + }); + test('timer primitive string and number', () => { + const t1 = new Timer(); + expect(t1.valueOf()).toBe(0); + expect(+t1).toBe(0); + expect(t1.toString()).toBe('0'); + expect(`${t1}`).toBe('0'); + const t2 = new Timer({ delay: 100 }); + expect(t2.valueOf()).toBePositive(); + expect(+t2).toBePositive(); + expect(t2.toString()).toMatch(/\d+/); + expect(`${t2}`).toMatch(/\d+/); + }); + test('timer with infinite delay', async () => { + const t1 = new Timer({ delay: Infinity }); + expect(t1.delay).toBe(Infinity); + expect(t1.scheduled).toBeUndefined(); + expect(t1.getTimeout()).toBe(Infinity); + expect(t1.valueOf()).toBe(Infinity); + expect(+t1).toBe(Infinity); + expect(t1.toString()).toBe('Infinity'); + expect(`${t1}`).toBe('Infinity'); + t1.cancel(); + await expect(t1).rejects.toThrow(timerErrors.ErrorTimerCancelled); + }); + test('timer does not keep event loop alive', async () => { + const f = async (timer: Timer | number = globalThis.maxTimeout) => { + timer = timer instanceof Timer ? timer : new Timer({ delay: timer }); + }; + const g = async (timer: Timer | number = Infinity) => { + timer = timer instanceof Timer ? timer : new Timer({ delay: timer }); + }; + await f(); + await f(); + await f(); + await g(); + await g(); + await g(); + }); + test('timer lifecycle', async () => { + const t1 = Timer.createTimer({ delay: 1000 }); + await t1.destroy('resolve'); + expect(t1.status).toBe('resolved'); + await expect(t1).resolves.toBeUndefined(); + const t2 = Timer.createTimer({ delay: 1000 }); + await t2.destroy('reject'); + expect(t2.status).toBe('rejected'); + await expect(t2).rejects.toThrow(timerErrors.ErrorTimerCancelled); + }); +}); From 9cd5c259270230e738ce14beaac07b012873e54d Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Sun, 11 Sep 2022 16:13:13 +1000 Subject: [PATCH 06/32] feat: introducing `timed` and `cancellable` decorators to automate asynchronous deadlines and asynchronous cancellation --- package-lock.json | 25 +++- package.json | 3 +- src/contexts/decorators/cancellable.ts | 4 + src/contexts/decorators/context.ts | 18 +++ src/contexts/decorators/index.ts | 4 + src/contexts/decorators/timed.ts | 151 ++++++++++++++++++++++ src/contexts/decorators/transactional.ts | 0 src/contexts/errors.ts | 10 ++ src/contexts/index.ts | 4 + src/contexts/types.ts | 16 +++ src/contexts/utils.ts | 3 + src/utils/utils.ts | 7 + tests/contexts/decorators/context.test.ts | 27 ++++ tests/contexts/decorators/timed.test.ts | 127 ++++++++++++++++++ 14 files changed, 391 insertions(+), 8 deletions(-) create mode 100644 src/contexts/decorators/cancellable.ts create mode 100644 src/contexts/decorators/context.ts create mode 100644 src/contexts/decorators/index.ts create mode 100644 src/contexts/decorators/timed.ts create mode 100644 src/contexts/decorators/transactional.ts create mode 100644 src/contexts/errors.ts create mode 100644 src/contexts/index.ts create mode 100644 src/contexts/types.ts create mode 100644 src/contexts/utils.ts create mode 100644 tests/contexts/decorators/context.test.ts create mode 100644 tests/contexts/decorators/timed.test.ts diff --git a/package-lock.json b/package-lock.json index 835225da2..20e91d198 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,6 +38,7 @@ "pako": "^1.0.11", "prompts": "^2.4.1", "readable-stream": "^3.6.0", + "real-cancellable-promise": "^1.1.1", "resource-counter": "^1.2.4", "threads": "^1.6.5", "utp-native": "^2.5.3", @@ -54,7 +55,7 @@ "@types/google-protobuf": "^3.7.4", "@types/jest": "^28.1.3", "@types/nexpect": "^0.4.31", - "@types/node": "^16.11.7", + "@types/node": "^16.11.49", "@types/node-forge": "^0.10.4", "@types/pako": "^1.0.2", "@types/prompts": "^2.0.13", @@ -3027,9 +3028,9 @@ } }, "node_modules/@types/node": { - "version": "16.11.39", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.39.tgz", - "integrity": "sha512-K0MsdV42vPwm9L6UwhIxMAOmcvH/1OoVkZyCgEtVu4Wx7sElGloy/W7kMBNe/oJ7V/jW9BVt1F6RahH6e7tPXw==" + "version": "16.11.49", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.49.tgz", + "integrity": "sha512-Abq9fBviLV93OiXMu+f6r0elxCzRwc0RC5f99cU892uBITL44pTvgvEqlRlPRi8EGcO1z7Cp8A4d0s/p3J/+Nw==" }, "node_modules/@types/node-forge": { "version": "0.10.10", @@ -9913,6 +9914,11 @@ "node": ">= 6" } }, + "node_modules/real-cancellable-promise": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/real-cancellable-promise/-/real-cancellable-promise-1.1.1.tgz", + "integrity": "sha512-vxanUX4Aff5sRX6Rb1CSeCDWhO20L0hKQXWTLOYbtRo9WYFMjlhEBX0E75iz3+7ucrmFdPpDolwLC7L65P7hag==" + }, "node_modules/rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", @@ -13749,9 +13755,9 @@ } }, "@types/node": { - "version": "16.11.39", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.39.tgz", - "integrity": "sha512-K0MsdV42vPwm9L6UwhIxMAOmcvH/1OoVkZyCgEtVu4Wx7sElGloy/W7kMBNe/oJ7V/jW9BVt1F6RahH6e7tPXw==" + "version": "16.11.49", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.49.tgz", + "integrity": "sha512-Abq9fBviLV93OiXMu+f6r0elxCzRwc0RC5f99cU892uBITL44pTvgvEqlRlPRi8EGcO1z7Cp8A4d0s/p3J/+Nw==" }, "@types/node-forge": { "version": "0.10.10", @@ -18882,6 +18888,11 @@ "util-deprecate": "^1.0.1" } }, + "real-cancellable-promise": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/real-cancellable-promise/-/real-cancellable-promise-1.1.1.tgz", + "integrity": "sha512-vxanUX4Aff5sRX6Rb1CSeCDWhO20L0hKQXWTLOYbtRo9WYFMjlhEBX0E75iz3+7ucrmFdPpDolwLC7L65P7hag==" + }, "rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", diff --git a/package.json b/package.json index 6815c25ae..3fd582a48 100644 --- a/package.json +++ b/package.json @@ -106,6 +106,7 @@ "pako": "^1.0.11", "prompts": "^2.4.1", "readable-stream": "^3.6.0", + "real-cancellable-promise": "^1.1.1", "resource-counter": "^1.2.4", "threads": "^1.6.5", "utp-native": "^2.5.3", @@ -118,7 +119,7 @@ "@types/google-protobuf": "^3.7.4", "@types/jest": "^28.1.3", "@types/nexpect": "^0.4.31", - "@types/node": "^16.11.7", + "@types/node": "^16.11.49", "@types/node-forge": "^0.10.4", "@types/pako": "^1.0.2", "@types/prompts": "^2.0.13", diff --git a/src/contexts/decorators/cancellable.ts b/src/contexts/decorators/cancellable.ts new file mode 100644 index 000000000..25d6bfe46 --- /dev/null +++ b/src/contexts/decorators/cancellable.ts @@ -0,0 +1,4 @@ +// Let's attempt the cancellable one as well +// it requires the promise +// we can avoid needing to use this in EFS for now +// it's specific to PK diff --git a/src/contexts/decorators/context.ts b/src/contexts/decorators/context.ts new file mode 100644 index 000000000..1b6df8a0f --- /dev/null +++ b/src/contexts/decorators/context.ts @@ -0,0 +1,18 @@ +import * as contextsUtils from '../utils'; + +/** + * Context parameter decorator + * It is only allowed to be used once + */ +function context(target: Object, key: string | symbol, index: number) { + const targetName = target['name'] ?? target.constructor.name; + const method = target[key]; + if (contextsUtils.contexts.has(method)) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` redeclares \`@context\` decorator`, + ); + } + contextsUtils.contexts.set(method, index); +} + +export default context; diff --git a/src/contexts/decorators/index.ts b/src/contexts/decorators/index.ts new file mode 100644 index 000000000..6441c4b5f --- /dev/null +++ b/src/contexts/decorators/index.ts @@ -0,0 +1,4 @@ +export { default as context } from './context'; +// Export { default as cancellable }, * from './cancellable'; +export { default as timed } from './timed'; +// Export { default as transactional }, * from './transactional'; diff --git a/src/contexts/decorators/timed.ts b/src/contexts/decorators/timed.ts new file mode 100644 index 000000000..24c7895d5 --- /dev/null +++ b/src/contexts/decorators/timed.ts @@ -0,0 +1,151 @@ +import * as contextsUtils from '../utils'; +import * as contextsErrors from '../errors'; +import Timer from '../../timer/Timer'; +import * as timerErrors from '../../timer/errors'; +import { + AsyncFunction, + GeneratorFunction, + AsyncGeneratorFunction, +} from '../../utils'; + +/** + * Timed method decorator + */ +function timed(delay: number = Infinity) { + return ( + target: any, + key: string | symbol, + descriptor: TypedPropertyDescriptor<(...params: any[]) => any>, + ): TypedPropertyDescriptor<(...params: any[]) => any> => { + const targetName = target['name'] ?? target.constructor.name; + const f = descriptor['value']; + if (typeof f !== 'function') { + throw new TypeError( + `\`${targetName}.${key.toString()}\` is not a function`, + ); + } + const contextIndex = contextsUtils.contexts.get(target[key]); + if (contextIndex == null) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` does not have a \`@context\` parameter decorator`, + ); + } + const wrap = (that: any, params: Array) => { + const context = params[contextIndex]; + if ( + context !== undefined && + (typeof context !== 'object' || context === null) + ) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter is not a context object`, + ); + } + if (context?.timer !== undefined && !(context.timer instanceof Timer)) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`timer\` property is not an instance of \`Timer\``, + ); + } + if ( + context?.signal !== undefined && + !(context.signal instanceof AbortSignal) + ) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`signal\` property is not an instance of \`AbortSignal\``, + ); + } + // Now `context: { timer: Timer | undefined; signal: AbortSignal | undefined } | undefined` + if ( + context === undefined || + (context.timer === undefined && context.signal === undefined) + ) { + const abortController = new AbortController(); + const timer = new Timer({ + delay, + handler: () => + void abortController.abort( + new contextsErrors.ErrorContextsTimerExpired(), + ), + }); + params[contextIndex] = context !== undefined ? context : {}; + params[contextIndex].signal = abortController.signal; + params[contextIndex].timer = timer; + const result = f.apply(that, params); + timer.catch((e) => { + // Ignore cancellation + if (!(e instanceof timerErrors.ErrorTimerCancelled)) { + throw e; + } + }); + timer.cancel(); + return result; + } else if ( + context.timer === undefined && + context.signal instanceof AbortSignal + ) { + const abortController = new AbortController(); + const timer = new Timer({ + delay, + handler: () => + void abortController.abort( + new contextsErrors.ErrorContextsTimerExpired(), + ), + }); + context.signal.onabort = () => + void abortController.abort(context.signal.reason); + params[contextIndex].signal = abortController.signal; + params[contextIndex].timer = timer; + const result = f.apply(that, params); + timer.catch((e) => { + // Ignore cancellation + if (!(e instanceof timerErrors.ErrorTimerCancelled)) { + throw e; + } + }); + timer.cancel(); + return result; + } else if ( + context.timer instanceof Timer && + context.signal === undefined + ) { + const abortController = new AbortController(); + context.timer.then( + () => + void abortController.abort( + new contextsErrors.ErrorContextsTimerExpired(), + ), + ); + params[contextIndex].signal = abortController.signal; + return f.apply(that, params); + } else if ( + context.timer instanceof Timer && + context.signal instanceof AbortSignal + ) { + return f.apply(that, params); + } + }; + if (f instanceof AsyncFunction) { + descriptor['value'] = async function (...params) { + return wrap(this, params); + }; + } else if (f instanceof GeneratorFunction) { + descriptor['value'] = function* (...params) { + return yield* wrap(this, params); + }; + } else if (f instanceof AsyncGeneratorFunction) { + descriptor['value'] = async function* (...params) { + return yield* wrap(this, params); + }; + } else { + descriptor['value'] = function (...params) { + return wrap(this, params); + }; + } + // Preserve the name + Object.defineProperty(descriptor['value'], 'name', { + value: typeof key === 'symbol' ? `[${key.description}]` : key, + }); + return descriptor; + }; +} + +export default timed; diff --git a/src/contexts/decorators/transactional.ts b/src/contexts/decorators/transactional.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/contexts/errors.ts b/src/contexts/errors.ts new file mode 100644 index 000000000..0b06168e5 --- /dev/null +++ b/src/contexts/errors.ts @@ -0,0 +1,10 @@ +import { ErrorPolykey, sysexits } from '../errors'; + +class ErrorContexts extends ErrorPolykey {} + +class ErrorContextsTimerExpired extends ErrorContexts { + static description = 'Aborted due to timer expiration'; + exitCode = sysexits.UNAVAILABLE; +} + +export { ErrorContexts, ErrorContextsTimerExpired }; diff --git a/src/contexts/index.ts b/src/contexts/index.ts new file mode 100644 index 000000000..9432815a9 --- /dev/null +++ b/src/contexts/index.ts @@ -0,0 +1,4 @@ +export * from './decorators'; +export * from './utils'; +export * as types from './types'; +export * as errors from './errors'; diff --git a/src/contexts/types.ts b/src/contexts/types.ts new file mode 100644 index 000000000..0fe6bad2e --- /dev/null +++ b/src/contexts/types.ts @@ -0,0 +1,16 @@ +import type { DBTransaction } from '@matrixai/db'; +import type Timer from '../timer/Timer'; + +type ContextCancellable = { + signal: AbortSignal; +}; + +type ContextTimed = ContextCancellable & { + timer: Timer; +}; + +type ContextTransactional = { + tran: DBTransaction; +}; + +export type { ContextCancellable, ContextTimed, ContextTransactional }; diff --git a/src/contexts/utils.ts b/src/contexts/utils.ts new file mode 100644 index 000000000..d4f675f9c --- /dev/null +++ b/src/contexts/utils.ts @@ -0,0 +1,3 @@ +const contexts = new WeakMap(); + +export { contexts }; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 65d3ee6e5..2e31d7c6c 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -309,6 +309,10 @@ function debounce

( }; } +const AsyncFunction = (async () => {}).constructor; +const GeneratorFunction = function* () {}.constructor; +const AsyncGeneratorFunction = async function* () {}.constructor; + export { getDefaultNodePath, never, @@ -331,4 +335,7 @@ export { asyncIterableArray, bufferSplit, debounce, + AsyncFunction, + GeneratorFunction, + AsyncGeneratorFunction, }; diff --git a/tests/contexts/decorators/context.test.ts b/tests/contexts/decorators/context.test.ts new file mode 100644 index 000000000..09627a359 --- /dev/null +++ b/tests/contexts/decorators/context.test.ts @@ -0,0 +1,27 @@ +import context from '@/contexts/decorators/context'; +import * as contextsUtils from '@/contexts/utils'; + +describe('contexts/utils', () => { + test('context parameter decorator', () => { + class C { + f(@context _a: any) {} + g(_a: any, @context _b: any) {} + h(_a: any, _b: any, @context ..._rest: Array) {} + } + expect(contextsUtils.contexts.get(C.prototype.f)).toBe(0); + expect(contextsUtils.contexts.get(C.prototype.g)).toBe(1); + expect(contextsUtils.contexts.get(C.prototype.h)).toBe(2); + const c = new C(); + expect(contextsUtils.contexts.get(c.f)).toBe(0); + expect(contextsUtils.contexts.get(c.g)).toBe(1); + expect(contextsUtils.contexts.get(c.h)).toBe(2); + }); + test('context parameter decorator can only be used once', () => { + expect(() => { + class C { + f(@context _a: any, @context _b: any) {} + } + new C(); + }).toThrow(TypeError); + }); +}); diff --git a/tests/contexts/decorators/timed.test.ts b/tests/contexts/decorators/timed.test.ts new file mode 100644 index 000000000..c0a3bdca3 --- /dev/null +++ b/tests/contexts/decorators/timed.test.ts @@ -0,0 +1,127 @@ +import context from '@/contexts/decorators/context'; +import timed from '@/contexts/decorators/timed'; +import Timer from '@/timer/Timer'; +import { + AsyncFunction, + GeneratorFunction, + AsyncGeneratorFunction, +} from '@/utils'; + +describe('context/decorators/timed', () => { + test('timed decorator', async () => { + const s = Symbol('sym'); + class X { + a( + ctx?: { signal?: AbortSignal; timer?: Timer }, + check?: (t: Timer) => any, + ): void; + @timed(1000) + a( + @context ctx: { signal: AbortSignal; timer: Timer }, + check?: (t: Timer) => any, + ): void { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(ctx.timer).toBeInstanceOf(Timer); + if (check != null) check(ctx.timer); + } + + b( + ctx?: { signal?: AbortSignal; timer?: Timer }, + check?: (t: Timer) => any, + ): Promise; + @timed(Infinity) + async b( + @context ctx: { signal: AbortSignal; timer: Timer }, + check?: (t: Timer) => any, + ): Promise { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(ctx.timer).toBeInstanceOf(Timer); + if (check != null) check(ctx.timer); + } + + c( + ctx?: { signal?: AbortSignal; timer?: Timer }, + check?: (t: Timer) => any, + ): Generator; + @timed(0) + *c( + @context ctx: { signal: AbortSignal; timer: Timer }, + check?: (t: Timer) => any, + ): Generator { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(ctx.timer).toBeInstanceOf(Timer); + if (check != null) check(ctx.timer); + } + + d( + ctx?: { signal?: AbortSignal; timer?: Timer }, + check?: (t: Timer) => any, + ): AsyncGenerator; + @timed(NaN) + async *d( + @context ctx: { signal: AbortSignal; timer: Timer }, + check?: (t: Timer) => any, + ): AsyncGenerator { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(ctx.timer).toBeInstanceOf(Timer); + if (check != null) check(ctx.timer); + } + + [s]( + ctx?: { signal?: AbortSignal; timer?: Timer }, + check?: (t: Timer) => any, + ): void; + @timed() + [s]( + @context ctx: { signal: AbortSignal; timer: Timer }, + check?: (t: Timer) => any, + ): void { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(ctx.timer).toBeInstanceOf(Timer); + if (check != null) check(ctx.timer); + } + } + const x = new X(); + x.a(); + x.a({}); + x.a({ timer: new Timer({ delay: 100 }) }, (t) => { + expect(t.delay).toBe(100); + }); + expect(x.a).toBeInstanceOf(Function); + expect(x.a.name).toBe('a'); + await x.b(); + await x.b({}); + await x.b({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + }); + expect(x.b).toBeInstanceOf(AsyncFunction); + expect(x.b.name).toBe('b'); + for (const _ of x.c()) { + } + for (const _ of x.c({})) { + } + for (const _ of x.c({ timer: new Timer({ delay: 150 }) }, (t) => { + expect(t.delay).toBe(150); + })) { + } + expect(x.c).toBeInstanceOf(GeneratorFunction); + expect(x.c.name).toBe('c'); + for await (const _ of x.d()) { + } + for await (const _ of x.d({})) { + } + for await (const _ of x.d({ timer: new Timer({ delay: 200 }) }, (t) => { + expect(t.delay).toBe(200); + })) { + } + expect(x.d).toBeInstanceOf(AsyncGeneratorFunction); + expect(x.d.name).toBe('d'); + x[s](); + x[s]({}); + x[s]({ timer: new Timer({ delay: 250 }) }, (t) => { + expect(t.delay).toBe(250); + }); + expect(x[s]).toBeInstanceOf(Function); + expect(x[s].name).toBe('[sym]'); + }); +}); From fb5189b1a546317e71a28906373ab10421d1de1a Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Thu, 25 Aug 2022 18:53:26 +1000 Subject: [PATCH 07/32] fix: integrating `Timer` with `PromiseCancellable` from `@matrixai/async-cancellable` * added `isPromise`, `isPromiseLike`, `isIterable`, `isAsyncIterable` to detect async and generator interfaces * timed decorator works for regular values, `PromiseLike` and `Iterable` and `AsyncIterable` * introduced `ContextTimed` type and other `Context*` types * stack trace is refers when construction time, so decorator takes error class constructor --- package-lock.json | 22 +- package.json | 2 +- src/contexts/decorators/cancellable.ts | 109 ++- src/contexts/decorators/index.ts | 3 +- src/contexts/decorators/timed.ts | 312 +++++--- src/contexts/decorators/transactional.ts | 0 src/contexts/errors.ts | 4 +- src/timer/Timer.ts | 197 +++-- src/timer/errors.ts | 10 - src/timer/index.ts | 1 - src/utils/utils.ts | 37 +- tests/contexts/decorators/cancellable.test.ts | 395 ++++++++++ tests/contexts/decorators/timed.test.ts | 730 ++++++++++++++++-- tests/timer/Timer.test.ts | 212 ++++- 14 files changed, 1726 insertions(+), 308 deletions(-) delete mode 100644 src/contexts/decorators/transactional.ts delete mode 100644 src/timer/errors.ts create mode 100644 tests/contexts/decorators/cancellable.test.ts diff --git a/package-lock.json b/package-lock.json index 20e91d198..d7de58ddd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "GPL-3.0", "dependencies": { "@grpc/grpc-js": "1.6.7", + "@matrixai/async-cancellable": "^1.0.2", "@matrixai/async-init": "^1.8.2", "@matrixai/async-locks": "^3.1.2", "@matrixai/db": "^5.0.3", @@ -38,7 +39,6 @@ "pako": "^1.0.11", "prompts": "^2.4.1", "readable-stream": "^3.6.0", - "real-cancellable-promise": "^1.1.1", "resource-counter": "^1.2.4", "threads": "^1.6.5", "utp-native": "^2.5.3", @@ -2624,6 +2624,11 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@matrixai/async-cancellable": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@matrixai/async-cancellable/-/async-cancellable-1.0.2.tgz", + "integrity": "sha512-ugMfKtp7MlhXfBP//jGEAEEDbkVlr1aw8pqe2NrEUyyfKrZlX2jib50YocQYf+CcP4XnFAEzBDIpTAmqjukCug==" + }, "node_modules/@matrixai/async-init": { "version": "1.8.2", "resolved": "https://registry.npmjs.org/@matrixai/async-init/-/async-init-1.8.2.tgz", @@ -9914,11 +9919,6 @@ "node": ">= 6" } }, - "node_modules/real-cancellable-promise": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/real-cancellable-promise/-/real-cancellable-promise-1.1.1.tgz", - "integrity": "sha512-vxanUX4Aff5sRX6Rb1CSeCDWhO20L0hKQXWTLOYbtRo9WYFMjlhEBX0E75iz3+7ucrmFdPpDolwLC7L65P7hag==" - }, "node_modules/rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", @@ -13394,6 +13394,11 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "@matrixai/async-cancellable": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@matrixai/async-cancellable/-/async-cancellable-1.0.2.tgz", + "integrity": "sha512-ugMfKtp7MlhXfBP//jGEAEEDbkVlr1aw8pqe2NrEUyyfKrZlX2jib50YocQYf+CcP4XnFAEzBDIpTAmqjukCug==" + }, "@matrixai/async-init": { "version": "1.8.2", "resolved": "https://registry.npmjs.org/@matrixai/async-init/-/async-init-1.8.2.tgz", @@ -18888,11 +18893,6 @@ "util-deprecate": "^1.0.1" } }, - "real-cancellable-promise": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/real-cancellable-promise/-/real-cancellable-promise-1.1.1.tgz", - "integrity": "sha512-vxanUX4Aff5sRX6Rb1CSeCDWhO20L0hKQXWTLOYbtRo9WYFMjlhEBX0E75iz3+7ucrmFdPpDolwLC7L65P7hag==" - }, "rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", diff --git a/package.json b/package.json index 3fd582a48..ff66caae9 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ }, "dependencies": { "@grpc/grpc-js": "1.6.7", + "@matrixai/async-cancellable": "^1.0.2", "@matrixai/async-init": "^1.8.2", "@matrixai/async-locks": "^3.1.2", "@matrixai/db": "^5.0.3", @@ -106,7 +107,6 @@ "pako": "^1.0.11", "prompts": "^2.4.1", "readable-stream": "^3.6.0", - "real-cancellable-promise": "^1.1.1", "resource-counter": "^1.2.4", "threads": "^1.6.5", "utp-native": "^2.5.3", diff --git a/src/contexts/decorators/cancellable.ts b/src/contexts/decorators/cancellable.ts index 25d6bfe46..ae4301256 100644 --- a/src/contexts/decorators/cancellable.ts +++ b/src/contexts/decorators/cancellable.ts @@ -1,4 +1,105 @@ -// Let's attempt the cancellable one as well -// it requires the promise -// we can avoid needing to use this in EFS for now -// it's specific to PK +import type { ContextCancellable } from '../types'; +import { PromiseCancellable } from '@matrixai/async-cancellable'; +import * as contextsUtils from '../utils'; + +function cancellable(lazy: boolean = false) { + return < + T extends TypedPropertyDescriptor< + (...params: Array) => PromiseLike + >, + >( + target: any, + key: string | symbol, + descriptor: T, + ): T => { + // Target is instance prototype for instance methods // or the class prototype for static methods + const targetName = target['name'] ?? target.constructor.name; + const f = descriptor['value']; + if (typeof f !== 'function') { + throw new TypeError( + `\`${targetName}.${key.toString()}\` is not a function`, + ); + } + const contextIndex = contextsUtils.contexts.get(target[key]); + if (contextIndex == null) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` does not have a \`@context\` parameter decorator`, + ); + } + descriptor['value'] = function (...params) { + let context: Partial = params[contextIndex]; + if (context === undefined) { + context = {}; + params[contextIndex] = context; + } + // Runtime type check on the context parameter + if (typeof context !== 'object' || context === null) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter is not a context object`, + ); + } + if ( + context.signal !== undefined && + !(context.signal instanceof AbortSignal) + ) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`signal\` property is not an instance of \`AbortSignal\``, + ); + } + // Mutating the `context` parameter + if (context.signal === undefined) { + const abortController = new AbortController(); + context.signal = abortController.signal; + const result = f.apply(this, params); + return new PromiseCancellable((resolve, reject, signal) => { + if (!lazy) { + signal.addEventListener('abort', () => { + reject(signal.reason); + }); + } + void result.then(resolve, reject); + }, abortController); + } else { + // In this case, `context.signal` is set + // and we chain the upsteam signal to the downstream signal + const abortController = new AbortController(); + const signalUpstream = context.signal; + const signalHandler = () => { + abortController.abort(signalUpstream.reason); + }; + if (signalUpstream.aborted) { + abortController.abort(signalUpstream.reason); + } else { + signalUpstream.addEventListener('abort', signalHandler); + } + // Overwrite the signal property with this context's `AbortController.signal` + context.signal = abortController.signal; + const result = f.apply(this, params); + // The `abortController` must be shared in the `finally` clause + // to link up final promise's cancellation with the target + // function's signal + return new PromiseCancellable((resolve, reject, signal) => { + if (!lazy) { + if (signal.aborted) { + reject(signal.reason); + } else { + signal.addEventListener('abort', () => { + reject(signal.reason); + }); + } + } + void result.then(resolve, reject); + }, abortController).finally(() => { + signalUpstream.removeEventListener('abort', signalHandler); + }, abortController); + } + }; + // Preserve the name + Object.defineProperty(descriptor['value'], 'name', { + value: typeof key === 'symbol' ? `[${key.description}]` : key, + }); + return descriptor; + }; +} + +export default cancellable; diff --git a/src/contexts/decorators/index.ts b/src/contexts/decorators/index.ts index 6441c4b5f..ca5692398 100644 --- a/src/contexts/decorators/index.ts +++ b/src/contexts/decorators/index.ts @@ -1,4 +1,3 @@ export { default as context } from './context'; -// Export { default as cancellable }, * from './cancellable'; +export { default as cancellable } from './cancellable'; export { default as timed } from './timed'; -// Export { default as transactional }, * from './transactional'; diff --git a/src/contexts/decorators/timed.ts b/src/contexts/decorators/timed.ts index 24c7895d5..218087411 100644 --- a/src/contexts/decorators/timed.ts +++ b/src/contexts/decorators/timed.ts @@ -1,22 +1,136 @@ +import type { ContextTimed } from '../types'; import * as contextsUtils from '../utils'; import * as contextsErrors from '../errors'; import Timer from '../../timer/Timer'; -import * as timerErrors from '../../timer/errors'; -import { - AsyncFunction, - GeneratorFunction, - AsyncGeneratorFunction, -} from '../../utils'; +import * as utils from '../../utils'; + +/** + * This sets up the context + * This will mutate the `params` parameter + * It returns a teardown function to be called + * when the target function is finished + */ +function setupContext( + delay: number, + errorTimeoutConstructor: new () => Error, + targetName: string, + key: string | symbol, + contextIndex: number, + params: Array, +): () => void { + let context: Partial = params[contextIndex]; + if (context === undefined) { + context = {}; + params[contextIndex] = context; + } + // Runtime type check on the context parameter + if (typeof context !== 'object' || context === null) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter is not a context object`, + ); + } + if (context.timer !== undefined && !(context.timer instanceof Timer)) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`timer\` property is not an instance of \`Timer\``, + ); + } + if ( + context.signal !== undefined && + !(context.signal instanceof AbortSignal) + ) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`signal\` property is not an instance of \`AbortSignal\``, + ); + } + // Mutating the `context` parameter + if (context.timer === undefined && context.signal === undefined) { + const abortController = new AbortController(); + const e = new errorTimeoutConstructor(); + const timer = new Timer(() => void abortController.abort(e), delay); + context.signal = abortController.signal; + context.timer = timer; + return () => { + timer.cancel(); + }; + } else if ( + context.timer === undefined && + context.signal instanceof AbortSignal + ) { + const abortController = new AbortController(); + const e = new errorTimeoutConstructor(); + const timer = new Timer(() => void abortController.abort(e), delay); + const signalUpstream = context.signal; + const signalHandler = () => { + timer.cancel(); + abortController.abort(signalUpstream.reason); + }; + // If already aborted, abort target and cancel the timer + if (signalUpstream.aborted) { + timer.cancel(); + abortController.abort(signalUpstream.reason); + } else { + signalUpstream.addEventListener('abort', signalHandler); + } + // Overwrite the signal property with this context's `AbortController.signal` + context.signal = abortController.signal; + context.timer = timer; + return () => { + signalUpstream.removeEventListener('abort', signalHandler); + timer.cancel(); + }; + } else if (context.timer instanceof Timer && context.signal === undefined) { + const abortController = new AbortController(); + const e = new errorTimeoutConstructor(); + let finished = false; + // If the timer resolves, then abort the target function + void context.timer.then( + (r: any, s: AbortSignal) => { + // If the timer is aborted after it resolves + // then don't bother aborting the target function + if (!finished && !s.aborted) { + abortController.abort(e); + } + return r; + }, + () => { + // Ignore any upstream cancellation + }, + ); + context.signal = abortController.signal; + return () => { + finished = true; + }; + } else { + // In this case, `context.timer` and `context.signal` are both instances of + // `Timer` and `AbortSignal` respectively + const signalHandler = () => { + context.timer!.cancel(); + }; + if (context.signal!.aborted) { + context.timer!.cancel(); + } else { + context.signal!.addEventListener('abort', signalHandler); + } + return () => { + context.signal!.removeEventListener('abort', signalHandler); + }; + } +} /** * Timed method decorator */ -function timed(delay: number = Infinity) { +function timed( + delay: number = Infinity, + errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedExpiry, +) { return ( target: any, key: string | symbol, - descriptor: TypedPropertyDescriptor<(...params: any[]) => any>, - ): TypedPropertyDescriptor<(...params: any[]) => any> => { + descriptor: TypedPropertyDescriptor<(...params: Array) => any>, + ) => { + // Target is instance prototype for instance methods + // or the class prototype for static methods const targetName = target['name'] ?? target.constructor.name; const f = descriptor['value']; if (typeof f !== 'function') { @@ -30,114 +144,96 @@ function timed(delay: number = Infinity) { `\`${targetName}.${key.toString()}\` does not have a \`@context\` parameter decorator`, ); } - const wrap = (that: any, params: Array) => { - const context = params[contextIndex]; - if ( - context !== undefined && - (typeof context !== 'object' || context === null) - ) { - throw new TypeError( - `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter is not a context object`, - ); - } - if (context?.timer !== undefined && !(context.timer instanceof Timer)) { - throw new TypeError( - `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`timer\` property is not an instance of \`Timer\``, - ); - } - if ( - context?.signal !== undefined && - !(context.signal instanceof AbortSignal) - ) { - throw new TypeError( - `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`signal\` property is not an instance of \`AbortSignal\``, - ); - } - // Now `context: { timer: Timer | undefined; signal: AbortSignal | undefined } | undefined` - if ( - context === undefined || - (context.timer === undefined && context.signal === undefined) - ) { - const abortController = new AbortController(); - const timer = new Timer({ - delay, - handler: () => - void abortController.abort( - new contextsErrors.ErrorContextsTimerExpired(), - ), - }); - params[contextIndex] = context !== undefined ? context : {}; - params[contextIndex].signal = abortController.signal; - params[contextIndex].timer = timer; - const result = f.apply(that, params); - timer.catch((e) => { - // Ignore cancellation - if (!(e instanceof timerErrors.ErrorTimerCancelled)) { - throw e; - } - }); - timer.cancel(); - return result; - } else if ( - context.timer === undefined && - context.signal instanceof AbortSignal - ) { - const abortController = new AbortController(); - const timer = new Timer({ + if (f instanceof utils.AsyncFunction) { + descriptor['value'] = async function (...params) { + const teardownContext = setupContext( delay, - handler: () => - void abortController.abort( - new contextsErrors.ErrorContextsTimerExpired(), - ), - }); - context.signal.onabort = () => - void abortController.abort(context.signal.reason); - params[contextIndex].signal = abortController.signal; - params[contextIndex].timer = timer; - const result = f.apply(that, params); - timer.catch((e) => { - // Ignore cancellation - if (!(e instanceof timerErrors.ErrorTimerCancelled)) { - throw e; - } - }); - timer.cancel(); - return result; - } else if ( - context.timer instanceof Timer && - context.signal === undefined - ) { - const abortController = new AbortController(); - context.timer.then( - () => - void abortController.abort( - new contextsErrors.ErrorContextsTimerExpired(), - ), + errorTimeoutConstructor, + targetName, + key, + contextIndex, + params, ); - params[contextIndex].signal = abortController.signal; - return f.apply(that, params); - } else if ( - context.timer instanceof Timer && - context.signal instanceof AbortSignal - ) { - return f.apply(that, params); - } - }; - if (f instanceof AsyncFunction) { - descriptor['value'] = async function (...params) { - return wrap(this, params); + try { + return await f.apply(this, params); + } finally { + teardownContext(); + } }; - } else if (f instanceof GeneratorFunction) { + } else if (f instanceof utils.GeneratorFunction) { descriptor['value'] = function* (...params) { - return yield* wrap(this, params); + const teardownContext = setupContext( + delay, + errorTimeoutConstructor, + targetName, + key, + contextIndex, + params, + ); + try { + return yield* f.apply(this, params); + } finally { + teardownContext(); + } }; - } else if (f instanceof AsyncGeneratorFunction) { + } else if (f instanceof utils.AsyncGeneratorFunction) { descriptor['value'] = async function* (...params) { - return yield* wrap(this, params); + const teardownContext = setupContext( + delay, + errorTimeoutConstructor, + targetName, + key, + contextIndex, + params, + ); + try { + return yield* f.apply(this, params); + } finally { + teardownContext(); + } }; } else { descriptor['value'] = function (...params) { - return wrap(this, params); + const teardownContext = setupContext( + delay, + errorTimeoutConstructor, + targetName, + key, + contextIndex, + params, + ); + const result = f.apply(this, params); + if (utils.isPromiseLike(result)) { + return result.then( + (r) => { + teardownContext(); + return r; + }, + (e) => { + teardownContext(); + throw e; + }, + ); + } else if (utils.isIterable(result)) { + return (function* () { + try { + return yield* result; + } finally { + teardownContext(); + } + })(); + } else if (utils.isAsyncIterable(result)) { + return (async function* () { + try { + return yield* result; + } finally { + teardownContext(); + } + })(); + } else { + teardownContext(); + return result; + } }; } // Preserve the name diff --git a/src/contexts/decorators/transactional.ts b/src/contexts/decorators/transactional.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/contexts/errors.ts b/src/contexts/errors.ts index 0b06168e5..0c29aa149 100644 --- a/src/contexts/errors.ts +++ b/src/contexts/errors.ts @@ -2,9 +2,9 @@ import { ErrorPolykey, sysexits } from '../errors'; class ErrorContexts extends ErrorPolykey {} -class ErrorContextsTimerExpired extends ErrorContexts { +class ErrorContextsTimedExpiry extends ErrorContexts { static description = 'Aborted due to timer expiration'; exitCode = sysexits.UNAVAILABLE; } -export { ErrorContexts, ErrorContextsTimerExpired }; +export { ErrorContexts, ErrorContextsTimedExpiry }; diff --git a/src/timer/Timer.ts b/src/timer/Timer.ts index 15287bd32..c9068004b 100644 --- a/src/timer/Timer.ts +++ b/src/timer/Timer.ts @@ -1,30 +1,26 @@ +import type { PromiseCancellableController } from '@matrixai/async-cancellable'; import { performance } from 'perf_hooks'; -import { CreateDestroy } from '@matrixai/async-init/dist/CreateDestroy'; -import * as timerErrors from './errors'; +import { PromiseCancellable } from '@matrixai/async-cancellable'; /** * Unlike `setTimeout` or `setInterval`, * this will not keep the NodeJS event loop alive */ -interface Timer extends CreateDestroy {} -@CreateDestroy() -class Timer implements Promise { - public static createTimer({ - handler, - delay = 0, - }: { - handler?: () => T; - delay?: number; - } = {}): Timer { - return new this({ handler, delay }); - } - +class Timer + implements Pick, keyof PromiseCancellable> +{ /** * Delay in milliseconds * This may be `Infinity` */ public readonly delay: number; + /** + * If it is lazy, the timer will not eagerly reject + * on cancellation if the handler has started executing + */ + public readonly lazy: boolean; + /** * Timestamp when this is constructed * Guaranteed to be weakly monotonic within the process lifetime @@ -42,12 +38,12 @@ class Timer implements Promise { /** * Handler to be executed */ - protected handler?: () => T; + protected handler?: (signal: AbortSignal) => T | PromiseLike; /** * Deconstructed promise */ - protected p: Promise; + protected p: PromiseCancellable; /** * Resolve deconstructed promise @@ -57,7 +53,12 @@ class Timer implements Promise { /** * Reject deconstructed promise */ - protected rejectP: (reason?: timerErrors.ErrorTimer) => void; + protected rejectP: (reason?: any) => void; + + /** + * Abort controller allows immediate cancellation + */ + protected abortController: AbortController; /** * Internal timeout reference @@ -65,34 +66,82 @@ class Timer implements Promise { protected timeoutRef?: ReturnType; /** - * Whether the timer has timed out - * This is only `true` when the timer resolves - * If the timer rejects, this stays `false` + * The status indicates when we have started settling or settled */ - protected _status: 'resolved' | 'rejected' | null = null; + protected _status: 'settling' | 'settled' | null = null; - constructor({ - handler, - delay = 0, - }: { - handler?: () => T; + /** + * Construct a Timer + * By default `lazy` is false, which means it will eagerly reject + * the timer, even if the handler has already started executing + * If `lazy` is true, this will make the timer wait for the handler + * to finish executing + * Note that passing a custom controller does not stop the default behaviour + */ + constructor( + handler?: (signal: AbortSignal) => T | PromiseLike, + delay?: number, + lazy?: boolean, + controller?: PromiseCancellableController, + ); + constructor(opts?: { + handler?: (signal: AbortSignal) => T | PromiseLike; delay?: number; - } = {}) { + lazy?: boolean; + controller?: PromiseCancellableController; + }); + constructor( + handlerOrOpts?: + | ((signal: AbortSignal) => T | PromiseLike) + | { + handler?: (signal: AbortSignal) => T | PromiseLike; + delay?: number; + lazy?: boolean; + controller?: PromiseCancellableController; + }, + delay: number = 0, + lazy: boolean = false, + controller?: PromiseCancellableController, + ) { + let handler: ((signal: AbortSignal) => T | PromiseLike) | undefined; + if (typeof handlerOrOpts === 'function') { + handler = handlerOrOpts; + } else if (typeof handlerOrOpts === 'object' && handlerOrOpts !== null) { + handler = handlerOrOpts.handler; + delay = handlerOrOpts.delay ?? delay; + lazy = handlerOrOpts.lazy ?? lazy; + controller = handlerOrOpts.controller ?? controller; + } // Clip to delay >= 0 delay = Math.max(delay, 0); // Coerce NaN to minimal delay of 0 if (isNaN(delay)) delay = 0; this.handler = handler; this.delay = delay; - this.p = new Promise((resolve, reject) => { + this.lazy = lazy; + let abortController: AbortController; + if (typeof controller === 'function') { + abortController = new AbortController(); + controller(abortController.signal); + } else if (controller != null) { + abortController = controller; + } else { + abortController = new AbortController(); + abortController.signal.addEventListener( + 'abort', + () => void this.reject(abortController.signal.reason), + ); + } + this.p = new PromiseCancellable((resolve, reject) => { this.resolveP = resolve.bind(this.p); this.rejectP = reject.bind(this.p); - }); + }, abortController); + this.abortController = abortController; // If the delay is Infinity, there is no `setTimeout` // therefore this promise will never resolve // it may still reject however if (isFinite(delay)) { - this.timeoutRef = setTimeout(() => void this.destroy('resolve'), delay); + this.timeoutRef = setTimeout(() => void this.fulfill(), delay); if (typeof this.timeoutRef.unref === 'function') { // Do not keep the event loop alive this.timeoutRef.unref(); @@ -110,30 +159,14 @@ class Timer implements Promise { return this.constructor.name; } - public get status(): 'resolved' | 'rejected' | null { + public get status(): 'settling' | 'settled' | null { return this._status; } - public async destroy(type: 'resolve' | 'reject' = 'resolve'): Promise { - clearTimeout(this.timeoutRef); - delete this.timeoutRef; - if (type === 'resolve') { - this._status = 'resolved'; - if (this.handler != null) { - this.resolveP(this.handler()); - } else { - this.resolveP(); - } - } else if (type === 'reject') { - this._status = 'rejected'; - this.rejectP(new timerErrors.ErrorTimerCancelled()); - } - } - /** * Gets the remaining time in milliseconds * This will return `Infinity` if `delay` is `Infinity` - * This will return `0` if status is `resolved` or `rejected` + * This will return `0` if status is `settling` or `settled` */ public getTimeout(): number { if (this._status !== null) return 0; @@ -149,6 +182,7 @@ class Timer implements Promise { /** * To remaining time as a string * This may return `'Infinity'` if `this.delay` is `Infinity` + * This will return `'0'` if status is `settling` or `settled` */ public toString(): string { return this.getTimeout().toString(); @@ -157,39 +191,82 @@ class Timer implements Promise { /** * To remaining time as a number * This may return `Infinity` if `this.delay` is `Infinity` + * This will return `0` if status is `settling` or `settled` */ public valueOf(): number { return this.getTimeout(); } + /** + * Cancels the timer + * Unlike `PromiseCancellable`, canceling the timer will not result + * in an unhandled promise rejection, all promise rejections are ignored + */ + public cancel(reason?: any): void { + void this.p.catch(() => {}); + this.p.cancel(reason); + } + public then( onFulfilled?: - | ((value: T) => TResult1 | PromiseLike) + | ((value: T, signal: AbortSignal) => TResult1 | PromiseLike) | undefined | null, onRejected?: - | ((reason: any) => TResult2 | PromiseLike) + | ((reason: any, signal: AbortSignal) => TResult2 | PromiseLike) | undefined | null, - ): Promise { - return this.p.then(onFulfilled, onRejected); + controller?: PromiseCancellableController, + ): PromiseCancellable { + return this.p.then(onFulfilled, onRejected, controller); } public catch( onRejected?: - | ((reason: any) => TResult | PromiseLike) + | ((reason: any, signal: AbortSignal) => TResult | PromiseLike) | undefined | null, - ): Promise { - return this.p.catch(onRejected); + controller?: PromiseCancellableController, + ): PromiseCancellable { + return this.p.catch(onRejected, controller); } - public finally(onFinally?: (() => void) | undefined | null): Promise { - return this.p.finally(onFinally); + public finally( + onFinally?: ((signal: AbortSignal) => void) | undefined | null, + controller?: PromiseCancellableController, + ): PromiseCancellable { + return this.p.finally(onFinally, controller); } - public cancel() { - void this.destroy('reject'); + protected async fulfill(): Promise { + this._status = 'settling'; + clearTimeout(this.timeoutRef); + delete this.timeoutRef; + if (this.handler != null) { + try { + const result = await this.handler(this.abortController.signal); + this.resolveP(result); + } catch (e) { + this.rejectP(e); + } + } else { + this.resolveP(); + } + this._status = 'settled'; + } + + protected async reject(reason?: any): Promise { + if ( + (this.lazy && (this._status == null || this._status === 'settling')) || + this._status === 'settled' + ) { + return; + } + this._status = 'settling'; + clearTimeout(this.timeoutRef); + delete this.timeoutRef; + this.rejectP(reason); + this._status = 'settled'; } } diff --git a/src/timer/errors.ts b/src/timer/errors.ts deleted file mode 100644 index b9767f636..000000000 --- a/src/timer/errors.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { ErrorPolykey, sysexits } from '../errors'; - -class ErrorTimer extends ErrorPolykey {} - -class ErrorTimerCancelled extends ErrorTimer { - static description = 'Timer is cancelled'; - exitCode = sysexits.USAGE; -} - -export { ErrorTimer, ErrorTimerCancelled }; diff --git a/src/timer/index.ts b/src/timer/index.ts index 641d7a25d..ed32c1af2 100644 --- a/src/timer/index.ts +++ b/src/timer/index.ts @@ -1,2 +1 @@ export { default as Timer } from './Timer'; -export * as errors from './errors'; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 2e31d7c6c..615dc15b4 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -9,6 +9,10 @@ import process from 'process'; import path from 'path'; import * as utilsErrors from './errors'; +const AsyncFunction = (async () => {}).constructor; +const GeneratorFunction = function* () {}.constructor; +const AsyncGeneratorFunction = async function* () {}.constructor; + function getDefaultNodePath(): string | undefined { const prefix = 'polykey'; const platform = os.platform(); @@ -309,11 +313,31 @@ function debounce

( }; } -const AsyncFunction = (async () => {}).constructor; -const GeneratorFunction = function* () {}.constructor; -const AsyncGeneratorFunction = async function* () {}.constructor; +function isPromise(v: any): v is Promise { + return v instanceof Promise || ( + v != null + && typeof v.then === 'function' + && typeof v.catch === 'function' + && typeof v.finally === 'function' + ); +} + +function isPromiseLike(v: any): v is PromiseLike { + return v != null && typeof v.then === 'function'; +} + +function isIterable(v: any): v is Iterable { + return v != null && typeof v[Symbol.iterator] === 'function'; +} + +function isAsyncIterable(v: any): v is AsyncIterable { + return v != null && typeof v[Symbol.asyncIterator] === 'function'; +} export { + AsyncFunction, + GeneratorFunction, + AsyncGeneratorFunction, getDefaultNodePath, never, mkdirExists, @@ -335,7 +359,8 @@ export { asyncIterableArray, bufferSplit, debounce, - AsyncFunction, - GeneratorFunction, - AsyncGeneratorFunction, + isPromise, + isPromiseLike, + isIterable, + isAsyncIterable, }; diff --git a/tests/contexts/decorators/cancellable.test.ts b/tests/contexts/decorators/cancellable.test.ts new file mode 100644 index 000000000..7c03304f7 --- /dev/null +++ b/tests/contexts/decorators/cancellable.test.ts @@ -0,0 +1,395 @@ +import type { ContextCancellable, ContextTransactional } from '@/contexts/types'; +import { PromiseCancellable } from '@matrixai/async-cancellable'; +import context from '@/contexts/decorators/context'; +import cancellable from '@/contexts/decorators/cancellable'; +import { AsyncFunction, sleep } from '@/utils'; + +describe('context/decorators/cancellable', () => { + describe('cancellable decorator runtime validation', () => { + test('cancellable decorator requires context decorator', async () => { + expect(() => { + class C { + @cancellable() + async f(_ctx: ContextCancellable): Promise { + return 'hello world'; + } + } + return C; + }).toThrow(TypeError); + }); + test('cancellable decorator fails on invalid context', async () => { + await expect(async () => { + class C { + @cancellable() + async f(@context _ctx: ContextCancellable): Promise { + return 'hello world'; + } + } + const c = new C(); + // @ts-ignore invalid context signal + await c.f({ signal: 'lol' }); + }).rejects.toThrow(TypeError); + }); + }); + describe('cancellable decorator syntax', () => { + // Decorators cannot change type signatures + // use overloading to change required context parameter to optional context parameter + const symbolFunction = Symbol('sym'); + class X { + functionPromise( + ctx?: Partial, + ): PromiseCancellable; + @cancellable() + functionPromise(@context ctx: ContextCancellable): Promise { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + return new Promise((resolve) => void resolve()); + } + + asyncFunction( + ctx?: Partial, + ): PromiseCancellable; + @cancellable(true) + async asyncFunction(@context ctx: ContextCancellable): Promise { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + } + + [symbolFunction]( + ctx?: Partial, + ): PromiseCancellable; + @cancellable(false) + [symbolFunction](@context ctx: ContextCancellable): Promise { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + return new Promise((resolve) => void resolve()); + } + } + const x = new X(); + test('functionPromise', async () => { + const pC = x.functionPromise(); + expect(pC).toBeInstanceOf(PromiseCancellable); + await pC; + await x.functionPromise({}); + await x.functionPromise({ signal: new AbortController().signal }); + expect(x.functionPromise).toBeInstanceOf(Function); + expect(x.functionPromise.name).toBe('functionPromise'); + }); + test('asyncFunction', async () => { + const pC = x.asyncFunction(); + expect(pC).toBeInstanceOf(PromiseCancellable); + await x.asyncFunction({}); + await x.asyncFunction({ signal: new AbortController().signal }); + expect(x.asyncFunction).toBeInstanceOf(Function); + expect(x.asyncFunction).not.toBeInstanceOf(AsyncFunction); + expect(x.asyncFunction.name).toBe('asyncFunction'); + }); + test('symbolFunction', async () => { + const pC = x[symbolFunction](); + expect(pC).toBeInstanceOf(PromiseCancellable); + await x[symbolFunction]({}); + await x[symbolFunction]({ signal: new AbortController().signal }); + expect(x[symbolFunction]).toBeInstanceOf(Function); + expect(x[symbolFunction].name).toBe('[sym]'); + }); + }); + describe('cancellable decorator cancellation', () => { + test('async function cancel and eager rejection', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @cancellable() + async f(@context ctx: ContextCancellable): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel(); + await expect(pC).rejects.toBeUndefined(); + }); + test('async function cancel and lazy rejection', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @cancellable(true) + async f(@context ctx: ContextCancellable): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel(); + await expect(pC).resolves.toBe('hello world'); + }); + test('async function cancel with custom error and eager rejection', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @cancellable() + async f(@context ctx: ContextCancellable): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('async function cancel with custom error and lazy rejection', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @cancellable(true) + async f(@context ctx: ContextCancellable): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw ctx.signal.reason; + } + await sleep(1); + } + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('promise cancellable function - eager rejection', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @cancellable() + f(@context ctx: ContextCancellable): PromiseCancellable { + const pC = new PromiseCancellable((resolve, reject, signal) => { + if (signal.aborted) { + reject('eager 2:' + signal.reason); + } else { + signal.onabort = () => { + reject('lazy 2:' + signal.reason); + }; + } + sleep(10).then(() => { + resolve('hello world'); + }); + }); + if (ctx.signal.aborted) { + pC.cancel('eager 1:' + ctx.signal.reason); + } else { + ctx.signal.onabort = () => { + pC.cancel('lazy 1:' + ctx.signal.reason); + } + } + return pC; + } + } + const c = new C(); + // Signal is aborted afterwards + const pC1 = c.f(); + pC1.cancel('cancel reason'); + await expect(pC1).rejects.toBe('cancel reason'); + // Signal is already aborted + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC2 = c.f({ signal: abortController.signal }); + await expect(pC2).rejects.toBe('cancel reason'); + }); + test('promise cancellable function - lazy rejection', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @cancellable(true) + f(@context ctx: ContextCancellable): PromiseCancellable { + const pC = new PromiseCancellable((resolve, reject, signal) => { + if (signal.aborted) { + reject('eager 2:' + signal.reason); + } else { + signal.onabort = () => { + reject('lazy 2:' + signal.reason); + }; + } + sleep(10).then(() => { + resolve('hello world'); + }); + }); + if (ctx.signal.aborted) { + pC.cancel('eager 1:' + ctx.signal.reason); + } else { + ctx.signal.onabort = () => { + pC.cancel('lazy 1:' + ctx.signal.reason); + } + } + return pC; + } + } + const c = new C(); + // Signal is aborted afterwards + const pC1 = c.f(); + pC1.cancel('cancel reason'); + await expect(pC1).rejects.toBe('lazy 2:lazy 1:cancel reason'); + // Signal is already aborted + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC2 = c.f({ signal: abortController.signal }); + await expect(pC2).rejects.toBe('lazy 2:eager 1:cancel reason'); + }); + }); + describe('cancellable decorator propagation', () => { + test('propagate signal', async () => { + let signal: AbortSignal; + class C { + f(ctx?: Partial): PromiseCancellable; + @cancellable(true) + async f(@context ctx: ContextCancellable): Promise { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + signal = ctx.signal; + return await this.g(ctx); + } + + g(ctx?: Partial): PromiseCancellable; + @cancellable(true) + g(@context ctx: ContextCancellable): Promise { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + // The signal is actually not the same + // it is chained instead + expect(signal).not.toBe(ctx.signal); + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject('early:' + ctx.signal.reason); + } else { + const timeout = setTimeout(() => { + resolve('g'); + }, 10); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject('during:' + ctx.signal.reason); + }); + } + }); + } + } + const c = new C(); + const pC1 = c.f(); + await expect(pC1).resolves.toBe('g'); + expect(signal!.aborted).toBe(false); + const pC2 = c.f(); + pC2.cancel('cancel reason'); + await expect(pC2).rejects.toBe('during:cancel reason'); + expect(signal!.aborted).toBe(true); + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC3 = c.f({ signal: abortController.signal }); + await expect(pC3).rejects.toBe('early:cancel reason'); + expect(signal!.aborted).toBe(true); + }); + test('nested cancellable - lazy then lazy', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @cancellable(true) + @cancellable(true) + async f(@context ctx: ContextCancellable): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('throw:cancel reason'); + }); + test('nested cancellable - lazy then eager', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @cancellable(true) + @cancellable(false) + async f(@context ctx: ContextCancellable): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('nested cancellable - eager then lazy', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @cancellable(false) + @cancellable(true) + async f(@context ctx: ContextCancellable): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('signal event listeners are removed', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @cancellable() + async f(@context ctx: ContextCancellable): Promise { + return 'hello world'; + } + } + const abortController = new AbortController(); + let listenerCount = 0; + const signal = new Proxy(abortController.signal, { + get(target, prop, receiver) { + if (prop === 'addEventListener') { + return function addEventListener(...args) { + listenerCount++; + return target[prop].apply(this, args); + }; + } else if (prop === 'removeEventListener') { + return function addEventListener(...args) { + listenerCount--; + return target[prop].apply(this, args); + }; + } else { + return Reflect.get(target, prop, receiver); + } + }, + }); + const c = new C(); + await c.f({ signal }); + await c.f({ signal }); + const pC = c.f({ signal }); + pC.cancel(); + await expect(pC).rejects.toBe(undefined); + expect(listenerCount).toBe(0); + }); + }); +}); diff --git a/tests/contexts/decorators/timed.test.ts b/tests/contexts/decorators/timed.test.ts index c0a3bdca3..382c5dac8 100644 --- a/tests/contexts/decorators/timed.test.ts +++ b/tests/contexts/decorators/timed.test.ts @@ -1,37 +1,95 @@ +import type { ContextTimed } from '@/contexts/types'; import context from '@/contexts/decorators/context'; import timed from '@/contexts/decorators/timed'; +import * as contextsErrors from '@/contexts/errors'; import Timer from '@/timer/Timer'; import { AsyncFunction, GeneratorFunction, AsyncGeneratorFunction, + sleep, } from '@/utils'; describe('context/decorators/timed', () => { - test('timed decorator', async () => { - const s = Symbol('sym'); + describe('timed decorator runtime validation', () => { + test('timed decorator requires context decorator', async () => { + expect(() => { + class C { + @timed(50) + async f(_ctx: ContextTimed): Promise { + return 'hello world'; + } + } + return C; + }).toThrow(TypeError); + }); + test('timed decorator fails on invalid context', async () => { + await expect(async () => { + class C { + @timed(50) + async f(@context _ctx: ContextTimed): Promise { + return 'hello world'; + } + } + const c = new C(); + // @ts-ignore invalid context timer + await c.f({ timer: 1 }); + }).rejects.toThrow(TypeError); + await expect(async () => { + class C { + @timed(50) + async f(@context _ctx: ContextTimed): Promise { + return 'hello world'; + } + } + const c = new C(); + // @ts-ignore invalid context signal + await c.f({ signal: 'lol' }); + }).rejects.toThrow(TypeError); + }); + }); + describe('timed decorator syntax', () => { + // Decorators cannot change type signatures + // use overloading to change required context parameter to optional context parameter + const symbolFunction = Symbol('sym'); class X { - a( - ctx?: { signal?: AbortSignal; timer?: Timer }, + functionValue( + ctx?: Partial, check?: (t: Timer) => any, ): void; @timed(1000) - a( - @context ctx: { signal: AbortSignal; timer: Timer }, + functionValue( + @context ctx: ContextTimed, check?: (t: Timer) => any, ): void { expect(ctx.signal).toBeInstanceOf(AbortSignal); expect(ctx.timer).toBeInstanceOf(Timer); if (check != null) check(ctx.timer); + return; + } + + functionPromise( + ctx?: Partial, + check?: (t: Timer) => any, + ): Promise; + @timed(1000) + functionPromise( + @context ctx: ContextTimed, + check?: (t: Timer) => any, + ): Promise { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(ctx.timer).toBeInstanceOf(Timer); + if (check != null) check(ctx.timer); + return new Promise((resolve) => void resolve()); } - b( - ctx?: { signal?: AbortSignal; timer?: Timer }, + asyncFunction( + ctx?: Partial, check?: (t: Timer) => any, ): Promise; @timed(Infinity) - async b( - @context ctx: { signal: AbortSignal; timer: Timer }, + async asyncFunction( + @context ctx: ContextTimed, check?: (t: Timer) => any, ): Promise { expect(ctx.signal).toBeInstanceOf(AbortSignal); @@ -39,13 +97,13 @@ describe('context/decorators/timed', () => { if (check != null) check(ctx.timer); } - c( - ctx?: { signal?: AbortSignal; timer?: Timer }, + generator( + ctx?: Partial, check?: (t: Timer) => any, ): Generator; @timed(0) - *c( - @context ctx: { signal: AbortSignal; timer: Timer }, + *generator( + @context ctx: ContextTimed, check?: (t: Timer) => any, ): Generator { expect(ctx.signal).toBeInstanceOf(AbortSignal); @@ -53,13 +111,25 @@ describe('context/decorators/timed', () => { if (check != null) check(ctx.timer); } - d( - ctx?: { signal?: AbortSignal; timer?: Timer }, + functionGenerator( + ctx?: Partial, + check?: (t: Timer) => any, + ): Generator; + @timed(0) + functionGenerator( + @context ctx: ContextTimed, + check?: (t: Timer) => any, + ): Generator { + return this.generator(ctx, check); + } + + asyncGenerator( + ctx?: Partial, check?: (t: Timer) => any, ): AsyncGenerator; @timed(NaN) - async *d( - @context ctx: { signal: AbortSignal; timer: Timer }, + async *asyncGenerator( + @context ctx: ContextTimed, check?: (t: Timer) => any, ): AsyncGenerator { expect(ctx.signal).toBeInstanceOf(AbortSignal); @@ -67,61 +137,593 @@ describe('context/decorators/timed', () => { if (check != null) check(ctx.timer); } - [s]( - ctx?: { signal?: AbortSignal; timer?: Timer }, + functionAsyncGenerator( + ctx?: Partial, check?: (t: Timer) => any, - ): void; + ): AsyncGenerator; + @timed(NaN) + functionAsyncGenerator( + @context ctx: ContextTimed, + check?: (t: Timer) => any, + ): AsyncGenerator { + return this.asyncGenerator(ctx, check); + } + + [symbolFunction]( + ctx?: Partial, + check?: (t: Timer) => any, + ): Promise; @timed() - [s]( - @context ctx: { signal: AbortSignal; timer: Timer }, + [symbolFunction]( + @context ctx: ContextTimed, check?: (t: Timer) => any, - ): void { + ): Promise { expect(ctx.signal).toBeInstanceOf(AbortSignal); expect(ctx.timer).toBeInstanceOf(Timer); if (check != null) check(ctx.timer); + return new Promise((resolve) => void resolve()); } } const x = new X(); - x.a(); - x.a({}); - x.a({ timer: new Timer({ delay: 100 }) }, (t) => { - expect(t.delay).toBe(100); - }); - expect(x.a).toBeInstanceOf(Function); - expect(x.a.name).toBe('a'); - await x.b(); - await x.b({}); - await x.b({ timer: new Timer({ delay: 50 }) }, (t) => { - expect(t.delay).toBe(50); - }); - expect(x.b).toBeInstanceOf(AsyncFunction); - expect(x.b.name).toBe('b'); - for (const _ of x.c()) { - } - for (const _ of x.c({})) { - } - for (const _ of x.c({ timer: new Timer({ delay: 150 }) }, (t) => { - expect(t.delay).toBe(150); - })) { - } - expect(x.c).toBeInstanceOf(GeneratorFunction); - expect(x.c.name).toBe('c'); - for await (const _ of x.d()) { - } - for await (const _ of x.d({})) { - } - for await (const _ of x.d({ timer: new Timer({ delay: 200 }) }, (t) => { - expect(t.delay).toBe(200); - })) { + test('functionValue', () => { + x.functionValue(); + x.functionValue({}); + x.functionValue({ timer: new Timer({ delay: 100 }) }, (t) => { + expect(t.delay).toBe(100); + }); + expect(x.functionValue).toBeInstanceOf(Function); + expect(x.functionValue.name).toBe('functionValue'); + }); + test('functionPromise', async () => { + await x.functionPromise(); + await x.functionPromise({}); + await x.functionPromise({ timer: new Timer({ delay: 100 }) }, (t) => { + expect(t.delay).toBe(100); + }); + expect(x.functionPromise).toBeInstanceOf(Function); + expect(x.functionPromise.name).toBe('functionPromise'); + }); + test('asyncFunction', async () => { + await x.asyncFunction(); + await x.asyncFunction({}); + await x.asyncFunction({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + }); + expect(x.asyncFunction).toBeInstanceOf(AsyncFunction); + expect(x.asyncFunction.name).toBe('asyncFunction'); + }); + test('generator', () => { + for (const _ of x.generator()) { + // NOOP + } + for (const _ of x.generator({})) { + // NOOP + } + for (const _ of x.generator({ timer: new Timer({ delay: 150 }) }, (t) => { + expect(t.delay).toBe(150); + })) { + // NOOP + } + expect(x.generator).toBeInstanceOf(GeneratorFunction); + expect(x.generator.name).toBe('generator'); + }); + test('functionGenerator', () => { + for (const _ of x.functionGenerator()) { + // NOOP + } + for (const _ of x.functionGenerator({})) { + // NOOP + } + for (const _ of x.functionGenerator( + { timer: new Timer({ delay: 150 }) }, + (t) => { + expect(t.delay).toBe(150); + }, + )) { + // NOOP + } + expect(x.functionGenerator).toBeInstanceOf(Function); + expect(x.functionGenerator.name).toBe('functionGenerator'); + }); + test('asyncGenerator', async () => { + for await (const _ of x.asyncGenerator()) { + // NOOP + } + for await (const _ of x.asyncGenerator({})) { + // NOOP + } + for await (const _ of x.asyncGenerator( + { timer: new Timer({ delay: 200 }) }, + (t) => { + expect(t.delay).toBe(200); + }, + )) { + // NOOP + } + expect(x.asyncGenerator).toBeInstanceOf(AsyncGeneratorFunction); + expect(x.asyncGenerator.name).toBe('asyncGenerator'); + }); + test('functionAsyncGenerator', async () => { + for await (const _ of x.functionAsyncGenerator()) { + // NOOP + } + for await (const _ of x.functionAsyncGenerator({})) { + // NOOP + } + for await (const _ of x.functionAsyncGenerator( + { timer: new Timer({ delay: 200 }) }, + (t) => { + expect(t.delay).toBe(200); + }, + )) { + // NOOP + } + expect(x.functionAsyncGenerator).toBeInstanceOf(Function); + expect(x.functionAsyncGenerator.name).toBe('functionAsyncGenerator'); + }); + test('symbolFunction', async () => { + await x[symbolFunction](); + await x[symbolFunction]({}); + await x[symbolFunction]({ timer: new Timer({ delay: 250 }) }, (t) => { + expect(t.delay).toBe(250); + }); + expect(x[symbolFunction]).toBeInstanceOf(Function); + expect(x[symbolFunction].name).toBe('[sym]'); + }); + }); + describe('timed decorator expiry', () => { + // Timed decorator does not automatically reject the promise + // it only signals that it is aborted + // it is up to the function to decide how to reject + test('async function expiry', async () => { + class C { + /** + * Async function + */ + f(ctx?: Partial): Promise; + @timed(50) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf( + contextsErrors.ErrorContextsTimedExpiry, + ); + return 'hello world'; + } + } + const c = new C(); + await expect(c.f()).resolves.toBe('hello world'); + }); + test('async function expiry with custom error', async () => { + class ErrorCustom extends Error {} + class C { + /** + * Async function + */ + f(ctx?: Partial): Promise; + @timed(50, ErrorCustom) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf(ErrorCustom); + throw ctx.signal.reason; + } + } + const c = new C(); + await expect(c.f()).rejects.toBeInstanceOf(ErrorCustom); + }); + test('promise function expiry', async () => { + class C { + /** + * Regular function returning promise + */ + f(ctx?: Partial): Promise; + @timed(50) + f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + return sleep(15) + .then(() => { + expect(ctx.signal.aborted).toBe(false); + }) + .then(() => sleep(40)) + .then(() => { + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf( + contextsErrors.ErrorContextsTimedExpiry, + ); + }) + .then(() => { + return 'hello world'; + }); + } + } + const c = new C(); + await expect(c.f()).resolves.toBe('hello world'); + }); + test('promise function expiry and late rejection', async () => { + let timeout: ReturnType | undefined; + class C { + /** + * Regular function that actually rejects + * when the signal is aborted + */ + f(ctx?: Partial): Promise; + @timed(50) + f(@context ctx: ContextTimed): Promise { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + } + timeout = setTimeout(() => { + resolve('hello world'); + }, 50000); + ctx.signal.onabort = () => { + clearTimeout(timeout); + timeout = undefined; + reject(ctx.signal.reason); + }; + }); + } + } + const c = new C(); + await expect(c.f()).rejects.toBeInstanceOf( + contextsErrors.ErrorContextsTimedExpiry, + ); + expect(timeout).toBeUndefined(); + }); + test('promise function expiry and early rejection', async () => { + let timeout: ReturnType | undefined; + class C { + /** + * Regular function that actually rejects immediately + */ + f(ctx?: Partial): Promise; + @timed(0) + f(@context ctx: ContextTimed): Promise { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + } + timeout = setTimeout(() => { + resolve('hello world'); + }, 50000); + ctx.signal.onabort = () => { + clearTimeout(timeout); + timeout = undefined; + reject(ctx.signal.reason); + }; + }); + } + } + const c = new C(); + await expect(c.f()).rejects.toBeInstanceOf( + contextsErrors.ErrorContextsTimedExpiry, + ); + expect(timeout).toBeUndefined(); + }); + test('async generator expiry', async () => { + class C { + f(ctx?: Partial): AsyncGenerator; + @timed(50) + async *f(@context ctx: ContextTimed): AsyncGenerator { + while (true) { + if (ctx.signal.aborted) { + throw ctx.signal.reason; + } + yield 'hello world'; + } + } + } + const c = new C(); + const g = c.f(); + await expect(g.next()).resolves.toEqual({ + value: 'hello world', + done: false, + }); + await expect(g.next()).resolves.toEqual({ + value: 'hello world', + done: false, + }); + await sleep(50); + await expect(g.next()).rejects.toThrow( + contextsErrors.ErrorContextsTimedExpiry, + ); + }); + test('generator expiry', async () => { + class C { + f(ctx?: Partial): Generator; + @timed(50) + *f(@context ctx: ContextTimed): Generator { + while (true) { + if (ctx.signal.aborted) { + throw ctx.signal.reason; + } + yield 'hello world'; + } + } + } + const c = new C(); + const g = c.f(); + expect(g.next()).toEqual({ value: 'hello world', done: false }); + expect(g.next()).toEqual({ value: 'hello world', done: false }); + await sleep(50); + expect(() => g.next()).toThrow(contextsErrors.ErrorContextsTimedExpiry); + }); + }); + describe('timed decorator propagation', () => { + test('propagate timer and signal', async () => { + let timer: Timer; + let signal: AbortSignal; + class C { + f(ctx?: Partial): Promise; + @timed(50) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await this.g(ctx); + } + + g(ctx?: Partial): Promise; + @timed(25) + async g(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + // Timer and signal will be propagated + expect(timer).toBe(ctx.timer); + expect(signal).toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(50); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + } + } + const c = new C(); + await expect(c.f()).resolves.toBe('g'); + }); + test('propagate timer only', async () => { + let timer: Timer; + let signal: AbortSignal; + class C { + f(ctx?: Partial): Promise; + @timed(50) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await this.g({ timer: ctx.timer }); + } + + g(ctx?: Partial): Promise; + @timed(25) + async g(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(timer).toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(50); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + } + } + const c = new C(); + await expect(c.f()).resolves.toBe('g'); + }); + test('propagate signal only', async () => { + let timer: Timer; + let signal: AbortSignal; + class C { + f(ctx?: Partial): Promise; + @timed(50) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await this.g({ signal: ctx.signal }); + } + + g(ctx?: Partial): Promise; + @timed(25) + async g(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + // Even though signal is propagated + // because the timer isn't, the signal here is chained + expect(timer).not.toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(25); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + } + } + const c = new C(); + await expect(c.f()).resolves.toBe('g'); + }); + test('propagate nothing', async () => { + let timer: Timer; + let signal: AbortSignal; + class C { + f(ctx?: Partial): Promise; + @timed(50) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await this.g(); + } + + g(ctx?: Partial): Promise; + @timed(25) + async g(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(timer).not.toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(25); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + } + } + const c = new C(); + await expect(c.f()).resolves.toBe('g'); + }); + test('propagated expiry', async () => { + class C { + f(ctx?: Partial): Promise; + @timed(25) + async f(@context ctx: ContextTimed): Promise { + // The `g` will use up all the remaining time + const counter = await this.g(ctx.timer.getTimeout()); + expect(counter).toBeGreaterThan(0); + // The `h` will reject eventually + // it may reject immediately + // it may reject after some time + await this.h(ctx); + return 'hello world'; + } + + async g(timeout: number): Promise { + const start = performance.now(); + let counter = 0; + while (true) { + if (performance.now() - start > timeout) { + break; + } + await sleep(1); + counter++; + } + return counter; + } + + h(ctx?: Partial): Promise; + @timed(25) + async h(@context ctx: ContextTimed): Promise { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + return; + } + const timeout = setTimeout(() => { + resolve('hello world'); + }, 25); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject(ctx.signal.reason); + }); + }); + } + } + const c = new C(); + await expect(c.f()).rejects.toThrow( + contextsErrors.ErrorContextsTimedExpiry, + ); + }); + }); + describe('timed decorator explicit timer cancellation or signal abortion', () => { + // If the timer is cancelled + // there will be no timeout error + let ctx_: ContextTimed | undefined; + class C { + f(ctx?: Partial): Promise; + @timed(50) + f(@context ctx: ContextTimed): Promise { + ctx_ = ctx; + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason + ' begin'); + return; + } + const timeout = setTimeout(() => { + resolve('hello world'); + }, 25); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject(ctx.signal.reason + ' during'); + }); + }); + } } - expect(x.d).toBeInstanceOf(AsyncGeneratorFunction); - expect(x.d.name).toBe('d'); - x[s](); - x[s]({}); - x[s]({ timer: new Timer({ delay: 250 }) }, (t) => { - expect(t.delay).toBe(250); - }); - expect(x[s]).toBeInstanceOf(Function); - expect(x[s].name).toBe('[sym]'); + const c = new C(); + beforeEach(() => { + ctx_ = undefined; + }); + test('explicit timer cancellation - begin', async () => { + const timer = new Timer({ delay: 100 }); + timer.cancel('reason'); + const p = c.f({ timer }); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit timer cancellation - during', async () => { + const timer = new Timer({ delay: 100 }); + const p = c.f({ timer }); + timer.cancel('reason'); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit timer cancellation - during after sleep', async () => { + const timer = new Timer({ delay: 20 }); + const p = c.f({ timer }); + await sleep(1); + timer.cancel('reason'); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit signal abortion - begin', async () => { + const abortController = new AbortController(); + abortController.abort('reason'); + const p = c.f({ signal: abortController.signal }); + expect(ctx_!.timer.status).toBe('settled'); + await expect(p).rejects.toBe('reason begin'); + }); + test('explicit signal abortion - during', async () => { + const abortController = new AbortController(); + const p = c.f({ signal: abortController.signal }); + abortController.abort('reason'); + // Timer is also cancelled immediately + expect(ctx_!.timer.status).toBe('settled'); + await expect(p).rejects.toBe('reason during'); + }); + test('explicit signal signal abortion with passed in timer - during', async () => { + const timer = new Timer({ delay: 100 }); + const abortController = new AbortController(); + const p = c.f({ timer, signal: abortController.signal }); + abortController.abort('abort reason'); + expect(ctx_!.timer.status).toBe('settled'); + expect(timer.status).toBe('settled'); + expect(ctx_!.signal.aborted).toBe(true); + await expect(p).rejects.toBe('abort reason during'); + }); + test('explicit timer cancellation and signal abortion - begin', async () => { + const timer = new Timer({ delay: 100 }); + timer.cancel('timer reason'); + const abortController = new AbortController(); + abortController.abort('abort reason'); + const p = c.f({ timer, signal: abortController.signal }); + expect(ctx_!.timer.status).toBe('settled'); + expect(ctx_!.signal.aborted).toBe(true); + await expect(p).rejects.toBe('abort reason begin'); + }); }); }); diff --git a/tests/timer/Timer.test.ts b/tests/timer/Timer.test.ts index be32b16c0..fe8621575 100644 --- a/tests/timer/Timer.test.ts +++ b/tests/timer/Timer.test.ts @@ -1,51 +1,29 @@ import { performance } from 'perf_hooks'; import { Timer } from '@/timer'; -import * as timerErrors from '@/timer/errors'; import { sleep } from '@/utils'; describe(Timer.name, () => { test('timer is thenable and awaitable', async () => { const t1 = new Timer(); expect(await t1).toBeUndefined(); - expect(t1.status).toBe('resolved'); + expect(t1.status).toBe('settled'); const t2 = new Timer(); await expect(t2).resolves.toBeUndefined(); - expect(t2.status).toBe('resolved'); + expect(t2.status).toBe('settled'); }); test('timer delays', async () => { const t1 = new Timer({ delay: 20, handler: () => 1 }); - const t2 = new Timer({ delay: 10, handler: () => 2 }); + const t2 = new Timer(() => 2, 10); const result = await Promise.any([t1, t2]); expect(result).toBe(2); }); test('timer handlers', async () => { - const t1 = new Timer({ handler: () => 123 }); + const t1 = new Timer(() => 123); expect(await t1).toBe(123); - expect(t1.status).toBe('resolved'); + expect(t1.status).toBe('settled'); const t2 = new Timer({ delay: 100, handler: () => '123' }); expect(await t2).toBe('123'); - expect(t2.status).toBe('resolved'); - }); - test('timer cancellation', async () => { - const t1 = new Timer({ delay: 100 }); - t1.cancel(); - await expect(t1).rejects.toThrow(timerErrors.ErrorTimerCancelled); - expect(t1.status).toBe('rejected'); - const t2 = new Timer({ delay: 100 }); - const results = await Promise.all([ - (async () => { - try { - await t2; - } catch (e) { - return e; - } - })(), - (async () => { - t2.cancel(); - })(), - ]); - expect(results[0]).toBeInstanceOf(timerErrors.ErrorTimerCancelled); - expect(t2.status).toBe('rejected'); + expect(t2.status).toBe('settled'); }); test('timer timestamps', async () => { const start = new Date(performance.timeOrigin + performance.now()); @@ -79,14 +57,16 @@ describe(Timer.name, () => { expect(+t1).toBe(Infinity); expect(t1.toString()).toBe('Infinity'); expect(`${t1}`).toBe('Infinity'); - t1.cancel(); - await expect(t1).rejects.toThrow(timerErrors.ErrorTimerCancelled); + t1.cancel(new Error('Oh No')); + await expect(t1).rejects.toThrow('Oh No'); }); test('timer does not keep event loop alive', async () => { const f = async (timer: Timer | number = globalThis.maxTimeout) => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars timer = timer instanceof Timer ? timer : new Timer({ delay: timer }); }; const g = async (timer: Timer | number = Infinity) => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars timer = timer instanceof Timer ? timer : new Timer({ delay: timer }); }; await f(); @@ -96,14 +76,168 @@ describe(Timer.name, () => { await g(); await g(); }); - test('timer lifecycle', async () => { - const t1 = Timer.createTimer({ delay: 1000 }); - await t1.destroy('resolve'); - expect(t1.status).toBe('resolved'); - await expect(t1).resolves.toBeUndefined(); - const t2 = Timer.createTimer({ delay: 1000 }); - await t2.destroy('reject'); - expect(t2.status).toBe('rejected'); - await expect(t2).rejects.toThrow(timerErrors.ErrorTimerCancelled); + test('custom signal handler ignores default rejection', async () => { + const onabort = jest.fn(); + const t = new Timer( + () => 1, + 50, + false, + (signal) => { + signal.onabort = onabort; + }, + ); + t.cancel('abort'); + await expect(t).resolves.toBe(1); + expect(onabort).toBeCalled(); + }); + test('custom abort controller ignores default rejection', async () => { + const onabort = jest.fn(); + const abortController = new AbortController(); + abortController.signal.onabort = onabort; + const t = new Timer(() => 1, 50, false, abortController); + t.cancel('abort'); + await expect(t).resolves.toBe(1); + expect(onabort).toBeCalled(); + }); + describe('timer cancellation', () => { + test('cancellation rejects the timer with the reason', async () => { + const t1 = new Timer(undefined, 100); + t1.cancel(); + await expect(t1).rejects.toBeUndefined(); + expect(t1.status).toBe('settled'); + const t2 = new Timer({ delay: 100 }); + const results = await Promise.all([ + (async () => { + try { + await t2; + } catch (e) { + return e; + } + })(), + (async () => { + t2.cancel('Surprise!'); + })(), + ]); + expect(results[0]).toBe('Surprise!'); + expect(t2.status).toBe('settled'); + }); + test('non-lazy cancellation is early/eager rejection', async () => { + let resolveHandlerCalledP; + const handlerCalledP = new Promise((resolve) => { + resolveHandlerCalledP = resolve; + }); + let p; + const handler = jest.fn().mockImplementation((signal: AbortSignal) => { + resolveHandlerCalledP(); + p = new Promise((resolve, reject) => { + if (signal.aborted) { + reject('handler abort start'); + return; + } + const timeout = setTimeout(() => resolve('handler result'), 100); + signal.addEventListener( + 'abort', + () => { + clearTimeout(timeout); + reject('handler abort during'); + }, + { once: true }, + ); + }); + return p; + }); + // Non-lazy means that it will do an early rejection + const t = new Timer({ + handler, + delay: 100, + lazy: false, + }); + await handlerCalledP; + expect(handler).toBeCalledWith(expect.any(AbortSignal)); + t.cancel('timer abort'); + await expect(t).rejects.toBe('timer abort'); + await expect(p).rejects.toBe('handler abort during'); + }); + test('lazy cancellation', async () => { + let resolveHandlerCalledP; + const handlerCalledP = new Promise((resolve) => { + resolveHandlerCalledP = resolve; + }); + let p; + const handler = jest.fn().mockImplementation((signal: AbortSignal) => { + resolveHandlerCalledP(); + p = new Promise((resolve, reject) => { + if (signal.aborted) { + reject('handler abort start'); + return; + } + const timeout = setTimeout(() => resolve('handler result'), 100); + signal.addEventListener( + 'abort', + () => { + clearTimeout(timeout); + reject('handler abort during'); + }, + { once: true }, + ); + }); + return p; + }); + // Lazy means that it will not do an early rejection + const t = new Timer({ + handler, + delay: 100, + lazy: true, + }); + await handlerCalledP; + expect(handler).toBeCalledWith(expect.any(AbortSignal)); + t.cancel('timer abort'); + await expect(t).rejects.toBe('handler abort during'); + await expect(p).rejects.toBe('handler abort during'); + }); + test('cancellation should not have an unhandled promise rejection', async () => { + const timer = new Timer(); + timer.cancel('reason'); + }); + test('multiple cancellations should have an unhandled promise rejection', async () => { + const timer = new Timer(); + timer.cancel('reason 1'); + timer.cancel('reason 2'); + }); + test('only the first reason is used in multiple cancellations', async () => { + const timer = new Timer(); + timer.cancel('reason 1'); + timer.cancel('reason 2'); + await expect(timer).rejects.toBe('reason 1'); + }); + test('lazy cancellation allows resolution if signal is ignored', async () => { + const timer = new Timer({ + handler: (signal) => { + expect(signal.aborted).toBe(true); + return new Promise((resolve) => { + setTimeout(() => { + resolve('result'); + }, 50); + }); + }, + lazy: true, + }); + timer.cancel('reason'); + expect(await timer).toBe('result'); + }); + test('lazy cancellation allows rejection if signal is ignored', async () => { + const timer = new Timer({ + handler: () => { + return new Promise((resolve, reject) => { + setTimeout(() => { + reject('error'); + }, 50); + }); + }, + lazy: true, + }); + timer.cancel('reason'); + await expect(timer).rejects.toBe('error'); + }); }); }); From 3ee78bfe908875f2072b4c1d2cb8875cbb3c6066 Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Fri, 9 Sep 2022 13:58:19 +1000 Subject: [PATCH 08/32] fix: `Timer` is clipped to maximum timeout if given finite delay --- src/timer/Timer.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/timer/Timer.ts b/src/timer/Timer.ts index c9068004b..a488d123b 100644 --- a/src/timer/Timer.ts +++ b/src/timer/Timer.ts @@ -112,10 +112,18 @@ class Timer lazy = handlerOrOpts.lazy ?? lazy; controller = handlerOrOpts.controller ?? controller; } - // Clip to delay >= 0 - delay = Math.max(delay, 0); // Coerce NaN to minimal delay of 0 - if (isNaN(delay)) delay = 0; + if (isNaN(delay)) { + delay = 0; + } else { + // Clip to delay >= 0 + delay = Math.max(delay, 0); + if (isFinite(delay)) { + // Clip to delay <= 2147483647 (maximum timeout) + // but only if delay is finite + delay = Math.min(delay, 2**31 - 1); + } + } this.handler = handler; this.delay = delay; this.lazy = lazy; From 4c63ac8bf955c298f5a0a8e74ff3316058be7373 Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Thu, 8 Sep 2022 02:11:30 +1000 Subject: [PATCH 09/32] feat(contexts): `timed` and `cancellable` higher order function operators --- src/contexts/decorators/index.ts | 1 + src/contexts/decorators/timedCancellable.ts | 18 + src/contexts/functions/cancellable.ts | 71 +++ src/contexts/functions/index.ts | 3 + src/contexts/functions/timed.ts | 204 +++++++ src/contexts/functions/timedCancellable.ts | 5 + src/contexts/types.ts | 7 +- tests/contexts/decorators/cancellable.test.ts | 6 +- tests/contexts/functions/cancellable.test.ts | 280 +++++++++ tests/contexts/functions/timed.test.ts | 541 ++++++++++++++++++ 10 files changed, 1127 insertions(+), 9 deletions(-) create mode 100644 src/contexts/decorators/timedCancellable.ts create mode 100644 src/contexts/functions/cancellable.ts create mode 100644 src/contexts/functions/index.ts create mode 100644 src/contexts/functions/timed.ts create mode 100644 src/contexts/functions/timedCancellable.ts create mode 100644 tests/contexts/functions/cancellable.test.ts create mode 100644 tests/contexts/functions/timed.test.ts diff --git a/src/contexts/decorators/index.ts b/src/contexts/decorators/index.ts index ca5692398..e8997e285 100644 --- a/src/contexts/decorators/index.ts +++ b/src/contexts/decorators/index.ts @@ -1,3 +1,4 @@ export { default as context } from './context'; export { default as cancellable } from './cancellable'; export { default as timed } from './timed'; +export { default as timedCancellable } from './timedCancellable'; diff --git a/src/contexts/decorators/timedCancellable.ts b/src/contexts/decorators/timedCancellable.ts new file mode 100644 index 000000000..8b6357dc3 --- /dev/null +++ b/src/contexts/decorators/timedCancellable.ts @@ -0,0 +1,18 @@ + +// equivalent to timed(cancellable()) +// timeout is always lazy +// it's only if you call cancel +// PLUS this only works with PromiseLike +// the timed just wraps that together +// and the result is a bit more efficient +// to avoid having to chain the signals up too much + +function timedCancellable( + lazy: boolean = false, + delay: number = Infinity, + errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedExpiry, +) { + +} + +export default timedCancellable; diff --git a/src/contexts/functions/cancellable.ts b/src/contexts/functions/cancellable.ts new file mode 100644 index 000000000..5194832c0 --- /dev/null +++ b/src/contexts/functions/cancellable.ts @@ -0,0 +1,71 @@ +import type { ContextCancellable } from "../types"; +import { PromiseCancellable } from '@matrixai/async-cancellable'; + +type ContextRemaining = Omit; + +type ContextAndParameters> = + keyof ContextRemaining extends never + ? [Partial?, ...P] + : [Partial & ContextRemaining, ...P]; + +function cancellable< + C extends ContextCancellable, + P extends Array, + R +>( + f: (ctx: C, ...params: P) => PromiseLike, + lazy: boolean = false, +): (...params: ContextAndParameters) => PromiseCancellable { + return (...params) => { + const ctx = params[0] ?? {}; + const args = params.slice(1) as P; + if (ctx.signal === undefined) { + const abortController = new AbortController(); + ctx.signal = abortController.signal; + const result = f(ctx as C, ...args); + return new PromiseCancellable((resolve, reject, signal) => { + if (!lazy) { + signal.addEventListener('abort', () => { + reject(signal.reason); + }); + } + void result.then(resolve, reject); + }, abortController); + } else { + // In this case, `context.signal` is set + // and we chain the upsteam signal to the downstream signal + const abortController = new AbortController(); + const signalUpstream = ctx.signal; + const signalHandler = () => { + abortController.abort(signalUpstream.reason); + }; + if (signalUpstream.aborted) { + abortController.abort(signalUpstream.reason); + } else { + signalUpstream.addEventListener('abort', signalHandler); + } + // Overwrite the signal property with this context's `AbortController.signal` + ctx.signal = abortController.signal; + const result = f(ctx as C, ...args); + // The `abortController` must be shared in the `finally` clause + // to link up final promise's cancellation with the target + // function's signal + return new PromiseCancellable((resolve, reject, signal) => { + if (!lazy) { + if (signal.aborted) { + reject(signal.reason); + } else { + signal.addEventListener('abort', () => { + reject(signal.reason); + }); + } + } + void result.then(resolve, reject); + }, abortController).finally(() => { + signalUpstream.removeEventListener('abort', signalHandler); + }, abortController); + } + }; +} + +export default cancellable; diff --git a/src/contexts/functions/index.ts b/src/contexts/functions/index.ts new file mode 100644 index 000000000..f3165cf18 --- /dev/null +++ b/src/contexts/functions/index.ts @@ -0,0 +1,3 @@ +export { default as cancellable } from './cancellable'; +export { default as timed } from './timed'; +export { default as timedCancellable } from './timedCancellable'; diff --git a/src/contexts/functions/timed.ts b/src/contexts/functions/timed.ts new file mode 100644 index 000000000..a94e90215 --- /dev/null +++ b/src/contexts/functions/timed.ts @@ -0,0 +1,204 @@ +import type { ContextTimed } from '../types'; +import * as contextsErrors from '../errors'; +import Timer from '../../timer/Timer'; +import * as utils from '../../utils'; + +function setupContext( + delay: number, + errorTimeoutConstructor: new () => Error, + ctx: Partial, +): () => void { + // Mutating the `context` parameter + if (ctx.timer === undefined && ctx.signal === undefined) { + const abortController = new AbortController(); + const e = new errorTimeoutConstructor(); + const timer = new Timer(() => void abortController.abort(e), delay); + ctx.signal = abortController.signal; + ctx.timer = timer; + return () => { + timer.cancel(); + }; + } else if ( + ctx.timer === undefined && + ctx.signal instanceof AbortSignal + ) { + const abortController = new AbortController(); + const e = new errorTimeoutConstructor(); + const timer = new Timer(() => void abortController.abort(e), delay); + const signalUpstream = ctx.signal; + const signalHandler = () => { + timer.cancel(); + abortController.abort(signalUpstream.reason); + }; + // If already aborted, abort target and cancel the timer + if (signalUpstream.aborted) { + timer.cancel(); + abortController.abort(signalUpstream.reason); + } else { + signalUpstream.addEventListener('abort', signalHandler); + } + // Overwrite the signal property with this ctx's `AbortController.signal` + ctx.signal = abortController.signal; + ctx.timer = timer; + return () => { + signalUpstream.removeEventListener('abort', signalHandler); + timer.cancel(); + }; + } else if (ctx.timer instanceof Timer && ctx.signal === undefined) { + const abortController = new AbortController(); + const e = new errorTimeoutConstructor(); + let finished = false; + // If the timer resolves, then abort the target function + void ctx.timer.then( + (r: any, s: AbortSignal) => { + // If the timer is aborted after it resolves + // then don't bother aborting the target function + if (!finished && !s.aborted) { + abortController.abort(e); + } + return r; + }, + () => { + // Ignore any upstream cancellation + }, + ); + ctx.signal = abortController.signal; + return () => { + finished = true; + }; + } else { + // In this case, `ctx.timer` and `ctx.signal` are both instances of + // `Timer` and `AbortSignal` respectively + const signalHandler = () => { + ctx!.timer!.cancel(); + }; + if (ctx.signal!.aborted) { + ctx.timer!.cancel(); + } else { + ctx.signal!.addEventListener('abort', signalHandler); + } + return () => { + ctx!.signal!.removeEventListener('abort', signalHandler); + }; + } +} + +type ContextRemaining = Omit; + +type ContextAndParameters> = + keyof ContextRemaining extends never + ? [Partial?, ...P] + : [Partial & ContextRemaining, ...P]; + +/** + * Timed HOF + * This overloaded signature is external signature + */ +function timed< + C extends ContextTimed, + P extends Array, + R +>( + f: (ctx: C, ...params: P) => R, + delay?: number, + errorTimeoutConstructor?: new () => Error, +): ( ...params: ContextAndParameters) => R; +function timed< + C extends ContextTimed, + P extends Array +>( + f: (ctx: C, ...params: P) => any, + delay: number = Infinity, + errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedExpiry, +): ( ...params: ContextAndParameters) => any { + if (f instanceof utils.AsyncFunction) { + return async (...params) => { + const ctx = params[0] ?? {}; + const args = params.slice(1) as P; + const teardownContext = setupContext( + delay, + errorTimeoutConstructor, + ctx, + ); + try { + return await f(ctx as C, ...args); + } finally { + teardownContext(); + } + }; + } else if (f instanceof utils.GeneratorFunction) { + return function* (...params) { + const ctx = params[0] ?? {}; + const args = params.slice(1) as P; + const teardownContext = setupContext( + delay, + errorTimeoutConstructor, + ctx, + ); + try { + return yield* f(ctx as C, ...args); + } finally { + teardownContext(); + } + }; + } else if (f instanceof utils.AsyncGeneratorFunction) { + return async function* (...params) { + const ctx = params[0] ?? {}; + const args = params.slice(1) as P; + const teardownContext = setupContext( + delay, + errorTimeoutConstructor, + ctx, + ); + try { + return yield* f(ctx as C, ...args); + } finally { + teardownContext(); + } + }; + } else { + return (...params) => { + const ctx = params[0] ?? {}; + const args = params.slice(1) as P; + const teardownContext = setupContext( + delay, + errorTimeoutConstructor, + ctx, + ); + const result = f(ctx as C, ...args); + if (utils.isPromiseLike(result)) { + return result.then( + (r) => { + teardownContext(); + return r; + }, + (e) => { + teardownContext(); + throw e; + }, + ); + } else if (utils.isIterable(result)) { + return (function* () { + try { + return yield* result; + } finally { + teardownContext(); + } + })(); + } else if (utils.isAsyncIterable(result)) { + return (async function* () { + try { + return yield* result; + } finally { + teardownContext(); + } + })(); + } else { + teardownContext(); + return result; + } + }; + } +} + +export default timed; diff --git a/src/contexts/functions/timedCancellable.ts b/src/contexts/functions/timedCancellable.ts new file mode 100644 index 000000000..4f54f8c8b --- /dev/null +++ b/src/contexts/functions/timedCancellable.ts @@ -0,0 +1,5 @@ +function timedCancellable() { + +} + +export default timedCancellable; diff --git a/src/contexts/types.ts b/src/contexts/types.ts index 0fe6bad2e..6160ef3da 100644 --- a/src/contexts/types.ts +++ b/src/contexts/types.ts @@ -1,4 +1,3 @@ -import type { DBTransaction } from '@matrixai/db'; import type Timer from '../timer/Timer'; type ContextCancellable = { @@ -9,8 +8,4 @@ type ContextTimed = ContextCancellable & { timer: Timer; }; -type ContextTransactional = { - tran: DBTransaction; -}; - -export type { ContextCancellable, ContextTimed, ContextTransactional }; +export type { ContextCancellable, ContextTimed }; diff --git a/tests/contexts/decorators/cancellable.test.ts b/tests/contexts/decorators/cancellable.test.ts index 7c03304f7..348fb8547 100644 --- a/tests/contexts/decorators/cancellable.test.ts +++ b/tests/contexts/decorators/cancellable.test.ts @@ -1,4 +1,4 @@ -import type { ContextCancellable, ContextTransactional } from '@/contexts/types'; +import type { ContextCancellable } from '@/contexts/types'; import { PromiseCancellable } from '@matrixai/async-cancellable'; import context from '@/contexts/decorators/context'; import cancellable from '@/contexts/decorators/cancellable'; @@ -91,7 +91,7 @@ describe('context/decorators/cancellable', () => { }); }); describe('cancellable decorator cancellation', () => { - test('async function cancel and eager rejection', async () => { + test('async function cancel - eager', async () => { class C { f(ctx?: Partial): PromiseCancellable; @cancellable() @@ -110,7 +110,7 @@ describe('context/decorators/cancellable', () => { pC.cancel(); await expect(pC).rejects.toBeUndefined(); }); - test('async function cancel and lazy rejection', async () => { + test('async function cancel - lazy', async () => { class C { f(ctx?: Partial): PromiseCancellable; @cancellable(true) diff --git a/tests/contexts/functions/cancellable.test.ts b/tests/contexts/functions/cancellable.test.ts new file mode 100644 index 000000000..06bad3e39 --- /dev/null +++ b/tests/contexts/functions/cancellable.test.ts @@ -0,0 +1,280 @@ +import type { ContextCancellable } from '@/contexts/types'; +import { PromiseCancellable } from '@matrixai/async-cancellable'; +import cancellable from '@/contexts/functions/cancellable'; +import { AsyncFunction, sleep } from '@/utils'; + +describe('context/functions/cancellable', () => { + describe('cancellable decorator syntax', () => { + test('async function', async () => { + const f = async function ( + ctx: ContextCancellable, + a: number, + b: number, + ): Promise { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + return a + b; + }; + const fCancellable = cancellable(f); + const pC = fCancellable(undefined, 1, 2); + expect(pC).toBeInstanceOf(PromiseCancellable); + await pC; + await fCancellable({}, 1, 2); + await fCancellable({ signal: new AbortController().signal }, 1, 2); + expect(fCancellable).toBeInstanceOf(Function); + expect(fCancellable).not.toBeInstanceOf(AsyncFunction); + }); + }); + describe('cancellable cancellation', () => { + test('async function cancel - eager', async () => { + const f = async (ctx: ContextCancellable): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + }; + const fCancellable = cancellable(f); + const pC = fCancellable(); + await sleep(1); + pC.cancel(); + await expect(pC).rejects.toBeUndefined(); + }); + test('async function cancel - lazy', async () => { + const f = async(ctx: ContextCancellable): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + }; + const fCancellable = cancellable(f, true); + const pC = fCancellable(); + await sleep(1); + pC.cancel(); + await expect(pC).resolves.toBe('hello world'); + }); + test('async function cancel with custom error and eager rejection', async () => { + const f = async (ctx: ContextCancellable): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + }; + const fCancellable = cancellable(f, false); + const pC = fCancellable(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('async function cancel with custom error and lazy rejection', async () => { + const f = async (ctx: ContextCancellable): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw ctx.signal.reason; + } + await sleep(1); + } + }; + const fCancellable = cancellable(f, true); + const pC = fCancellable(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('promise cancellable function - eager rejection', async () => { + const f = (ctx: ContextCancellable): PromiseCancellable => { + const pC = new PromiseCancellable((resolve, reject, signal) => { + if (signal.aborted) { + reject('eager 2:' + signal.reason); + } else { + signal.onabort = () => { + reject('lazy 2:' + signal.reason); + }; + } + sleep(10).then(() => { + resolve('hello world'); + }); + }); + if (ctx.signal.aborted) { + pC.cancel('eager 1:' + ctx.signal.reason); + } else { + ctx.signal.onabort = () => { + pC.cancel('lazy 1:' + ctx.signal.reason); + } + } + return pC; + }; + const fCancellable = cancellable(f); + // Signal is aborted afterwards + const pC1 = fCancellable(); + pC1.cancel('cancel reason'); + await expect(pC1).rejects.toBe('cancel reason'); + // Signal is already aborted + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC2 = fCancellable({ signal: abortController.signal }); + await expect(pC2).rejects.toBe('cancel reason'); + }); + test('promise cancellable function - lazy rejection', async () => { + const f = (ctx: ContextCancellable): PromiseCancellable => { + const pC = new PromiseCancellable((resolve, reject, signal) => { + if (signal.aborted) { + reject('eager 2:' + signal.reason); + } else { + signal.onabort = () => { + reject('lazy 2:' + signal.reason); + }; + } + sleep(10).then(() => { + resolve('hello world'); + }); + }); + if (ctx.signal.aborted) { + pC.cancel('eager 1:' + ctx.signal.reason); + } else { + ctx.signal.onabort = () => { + pC.cancel('lazy 1:' + ctx.signal.reason); + } + } + return pC; + }; + const fCancellable = cancellable(f, true); + // Signal is aborted afterwards + const pC1 = fCancellable(); + pC1.cancel('cancel reason'); + await expect(pC1).rejects.toBe('lazy 2:lazy 1:cancel reason'); + // Signal is already aborted + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC2 = fCancellable({ signal: abortController.signal }); + await expect(pC2).rejects.toBe('lazy 2:eager 1:cancel reason'); + }); + }); + describe('cancellable propagation', () => { + test('propagate signal', async () => { + let signal: AbortSignal; + const g = async (ctx: ContextCancellable): Promise => { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + // The signal is actually not the same + // it is chained instead + expect(signal).not.toBe(ctx.signal); + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject('early:' + ctx.signal.reason); + } else { + const timeout = setTimeout(() => { + resolve('g'); + }, 10); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject('during:' + ctx.signal.reason); + }); + } + }); + }; + const gCancellable = cancellable(g, true); + const f = async (ctx: ContextCancellable): Promise => { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + signal = ctx.signal; + return await gCancellable(ctx); + }; + const fCancellable = cancellable(f, true); + const pC1 = fCancellable(); + await expect(pC1).resolves.toBe('g'); + expect(signal!.aborted).toBe(false); + const pC2 = fCancellable(); + pC2.cancel('cancel reason'); + await expect(pC2).rejects.toBe('during:cancel reason'); + expect(signal!.aborted).toBe(true); + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC3 = fCancellable({ signal: abortController.signal }); + await expect(pC3).rejects.toBe('early:cancel reason'); + expect(signal!.aborted).toBe(true); + }); + test('nested cancellable - lazy then lazy', async () => { + const f = async(ctx: ContextCancellable): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + }; + const fCancellable = cancellable(cancellable(f, true), true); + const pC = fCancellable(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('throw:cancel reason'); + }); + test('nested cancellable - lazy then eager', async () => { + const f = async(ctx: ContextCancellable): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + }; + const fCancellable = cancellable(cancellable(f, true), false); + const pC = fCancellable(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('nested cancellable - eager then lazy', async () => { + const f = async(ctx: ContextCancellable): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + }; + const fCancellable = cancellable(cancellable(f, false), true); + const pC = fCancellable(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('signal event listeners are removed', async () => { + const f = async (ctx: ContextCancellable): Promise => { + return 'hello world'; + }; + const abortController = new AbortController(); + let listenerCount = 0; + const signal = new Proxy(abortController.signal, { + get(target, prop, receiver) { + if (prop === 'addEventListener') { + return function addEventListener(...args) { + listenerCount++; + return target[prop].apply(this, args); + }; + } else if (prop === 'removeEventListener') { + return function addEventListener(...args) { + listenerCount--; + return target[prop].apply(this, args); + }; + } else { + return Reflect.get(target, prop, receiver); + } + }, + }); + const fCancellable = cancellable(f); + await fCancellable({ signal }); + await fCancellable({ signal }); + const pC = fCancellable({ signal }); + pC.cancel(); + await expect(pC).rejects.toBe(undefined); + expect(listenerCount).toBe(0); + }); + }); +}); diff --git a/tests/contexts/functions/timed.test.ts b/tests/contexts/functions/timed.test.ts new file mode 100644 index 000000000..72dc62cb4 --- /dev/null +++ b/tests/contexts/functions/timed.test.ts @@ -0,0 +1,541 @@ +import type { ContextTimed } from '@/contexts/types'; +import timed from '@/contexts/functions/timed'; +import * as contextsErrors from '@/contexts/errors'; +import Timer from '@/timer/Timer'; +import { + AsyncFunction, + GeneratorFunction, + AsyncGeneratorFunction, + sleep +} from '@/utils'; + +describe('context/functions/timed', () => { + describe('timed syntax', () => { + test('function value', () => { + const f = function ( + ctx: ContextTimed, + check?: (t: Timer) => any, + ): string { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + if (check != null) check(ctx.timer); + return 'hello world'; + }; + const fTimed = timed(f); + fTimed(undefined); + fTimed({}); + fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + }); + expect(fTimed).toBeInstanceOf(Function); + }); + test('function promise', async () => { + const f = function ( + ctx: ContextTimed, + check?: (t: Timer) => any, + ): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + if (check != null) check(ctx.timer); + return new Promise((resolve) => void resolve()); + }; + const fTimed = timed(f); + await fTimed(undefined); + await fTimed({}); + await fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + }); + expect(fTimed).toBeInstanceOf(Function); + }); + test('async function', async () => { + const f = async function ( + ctx: ContextTimed, + check?: (t: Timer) => any, + ): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + if (check != null) check(ctx.timer); + return; + }; + const fTimed = timed(f); + await fTimed(undefined); + await fTimed({}); + await fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + }); + expect(fTimed).toBeInstanceOf(AsyncFunction); + }); + test('generator', () => { + const f = function* ( + ctx: ContextTimed, + check?: (t: Timer) => any, + ): Generator { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + if (check != null) check(ctx.timer); + return; + }; + const fTimed = timed(f); + for (const _ of fTimed()) { + // NOOP + } + for (const _ of fTimed({})) { + // NOOP + } + for (const _ of fTimed({ timer: new Timer({ delay: 150 }) }, (t) => { + expect(t.delay).toBe(150); + })) { + // NOOP + } + expect(fTimed).toBeInstanceOf(GeneratorFunction); + const g = (ctx: ContextTimed, check?: (t: Timer) => any) => f(ctx, check); + const gTimed = timed(g); + for (const _ of gTimed()) { + // NOOP + } + for (const _ of gTimed({})) { + // NOOP + } + for (const _ of gTimed({ timer: new Timer({ delay: 150 }) }, (t) => { + expect(t.delay).toBe(150); + })) { + // NOOP + } + expect(gTimed).not.toBeInstanceOf(GeneratorFunction); + }); + test('async generator', async () => { + const f = async function* ( + ctx: ContextTimed, + check?: (t: Timer) => any, + ): AsyncGenerator { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + if (check != null) check(ctx.timer); + return; + }; + const fTimed = timed(f); + for await (const _ of fTimed()) { + // NOOP + } + for await (const _ of fTimed({})) { + // NOOP + } + for await (const _ of fTimed( + { timer: new Timer({ delay: 200 }) }, + (t) => { + expect(t.delay).toBe(200); + }, + )) { + // NOOP + } + expect(fTimed).toBeInstanceOf(AsyncGeneratorFunction); + const g = (ctx: ContextTimed, check?: (t: Timer) => any) => f(ctx, check); + const gTimed = timed(g); + for await (const _ of gTimed()) { + // NOOP + } + for await (const _ of gTimed({})) { + // NOOP + } + for await (const _ of gTimed( + { timer: new Timer({ delay: 200 }) }, + (t) => { + expect(t.delay).toBe(200); + }, + )) { + // NOOP + } + expect(gTimed).not.toBeInstanceOf(AsyncGeneratorFunction); + }); + }); + describe('timed expiry', () => { + // Timed decorator does not automatically reject the promise + // it only signals that it is aborted + // it is up to the function to decide how to reject + test('async function expiry', async () => { + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf( + contextsErrors.ErrorContextsTimedExpiry, + ); + return 'hello world'; + } + const fTimed = timed(f, 50); + await expect(fTimed()).resolves.toBe('hello world'); + }); + test('async function expiry with custom error', async () => { + class ErrorCustom extends Error {} + /** + * Async function + */ + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf(ErrorCustom); + throw ctx.signal.reason; + }; + const fTimed = timed(f, 50, ErrorCustom); + await expect(fTimed()).rejects.toBeInstanceOf(ErrorCustom); + }); + test('promise function expiry', async () => { + /** + * Regular function returning promise + */ + const f = (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + return sleep(15) + .then(() => { + expect(ctx.signal.aborted).toBe(false); + }) + .then(() => sleep(40)) + .then(() => { + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf( + contextsErrors.ErrorContextsTimedExpiry, + ); + }) + .then(() => { + return 'hello world'; + }); + }; + const fTimed = timed(f, 50); + // const c = new C(); + await expect(fTimed()).resolves.toBe('hello world'); + }); + test('promise function expiry and late rejection', async () => { + let timeout: ReturnType | undefined; + /** + * Regular function that actually rejects + * when the signal is aborted + */ + const f = (ctx: ContextTimed): Promise => { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + } + timeout = setTimeout(() => { + resolve('hello world'); + }, 50000); + ctx.signal.onabort = () => { + clearTimeout(timeout); + timeout = undefined; + reject(ctx.signal.reason); + }; + }); + }; + const fTimed = timed(f, 50); + await expect(fTimed()).rejects.toBeInstanceOf( + contextsErrors.ErrorContextsTimedExpiry, + ); + expect(timeout).toBeUndefined(); + }); + test('promise function expiry and early rejection', async () => { + let timeout: ReturnType | undefined; + /** + * Regular function that actually rejects immediately + */ + const f = (ctx: ContextTimed): Promise => { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + } + timeout = setTimeout(() => { + resolve('hello world'); + }, 50000); + ctx.signal.onabort = () => { + clearTimeout(timeout); + timeout = undefined; + reject(ctx.signal.reason); + }; + }); + }; + const fTimed = timed(f, 0); + await expect(fTimed()).rejects.toBeInstanceOf( + contextsErrors.ErrorContextsTimedExpiry, + ); + expect(timeout).toBeUndefined(); + }); + test('async generator expiry', async () => { + const f = async function *(ctx: ContextTimed): AsyncGenerator { + while (true) { + if (ctx.signal.aborted) { + throw ctx.signal.reason; + } + yield 'hello world'; + } + }; + const fTimed = timed(f, 50); + const g = fTimed(); + await expect(g.next()).resolves.toEqual({ + value: 'hello world', + done: false, + }); + await expect(g.next()).resolves.toEqual({ + value: 'hello world', + done: false, + }); + await sleep(50); + await expect(g.next()).rejects.toThrow( + contextsErrors.ErrorContextsTimedExpiry, + ); + }); + test('generator expiry', async () => { + const f = function* (ctx: ContextTimed): Generator { + while (true) { + if (ctx.signal.aborted) { + throw ctx.signal.reason; + } + yield 'hello world'; + } + }; + const fTimed = timed(f, 50); + const g = fTimed(); + expect(g.next()).toEqual({ value: 'hello world', done: false }); + expect(g.next()).toEqual({ value: 'hello world', done: false }); + await sleep(50); + expect(() => g.next()).toThrow(contextsErrors.ErrorContextsTimedExpiry); + }); + }); + describe('timed propagation', () => { + test('propagate timer and signal', async () => { + let timer: Timer; + let signal: AbortSignal; + const g = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + // Timer and signal will be propagated + expect(timer).toBe(ctx.timer); + expect(signal).toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(50); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + }; + const gTimed = timed(g, 25); + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await gTimed(ctx); + }; + const fTimed = timed(f, 50); + await expect(fTimed()).resolves.toBe('g'); + }); + test('propagate timer only', async () => { + let timer: Timer; + let signal: AbortSignal; + const g = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(timer).toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(50); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + } + const gTimed = timed(g, 25); + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await gTimed({ timer: ctx.timer }); + }; + const fTimed = timed(f, 50); + await expect(fTimed()).resolves.toBe('g'); + }); + test('propagate signal only', async () => { + let timer: Timer; + let signal: AbortSignal; + const g = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + // Even though signal is propagated + // because the timer isn't, the signal here is chained + expect(timer).not.toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(25); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + } + const gTimed = timed(g, 25); + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await gTimed({ signal: ctx.signal }); + }; + const fTimed = timed(f, 50); + await expect(fTimed()).resolves.toBe('g'); + }); + test('propagate nothing', async () => { + let timer: Timer; + let signal: AbortSignal; + const g = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(timer).not.toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(25); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + } + const gTimed = timed(g, 25); + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await gTimed(); + }; + const fTimed = timed(f, 50); + await expect(fTimed()).resolves.toBe('g'); + }); + test('propagated expiry', async () => { + const g = async (timeout: number): Promise => { + const start = performance.now(); + let counter = 0; + while (true) { + if (performance.now() - start > timeout) { + break; + } + await sleep(1); + counter++; + } + return counter; + }; + const h = async (ctx: ContextTimed): Promise => { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + return; + } + const timeout = setTimeout(() => { + resolve('hello world'); + }, 25); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject(ctx.signal.reason); + }); + }); + }; + const hTimed = timed(h, 25); + const f = async (ctx: ContextTimed): Promise => { + // The `g` will use up all the remaining time + const counter = await g(ctx.timer.getTimeout()); + expect(counter).toBeGreaterThan(0); + // The `h` will reject eventually + // it may reject immediately + // it may reject after some time + await hTimed(ctx); + return 'hello world'; + } + const fTimed = timed(f, 25); + await expect(fTimed()).rejects.toThrow( + contextsErrors.ErrorContextsTimedExpiry, + ); + }); + }); + describe('timed explicit timer cancellation or signal abortion', () => { + // If the timer is cancelled + // there will be no timeout error + let ctx_: ContextTimed | undefined; + const f = (ctx: ContextTimed): Promise => { + ctx_ = ctx; + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason + ' begin'); + return; + } + const timeout = setTimeout(() => { + resolve('hello world'); + }, 25); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject(ctx.signal.reason + ' during'); + }); + }); + }; + const fTimed = timed(f, 50); + beforeEach(() => { + ctx_ = undefined; + }); + test('explicit timer cancellation - begin', async () => { + const timer = new Timer({ delay: 100 }); + timer.cancel('reason'); + const p = fTimed({ timer }); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit timer cancellation - during', async () => { + const timer = new Timer({ delay: 100 }); + const p = fTimed({ timer }); + timer.cancel('reason'); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit timer cancellation - during after sleep', async () => { + const timer = new Timer({ delay: 20 }); + const p = fTimed({ timer }); + await sleep(1); + timer.cancel('reason'); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit signal abortion - begin', async () => { + const abortController = new AbortController(); + abortController.abort('reason'); + const p = fTimed({ signal: abortController.signal }); + expect(ctx_!.timer.status).toBe('settled'); + await expect(p).rejects.toBe('reason begin'); + }); + test('explicit signal abortion - during', async () => { + const abortController = new AbortController(); + const p = fTimed({ signal: abortController.signal }); + abortController.abort('reason'); + // Timer is also cancelled immediately + expect(ctx_!.timer.status).toBe('settled'); + await expect(p).rejects.toBe('reason during'); + }); + test('explicit signal signal abortion with passed in timer - during', async () => { + const timer = new Timer({ delay: 100 }); + const abortController = new AbortController(); + const p = fTimed({ timer, signal: abortController.signal }); + abortController.abort('abort reason'); + expect(ctx_!.timer.status).toBe('settled'); + expect(timer.status).toBe('settled'); + expect(ctx_!.signal.aborted).toBe(true); + await expect(p).rejects.toBe('abort reason during'); + }); + test('explicit timer cancellation and signal abortion - begin', async () => { + const timer = new Timer({ delay: 100 }); + timer.cancel('timer reason'); + const abortController = new AbortController(); + abortController.abort('abort reason'); + const p = fTimed({ timer, signal: abortController.signal }); + expect(ctx_!.timer.status).toBe('settled'); + expect(ctx_!.signal.aborted).toBe(true); + await expect(p).rejects.toBe('abort reason begin'); + }); + }); +}); From 82eb0291a949685da317266e9e515edb4175ed04 Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Fri, 9 Sep 2022 17:34:11 +1000 Subject: [PATCH 10/32] fix(contexts): changed timeout exception to `ErrorContextsTimedTimeout` --- src/contexts/decorators/timed.ts | 2 +- src/contexts/decorators/timedCancellable.ts | 2 +- src/contexts/errors.ts | 4 ++-- src/contexts/functions/timed.ts | 2 +- tests/contexts/decorators/timed.test.ts | 14 +++++++------- tests/contexts/functions/timed.test.ts | 14 +++++++------- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/contexts/decorators/timed.ts b/src/contexts/decorators/timed.ts index 218087411..038b9ebaf 100644 --- a/src/contexts/decorators/timed.ts +++ b/src/contexts/decorators/timed.ts @@ -122,7 +122,7 @@ function setupContext( */ function timed( delay: number = Infinity, - errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedExpiry, + errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedTimeOut, ) { return ( target: any, diff --git a/src/contexts/decorators/timedCancellable.ts b/src/contexts/decorators/timedCancellable.ts index 8b6357dc3..995482b27 100644 --- a/src/contexts/decorators/timedCancellable.ts +++ b/src/contexts/decorators/timedCancellable.ts @@ -10,7 +10,7 @@ function timedCancellable( lazy: boolean = false, delay: number = Infinity, - errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedExpiry, + errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedTimeOut, ) { } diff --git a/src/contexts/errors.ts b/src/contexts/errors.ts index 0c29aa149..78c5b5af6 100644 --- a/src/contexts/errors.ts +++ b/src/contexts/errors.ts @@ -2,9 +2,9 @@ import { ErrorPolykey, sysexits } from '../errors'; class ErrorContexts extends ErrorPolykey {} -class ErrorContextsTimedExpiry extends ErrorContexts { +class ErrorContextsTimedTimeOut extends ErrorContexts { static description = 'Aborted due to timer expiration'; exitCode = sysexits.UNAVAILABLE; } -export { ErrorContexts, ErrorContextsTimedExpiry }; +export { ErrorContexts, ErrorContextsTimedTimeOut }; diff --git a/src/contexts/functions/timed.ts b/src/contexts/functions/timed.ts index a94e90215..07e66970d 100644 --- a/src/contexts/functions/timed.ts +++ b/src/contexts/functions/timed.ts @@ -109,7 +109,7 @@ function timed< >( f: (ctx: C, ...params: P) => any, delay: number = Infinity, - errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedExpiry, + errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedTimeOut, ): ( ...params: ContextAndParameters) => any { if (f instanceof utils.AsyncFunction) { return async (...params) => { diff --git a/tests/contexts/decorators/timed.test.ts b/tests/contexts/decorators/timed.test.ts index 382c5dac8..f0c8e790d 100644 --- a/tests/contexts/decorators/timed.test.ts +++ b/tests/contexts/decorators/timed.test.ts @@ -289,7 +289,7 @@ describe('context/decorators/timed', () => { await sleep(40); expect(ctx.signal.aborted).toBe(true); expect(ctx.signal.reason).toBeInstanceOf( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); return 'hello world'; } @@ -335,7 +335,7 @@ describe('context/decorators/timed', () => { .then(() => { expect(ctx.signal.aborted).toBe(true); expect(ctx.signal.reason).toBeInstanceOf( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); }) .then(() => { @@ -373,7 +373,7 @@ describe('context/decorators/timed', () => { } const c = new C(); await expect(c.f()).rejects.toBeInstanceOf( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); expect(timeout).toBeUndefined(); }); @@ -403,7 +403,7 @@ describe('context/decorators/timed', () => { } const c = new C(); await expect(c.f()).rejects.toBeInstanceOf( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); expect(timeout).toBeUndefined(); }); @@ -432,7 +432,7 @@ describe('context/decorators/timed', () => { }); await sleep(50); await expect(g.next()).rejects.toThrow( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); }); test('generator expiry', async () => { @@ -453,7 +453,7 @@ describe('context/decorators/timed', () => { expect(g.next()).toEqual({ value: 'hello world', done: false }); expect(g.next()).toEqual({ value: 'hello world', done: false }); await sleep(50); - expect(() => g.next()).toThrow(contextsErrors.ErrorContextsTimedExpiry); + expect(() => g.next()).toThrow(contextsErrors.ErrorContextsTimedTimeOut); }); }); describe('timed decorator propagation', () => { @@ -636,7 +636,7 @@ describe('context/decorators/timed', () => { } const c = new C(); await expect(c.f()).rejects.toThrow( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); }); }); diff --git a/tests/contexts/functions/timed.test.ts b/tests/contexts/functions/timed.test.ts index 72dc62cb4..ca75a1771 100644 --- a/tests/contexts/functions/timed.test.ts +++ b/tests/contexts/functions/timed.test.ts @@ -160,7 +160,7 @@ describe('context/functions/timed', () => { await sleep(40); expect(ctx.signal.aborted).toBe(true); expect(ctx.signal.reason).toBeInstanceOf( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); return 'hello world'; } @@ -198,7 +198,7 @@ describe('context/functions/timed', () => { .then(() => { expect(ctx.signal.aborted).toBe(true); expect(ctx.signal.reason).toBeInstanceOf( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); }) .then(() => { @@ -232,7 +232,7 @@ describe('context/functions/timed', () => { }; const fTimed = timed(f, 50); await expect(fTimed()).rejects.toBeInstanceOf( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); expect(timeout).toBeUndefined(); }); @@ -258,7 +258,7 @@ describe('context/functions/timed', () => { }; const fTimed = timed(f, 0); await expect(fTimed()).rejects.toBeInstanceOf( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); expect(timeout).toBeUndefined(); }); @@ -283,7 +283,7 @@ describe('context/functions/timed', () => { }); await sleep(50); await expect(g.next()).rejects.toThrow( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); }); test('generator expiry', async () => { @@ -300,7 +300,7 @@ describe('context/functions/timed', () => { expect(g.next()).toEqual({ value: 'hello world', done: false }); expect(g.next()).toEqual({ value: 'hello world', done: false }); await sleep(50); - expect(() => g.next()).toThrow(contextsErrors.ErrorContextsTimedExpiry); + expect(() => g.next()).toThrow(contextsErrors.ErrorContextsTimedTimeOut); }); }); describe('timed propagation', () => { @@ -452,7 +452,7 @@ describe('context/functions/timed', () => { } const fTimed = timed(f, 25); await expect(fTimed()).rejects.toThrow( - contextsErrors.ErrorContextsTimedExpiry, + contextsErrors.ErrorContextsTimedTimeOut, ); }); }); From e0d7182e754ff69a3924eccecac1638708e05a3f Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Wed, 7 Sep 2022 22:09:36 +1000 Subject: [PATCH 11/32] chore: updated `@types/node` and typescript --- package-lock.json | 28 ++++++++++++++-------------- package.json | 4 ++-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index d7de58ddd..f9aa13d13 100644 --- a/package-lock.json +++ b/package-lock.json @@ -55,7 +55,7 @@ "@types/google-protobuf": "^3.7.4", "@types/jest": "^28.1.3", "@types/nexpect": "^0.4.31", - "@types/node": "^16.11.49", + "@types/node": "^16.11.57", "@types/node-forge": "^0.10.4", "@types/pako": "^1.0.2", "@types/prompts": "^2.0.13", @@ -89,7 +89,7 @@ "ts-node": "^10.9.1", "tsconfig-paths": "^3.9.0", "typedoc": "^0.22.15", - "typescript": "^4.5.2" + "typescript": "^4.7.4" } }, "node_modules/@ampproject/remapping": { @@ -3033,9 +3033,9 @@ } }, "node_modules/@types/node": { - "version": "16.11.49", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.49.tgz", - "integrity": "sha512-Abq9fBviLV93OiXMu+f6r0elxCzRwc0RC5f99cU892uBITL44pTvgvEqlRlPRi8EGcO1z7Cp8A4d0s/p3J/+Nw==" + "version": "16.11.57", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.57.tgz", + "integrity": "sha512-diBb5AE2V8h9Fs9zEDtBwSeLvIACng/aAkdZ3ujMV+cGuIQ9Nc/V+wQqurk9HJp8ni5roBxQHW21z/ZYbGDivg==" }, "node_modules/@types/node-forge": { "version": "0.10.10", @@ -11140,9 +11140,9 @@ } }, "node_modules/typescript": { - "version": "4.7.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.3.tgz", - "integrity": "sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA==", + "version": "4.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz", + "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -13760,9 +13760,9 @@ } }, "@types/node": { - "version": "16.11.49", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.49.tgz", - "integrity": "sha512-Abq9fBviLV93OiXMu+f6r0elxCzRwc0RC5f99cU892uBITL44pTvgvEqlRlPRi8EGcO1z7Cp8A4d0s/p3J/+Nw==" + "version": "16.11.57", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.57.tgz", + "integrity": "sha512-diBb5AE2V8h9Fs9zEDtBwSeLvIACng/aAkdZ3ujMV+cGuIQ9Nc/V+wQqurk9HJp8ni5roBxQHW21z/ZYbGDivg==" }, "@types/node-forge": { "version": "0.10.10", @@ -19760,9 +19760,9 @@ } }, "typescript": { - "version": "4.7.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.3.tgz", - "integrity": "sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA==", + "version": "4.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz", + "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==", "dev": true }, "uglify-js": { diff --git a/package.json b/package.json index ff66caae9..ce5da85c3 100644 --- a/package.json +++ b/package.json @@ -119,7 +119,7 @@ "@types/google-protobuf": "^3.7.4", "@types/jest": "^28.1.3", "@types/nexpect": "^0.4.31", - "@types/node": "^16.11.49", + "@types/node": "^16.11.57", "@types/node-forge": "^0.10.4", "@types/pako": "^1.0.2", "@types/prompts": "^2.0.13", @@ -153,6 +153,6 @@ "ts-node": "^10.9.1", "tsconfig-paths": "^3.9.0", "typedoc": "^0.22.15", - "typescript": "^4.5.2" + "typescript": "^4.7.4" } } From 4f8e8346e3bd9fb01fd23203f90ea996d6a34c83 Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Sun, 11 Sep 2022 16:13:53 +1000 Subject: [PATCH 12/32] feat: introducing `tasks` domain for managing background asynchronous tasks * Created a `Plug` class for managing the locking of scheduling and queuing loops * Using `performance.now()` for acquiring the current time --- src/tasks/Queue.ts | 692 ++++++++++++++++++++++++++++++++++ src/tasks/Scheduler.ts | 442 ++++++++++++++++++++++ src/tasks/Task.ts | 101 +++++ src/tasks/errors.ts | 80 ++++ src/tasks/index.ts | 4 + src/tasks/types.ts | 110 ++++++ src/tasks/utils.ts | 98 +++++ src/types.ts | 6 + src/utils/Plug.ts | 36 ++ src/utils/index.ts | 1 + src/utils/utils.ts | 11 +- tests/tasks/Queue.test.ts | 415 ++++++++++++++++++++ tests/tasks/Scheduler.test.ts | 119 ++++++ tests/tasks/utils.test.ts | 29 ++ tests/utils/Plug.test.ts | 19 + 15 files changed, 2158 insertions(+), 5 deletions(-) create mode 100644 src/tasks/Queue.ts create mode 100644 src/tasks/Scheduler.ts create mode 100644 src/tasks/Task.ts create mode 100644 src/tasks/errors.ts create mode 100644 src/tasks/index.ts create mode 100644 src/tasks/types.ts create mode 100644 src/tasks/utils.ts create mode 100644 src/utils/Plug.ts create mode 100644 tests/tasks/Queue.test.ts create mode 100644 tests/tasks/Scheduler.test.ts create mode 100644 tests/tasks/utils.test.ts create mode 100644 tests/utils/Plug.test.ts diff --git a/src/tasks/Queue.ts b/src/tasks/Queue.ts new file mode 100644 index 000000000..35d90a6f8 --- /dev/null +++ b/src/tasks/Queue.ts @@ -0,0 +1,692 @@ +import type { DB, LevelPath, KeyPath } from '@matrixai/db'; +import type { + TaskData, + TaskHandlerId, + TaskHandler, + TaskTimestamp, + TaskParameters, + TaskIdEncoded, +} from './types'; +import type KeyManager from '../keys/KeyManager'; +import type { DBTransaction } from '@matrixai/db'; +import type { TaskId, TaskGroup } from './types'; +import EventEmitter from 'events'; +import Logger from '@matrixai/logger'; +import { + CreateDestroyStartStop, + ready, +} from '@matrixai/async-init/dist/CreateDestroyStartStop'; +import { IdInternal } from '@matrixai/id'; +import { RWLockReader } from '@matrixai/async-locks'; +import { extractTs } from '@matrixai/id/dist/IdSortable'; +import * as tasksErrors from './errors'; +import * as tasksUtils from './utils'; +import Task from './Task'; +import { Plug } from '../utils/index'; + +interface Queue extends CreateDestroyStartStop {} +@CreateDestroyStartStop( + new tasksErrors.ErrorQueueRunning(), + new tasksErrors.ErrorQueueDestroyed(), +) +class Queue { + public static async createQueue({ + db, + keyManager, + handlers = {}, + delay = false, + concurrencyLimit = Number.POSITIVE_INFINITY, + logger = new Logger(this.name), + fresh = false, + }: { + db: DB; + keyManager: KeyManager; + handlers?: Record; + delay?: boolean; + concurrencyLimit?: number; + logger?: Logger; + fresh?: boolean; + }) { + logger.info(`Creating ${this.name}`); + const queue = new this({ db, keyManager, concurrencyLimit, logger }); + await queue.start({ handlers, delay, fresh }); + logger.info(`Created ${this.name}`); + return queue; + } + + // Concurrency variables + public concurrencyLimit: number; + protected concurrencyCount: number = 0; + protected concurrencyPlug: Plug = new Plug(); + protected activeTasksPlug: Plug = new Plug(); + + protected logger: Logger; + protected db: DB; + protected queueDbPath: LevelPath = [this.constructor.name]; + /** + * Tasks collection + * `tasks/{TaskId} -> {json(Task)}` + */ + public readonly queueTasksDbPath: LevelPath = [...this.queueDbPath, 'tasks']; + public readonly queueStartTimeDbPath: LevelPath = [ + ...this.queueDbPath, + 'startTime', + ]; + /** + * This is used to track pending tasks in order of start time + */ + protected queueDbTimestampPath: LevelPath = [ + ...this.queueDbPath, + 'timestamp', + ]; + // FIXME: remove this path, data is part of the task data record + protected queueDbMetadataPath: LevelPath = [...this.queueDbPath, 'metadata']; + /** + * Tracks actively running tasks + */ + protected queueDbActivePath: LevelPath = [...this.queueDbPath, 'active']; + /** + * Tasks by groups + * `groups/...taskGroup: Array -> {raw(TaskId)}` + */ + public readonly queueGroupsDbPath: LevelPath = [ + ...this.queueDbPath, + 'groups', + ]; + /** + * Last Task Id + */ + public readonly queueLastTaskIdPath: KeyPath = [ + ...this.queueDbPath, + 'lastTaskId', + ]; + + // /** + // * Listeners for task execution + // * When a task is executed, these listeners are synchronously executed + // * The listeners are intended for resolving or rejecting task promises + // */ + // protected listeners: Map> = new Map(); + + // variables to consuming tasks + protected activeTaskLoop: Promise | null = null; + protected taskLoopPlug: Plug = new Plug(); + protected taskLoopEnding: boolean; + // FIXME: might not be needed + protected cleanUpLock: RWLockReader = new RWLockReader(); + + protected handlers: Map = new Map(); + protected taskPromises: Map> = new Map(); + protected taskEvents: EventEmitter = new EventEmitter(); + protected keyManager: KeyManager; + protected generateTaskId: () => TaskId; + + public constructor({ + db, + keyManager, + concurrencyLimit, + logger, + }: { + db: DB; + keyManager: KeyManager; + concurrencyLimit: number; + logger: Logger; + }) { + this.logger = logger; + this.concurrencyLimit = concurrencyLimit; + this.db = db; + this.keyManager = keyManager; + } + + public async start({ + handlers = {}, + delay = false, + fresh = false, + }: { + handlers?: Record; + delay?: boolean; + fresh?: boolean; + } = {}): Promise { + this.logger.info(`Starting ${this.constructor.name}`); + if (fresh) { + this.handlers.clear(); + await this.db.clear(this.queueDbPath); + } + const lastTaskId = await this.getLastTaskId(); + this.generateTaskId = tasksUtils.createTaskIdGenerator( + this.keyManager.getNodeId(), + lastTaskId, + ); + for (const taskHandlerId in handlers) { + this.handlers.set( + taskHandlerId as TaskHandlerId, + handlers[taskHandlerId], + ); + } + if (!delay) await this.startTasks(); + this.logger.info(`Started ${this.constructor.name}`); + } + + public async stop(): Promise { + this.logger.info(`Stopping ${this.constructor.name}`); + await this.stopTasks(); + this.logger.info(`Stopped ${this.constructor.name}`); + } + + public async destroy() { + this.logger.info(`Destroying ${this.constructor.name}`); + this.handlers.clear(); + await this.db.clear(this.queueDbPath); + this.logger.info(`Destroyed ${this.constructor.name}`); + } + + // Promises are "connected" to events + // + // when tasks are "dispatched" to the queue + // they are actually put into a persistent system + // then we proceed to execution + // + // a task here is a function + // this is already managed by the Scheduler + // along with the actual function itself? + // we also have a priority + // + // t is a task + // but it's actually just a function + // and in this case + // note that we are "passing" in the parameters at this point + // but it is any function + // () => taskHandler(parameters) + // + // it returns a "task" + // that should be used like a "lazy" promise + // the actual task function depends on the situation + // don't we need to know actual metadata + // wait a MINUTE + // if we are "persisting" it + // do we persist it here? + + /** + * Pushes tasks into the persistent database + */ + @ready(new tasksErrors.ErrorQueueNotRunning()) + public async pushTask( + taskId: TaskId, + taskTimestampKey: Buffer, + tran?: DBTransaction, + ): Promise { + if (tran == null) { + return this.db.withTransactionF((tran) => + this.pushTask(taskId, taskTimestampKey, tran), + ); + } + + this.logger.info('adding task'); + await tran.lock([ + [...this.queueDbTimestampPath, 'loopSerialisation'].join(''), + 'read', + ]); + await tran.put( + [...this.queueStartTimeDbPath, taskId.toBuffer()], + taskTimestampKey, + true, + ); + await tran.put( + [...this.queueDbTimestampPath, taskTimestampKey], + taskId.toBuffer(), + true, + ); + await tran.put( + [...this.queueDbMetadataPath, taskId.toBuffer()], + taskTimestampKey, + true, + ); + tran.queueSuccess(async () => await this.taskLoopPlug.unplug()); + } + + /** + * Removes a task from the persistent database + */ + // @ready(new tasksErrors.ErrorQueueNotRunning(), false, ['stopping', 'starting']) + public async removeTask(taskId: TaskId, tran?: DBTransaction) { + if (tran == null) { + return this.db.withTransactionF((tran) => this.removeTask(taskId, tran)); + } + + this.logger.info('removing task'); + await tran.lock([ + [...this.queueDbTimestampPath, 'loopSerialisation'].join(''), + 'read', + ]); + const timestampBuffer = await tran.get( + [...this.queueDbMetadataPath, taskId.toBuffer()], + true, + ); + // Noop + if (timestampBuffer == null) return; + // Removing records + await tran.del([...this.queueDbTimestampPath, timestampBuffer]); + await tran.del([...this.queueDbMetadataPath, taskId.toBuffer()]); + await tran.del([...this.queueDbActivePath, taskId.toBuffer()]); + } + + /** + * This will get the next task based on priority + */ + protected async getNextTask( + tran?: DBTransaction, + ): Promise { + if (tran == null) { + return this.db.withTransactionF((tran) => this.getNextTask(tran)); + } + + await tran.lock([ + [...this.queueDbTimestampPath, 'loopSerialisation'].join(''), + 'write', + ]); + // Read out the database until we read a task not already executing + let taskId: TaskId | undefined; + for await (const [, taskIdBuffer] of tran.iterator( + this.queueDbTimestampPath, + )) { + taskId = IdInternal.fromBuffer(taskIdBuffer); + const exists = await tran.get( + [...this.queueDbActivePath, taskId.toBuffer()], + true, + ); + // Looking for an inactive task + if (exists == null) break; + taskId = undefined; + } + if (taskId == null) return; + await tran.put( + [...this.queueDbActivePath, taskId.toBuffer()], + Buffer.alloc(0, 0), + true, + ); + return taskId; + } + + @ready(new tasksErrors.ErrorQueueNotRunning(), false, ['starting']) + public async startTasks() { + // Nop if running + if (this.activeTaskLoop != null) return; + + this.activeTaskLoop = this.initTaskLoop(); + // Unplug if tasks exist to be consumed + for await (const _ of this.db.iterator(this.queueDbTimestampPath, { + limit: 1, + })) { + // Unplug if tasks exist + await this.taskLoopPlug.unplug(); + } + } + + @ready(new tasksErrors.ErrorQueueNotRunning(), false, ['stopping']) + public async stopTasks() { + this.taskLoopEnding = true; + await this.taskLoopPlug.unplug(); + await this.concurrencyPlug.unplug(); + await this.activeTaskLoop; + this.activeTaskLoop = null; + // FIXME: likely not needed, remove + await this.cleanUpLock.waitForUnlock(); + } + + protected async initTaskLoop() { + this.logger.info('initializing task loop'); + this.taskLoopEnding = false; + await this.taskLoopPlug.plug(); + const pace = async () => { + if (this.taskLoopEnding) return false; + await this.taskLoopPlug.waitForUnplug(); + await this.concurrencyPlug.waitForUnplug(); + return !this.taskLoopEnding; + }; + while (await pace()) { + // Check for task + const nextTaskId = await this.getNextTask(); + if (nextTaskId == null) { + this.logger.info('no task found, waiting'); + await this.taskLoopPlug.plug(); + continue; + } + + // Do the task with concurrency here. + // We need to call whatever dispatches tasks here + // and hook lifecycle to the promise. + // call scheduler. handleTask? + const taskIdEncoded = tasksUtils.encodeTaskId(nextTaskId); + await this.concurrencyIncrement(); + const prom = this.handleTask(nextTaskId); + this.logger.info(`started task ${taskIdEncoded}`); + + const [cleanupRelease] = await this.cleanUpLock.read()(); + const onFinally = async () => { + await this.concurrencyDecrement(); + await cleanupRelease(); + }; + + void prom.then( + async () => { + await this.removeTask(nextTaskId); + // TODO: emit an event for completed task + await onFinally(); + }, + async () => { + // FIXME: should only remove failed tasks but not cancelled + await this.removeTask(nextTaskId); + // TODO: emit an event for a failed or cancelled task + await onFinally(); + }, + ); + } + await this.activeTasksPlug.waitForUnplug(); + this.logger.info('dispatching ending'); + } + + // Concurrency limiting methods + /** + * Awaits an open slot in the concurrency. + * Must be paired with `concurrencyDecrement` when operation is done. + */ + + /** + * Increment and concurrencyPlug if full + */ + protected async concurrencyIncrement() { + if (this.concurrencyCount < this.concurrencyLimit) { + this.concurrencyCount += 1; + await this.activeTasksPlug.plug(); + if (this.concurrencyCount >= this.concurrencyLimit) { + await this.concurrencyPlug.plug(); + } + } + } + + /** + * Decrement and unplugs, resolves concurrencyActivePromise if empty + */ + protected async concurrencyDecrement() { + this.concurrencyCount -= 1; + if (this.concurrencyCount < this.concurrencyLimit) { + await this.concurrencyPlug.unplug(); + } + if (this.concurrencyCount === 0) { + await this.activeTasksPlug.unplug(); + } + } + + /** + * Will resolve when the concurrency counter reaches 0 + */ + public async allActiveTasksSettled() { + await this.activeTasksPlug.waitForUnplug(); + } + + /** + * IF a handler does not exist + * if the task is executed + * then an exception is thrown + * if listener exists, the exception is passed into this listener function + * if it doesn't exist, then it's just a reference exception in general, this can be logged + * There's nothing else to do + */ + // @ready(new tasksErrors.ErrorSchedulerNotRunning()) + // protected registerListener( + // taskId: TaskId, + // taskListener: TaskListener + // ): void { + // const taskIdString = taskId.toString() as TaskIdString; + // const taskListeners = this.listeners.get(taskIdString); + // if (taskListeners != null) { + // taskListeners.push(taskListener); + // } else { + // this.listeners.set(taskIdString, [taskListener]); + // } + // } + + // @ready(new tasksErrors.ErrorSchedulerNotRunning()) + // protected deregisterListener( + // taskId: TaskId, + // taskListener: TaskListener + // ): void { + // const taskIdString = taskId.toString() as TaskIdString; + // const taskListeners = this.listeners.get(taskIdString); + // if (taskListeners == null || taskListeners.length < 1) return; + // const index = taskListeners.indexOf(taskListener); + // if (index !== -1) { + // taskListeners.splice(index, 1); + // } + // } + + protected async handleTask(taskId: TaskId) { + // Get the task information and use the relevant handler + // throw and error if the task does not exist + // throw an error if the handler does not exist. + + return await this.db.withTransactionF(async (tran) => { + // Getting task information + const taskData = await tran.get([ + ...this.queueTasksDbPath, + taskId.toBuffer(), + ]); + if (taskData == null) throw Error('TEMP task not found'); + // Getting handler + const handler = this.getHandler(taskData.handlerId); + if (handler == null) throw Error('TEMP handler not found'); + + const prom = handler(...taskData.parameters); + + // Add the promise to the map and hook any lifecycle stuff + const taskIdEncoded = tasksUtils.encodeTaskId(taskId); + return prom + .finally(async () => { + // Cleaning up is a separate transaction + await this.db.withTransactionF(async (tran) => { + const taskTimestampKeybuffer = await tran.get( + [...this.queueStartTimeDbPath, taskId.toBuffer()], + true, + ); + await tran.del([...this.queueTasksDbPath, taskId.toBuffer()]); + await tran.del([...this.queueStartTimeDbPath, taskId.toBuffer()]); + if (taskData.taskGroup != null) { + await tran.del([ + ...this.queueGroupsDbPath, + ...taskData.taskGroup, + taskTimestampKeybuffer!, + ]); + } + }); + }) + .then( + (value) => { + this.taskEvents.emit(taskIdEncoded, value); + return value; + }, + (reason) => { + this.taskEvents.emit(taskIdEncoded, reason); + throw reason; + }, + ); + }); + } + + public getHandler(handlerId: TaskHandlerId): TaskHandler | undefined { + return this.handlers.get(handlerId); + } + + public getHandlers(): Record { + return Object.fromEntries(this.handlers); + } + + /** + * Registers a handler for tasks with the same `TaskHandlerId` + * If tasks are dispatched without their respective handler, + * the scheduler will throw `tasksErrors.ErrorSchedulerHandlerMissing` + */ + public registerHandler(handlerId: TaskHandlerId, handler: TaskHandler) { + this.handlers.set(handlerId, handler); + } + + /** + * Deregisters a handler + */ + public deregisterHandler(handlerId: TaskHandlerId) { + this.handlers.delete(handlerId); + } + + @ready(new tasksErrors.ErrorSchedulerNotRunning()) + public getTaskP(taskId: TaskId, tran?: DBTransaction): Promise { + const taskIdEncoded = tasksUtils.encodeTaskId(taskId); + // This will return a task promise if it already exists + const existingTaskPromise = this.taskPromises.get(taskIdEncoded); + if (existingTaskPromise != null) return existingTaskPromise; + + // If the task exist then it will create the task promise and return that + const newTaskPromise = new Promise((resolve, reject) => { + const resultListener = (result) => { + if (result instanceof Error) reject(result); + else resolve(result); + }; + this.taskEvents.once(taskIdEncoded, resultListener); + // If not task promise exists then with will check if the task exists + void (tran ?? this.db) + .get([...this.queueTasksDbPath, taskId.toBuffer()], true) + .then( + (taskData) => { + if (taskData == null) { + this.taskEvents.removeListener(taskIdEncoded, resultListener); + reject(Error('TEMP task not found')); + } + }, + (reason) => reject(reason), + ); + }).finally(() => { + this.taskPromises.delete(taskIdEncoded); + }); + this.taskPromises.set(taskIdEncoded, newTaskPromise); + return newTaskPromise; + } + + @ready(new tasksErrors.ErrorSchedulerNotRunning()) + public async *getGroupTasks( + taskGroup: TaskGroup, + tran?: DBTransaction, + ): AsyncGenerator { + if (tran == null) { + return yield* this.db.withTransactionG((tran) => + this.getGroupTasks(taskGroup, tran), + ); + } + + for await (const [, taskIdBuffer] of tran.iterator([ + ...this.queueGroupsDbPath, + ...taskGroup, + ])) { + yield IdInternal.fromBuffer(taskIdBuffer); + } + } + + @ready(new tasksErrors.ErrorSchedulerNotRunning(), false, ['starting']) + public async getLastTaskId( + tran?: DBTransaction, + ): Promise { + const lastTaskIdBuffer = await (tran ?? this.db).get( + this.queueLastTaskIdPath, + true, + ); + if (lastTaskIdBuffer == null) return; + return IdInternal.fromBuffer(lastTaskIdBuffer); + } + + public async createTask( + handlerId: TaskHandlerId, + parameters: TaskParameters = [], + priority: number = 0, + taskGroup?: TaskGroup, + lazy: boolean = false, + tran?: DBTransaction, + ): Promise> { + if (tran == null) { + return this.db.withTransactionF((tran) => + this.createTask(handlerId, parameters, priority, taskGroup, lazy, tran), + ); + } + + // This does a combination of things + // 1. create save the new task within the DB + // 2. if timer exist and new delay is longer then just return the task + // 3. else cancel the timer and create a new one with the delay + const taskId = this.generateTaskId(); + // Timestamp extracted from `IdSortable` is a floating point in seconds + // with subsecond fractionals, multiply it by 1000 gives us milliseconds + const taskTimestamp = Math.trunc(extractTs(taskId) * 1000) as TaskTimestamp; + const taskPriority = tasksUtils.toPriority(priority); + const taskData: TaskData = { + handlerId, + parameters, + timestamp: taskTimestamp, + taskGroup, + priority: taskPriority, + }; + const taskIdBuffer = taskId.toBuffer(); + // Save the task + await tran.put([...this.queueTasksDbPath, taskIdBuffer], taskData); + // Save the last task ID + await tran.put(this.queueLastTaskIdPath, taskIdBuffer, true); + + // Adding to group + if (taskGroup != null) { + await tran.put( + [...this.queueGroupsDbPath, ...taskGroup, taskIdBuffer], + taskIdBuffer, + true, + ); + } + let taskPromise: Promise | null = null; + if (!lazy) { + taskPromise = this.getTaskP(taskId, tran); + } + return new Task( + this, + taskId, + handlerId, + parameters, + taskTimestamp, + // Delay, + taskGroup, + taskPriority, + taskPromise, + ); + } +} + +export default Queue; + +// Epic queue +// need to do a couple things: +// 1. integrate fast-check +// 2. integrate span checks +// 3. might also consider span logs? +// 4. open tracing observability +// 5. structured logging +// 6. async hooks to get traced promises to understand the situation +// 7. do we also get fantasy land promises? and async cancellable stuff? +// 8. task abstractions? +// need to use the db for this +// 9. priority structure +// 10. timers +// abort controller + +// kinetic data structure +// the priority grows as a function of time +// order by priority <- this thing has a static value +// in a key value DB, you can maintain sorted index of values +// IDs can be lexicographically sortable + +// this is a persistent queue +// of tasks that should be EXECUTED right now +// the scheduler is a persistent scheduler of scheduled tasks +// tasks get pushed from the scheduler into the queue +// the queue connects to the WorkerManager diff --git a/src/tasks/Scheduler.ts b/src/tasks/Scheduler.ts new file mode 100644 index 000000000..56a90e000 --- /dev/null +++ b/src/tasks/Scheduler.ts @@ -0,0 +1,442 @@ +import type { DB, LevelPath } from '@matrixai/db'; +import type { TaskData, TaskIdString } from './types'; +import type KeyManager from '../keys/KeyManager'; +import type Task from './Task'; +import type Queue from './Queue'; +import type { DBTransaction } from '@matrixai/db'; +import type { + TaskDelay, + TaskHandlerId, + TaskId, + TaskParameters, + TaskGroup, +} from './types'; +import Logger, { LogLevel } from '@matrixai/logger'; +import { IdInternal } from '@matrixai/id'; +import { + CreateDestroyStartStop, + ready, +} from '@matrixai/async-init/dist/CreateDestroyStartStop'; +import lexi from 'lexicographic-integer'; +import * as tasksUtils from './utils'; +import * as tasksErrors from './errors'; +import { Plug } from '../utils/index'; + +interface Scheduler extends CreateDestroyStartStop {} +@CreateDestroyStartStop( + new tasksErrors.ErrorSchedulerRunning(), + new tasksErrors.ErrorSchedulerDestroyed(), +) +class Scheduler { + /** + * Create the scheduler, which will create its own Queue + * This will automatically start the scheduler + * If the scheduler needs to be started after the fact + * Make sure to construct it, and then call `start` manually + */ + public static async createScheduler({ + db, + queue, + logger = new Logger(this.name), + delay = false, + fresh = false, + }: { + db: DB; + queue: Queue; + logger?: Logger; + delay?: boolean; + fresh?: boolean; + }): Promise { + logger.info(`Creating ${this.name}`); + const scheduler = new this({ db, queue, logger }); + await scheduler.start({ delay, fresh }); + logger.info(`Created ${this.name}`); + return scheduler; + } + + protected logger: Logger; + protected db: DB; + protected keyManager: KeyManager; + protected queue: Queue; + // TODO: remove this? + protected promises: Map> = new Map(); + + // TODO: swap this out for the timer system later + + protected dispatchTimer?: ReturnType; + protected dispatchTimerTimestamp: number = Number.POSITIVE_INFINITY; + protected pendingDispatch: Promise | null = null; + protected dispatchPlug: Plug = new Plug(); + protected dispatchEnding: boolean = false; + + protected schedulerDbPath: LevelPath = [this.constructor.name]; + + /** + * Tasks scheduled by time + * `time/{lexi(TaskTimestamp + TaskDelay)} -> {raw(TaskId)}` + */ + protected schedulerTimeDbPath: LevelPath = [...this.schedulerDbPath, 'time']; + + // /** + // * Tasks queued for execution + // * `pending/{lexi(TaskPriority)}/{lexi(TaskTimestamp + TaskDelay)} -> {raw(TaskId)}` + // */ + // protected schedulerPendingDbPath: LevelPath = [ + // ...this.schedulerDbPath, + // 'pending', + // ]; + + // /** + // * Task handlers + // * `handlers/{TaskHandlerId}/{TaskId} -> {raw(TaskId)}` + // */ + // protected schedulerHandlersDbPath: LevelPath = [ + // ...this.schedulerDbPath, + // 'handlers', + // ]; + + public constructor({ + db, + queue, + logger, + }: { + db: DB; + queue: Queue; + logger: Logger; + }) { + this.logger = logger; + this.db = db; + this.queue = queue; + } + + public get isDispatching(): boolean { + return this.dispatchTimer != null; + } + + public async start({ + delay = false, + fresh = false, + }: { + delay?: boolean; + fresh?: boolean; + } = {}): Promise { + this.logger.setLevel(LogLevel.INFO); + this.logger.setLevel(LogLevel.INFO); + this.logger.info(`Starting ${this.constructor.name}`); + if (fresh) { + await this.db.clear(this.schedulerDbPath); + } + // Don't start dispatching if we still want to register handlers + if (!delay) { + await this.startDispatching(); + } + this.logger.info(`Started ${this.constructor.name}`); + } + + /** + * Stop the scheduler + * This does not clear the handlers nor promises + * This maintains any registered handlers and awaiting promises + */ + public async stop(): Promise { + this.logger.info(`Stopping ${this.constructor.name}`); + await this.stopDispatching(); + this.logger.info(`Stopped ${this.constructor.name}`); + } + + /** + * Destroys the scheduler + * This must first clear all handlers + * Then it needs to cancel all promises + * Finally destroys all underlying state + */ + public async destroy() { + this.logger.info(`Destroying ${this.constructor.name}`); + await this.db.clear(this.schedulerDbPath); + this.logger.info(`Destroyed ${this.constructor.name}`); + } + + protected updateTimer(startTime: number) { + if (startTime >= this.dispatchTimerTimestamp) return; + const delay = Math.max(startTime - tasksUtils.getPerformanceTime(), 0); + clearTimeout(this.dispatchTimer); + this.dispatchTimer = setTimeout(async () => { + // This.logger.info('consuming pending tasks'); + await this.dispatchPlug.unplug(); + this.dispatchTimerTimestamp = Number.POSITIVE_INFINITY; + }, delay); + this.dispatchTimerTimestamp = startTime; + this.logger.info(`Timer was updated to ${delay} to end at ${startTime}`); + } + + /** + * Starts the dispatching of tasks + */ + @ready(new tasksErrors.ErrorSchedulerNotRunning(), false, ['starting']) + public async startDispatching(): Promise { + // Starting queue + await this.queue.startTasks(); + // If already started, do nothing + if (this.pendingDispatch == null) { + this.pendingDispatch = this.dispatchTaskLoop(); + } + } + + @ready(new tasksErrors.ErrorSchedulerNotRunning(), false, ['stopping']) + public async stopDispatching(): Promise { + const stopQueueP = this.queue.stopTasks(); + clearTimeout(this.dispatchTimer); + delete this.dispatchTimer; + this.dispatchEnding = true; + await this.dispatchPlug.unplug(); + await this.pendingDispatch; + this.pendingDispatch = null; + await stopQueueP; + } + + protected async dispatchTaskLoop(): Promise { + // This will pop tasks from the queue and put the where they need to go + this.logger.info('dispatching set up'); + this.dispatchEnding = false; + this.dispatchTimerTimestamp = Number.POSITIVE_INFINITY; + while (true) { + if (this.dispatchEnding) break; + // Setting up and waiting for plug + this.logger.info('dispatch waiting'); + await this.dispatchPlug.plug(); + // Get the next time to delay for + await this.db.withTransactionF(async (tran) => { + for await (const [keyPath] of tran.iterator(this.schedulerTimeDbPath, { + limit: 1, + })) { + const [taskTimestampKeyBuffer] = tasksUtils.splitTaskTimestampKey( + keyPath[0] as Buffer, + ); + const time = lexi.unpack(Array.from(taskTimestampKeyBuffer)); + this.updateTimer(time); + } + }); + await this.dispatchPlug.waitForUnplug(); + if (this.dispatchEnding) break; + this.logger.info('dispatch continuing'); + const time = tasksUtils.getPerformanceTime(); + // Peek ahead by 100 ms + const targetTimestamp = Buffer.from(lexi.pack(time + 100)); + await this.db.withTransactionF(async (tran) => { + for await (const [keyPath, taskIdBuffer] of tran.iterator( + this.schedulerTimeDbPath, + { + lte: targetTimestamp, + }, + )) { + const taskTimestampKeyBuffer = keyPath[0] as Buffer; + // Dispatch the task now and remove it from the scheduler + this.logger.info('dispatching task'); + await tran.del([...this.schedulerTimeDbPath, taskTimestampKeyBuffer]); + const taskId = IdInternal.fromBuffer(taskIdBuffer); + await this.queue.pushTask(taskId, taskTimestampKeyBuffer, tran); + } + }); + } + this.logger.info('dispatching ending'); + } + + /** + * Gets a scheduled task data + */ + @ready(new tasksErrors.ErrorSchedulerNotRunning()) + public async getTaskData( + taskId: TaskId, + tran?: DBTransaction, + ): Promise { + return await this.getTaskData_(taskId, tran); + } + + protected async getTaskData_( + taskId: TaskId, + tran?: DBTransaction, + ): Promise { + return await (tran ?? this.db).get([ + ...this.queue.queueTasksDbPath, + taskId.toBuffer(), + ]); + } + + /** + * Gets all scheduled task datas + * Tasks are sorted by the `TaskId` + */ + @ready(new tasksErrors.ErrorSchedulerNotRunning()) + public async *getTaskDatas( + order: 'asc' | 'desc' = 'asc', + tran?: DBTransaction, + ): AsyncGenerator<[TaskId, TaskData]> { + if (tran == null) { + return yield* this.db.withTransactionG((tran) => + this.getTaskDatas(...arguments, tran), + ); + } + for await (const [keyPath, taskData] of tran.iterator( + this.queue.queueTasksDbPath, + { valueAsBuffer: false, reverse: order !== 'asc' }, + )) { + const taskId = IdInternal.fromBuffer(keyPath[0] as Buffer); + yield [taskId, taskData]; + } + } + + // /** + // * Gets a task abstraction + // */ + // @ready(new tasksErrors.ErrorSchedulerNotRunning()) + // public async getTask(id: TaskId, tran?: DBTransaction) { + // const taskData = await (tran ?? this.db).get([...this.queueTasksDbPath, id.toBuffer()]); + // if (taskData == null) { + // return; + // } + // const { p: taskP, resolveP, rejectP } = utils.promise(); + // + // // can we standardise on the unified listener + // // that is 1 listener for every task is created automatically + // // if 1000 tasks are inserted into the DB + // // 1000 listeners are created automatically? + // + // // we can either... + // // A standardise on the listener + // // B standardise on the promise + // + // // if the creation of the promise is lazy + // // then one can standardise on the promise + // // the idea being if the promise exists, just return the promise + // // if it doesn't exist, then first check if the task id still exists + // // if so, create a promise out of that lazily + // // now you need an object map locking to prevent race conditions on promise creation + // // then there's only ever 1 promise for a given task + // // any other cases, they always give back the same promise + // + // + // const listener = (taskError, taskResult) => { + // if (taskError != null) { + // rejectP(taskError); + // } else { + // resolveP(taskResult); + // } + // this.deregisterListener(id, listener); + // }; + // this.registerListener(id, listener); + // return taskP; + // } + + /* + Const task = await scheduleTask(...); + await task; // <- any + + const task = scheduleTask(...); + await task; // <- Promise + + + const task = scheduleTask(...); + await task; // <- Task (you are actually waiting for both scheduling + task execution) + + const task = scheduleTask(..., lazy=true); + await task; // <- Task you are only awaiting the scheduling + await task.task; + + const task = scheduleTask(delay=10hrs, lazy=True); + + waited 68 hrs + + await task; <- there's no information about the task - ErrorTasksTaskMissing + + + const task = scheduleTask(delay=10hrs, lazy=True); + + waited 5 hrs + + await task; - it can register an event handler for this task + + for loop: + scheduleTask(delay=10hrs); + + + const task = await scheduler.scheduleTask(lazy=false); + await task.promise; + + const task = await scheduler.getTask(lazy=false); // this is natu + await task.promise; + + */ + + @ready(new tasksErrors.ErrorSchedulerNotRunning()) + public async scheduleTask( + handlerId: TaskHandlerId, + parameters: TaskParameters = [], + delay: TaskDelay = 0, + priority: number = 0, + taskGroup?: TaskGroup, + lazy: boolean = false, + tran?: DBTransaction, + ): Promise | undefined> { + if (tran == null) { + return this.db.withTransactionF((tran) => + this.scheduleTask( + handlerId, + parameters, + delay, + priority, + taskGroup, + lazy, + tran, + ), + ); + } + + // This does a combination of things + // 1. create save the new task within the DB + // 2. if timer exist and new delay is longer then just return the task + // 3. else cancel the timer and create a new one with the delay + + const task = await this.queue.createTask( + handlerId, + parameters, + priority, + taskGroup, + lazy, + tran, + ); + const taskIdBuffer = task.id.toBuffer(); + const startTime = task.timestamp + delay; + const taskTimestampKeyBuffer = tasksUtils.makeTaskTimestampKey( + startTime, + task.id, + ); + await tran.put( + [...this.queue.queueStartTimeDbPath, taskIdBuffer], + startTime, + ); + await tran.put( + [...this.queue.queueStartTimeDbPath, taskIdBuffer], + taskTimestampKeyBuffer, + true, + ); + await tran.put( + [...this.schedulerTimeDbPath, taskTimestampKeyBuffer], + taskIdBuffer, + true, + ); + + // Only update timer if transaction succeeds + tran.queueSuccess(() => { + this.updateTimer(startTime); + this.logger.info( + `Task ${tasksUtils.encodeTaskId( + task.id, + )} was scheduled for ${startTime}`, + ); + }); + + return task; + } +} + +export default Scheduler; diff --git a/src/tasks/Task.ts b/src/tasks/Task.ts new file mode 100644 index 000000000..ae3b38bf4 --- /dev/null +++ b/src/tasks/Task.ts @@ -0,0 +1,101 @@ +import type { + TaskId, + TaskData, + TaskHandlerId, + TaskTimestamp, + TaskDelay, + TaskPriority, + TaskParameters, + TaskGroup, +} from './types'; +import type { DeepReadonly } from '../types'; +import type Queue from './Queue'; + +class Task { + public readonly id: TaskId; + public readonly handlerId: TaskHandlerId; + public readonly parameters: DeepReadonly; + public readonly timestamp: TaskTimestamp; + // Public readonly delay: TaskDelay; + public readonly taskGroup: TaskGroup | undefined; + public readonly priority: TaskPriority; + + protected taskPromise: Promise | null; + protected queue: Queue; + + constructor( + queue: Queue, + id: TaskId, + handlerId: TaskHandlerId, + parameters: TaskParameters, + timestamp: TaskTimestamp, + // Delay: TaskDelay, + taskGroup: TaskGroup | undefined, + priority: TaskPriority, + taskPromise: Promise | null, + ) { + // I'm not sure about the queue + // but if this is the reference here + // then we need to add the event handler into the queue to wait for this + // this.queue = queue; + + this.id = id; + this.handlerId = handlerId; + this.parameters = parameters; + this.timestamp = timestamp; + // This.delay = delay; + this.taskGroup = taskGroup; + this.priority = priority; + this.queue = queue; + this.taskPromise = taskPromise; + } + + public toJSON(): TaskData & { id: TaskId } { + return { + id: this.id, + handlerId: this.handlerId, + // TODO: change this to `structuredClone` when available + parameters: JSON.parse(JSON.stringify(this.parameters)), + timestamp: this.timestamp, + // Delay: this.delay, + taskGroup: this.taskGroup, + priority: this.priority, + }; + } + + get promise() { + if (this.taskPromise != null) return this.taskPromise; + this.taskPromise = this.queue.getTaskP(this.id); + return this.taskPromise; + } +} + +// Const t = new Task(); +// +// const p = new Promise((resolve, reject) => { +// resolve(); +// }); +// +// p.then; +// P.catch +// p.finally +// /** +// * Represents the completion of an asynchronous operation +// */ +// interface Promise { +// /** +// * Attaches callbacks for the resolution and/or rejection of the Promise. +// * @param onfulfilled The callback to execute when the Promise is resolved. +// * @param onrejected The callback to execute when the Promise is rejected. +// * @returns A Promise for the completion of which ever callback is executed. +// */ + +// /** +// * Attaches a callback for only the rejection of the Promise. +// * @param onrejected The callback to execute when the Promise is rejected. +// * @returns A Promise for the completion of the callback. +// */ +// catch(onrejected?: ((reason: any) => TResult | PromiseLike) | undefined | null): Promise; +// } + +export default Task; diff --git a/src/tasks/errors.ts b/src/tasks/errors.ts new file mode 100644 index 000000000..5f85cfc47 --- /dev/null +++ b/src/tasks/errors.ts @@ -0,0 +1,80 @@ +import { ErrorPolykey, sysexits } from '../errors'; + +class ErrorTasks extends ErrorPolykey {} + +class ErrorScheduler extends ErrorTasks {} + +class ErrorSchedulerRunning extends ErrorScheduler { + static description = 'Scheduler is running'; + exitCode = sysexits.USAGE; +} + +class ErrorSchedulerNotRunning extends ErrorScheduler { + static description = 'Scheduler is not running'; + exitCode = sysexits.USAGE; +} + +class ErrorSchedulerDestroyed extends ErrorScheduler { + static description = 'Scheduler is destroyed'; + exitCode = sysexits.USAGE; +} + +class ErrorSchedulerHandlerMissing extends ErrorScheduler { + static description = 'Scheduler task handler is not registered'; + exitCode = sysexits.USAGE; +} + +class ErrorQueue extends ErrorTasks {} + +class ErrorQueueRunning extends ErrorQueue { + static description = 'Queue is running'; + exitCode = sysexits.USAGE; +} + +class ErrorQueueNotRunning extends ErrorQueue { + static description = 'Queue is not running'; + exitCode = sysexits.USAGE; +} + +class ErrorQueueDestroyed extends ErrorQueue { + static description = 'Queue is destroyed'; + exitCode = sysexits.USAGE; +} + +class ErrorTask extends ErrorTasks { + static description = 'Task error'; + exitCode = sysexits.USAGE; +} + +class ErrorTaskRejected extends ErrorTask { + static description = 'Task handler threw an exception'; + exitCode = sysexits.USAGE; +} + +class ErrorTaskCancelled extends ErrorTask { + static description = 'Task has been cancelled'; + exitCode = sysexits.USAGE; +} + +class ErrorTaskMissing extends ErrorTask { + static description = + 'Task does not (or never) existed anymore, it may have been fulfilled or cancelled'; + exitCode = sysexits.USAGE; +} + +export { + ErrorTasks, + ErrorScheduler, + ErrorSchedulerRunning, + ErrorSchedulerNotRunning, + ErrorSchedulerDestroyed, + ErrorSchedulerHandlerMissing, + ErrorQueue, + ErrorQueueRunning, + ErrorQueueNotRunning, + ErrorQueueDestroyed, + ErrorTask, + ErrorTaskRejected, + ErrorTaskCancelled, + ErrorTaskMissing, +}; diff --git a/src/tasks/index.ts b/src/tasks/index.ts new file mode 100644 index 000000000..ae900e45b --- /dev/null +++ b/src/tasks/index.ts @@ -0,0 +1,4 @@ +export { default as Scheduler } from './Scheduler'; +export * as types from './types'; +export * as utils from './utils'; +export * as errors from './errors'; diff --git a/src/tasks/types.ts b/src/tasks/types.ts new file mode 100644 index 000000000..260007480 --- /dev/null +++ b/src/tasks/types.ts @@ -0,0 +1,110 @@ +import type { Id } from '@matrixai/id'; +import type { POJO, Opaque, Callback } from '../types'; + +type TaskId = Opaque<'TaskId', Id>; +type TaskIdString = Opaque<'TaskIdString', string>; +type TaskIdEncoded = Opaque<'TaskIdEncoded', string>; + +/** + * Timestamp unix time in milliseconds + */ +type TaskTimestamp = number; + +/** + * Timestamp is millisecond number >= 0 + */ +type TaskDelay = number; + +type TaskParameters = Array; + +/** + * Task priority is an `uint8` [0 to 255] + * Where `0` is the highest priority and `255` is the lowest priority + */ +type TaskPriority = Opaque<'TaskPriority', number>; + +/** + * Task group, array of strings + */ +type TaskGroup = Array; + +/** + * Task data to be persisted + */ +type TaskData = { + handlerId: TaskHandlerId; + parameters: TaskParameters; + timestamp: TaskTimestamp; + // Delay: TaskDelay; + taskGroup: TaskGroup | undefined; + priority: TaskPriority; +}; + +/** + * Task information that is returned to the user + */ +type TaskInfo = TaskData & { + id: TaskId; +}; + +type TaskHandlerId = Opaque<'TaskHandlerId', string>; + +// Type TaskHandler

= [], R = any> = ( +// ...params: P +// ) => Promise; + +type TaskHandler = (...params: Array) => Promise; + +/** + * Task function is the result of a lambda abstraction of applying + * `TaskHandler` to its respective parameters + * This is what gets executed + */ +type TaskFunction = () => Promise; + +// Type TaskListener = Callback<[taskResult: any], void>; +// Make Task something that can be awaited on +// but when you "make" a promise or reference it +// you're for a promise +// that will resolve an event occurs +// or reject when an event occurs +// and the result of the execution +// now the exeuction of the event itself is is going to return ap romise +// something must be lisetning to it +// If you have a Record +// it has to be TaskIdString +// you can store things in it +// type X = Record; +// Task is the lowest level +// TaskData is low level +// TaskInfo is high level +// TaskId +// Task <- lazy promise +// TaskData <- low level data of a task (does not include id) +// TaskInfo <- high level (includes id) +// This is a lazy promise +// it's a promise of something that may not yet immediately executed +// type TaskPromise = Promise; +// Consider these variants... (should standardise what these are to be used) +// Task +// Tasks (usually a record, sometimes an array) +// TaskData - lower level data of a task +// TaskInfo - higher level information that is inclusive of data +// type TaskData = Record; + +export type { + TaskId, + TaskIdString, + TaskIdEncoded, + // Task, + TaskGroup, + TaskData, + TaskInfo, + TaskHandlerId, + TaskHandler, + TaskPriority, + // TaskListener + TaskParameters, + TaskTimestamp, + TaskDelay, +}; diff --git a/src/tasks/utils.ts b/src/tasks/utils.ts new file mode 100644 index 000000000..15e8330c6 --- /dev/null +++ b/src/tasks/utils.ts @@ -0,0 +1,98 @@ +import type { TaskId, TaskIdEncoded, TaskPriority } from './types'; +import type { NodeId } from '../nodes/types'; +import { IdInternal, IdSortable } from '@matrixai/id'; +import lexi from 'lexicographic-integer'; + +/** + * Generates TaskId + * TaskIds are lexicographically sortable 128 bit IDs + * They are strictly monotonic and unique with respect to the `nodeId` + * When the `NodeId` changes, make sure to regenerate this generator + */ +function createTaskIdGenerator(nodeId: NodeId, lastTaskId?: TaskId) { + const generator = new IdSortable({ + lastId: lastTaskId, + nodeId, + }); + return () => generator.get(); +} + +/** + * Converts `int8` to flipped `uint8` task priority + * Clips number to between -128 to 127 inclusive + */ +function toPriority(n: number): TaskPriority { + n = Math.min(n, 127); + n = Math.max(n, -128); + n *= -1; + n -= 1; + n += 128; + return n as TaskPriority; +} + +/** + * Converts flipped `uint8` task priority to `int8` + */ +function fromPriority(p: TaskPriority): number { + let n = p - 128; + n += 1; + // Prevent returning `-0` + if (n !== 0) n *= -1; + return n; +} + +function makeTaskTimestampKey(time: number, taskId: TaskId): Buffer { + const timestampBuffer = Buffer.from(lexi.pack(time)); + return Buffer.concat([timestampBuffer, taskId.toBuffer()]); +} + +/** + * Returns [taskTimestampBuffer, taskIdBuffer] + */ +function splitTaskTimestampKey(timestampBuffer: Buffer) { + // Last 16 bytes are TaskId + const splitPoint = timestampBuffer.length - 16; + const timeBuffer = timestampBuffer.slice(0, splitPoint); + const idBuffer = timestampBuffer.slice(splitPoint); + return [timeBuffer, idBuffer]; +} + +function getPerformanceTime(): number { + return performance.timeOrigin + performance.now(); +} + +/** + * Encodes the TaskId as a `base32hex` string + */ +function encodeTaskId(taskId: TaskId): TaskIdEncoded { + return taskId.toMultibase('base32hex') as TaskIdEncoded; +} + +/** + * Decodes an encoded TaskId string into a TaskId + */ +function decodeTaskId(taskIdEncoded: any): TaskId | undefined { + if (typeof taskIdEncoded !== 'string') { + return; + } + const taskId = IdInternal.fromMultibase(taskIdEncoded); + if (taskId == null) { + return; + } + // All TaskIds are 16 bytes long + if (taskId.length !== 16) { + return; + } + return taskId; +} + +export { + createTaskIdGenerator, + toPriority, + fromPriority, + makeTaskTimestampKey, + splitTaskTimestampKey, + getPerformanceTime, + encodeTaskId, + decodeTaskId, +}; diff --git a/src/types.ts b/src/types.ts index d0d73eef5..216f4fc49 100644 --- a/src/types.ts +++ b/src/types.ts @@ -45,6 +45,11 @@ interface ToString { toString(): string; } +/** + * Recursive readonly + */ +type DeepReadonly = { readonly [K in keyof T]: DeepReadonly }; + /** * Wrap a type to be reference counted * Useful for when we need to garbage collect data @@ -122,6 +127,7 @@ export type { Initial, InitialParameters, ToString, + DeepReadonly, Ref, Timer, PromiseDeconstructed, diff --git a/src/utils/Plug.ts b/src/utils/Plug.ts new file mode 100644 index 000000000..bde43ea38 --- /dev/null +++ b/src/utils/Plug.ts @@ -0,0 +1,36 @@ +import { Lock } from '@matrixai/async-locks'; + +/** + * Abstraction for using a Lock as a plug for asynchronous pausing of loops + */ +class Plug { + protected lock: Lock = new Lock(); + protected lockReleaser: (e?: Error) => Promise = async () => {}; + + /** + * Will cause waitForUnplug to block + */ + public async plug() { + if (this.lock.isLocked()) return; + [this.lockReleaser] = await this.lock.lock(0)(); + } + /** + * Will release waitForUnplug from blocking + */ + public async unplug() { + await this.lockReleaser(); + } + + /** + * Will block if plugged + */ + public async waitForUnplug() { + await this.lock.waitForUnlock(); + } + + public isPlugged() { + return this.lock.isLocked(); + } +} + +export default Plug; diff --git a/src/utils/index.ts b/src/utils/index.ts index 2ee8414ff..c1d5c537b 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,4 +1,5 @@ export { default as sysexits } from './sysexits'; +export { default as Plug } from './Plug'; export * from './utils'; export * from './matchers'; export * from './binary'; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 615dc15b4..066e69d7b 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -314,11 +314,12 @@ function debounce

( } function isPromise(v: any): v is Promise { - return v instanceof Promise || ( - v != null - && typeof v.then === 'function' - && typeof v.catch === 'function' - && typeof v.finally === 'function' + return ( + v instanceof Promise || + (v != null && + typeof v.then === 'function' && + typeof v.catch === 'function' && + typeof v.finally === 'function') ); } diff --git a/tests/tasks/Queue.test.ts b/tests/tasks/Queue.test.ts new file mode 100644 index 000000000..0c16f8389 --- /dev/null +++ b/tests/tasks/Queue.test.ts @@ -0,0 +1,415 @@ +import type { TaskHandlerId, TaskId } from '../../src/tasks/types'; +import type { TaskGroup } from '../../src/tasks/types'; +import os from 'os'; +import path from 'path'; +import fs from 'fs'; +import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; +import { DB } from '@matrixai/db'; +import { sleep } from '@matrixai/async-locks/dist/utils'; +import { IdInternal } from '@matrixai/id'; +import { promise } from 'encryptedfs/dist/utils'; +import Scheduler from '@/tasks/Scheduler'; +import Queue from '@/tasks/Queue'; +import * as keysUtils from '@/keys/utils'; +import * as tasksUtils from '@/tasks/utils'; +import KeyManager from '@/keys/KeyManager'; +import { globalRootKeyPems } from '../fixtures/globalRootKeyPems'; + +describe(Queue.name, () => { + const password = 'password'; + const logger = new Logger(`${Scheduler.name} test`, LogLevel.INFO, [ + new StreamHandler(), + ]); + let dbKey: Buffer; + let dbPath: string; + let db: DB; + let keyManager: KeyManager; + const handlerId = 'testId' as TaskHandlerId; + + const pushTask = async ( + queue: Queue, + handlerId, + params: Array, + lazy = true, + ) => { + const task = await queue.createTask( + handlerId, + params, + undefined, + undefined, + lazy, + ); + const timestampBuffer = tasksUtils.makeTaskTimestampKey( + task.timestamp, + task.id, + ); + await queue.pushTask(task.id, timestampBuffer); + return task; + }; + + beforeAll(async () => { + dataDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'polykey-test-'), + ); + const keysPath = `${dataDir}/keys`; + keyManager = await KeyManager.createKeyManager({ + password, + keysPath, + logger, + privateKeyPemOverride: globalRootKeyPems[0], + }); + dbKey = await keysUtils.generateKey(); + dbPath = `${dataDir}/db`; + }); + beforeEach(async () => { + db = await DB.createDB({ + dbPath, + logger, + crypto: { + key: dbKey, + ops: { + encrypt: keysUtils.encryptWithKey, + decrypt: keysUtils.decryptWithKey, + }, + }, + }); + }); + afterEach(async () => { + await db.stop(); + await db.destroy(); + }); + + test('can start and stop', async () => { + const queue = await Queue.createQueue({ + db, + keyManager, + concurrencyLimit: 2, + logger, + }); + await queue.stop(); + await queue.start(); + await queue.stop(); + }); + test('can consume tasks', async () => { + const handler = jest.fn(); + handler.mockImplementation(async () => {}); + const queue = await Queue.createQueue({ + db, + keyManager, + handlers: { [handlerId]: handler }, + concurrencyLimit: 2, + logger, + }); + await queue.startTasks(); + await pushTask(queue, handlerId, [0]); + await pushTask(queue, handlerId, [1]); + await queue.allActiveTasksSettled(); + await queue.stop(); + expect(handler).toHaveBeenCalled(); + }); + test('tasks persist', async () => { + const handler = jest.fn(); + handler.mockImplementation(async () => sleep(0)); + let queue = await Queue.createQueue({ + db, + keyManager, + delay: true, + concurrencyLimit: 2, + logger, + }); + + await pushTask(queue, handlerId, [0]); + await pushTask(queue, handlerId, [1]); + await pushTask(queue, handlerId, [2]); + await queue.stop(); + + queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + concurrencyLimit: 2, + logger, + }); + // Time for tasks to start processing + await sleep(100); + await queue.allActiveTasksSettled(); + await queue.stop(); + expect(handler).toHaveBeenCalled(); + }); + test('concurrency is enforced', async () => { + const handler = jest.fn(); + const prom = promise(); + handler.mockImplementation(async () => { + await prom.p; + }); + const queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + concurrencyLimit: 2, + logger, + }); + + await queue.startTasks(); + await pushTask(queue, handlerId, [0]); + await pushTask(queue, handlerId, [1]); + await pushTask(queue, handlerId, [2]); + await pushTask(queue, handlerId, [3]); + await sleep(200); + expect(handler).toHaveBeenCalledTimes(2); + prom.resolveP(); + await sleep(200); + await queue.allActiveTasksSettled(); + await queue.stop(); + expect(handler).toHaveBeenCalledTimes(4); + }); + test('called exactly 4 times', async () => { + const handler = jest.fn(); + handler.mockImplementation(async () => {}); + const queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + logger, + }); + + await queue.startTasks(); + await pushTask(queue, handlerId, [0]); + await pushTask(queue, handlerId, [1]); + await pushTask(queue, handlerId, [2]); + await pushTask(queue, handlerId, [3]); + await sleep(100); + await queue.stop(); + expect(handler).toHaveBeenCalledTimes(4); + }); + test('tasks can have an optional group', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (nextTaskId) => { + // Await sleep(1000); + logger.info(`task complete ${tasksUtils.encodeTaskId(nextTaskId)}`); + }); + const queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + delay: true, + concurrencyLimit: 2, + logger, + }); + + await queue.createTask(handlerId, [1], undefined, ['one'], true); + await queue.createTask(handlerId, [2], undefined, ['two'], true); + await queue.createTask(handlerId, [3], undefined, ['two'], true); + await queue.createTask( + handlerId, + [4], + undefined, + ['group1', 'three'], + true, + ); + await queue.createTask(handlerId, [5], undefined, ['group1', 'four'], true); + await queue.createTask(handlerId, [6], undefined, ['group1', 'four'], true); + await queue.createTask(handlerId, [7], undefined, ['group2', 'five'], true); + await queue.createTask(handlerId, [8], undefined, ['group2', 'six'], true); + + const listTasks = async (taskGroup: TaskGroup) => { + const tasks: Array = []; + for await (const task of queue.getGroupTasks(taskGroup)) { + tasks.push(task); + } + return tasks; + }; + + expect(await listTasks(['one'])).toHaveLength(1); + expect(await listTasks(['two'])).toHaveLength(2); + expect(await listTasks(['group1'])).toHaveLength(3); + expect(await listTasks(['group1', 'four'])).toHaveLength(2); + expect(await listTasks(['group2'])).toHaveLength(2); + expect(await listTasks([])).toHaveLength(8); + + await queue.stop(); + }); + test('completed tasks emit events', async () => { + const handler = jest.fn(); + handler.mockImplementation(async () => { + return 'completed'; + }); + const queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + concurrencyLimit: 2, + logger, + }); + + await pushTask(queue, handlerId, [0]); + await pushTask(queue, handlerId, [1]); + await pushTask(queue, handlerId, [2]); + await pushTask(queue, handlerId, [4]); + await queue.startTasks(); + await sleep(200); + await queue.allActiveTasksSettled(); + await queue.stop(); + expect(handler).toHaveBeenCalledTimes(4); + }); + test('can await a task promise resolve', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (fail) => { + if (!fail) throw Error('three'); + return fail; + }); + const queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + concurrencyLimit: 2, + logger, + }); + + const taskSucceed = await pushTask(queue, handlerId, [true], false); + + // Promise should succeed with result + const taskSucceedP = taskSucceed!.promise; + await expect(taskSucceedP).resolves.toBe(true); + + await queue.stop(); + }); + test('can await a task promise reject', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (fail) => { + if (!fail) throw Error('three'); + return fail; + }); + const queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + concurrencyLimit: 2, + logger, + }); + + const taskFail = await pushTask(queue, handlerId, [false], false); + // Promise should fail + const taskFailP = taskFail!.promise; + await expect(taskFailP).rejects.toBeInstanceOf(Error); + + await queue.stop(); + }); + test('getting multiple promises for a task should be the same promise', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (fail) => { + if (!fail) throw Error('three'); + return fail; + }); + const queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + delay: true, + concurrencyLimit: 2, + logger, + }); + + const taskSucceed = await pushTask(queue, handlerId, [true], false); + // If we get a 2nd task promise, it should be the same promise + const prom1 = queue.getTaskP(taskSucceed.id); + const prom2 = queue.getTaskP(taskSucceed.id); + expect(prom1).toBe(prom2); + expect(prom1).toBe(taskSucceed!.promise); + + await queue.stop(); + }); + test('task promise for invalid task should throw', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (fail) => { + if (!fail) throw Error('three'); + return fail; + }); + const queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + delay: true, + concurrencyLimit: 2, + logger, + }); + + // Getting task promise should not throw + const invalidTask = queue.getTaskP( + IdInternal.fromBuffer(Buffer.alloc(16, 0)), + ); + // Task promise will throw an error if task not found + await expect(invalidTask).rejects.toThrow(); + + await queue.stop(); + }); + test('lazy task promise for completed task should throw', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (fail) => { + if (!fail) throw Error('three'); + return fail; + }); + const queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + delay: true, + concurrencyLimit: 2, + logger, + }); + + const taskSucceed = await pushTask(queue, handlerId, [true], true); + const prom = queue.getTaskP(taskSucceed.id); + await queue.startTasks(); + await prom; + // Finished tasks should throw + await expect(taskSucceed?.promise).rejects.toThrow(); + + await queue.stop(); + }); + test('eager task promise for completed task should resolve', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (fail) => { + if (!fail) throw Error('three'); + return fail; + }); + const queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + delay: true, + concurrencyLimit: 2, + logger, + }); + + await queue.startTasks(); + const taskSucceed = await pushTask(queue, handlerId, [true], false); + await expect(taskSucceed?.promise).resolves.toBe(true); + + await queue.stop(); + }); + + test('template', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (nextTaskId) => { + // Await sleep(1000); + logger.info(`task complete ${tasksUtils.encodeTaskId(nextTaskId)}`); + }); + const queue = await Queue.createQueue({ + db, + handlers: { [handlerId]: handler }, + keyManager, + concurrencyLimit: 2, + logger, + }); + + await pushTask(queue, handlerId, [0]); + await pushTask(queue, handlerId, [1]); + await pushTask(queue, handlerId, [2]); + + await queue.startTasks(); + await sleep(100); + await queue.stop(); + expect(handler).toHaveBeenCalledTimes(3); + }); +}); diff --git a/tests/tasks/Scheduler.test.ts b/tests/tasks/Scheduler.test.ts new file mode 100644 index 000000000..1145789b7 --- /dev/null +++ b/tests/tasks/Scheduler.test.ts @@ -0,0 +1,119 @@ +import type { TaskHandlerId } from '../../src/tasks/types'; +import os from 'os'; +import path from 'path'; +import fs from 'fs'; +import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; +import { DB } from '@matrixai/db'; +import { sleep } from '@matrixai/async-locks/dist/utils'; +import KeyManager from '@/keys/KeyManager'; +import Scheduler from '@/tasks/Scheduler'; +import * as keysUtils from '@/keys/utils'; +import Queue from '@/tasks/Queue'; +import { globalRootKeyPems } from '../fixtures/globalRootKeyPems'; + +describe(Scheduler.name, () => { + const password = 'password'; + const logger = new Logger(`${Scheduler.name} test`, LogLevel.INFO, [ + new StreamHandler(), + ]); + let keyManager: KeyManager; + let dbKey: Buffer; + let dbPath: string; + let db: DB; + beforeAll(async () => { + dataDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'polykey-test-'), + ); + const keysPath = `${dataDir}/keys`; + keyManager = await KeyManager.createKeyManager({ + password, + keysPath, + logger, + privateKeyPemOverride: globalRootKeyPems[0], + }); + dbKey = await keysUtils.generateKey(); + dbPath = `${dataDir}/db`; + }); + beforeEach(async () => { + db = await DB.createDB({ + dbPath, + logger, + crypto: { + key: dbKey, + ops: { + encrypt: keysUtils.encryptWithKey, + decrypt: keysUtils.decryptWithKey, + }, + }, + }); + }); + afterEach(async () => { + await db.stop(); + await db.destroy(); + }); + test('can add tasks with scheduled delay', async () => { + const queue = await Queue.createQueue({ + db, + keyManager, + logger, + }); + const scheduler = await Scheduler.createScheduler({ + db, + queue, + logger, + }); + const taskHandler = 'asd' as TaskHandlerId; + const handler = jest.fn(); + handler.mockImplementation(async () => sleep(100)); + queue.registerHandler(taskHandler, handler); + + await scheduler.scheduleTask(taskHandler, [1], 1000); + await scheduler.scheduleTask(taskHandler, [2], 100); + await scheduler.scheduleTask(taskHandler, [3], 2000); + await scheduler.scheduleTask(taskHandler, [4], 10); + await scheduler.scheduleTask(taskHandler, [5], 10); + await scheduler.scheduleTask(taskHandler, [6], 10); + await scheduler.scheduleTask(taskHandler, [7], 3000); + await sleep(4000); + await scheduler.stop(); + expect(handler).toHaveBeenCalledTimes(7); + }); + test('scheduled tasks persist', async () => { + const queue = await Queue.createQueue({ + db, + keyManager, + logger, + }); + const scheduler = await Scheduler.createScheduler({ + db, + queue, + logger, + }); + const taskHandler = 'asd' as TaskHandlerId; + const handler = jest.fn(); + handler.mockImplementation(async () => sleep(100)); + queue.registerHandler(taskHandler, handler); + + await scheduler.start(); + await scheduler.scheduleTask(taskHandler, [1], 1000); + await scheduler.scheduleTask(taskHandler, [2], 100); + await scheduler.scheduleTask(taskHandler, [3], 2000); + await scheduler.scheduleTask(taskHandler, [4], 10); + await scheduler.scheduleTask(taskHandler, [5], 10); + await scheduler.scheduleTask(taskHandler, [6], 10); + await scheduler.scheduleTask(taskHandler, [7], 3000); + await sleep(500); + await scheduler.stop(); + + logger.info('intermission!!!!'); + + await scheduler.start(); + await sleep(4000); + await scheduler.stop(); + expect(handler).toHaveBeenCalledTimes(7); + }); + test.todo('Scheculed tasks get moved to queue after delay'); + test.todo('tasks timestamps are unique on taskId'); + test.todo('can remove scheduled tasks'); + test.todo('can not remove active tasks'); +}); diff --git a/tests/tasks/utils.test.ts b/tests/tasks/utils.test.ts new file mode 100644 index 000000000..9bf3e1cab --- /dev/null +++ b/tests/tasks/utils.test.ts @@ -0,0 +1,29 @@ +import type { TaskPriority } from '@/tasks/types'; +import * as tasksUtils from '@/tasks/utils'; + +describe('tasks/utils', () => { + test('encode priority from `int8` to flipped `uint8`', () => { + expect(tasksUtils.toPriority(128)).toBe(0); + expect(tasksUtils.toPriority(127)).toBe(0); + expect(tasksUtils.toPriority(126)).toBe(1); + expect(tasksUtils.toPriority(2)).toBe(125); + expect(tasksUtils.toPriority(1)).toBe(126); + expect(tasksUtils.toPriority(0)).toBe(127); + expect(tasksUtils.toPriority(-1)).toBe(128); + expect(tasksUtils.toPriority(-2)).toBe(129); + expect(tasksUtils.toPriority(-127)).toBe(254); + expect(tasksUtils.toPriority(-128)).toBe(255); + expect(tasksUtils.toPriority(-129)).toBe(255); + }); + test('decode from priority from flipped `uint8` to `int8`', () => { + expect(tasksUtils.fromPriority(0 as TaskPriority)).toBe(127); + expect(tasksUtils.fromPriority(1 as TaskPriority)).toBe(126); + expect(tasksUtils.fromPriority(125 as TaskPriority)).toBe(2); + expect(tasksUtils.fromPriority(126 as TaskPriority)).toBe(1); + expect(tasksUtils.fromPriority(127 as TaskPriority)).toBe(0); + expect(tasksUtils.fromPriority(128 as TaskPriority)).toBe(-1); + expect(tasksUtils.fromPriority(129 as TaskPriority)).toBe(-2); + expect(tasksUtils.fromPriority(254 as TaskPriority)).toBe(-127); + expect(tasksUtils.fromPriority(255 as TaskPriority)).toBe(-128); + }); +}); diff --git a/tests/utils/Plug.test.ts b/tests/utils/Plug.test.ts new file mode 100644 index 000000000..a1effeefd --- /dev/null +++ b/tests/utils/Plug.test.ts @@ -0,0 +1,19 @@ +import Plug from '@/utils/Plug'; + +describe(Plug.name, () => { + test('can plug and unplug', async () => { + const plug = new Plug(); + + // Calls are idempotent + await plug.plug(); + await plug.plug(); + await plug.plug(); + expect(plug.isPlugged()).toBeTrue(); + + // Calls are idempotent + await plug.unplug(); + await plug.unplug(); + await plug.unplug(); + expect(plug.isPlugged()).toBeFalse(); + }); +}); From ad8bffd23dec9c4d50235db8f5905c46112b23c1 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Fri, 2 Sep 2022 14:16:04 +1000 Subject: [PATCH 13/32] fix: `Queue` using `EventTarget` for task promises --- src/tasks/Queue.ts | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/tasks/Queue.ts b/src/tasks/Queue.ts index 35d90a6f8..f4bab5bf5 100644 --- a/src/tasks/Queue.ts +++ b/src/tasks/Queue.ts @@ -24,6 +24,14 @@ import * as tasksUtils from './utils'; import Task from './Task'; import { Plug } from '../utils/index'; +class TaskEvent extends Event { + detail?: any; + constructor(type: string, options?: CustomEventInit) { + super(type, options); + this.detail = options?.detail; + } +} + interface Queue extends CreateDestroyStartStop {} @CreateDestroyStartStop( new tasksErrors.ErrorQueueRunning(), @@ -117,7 +125,7 @@ class Queue { protected handlers: Map = new Map(); protected taskPromises: Map> = new Map(); - protected taskEvents: EventEmitter = new EventEmitter(); + protected taskEvents: EventTarget = new EventTarget(); protected keyManager: KeyManager; protected generateTaskId: () => TaskId; @@ -500,12 +508,12 @@ class Queue { }); }) .then( - (value) => { - this.taskEvents.emit(taskIdEncoded, value); - return value; + (result) => { + this.taskEvents.dispatchEvent(new TaskEvent(taskIdEncoded, {detail: [undefined, result]})); + return result; }, (reason) => { - this.taskEvents.emit(taskIdEncoded, reason); + this.taskEvents.dispatchEvent(new TaskEvent(taskIdEncoded, {detail: [reason]})); throw reason; }, ); @@ -545,18 +553,19 @@ class Queue { // If the task exist then it will create the task promise and return that const newTaskPromise = new Promise((resolve, reject) => { - const resultListener = (result) => { - if (result instanceof Error) reject(result); + const resultListener = (event: TaskEvent) => { + const [e, result] = event.detail; + if (e != null) reject(e); else resolve(result); }; - this.taskEvents.once(taskIdEncoded, resultListener); + this.taskEvents.addEventListener(taskIdEncoded, resultListener, {once: true}); // If not task promise exists then with will check if the task exists void (tran ?? this.db) .get([...this.queueTasksDbPath, taskId.toBuffer()], true) .then( (taskData) => { if (taskData == null) { - this.taskEvents.removeListener(taskIdEncoded, resultListener); + this.taskEvents.removeEventListener(taskIdEncoded, resultListener); reject(Error('TEMP task not found')); } }, From ca4fee32dd843806d39c311360f7155820aea9d1 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Fri, 2 Sep 2022 14:57:50 +1000 Subject: [PATCH 14/32] fix(tasks): adding `Task` getter functions `getTask` `getTasks` and `getGroupTasks` --- src/tasks/Queue.ts | 100 +++++++++++++++++++++++++++++++++++--- src/tasks/Scheduler.ts | 97 +++++++++++++++++++----------------- tests/tasks/Queue.test.ts | 3 +- 3 files changed, 146 insertions(+), 54 deletions(-) diff --git a/src/tasks/Queue.ts b/src/tasks/Queue.ts index f4bab5bf5..78e7b8636 100644 --- a/src/tasks/Queue.ts +++ b/src/tasks/Queue.ts @@ -10,7 +10,6 @@ import type { import type KeyManager from '../keys/KeyManager'; import type { DBTransaction } from '@matrixai/db'; import type { TaskId, TaskGroup } from './types'; -import EventEmitter from 'events'; import Logger from '@matrixai/logger'; import { CreateDestroyStartStop, @@ -509,11 +508,15 @@ class Queue { }) .then( (result) => { - this.taskEvents.dispatchEvent(new TaskEvent(taskIdEncoded, {detail: [undefined, result]})); + this.taskEvents.dispatchEvent( + new TaskEvent(taskIdEncoded, { detail: [undefined, result] }), + ); return result; }, (reason) => { - this.taskEvents.dispatchEvent(new TaskEvent(taskIdEncoded, {detail: [reason]})); + this.taskEvents.dispatchEvent( + new TaskEvent(taskIdEncoded, { detail: [reason] }), + ); throw reason; }, ); @@ -558,14 +561,19 @@ class Queue { if (e != null) reject(e); else resolve(result); }; - this.taskEvents.addEventListener(taskIdEncoded, resultListener, {once: true}); + this.taskEvents.addEventListener(taskIdEncoded, resultListener, { + once: true, + }); // If not task promise exists then with will check if the task exists void (tran ?? this.db) .get([...this.queueTasksDbPath, taskId.toBuffer()], true) .then( (taskData) => { if (taskData == null) { - this.taskEvents.removeEventListener(taskIdEncoded, resultListener); + this.taskEvents.removeEventListener( + taskIdEncoded, + resultListener, + ); reject(Error('TEMP task not found')); } }, @@ -578,14 +586,89 @@ class Queue { return newTaskPromise; } + @ready(new tasksErrors.ErrorSchedulerNotRunning()) + public async getTask( + taskId: TaskId, + lazy: boolean = false, + tran?: DBTransaction, + ): Promise> { + if (tran == null) { + return this.db.withTransactionF((tran) => + this.getTask(taskId, lazy, tran), + ); + } + + const taskData = await tran.get([ + ...this.queueTasksDbPath, + taskId.toBuffer(), + ]); + if (taskData == null) throw Error('TMP task not found'); + + let taskPromise: Promise | null = null; + if (!lazy) { + taskPromise = this.getTaskP(taskId, tran); + } + return new Task( + this, + taskId, + taskData.handlerId, + taskData.parameters, + taskData.timestamp, + // Delay, + taskData.taskGroup, + taskData.priority, + taskPromise, + ); + } + + /** + * Gets all scheduled tasks. + * Tasks are sorted by the `TaskId` + */ + @ready(new tasksErrors.ErrorSchedulerNotRunning()) + public async *getTasks( + order: 'asc' | 'desc' = 'asc', + lazy: boolean = false, + tran?: DBTransaction, + ): AsyncGenerator> { + if (tran == null) { + return yield* this.db.withTransactionG((tran) => + this.getTasks(order, lazy, tran), + ); + } + + for await (const [keyPath, taskData] of tran.iterator( + this.queueTasksDbPath, + { valueAsBuffer: false, reverse: order !== 'asc' }, + )) { + const taskId = IdInternal.fromBuffer(keyPath[0] as Buffer); + let taskPromise: Promise | null = null; + if (!lazy) { + taskPromise = this.getTaskP(taskId, tran); + } + yield new Task( + this, + taskId, + taskData.handlerId, + taskData.parameters, + taskData.timestamp, + // Delay, + taskData.taskGroup, + taskData.priority, + taskPromise, + ); + } + } + @ready(new tasksErrors.ErrorSchedulerNotRunning()) public async *getGroupTasks( taskGroup: TaskGroup, + lazy: boolean = false, tran?: DBTransaction, - ): AsyncGenerator { + ): AsyncGenerator> { if (tran == null) { return yield* this.db.withTransactionG((tran) => - this.getGroupTasks(taskGroup, tran), + this.getGroupTasks(taskGroup, lazy, tran), ); } @@ -593,7 +676,8 @@ class Queue { ...this.queueGroupsDbPath, ...taskGroup, ])) { - yield IdInternal.fromBuffer(taskIdBuffer); + const taskId = IdInternal.fromBuffer(taskIdBuffer); + yield this.getTask(taskId, lazy, tran); } } diff --git a/src/tasks/Scheduler.ts b/src/tasks/Scheduler.ts index 56a90e000..6d040fa7d 100644 --- a/src/tasks/Scheduler.ts +++ b/src/tasks/Scheduler.ts @@ -1,5 +1,5 @@ import type { DB, LevelPath } from '@matrixai/db'; -import type { TaskData, TaskIdString } from './types'; +import type { TaskIdString } from './types'; import type KeyManager from '../keys/KeyManager'; import type Task from './Task'; import type Queue from './Queue'; @@ -20,6 +20,7 @@ import { import lexi from 'lexicographic-integer'; import * as tasksUtils from './utils'; import * as tasksErrors from './errors'; +import { TaskData } from './types'; import { Plug } from '../utils/index'; interface Scheduler extends CreateDestroyStartStop {} @@ -241,50 +242,6 @@ class Scheduler { this.logger.info('dispatching ending'); } - /** - * Gets a scheduled task data - */ - @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public async getTaskData( - taskId: TaskId, - tran?: DBTransaction, - ): Promise { - return await this.getTaskData_(taskId, tran); - } - - protected async getTaskData_( - taskId: TaskId, - tran?: DBTransaction, - ): Promise { - return await (tran ?? this.db).get([ - ...this.queue.queueTasksDbPath, - taskId.toBuffer(), - ]); - } - - /** - * Gets all scheduled task datas - * Tasks are sorted by the `TaskId` - */ - @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public async *getTaskDatas( - order: 'asc' | 'desc' = 'asc', - tran?: DBTransaction, - ): AsyncGenerator<[TaskId, TaskData]> { - if (tran == null) { - return yield* this.db.withTransactionG((tran) => - this.getTaskDatas(...arguments, tran), - ); - } - for await (const [keyPath, taskData] of tran.iterator( - this.queue.queueTasksDbPath, - { valueAsBuffer: false, reverse: order !== 'asc' }, - )) { - const taskId = IdInternal.fromBuffer(keyPath[0] as Buffer); - yield [taskId, taskData]; - } - } - // /** // * Gets a task abstraction // */ @@ -437,6 +394,56 @@ class Scheduler { return task; } + + @ready(new tasksErrors.ErrorSchedulerNotRunning()) + public async getTask( + taskId: TaskId, + lazy: boolean = false, + tran?: DBTransaction, + ): Promise> { + if (tran == null) { + return this.db.withTransactionF((tran) => + this.getTask(taskId, lazy, tran), + ); + } + + // Wrapping `queue.getTask`, may want to filter for only scheduled tasks + return this.queue.getTask(taskId, lazy, tran); + } + + /** + * Gets all scheduled tasks. + * Tasks are sorted by the `TaskId` + */ + @ready(new tasksErrors.ErrorSchedulerNotRunning()) + public async *getTasks( + order: 'asc' | 'desc' = 'asc', + lazy: boolean = false, + tran?: DBTransaction, + ): AsyncGenerator> { + if (tran == null) { + return yield* this.db.withTransactionG((tran) => + this.getTasks(order, lazy, tran), + ); + } + + return yield* this.queue.getTasks(order, lazy, tran); + } + + @ready(new tasksErrors.ErrorSchedulerNotRunning()) + public async *getGroupTasks( + path: TaskPath, + lazy: boolean = false, + tran?: DBTransaction, + ): AsyncGenerator> { + if (tran == null) { + return yield* this.db.withTransactionG((tran) => + this.getGroupTasks(path, lazy, tran), + ); + } + + return yield* this.queue.getGroupTasks(path, lazy, tran); + } } export default Scheduler; diff --git a/tests/tasks/Queue.test.ts b/tests/tasks/Queue.test.ts index 0c16f8389..58a3d6fcf 100644 --- a/tests/tasks/Queue.test.ts +++ b/tests/tasks/Queue.test.ts @@ -1,5 +1,6 @@ import type { TaskHandlerId, TaskId } from '../../src/tasks/types'; import type { TaskGroup } from '../../src/tasks/types'; +import type Task from '@/tasks/Task'; import os from 'os'; import path from 'path'; import fs from 'fs'; @@ -213,7 +214,7 @@ describe(Queue.name, () => { await queue.createTask(handlerId, [8], undefined, ['group2', 'six'], true); const listTasks = async (taskGroup: TaskGroup) => { - const tasks: Array = []; + const tasks: Array> = []; for await (const task of queue.getGroupTasks(taskGroup)) { tasks.push(task); } From 462c4f8f6b7d66260ab9eb315ee9f92677330aa0 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Fri, 2 Sep 2022 15:10:27 +1000 Subject: [PATCH 15/32] fix(tasks): changing `TaskGroup` to `TaskPath` as a alias for `LevelPath`, `taskGroup` is refered to as `path` now --- src/tasks/Queue.ts | 43 ++++++++++++++++++--------------------- src/tasks/Scheduler.ts | 14 ++++++------- src/tasks/Task.ts | 10 ++++----- src/tasks/types.ts | 9 ++++---- tests/tasks/Queue.test.ts | 6 +++--- 5 files changed, 40 insertions(+), 42 deletions(-) diff --git a/src/tasks/Queue.ts b/src/tasks/Queue.ts index 78e7b8636..f1a3d7c03 100644 --- a/src/tasks/Queue.ts +++ b/src/tasks/Queue.ts @@ -9,7 +9,7 @@ import type { } from './types'; import type KeyManager from '../keys/KeyManager'; import type { DBTransaction } from '@matrixai/db'; -import type { TaskId, TaskGroup } from './types'; +import type { TaskId, TaskPath } from './types'; import Logger from '@matrixai/logger'; import { CreateDestroyStartStop, @@ -93,13 +93,10 @@ class Queue { */ protected queueDbActivePath: LevelPath = [...this.queueDbPath, 'active']; /** - * Tasks by groups - * `groups/...taskGroup: Array -> {raw(TaskId)}` + * Tasks by Path + * `groups/...taskPath: LevelPath -> {raw(TaskId)}` */ - public readonly queueGroupsDbPath: LevelPath = [ - ...this.queueDbPath, - 'groups', - ]; + public readonly queuePathDbPath: LevelPath = [...this.queueDbPath, 'groups']; /** * Last Task Id */ @@ -497,10 +494,10 @@ class Queue { ); await tran.del([...this.queueTasksDbPath, taskId.toBuffer()]); await tran.del([...this.queueStartTimeDbPath, taskId.toBuffer()]); - if (taskData.taskGroup != null) { + if (taskData.path != null) { await tran.del([ - ...this.queueGroupsDbPath, - ...taskData.taskGroup, + ...this.queuePathDbPath, + ...taskData.path, taskTimestampKeybuffer!, ]); } @@ -615,7 +612,7 @@ class Queue { taskData.parameters, taskData.timestamp, // Delay, - taskData.taskGroup, + taskData.path, taskData.priority, taskPromise, ); @@ -653,7 +650,7 @@ class Queue { taskData.parameters, taskData.timestamp, // Delay, - taskData.taskGroup, + taskData.path, taskData.priority, taskPromise, ); @@ -661,20 +658,20 @@ class Queue { } @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public async *getGroupTasks( - taskGroup: TaskGroup, + public async *getTasksByPath( + path: TaskPath, lazy: boolean = false, tran?: DBTransaction, ): AsyncGenerator> { if (tran == null) { return yield* this.db.withTransactionG((tran) => - this.getGroupTasks(taskGroup, lazy, tran), + this.getTasksByPath(path, lazy, tran), ); } for await (const [, taskIdBuffer] of tran.iterator([ - ...this.queueGroupsDbPath, - ...taskGroup, + ...this.queuePathDbPath, + ...path, ])) { const taskId = IdInternal.fromBuffer(taskIdBuffer); yield this.getTask(taskId, lazy, tran); @@ -697,13 +694,13 @@ class Queue { handlerId: TaskHandlerId, parameters: TaskParameters = [], priority: number = 0, - taskGroup?: TaskGroup, + path?: TaskPath, lazy: boolean = false, tran?: DBTransaction, ): Promise> { if (tran == null) { return this.db.withTransactionF((tran) => - this.createTask(handlerId, parameters, priority, taskGroup, lazy, tran), + this.createTask(handlerId, parameters, priority, path, lazy, tran), ); } @@ -720,7 +717,7 @@ class Queue { handlerId, parameters, timestamp: taskTimestamp, - taskGroup, + path: path, priority: taskPriority, }; const taskIdBuffer = taskId.toBuffer(); @@ -730,9 +727,9 @@ class Queue { await tran.put(this.queueLastTaskIdPath, taskIdBuffer, true); // Adding to group - if (taskGroup != null) { + if (path != null) { await tran.put( - [...this.queueGroupsDbPath, ...taskGroup, taskIdBuffer], + [...this.queuePathDbPath, ...path, taskIdBuffer], taskIdBuffer, true, ); @@ -748,7 +745,7 @@ class Queue { parameters, taskTimestamp, // Delay, - taskGroup, + path, taskPriority, taskPromise, ); diff --git a/src/tasks/Scheduler.ts b/src/tasks/Scheduler.ts index 6d040fa7d..0233560cd 100644 --- a/src/tasks/Scheduler.ts +++ b/src/tasks/Scheduler.ts @@ -9,7 +9,7 @@ import type { TaskHandlerId, TaskId, TaskParameters, - TaskGroup, + TaskPath, } from './types'; import Logger, { LogLevel } from '@matrixai/logger'; import { IdInternal } from '@matrixai/id'; @@ -330,7 +330,7 @@ class Scheduler { parameters: TaskParameters = [], delay: TaskDelay = 0, priority: number = 0, - taskGroup?: TaskGroup, + path?: TaskPath, lazy: boolean = false, tran?: DBTransaction, ): Promise | undefined> { @@ -341,7 +341,7 @@ class Scheduler { parameters, delay, priority, - taskGroup, + path, lazy, tran, ), @@ -357,7 +357,7 @@ class Scheduler { handlerId, parameters, priority, - taskGroup, + path, lazy, tran, ); @@ -431,18 +431,18 @@ class Scheduler { } @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public async *getGroupTasks( + public async *getTasksByPath( path: TaskPath, lazy: boolean = false, tran?: DBTransaction, ): AsyncGenerator> { if (tran == null) { return yield* this.db.withTransactionG((tran) => - this.getGroupTasks(path, lazy, tran), + this.getTasksByPath(path, lazy, tran), ); } - return yield* this.queue.getGroupTasks(path, lazy, tran); + return yield* this.queue.getTasksByPath(path, lazy, tran); } } diff --git a/src/tasks/Task.ts b/src/tasks/Task.ts index ae3b38bf4..fb0b0eab1 100644 --- a/src/tasks/Task.ts +++ b/src/tasks/Task.ts @@ -6,7 +6,7 @@ import type { TaskDelay, TaskPriority, TaskParameters, - TaskGroup, + TaskPath, } from './types'; import type { DeepReadonly } from '../types'; import type Queue from './Queue'; @@ -17,7 +17,7 @@ class Task { public readonly parameters: DeepReadonly; public readonly timestamp: TaskTimestamp; // Public readonly delay: TaskDelay; - public readonly taskGroup: TaskGroup | undefined; + public readonly path: TaskPath | undefined; public readonly priority: TaskPriority; protected taskPromise: Promise | null; @@ -30,7 +30,7 @@ class Task { parameters: TaskParameters, timestamp: TaskTimestamp, // Delay: TaskDelay, - taskGroup: TaskGroup | undefined, + path: TaskPath | undefined, priority: TaskPriority, taskPromise: Promise | null, ) { @@ -44,7 +44,7 @@ class Task { this.parameters = parameters; this.timestamp = timestamp; // This.delay = delay; - this.taskGroup = taskGroup; + this.path = path; this.priority = priority; this.queue = queue; this.taskPromise = taskPromise; @@ -58,7 +58,7 @@ class Task { parameters: JSON.parse(JSON.stringify(this.parameters)), timestamp: this.timestamp, // Delay: this.delay, - taskGroup: this.taskGroup, + path: this.path, priority: this.priority, }; } diff --git a/src/tasks/types.ts b/src/tasks/types.ts index 260007480..170b0619f 100644 --- a/src/tasks/types.ts +++ b/src/tasks/types.ts @@ -1,5 +1,6 @@ import type { Id } from '@matrixai/id'; import type { POJO, Opaque, Callback } from '../types'; +import type { LevelPath } from '@matrixai/db'; type TaskId = Opaque<'TaskId', Id>; type TaskIdString = Opaque<'TaskIdString', string>; @@ -24,9 +25,9 @@ type TaskParameters = Array; type TaskPriority = Opaque<'TaskPriority', number>; /** - * Task group, array of strings + * Task Path, a LevelPath */ -type TaskGroup = Array; +type TaskPath = LevelPath; /** * Task data to be persisted @@ -36,7 +37,7 @@ type TaskData = { parameters: TaskParameters; timestamp: TaskTimestamp; // Delay: TaskDelay; - taskGroup: TaskGroup | undefined; + path: TaskPath | undefined; priority: TaskPriority; }; @@ -97,7 +98,7 @@ export type { TaskIdString, TaskIdEncoded, // Task, - TaskGroup, + TaskPath, TaskData, TaskInfo, TaskHandlerId, diff --git a/tests/tasks/Queue.test.ts b/tests/tasks/Queue.test.ts index 58a3d6fcf..140234d42 100644 --- a/tests/tasks/Queue.test.ts +++ b/tests/tasks/Queue.test.ts @@ -1,5 +1,5 @@ import type { TaskHandlerId, TaskId } from '../../src/tasks/types'; -import type { TaskGroup } from '../../src/tasks/types'; +import type { TaskPath } from '../../src/tasks/types'; import type Task from '@/tasks/Task'; import os from 'os'; import path from 'path'; @@ -213,9 +213,9 @@ describe(Queue.name, () => { await queue.createTask(handlerId, [7], undefined, ['group2', 'five'], true); await queue.createTask(handlerId, [8], undefined, ['group2', 'six'], true); - const listTasks = async (taskGroup: TaskGroup) => { + const listTasks = async (taskGroup: TaskPath) => { const tasks: Array> = []; - for await (const task of queue.getGroupTasks(taskGroup)) { + for await (const task of queue.getTasksByPath(taskGroup)) { tasks.push(task); } return tasks; From 87721575a1e337f282aca3ac23e9659da4fd0cec Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Fri, 2 Sep 2022 16:59:06 +1000 Subject: [PATCH 16/32] fix(tasks): making `Task` type a POJO of certain task data and a promise --- src/tasks/Queue.ts | 111 +++++++++++++++++++++----------------- src/tasks/Scheduler.ts | 11 ++-- src/tasks/Task.ts | 2 +- src/tasks/types.ts | 8 ++- tests/tasks/Queue.test.ts | 15 +++--- 5 files changed, 82 insertions(+), 65 deletions(-) diff --git a/src/tasks/Queue.ts b/src/tasks/Queue.ts index f1a3d7c03..07bdfc07d 100644 --- a/src/tasks/Queue.ts +++ b/src/tasks/Queue.ts @@ -1,15 +1,17 @@ import type { DB, LevelPath, KeyPath } from '@matrixai/db'; import type { + Task, TaskData, TaskHandlerId, TaskHandler, TaskTimestamp, TaskParameters, TaskIdEncoded, + TaskId, + TaskPath, } from './types'; import type KeyManager from '../keys/KeyManager'; import type { DBTransaction } from '@matrixai/db'; -import type { TaskId, TaskPath } from './types'; import Logger from '@matrixai/logger'; import { CreateDestroyStartStop, @@ -20,7 +22,6 @@ import { RWLockReader } from '@matrixai/async-locks'; import { extractTs } from '@matrixai/id/dist/IdSortable'; import * as tasksErrors from './errors'; import * as tasksUtils from './utils'; -import Task from './Task'; import { Plug } from '../utils/index'; class TaskEvent extends Event { @@ -588,34 +589,38 @@ class Queue { taskId: TaskId, lazy: boolean = false, tran?: DBTransaction, - ): Promise> { + ): Promise { if (tran == null) { return this.db.withTransactionF((tran) => this.getTask(taskId, lazy, tran), ); } - const taskData = await tran.get([ ...this.queueTasksDbPath, taskId.toBuffer(), ]); if (taskData == null) throw Error('TMP task not found'); - - let taskPromise: Promise | null = null; - if (!lazy) { - taskPromise = this.getTaskP(taskId, tran); + const taskStartTime = await tran.get([ + ...this.queueStartTimeDbPath, + taskId.toBuffer(), + ]); + let promise: () => Promise; + if (lazy) { + promise = () => this.getTaskP(taskId); + } else { + const prom = this.getTaskP(taskId, tran); + promise = () => prom; } - return new Task( - this, - taskId, - taskData.handlerId, - taskData.parameters, - taskData.timestamp, - // Delay, - taskData.path, - taskData.priority, - taskPromise, - ); + return { + id: taskId, + handlerId: taskData.handlerId, + parameters: taskData.parameters, + timestamp: taskData.timestamp, + startTime: taskStartTime, + path: taskData.path, + priority: taskData.priority, + promise, + }; } /** @@ -627,33 +632,39 @@ class Queue { order: 'asc' | 'desc' = 'asc', lazy: boolean = false, tran?: DBTransaction, - ): AsyncGenerator> { + ): AsyncGenerator { if (tran == null) { return yield* this.db.withTransactionG((tran) => this.getTasks(order, lazy, tran), ); } - for await (const [keyPath, taskData] of tran.iterator( + for await (const [taskIdPath, taskData] of tran.iterator( this.queueTasksDbPath, { valueAsBuffer: false, reverse: order !== 'asc' }, )) { - const taskId = IdInternal.fromBuffer(keyPath[0] as Buffer); - let taskPromise: Promise | null = null; - if (!lazy) { - taskPromise = this.getTaskP(taskId, tran); + const taskId = IdInternal.fromBuffer(taskIdPath[0] as Buffer); + const taskStartTime = await tran.get([ + ...this.queueStartTimeDbPath, + ...taskIdPath, + ]); + let promise: () => Promise; + if (lazy) { + promise = () => this.getTaskP(taskId); + } else { + const prom = this.getTaskP(taskId, tran); + promise = () => prom; } - yield new Task( - this, - taskId, - taskData.handlerId, - taskData.parameters, - taskData.timestamp, - // Delay, - taskData.path, - taskData.priority, - taskPromise, - ); + yield { + id: taskId, + handlerId: taskData.handlerId, + parameters: taskData.parameters, + timestamp: taskData.timestamp, + startTime: taskStartTime, + path: taskData.path, + priority: taskData.priority, + promise, + }; } } @@ -662,7 +673,7 @@ class Queue { path: TaskPath, lazy: boolean = false, tran?: DBTransaction, - ): AsyncGenerator> { + ): AsyncGenerator { if (tran == null) { return yield* this.db.withTransactionG((tran) => this.getTasksByPath(path, lazy, tran), @@ -697,7 +708,7 @@ class Queue { path?: TaskPath, lazy: boolean = false, tran?: DBTransaction, - ): Promise> { + ): Promise { if (tran == null) { return this.db.withTransactionF((tran) => this.createTask(handlerId, parameters, priority, path, lazy, tran), @@ -734,21 +745,23 @@ class Queue { true, ); } - let taskPromise: Promise | null = null; - if (!lazy) { - taskPromise = this.getTaskP(taskId, tran); + let promise: () => Promise; + if (lazy) { + promise = () => this.getTaskP(taskId); + } else { + const prom = this.getTaskP(taskId, tran); + promise = () => prom; } - return new Task( - this, - taskId, + return { + id: taskId, handlerId, parameters, - taskTimestamp, - // Delay, path, - taskPriority, - taskPromise, - ); + priority: taskPriority, + timestamp: taskTimestamp, + startTime: undefined, + promise, + }; } } diff --git a/src/tasks/Scheduler.ts b/src/tasks/Scheduler.ts index 0233560cd..eb39c993a 100644 --- a/src/tasks/Scheduler.ts +++ b/src/tasks/Scheduler.ts @@ -1,10 +1,10 @@ import type { DB, LevelPath } from '@matrixai/db'; import type { TaskIdString } from './types'; import type KeyManager from '../keys/KeyManager'; -import type Task from './Task'; import type Queue from './Queue'; import type { DBTransaction } from '@matrixai/db'; import type { + Task, TaskDelay, TaskHandlerId, TaskId, @@ -20,7 +20,6 @@ import { import lexi from 'lexicographic-integer'; import * as tasksUtils from './utils'; import * as tasksErrors from './errors'; -import { TaskData } from './types'; import { Plug } from '../utils/index'; interface Scheduler extends CreateDestroyStartStop {} @@ -333,7 +332,7 @@ class Scheduler { path?: TaskPath, lazy: boolean = false, tran?: DBTransaction, - ): Promise | undefined> { + ): Promise { if (tran == null) { return this.db.withTransactionF((tran) => this.scheduleTask( @@ -400,7 +399,7 @@ class Scheduler { taskId: TaskId, lazy: boolean = false, tran?: DBTransaction, - ): Promise> { + ): Promise { if (tran == null) { return this.db.withTransactionF((tran) => this.getTask(taskId, lazy, tran), @@ -420,7 +419,7 @@ class Scheduler { order: 'asc' | 'desc' = 'asc', lazy: boolean = false, tran?: DBTransaction, - ): AsyncGenerator> { + ): AsyncGenerator { if (tran == null) { return yield* this.db.withTransactionG((tran) => this.getTasks(order, lazy, tran), @@ -435,7 +434,7 @@ class Scheduler { path: TaskPath, lazy: boolean = false, tran?: DBTransaction, - ): AsyncGenerator> { + ): AsyncGenerator { if (tran == null) { return yield* this.db.withTransactionG((tran) => this.getTasksByPath(path, lazy, tran), diff --git a/src/tasks/Task.ts b/src/tasks/Task.ts index fb0b0eab1..e88702847 100644 --- a/src/tasks/Task.ts +++ b/src/tasks/Task.ts @@ -3,7 +3,6 @@ import type { TaskData, TaskHandlerId, TaskTimestamp, - TaskDelay, TaskPriority, TaskParameters, TaskPath, @@ -11,6 +10,7 @@ import type { import type { DeepReadonly } from '../types'; import type Queue from './Queue'; +// FIXME: this file isn't needed anymore? class Task { public readonly id: TaskId; public readonly handlerId: TaskHandlerId; diff --git a/src/tasks/types.ts b/src/tasks/types.ts index 170b0619f..ab64dbdd5 100644 --- a/src/tasks/types.ts +++ b/src/tasks/types.ts @@ -41,6 +41,12 @@ type TaskData = { priority: TaskPriority; }; +type Task = TaskData & { + id: TaskId; + startTime: TaskTimestamp | undefined; + promise: () => Promise | undefined; +}; + /** * Task information that is returned to the user */ @@ -97,7 +103,7 @@ export type { TaskId, TaskIdString, TaskIdEncoded, - // Task, + Task, TaskPath, TaskData, TaskInfo, diff --git a/tests/tasks/Queue.test.ts b/tests/tasks/Queue.test.ts index 140234d42..65f54648a 100644 --- a/tests/tasks/Queue.test.ts +++ b/tests/tasks/Queue.test.ts @@ -1,6 +1,5 @@ import type { TaskHandlerId, TaskId } from '../../src/tasks/types'; -import type { TaskPath } from '../../src/tasks/types'; -import type Task from '@/tasks/Task'; +import type { TaskPath, Task } from '../../src/tasks/types'; import os from 'os'; import path from 'path'; import fs from 'fs'; @@ -214,7 +213,7 @@ describe(Queue.name, () => { await queue.createTask(handlerId, [8], undefined, ['group2', 'six'], true); const listTasks = async (taskGroup: TaskPath) => { - const tasks: Array> = []; + const tasks: Array = []; for await (const task of queue.getTasksByPath(taskGroup)) { tasks.push(task); } @@ -270,7 +269,7 @@ describe(Queue.name, () => { const taskSucceed = await pushTask(queue, handlerId, [true], false); // Promise should succeed with result - const taskSucceedP = taskSucceed!.promise; + const taskSucceedP = taskSucceed!.promise(); await expect(taskSucceedP).resolves.toBe(true); await queue.stop(); @@ -291,7 +290,7 @@ describe(Queue.name, () => { const taskFail = await pushTask(queue, handlerId, [false], false); // Promise should fail - const taskFailP = taskFail!.promise; + const taskFailP = taskFail!.promise(); await expect(taskFailP).rejects.toBeInstanceOf(Error); await queue.stop(); @@ -316,7 +315,7 @@ describe(Queue.name, () => { const prom1 = queue.getTaskP(taskSucceed.id); const prom2 = queue.getTaskP(taskSucceed.id); expect(prom1).toBe(prom2); - expect(prom1).toBe(taskSucceed!.promise); + expect(prom1).toBe(taskSucceed!.promise()); await queue.stop(); }); @@ -364,7 +363,7 @@ describe(Queue.name, () => { await queue.startTasks(); await prom; // Finished tasks should throw - await expect(taskSucceed?.promise).rejects.toThrow(); + await expect(taskSucceed?.promise()).rejects.toThrow(); await queue.stop(); }); @@ -385,7 +384,7 @@ describe(Queue.name, () => { await queue.startTasks(); const taskSucceed = await pushTask(queue, handlerId, [true], false); - await expect(taskSucceed?.promise).resolves.toBe(true); + await expect(taskSucceed?.promise()).resolves.toBe(true); await queue.stop(); }); From af446592a6d3d1d6b9e5c5d67d4de130e43cba4c Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Mon, 12 Sep 2022 01:49:05 +1000 Subject: [PATCH 17/32] fix(timer): Timer now maintains event loop reference, and can have infinite timers --- src/timer/Timer.ts | 12 ++++-------- tests/timer/Timer.test.ts | 16 ---------------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/src/timer/Timer.ts b/src/timer/Timer.ts index a488d123b..ad14b316a 100644 --- a/src/timer/Timer.ts +++ b/src/timer/Timer.ts @@ -145,20 +145,16 @@ class Timer this.rejectP = reject.bind(this.p); }, abortController); this.abortController = abortController; - // If the delay is Infinity, there is no `setTimeout` - // therefore this promise will never resolve + // If the delay is Infinity, this promise will never resolve // it may still reject however if (isFinite(delay)) { this.timeoutRef = setTimeout(() => void this.fulfill(), delay); - if (typeof this.timeoutRef.unref === 'function') { - // Do not keep the event loop alive - this.timeoutRef.unref(); - } this.timestamp = new Date(performance.timeOrigin + performance.now()); this.scheduled = new Date(this.timestamp.getTime() + delay); } else { - // There is no `setTimeout` nor `setInterval` - // so the event loop will not be kept alive + // Infinite interval, make sure you are cancelling the `Timer` + // otherwise you will keep the process alive + this.timeoutRef = setInterval(() => {}, 2**31 - 1); this.timestamp = new Date(performance.timeOrigin + performance.now()); } } diff --git a/tests/timer/Timer.test.ts b/tests/timer/Timer.test.ts index fe8621575..9b43cdd32 100644 --- a/tests/timer/Timer.test.ts +++ b/tests/timer/Timer.test.ts @@ -60,22 +60,6 @@ describe(Timer.name, () => { t1.cancel(new Error('Oh No')); await expect(t1).rejects.toThrow('Oh No'); }); - test('timer does not keep event loop alive', async () => { - const f = async (timer: Timer | number = globalThis.maxTimeout) => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - timer = timer instanceof Timer ? timer : new Timer({ delay: timer }); - }; - const g = async (timer: Timer | number = Infinity) => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - timer = timer instanceof Timer ? timer : new Timer({ delay: timer }); - }; - await f(); - await f(); - await f(); - await g(); - await g(); - await g(); - }); test('custom signal handler ignores default rejection', async () => { const onabort = jest.fn(); const t = new Timer( From 5ffa6dd5c88d6b2f6a60f30254d8455cf248dc74 Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Mon, 12 Sep 2022 01:56:57 +1000 Subject: [PATCH 18/32] fix(contexts): decorators should check for generators, not iterables --- src/contexts/decorators/timed.ts | 4 +-- src/contexts/functions/timed.ts | 4 +-- src/utils/utils.ts | 34 ++++++++++++++++++---- tests/contexts/decorators/timed.test.ts | 38 ++++++++++++++++++++----- tests/contexts/functions/timed.test.ts | 34 ++++++++++++++++------ 5 files changed, 89 insertions(+), 25 deletions(-) diff --git a/src/contexts/decorators/timed.ts b/src/contexts/decorators/timed.ts index 038b9ebaf..875aa1363 100644 --- a/src/contexts/decorators/timed.ts +++ b/src/contexts/decorators/timed.ts @@ -214,7 +214,7 @@ function timed( throw e; }, ); - } else if (utils.isIterable(result)) { + } else if (utils.isGenerator(result)) { return (function* () { try { return yield* result; @@ -222,7 +222,7 @@ function timed( teardownContext(); } })(); - } else if (utils.isAsyncIterable(result)) { + } else if (utils.isAsyncGenerator(result)) { return (async function* () { try { return yield* result; diff --git a/src/contexts/functions/timed.ts b/src/contexts/functions/timed.ts index 07e66970d..5c60c6b69 100644 --- a/src/contexts/functions/timed.ts +++ b/src/contexts/functions/timed.ts @@ -177,7 +177,7 @@ function timed< throw e; }, ); - } else if (utils.isIterable(result)) { + } else if (utils.isGenerator(result)) { return (function* () { try { return yield* result; @@ -185,7 +185,7 @@ function timed< teardownContext(); } })(); - } else if (utils.isAsyncIterable(result)) { + } else if (utils.isAsyncGenerator(result)) { return (async function* () { try { return yield* result; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 066e69d7b..03058031e 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -327,12 +327,34 @@ function isPromiseLike(v: any): v is PromiseLike { return v != null && typeof v.then === 'function'; } -function isIterable(v: any): v is Iterable { - return v != null && typeof v[Symbol.iterator] === 'function'; +/** + * Is generator object + * Use this to check for generators + */ +function isGenerator(v: any): v is Generator { + return ( + v != null && + typeof v[Symbol.iterator] === 'function' && + typeof v.next === 'function' && + typeof v.return === 'function' && + typeof v.throw === 'function' + ); } -function isAsyncIterable(v: any): v is AsyncIterable { - return v != null && typeof v[Symbol.asyncIterator] === 'function'; +/** + * Is async generator object + * Use this to check for async generators + */ +function isAsyncGenerator(v: any): v is AsyncGenerator { + return ( + v != null && + typeof v === 'object' && + typeof v[Symbol.asyncIterator] === 'function' && + typeof v.next === 'function' && + typeof v.return === 'function' && + typeof v.throw === 'function' + ); +} } export { @@ -362,6 +384,6 @@ export { debounce, isPromise, isPromiseLike, - isIterable, - isAsyncIterable, + isGenerator, + isAsyncGenerator, }; diff --git a/tests/contexts/decorators/timed.test.ts b/tests/contexts/decorators/timed.test.ts index f0c8e790d..aee7af5a5 100644 --- a/tests/contexts/decorators/timed.test.ts +++ b/tests/contexts/decorators/timed.test.ts @@ -56,16 +56,31 @@ describe('context/decorators/timed', () => { functionValue( ctx?: Partial, check?: (t: Timer) => any, - ): void; + ): string; @timed(1000) functionValue( @context ctx: ContextTimed, check?: (t: Timer) => any, - ): void { + ): string { expect(ctx.signal).toBeInstanceOf(AbortSignal); expect(ctx.timer).toBeInstanceOf(Timer); if (check != null) check(ctx.timer); - return; + return 'hello world'; + } + + functionValueArray( + ctx?: Partial, + check?: (t: Timer) => any, + ): Array; + @timed(1000) + functionValueArray( + @context ctx: ContextTimed, + check?: (t: Timer) => any, + ): Array { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(ctx.timer).toBeInstanceOf(Timer); + if (check != null) check(ctx.timer); + return [1,2,3,4]; } functionPromise( @@ -166,14 +181,23 @@ describe('context/decorators/timed', () => { } const x = new X(); test('functionValue', () => { - x.functionValue(); - x.functionValue({}); - x.functionValue({ timer: new Timer({ delay: 100 }) }, (t) => { + expect(x.functionValue()).toBe('hello world'); + expect(x.functionValue({})).toBe('hello world'); + expect(x.functionValue({ timer: new Timer({ delay: 100 }) }, (t) => { expect(t.delay).toBe(100); - }); + })).toBe('hello world'); expect(x.functionValue).toBeInstanceOf(Function); expect(x.functionValue.name).toBe('functionValue'); }); + test('functionValueArray', () => { + expect(x.functionValueArray()).toStrictEqual([1,2,3,4]); + expect(x.functionValueArray({})).toStrictEqual([1,2,3,4]); + expect(x.functionValueArray({ timer: new Timer({ delay: 100 }) }, (t) => { + expect(t.delay).toBe(100); + })).toStrictEqual([1,2,3,4]); + expect(x.functionValueArray).toBeInstanceOf(Function); + expect(x.functionValueArray.name).toBe('functionValueArray'); + }); test('functionPromise', async () => { await x.functionPromise(); await x.functionPromise({}); diff --git a/tests/contexts/functions/timed.test.ts b/tests/contexts/functions/timed.test.ts index ca75a1771..d9a4d0bac 100644 --- a/tests/contexts/functions/timed.test.ts +++ b/tests/contexts/functions/timed.test.ts @@ -22,11 +22,29 @@ describe('context/functions/timed', () => { return 'hello world'; }; const fTimed = timed(f); - fTimed(undefined); - fTimed({}); - fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(fTimed(undefined)).toBe('hello world'); + expect(fTimed({})).toBe('hello world'); + expect(fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { expect(t.delay).toBe(50); - }); + })).toBe('hello world'); + expect(fTimed).toBeInstanceOf(Function); + }); + test('function value array', () => { + const f = function ( + ctx: ContextTimed, + check?: (t: Timer) => any, + ): Array { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + if (check != null) check(ctx.timer); + return [1,2,3,4]; + }; + const fTimed = timed(f); + expect(fTimed(undefined)).toStrictEqual([1,2,3,4]); + expect(fTimed({})).toStrictEqual([1,2,3,4]); + expect(fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + })).toStrictEqual([1,2,3,4]); expect(fTimed).toBeInstanceOf(Function); }); test('function promise', async () => { @@ -40,11 +58,11 @@ describe('context/functions/timed', () => { return new Promise((resolve) => void resolve()); }; const fTimed = timed(f); - await fTimed(undefined); - await fTimed({}); - await fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(await fTimed(undefined)).toBeUndefined(); + expect(await fTimed({})).toBeUndefined(); + expect(await fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { expect(t.delay).toBe(50); - }); + })).toBeUndefined(); expect(fTimed).toBeInstanceOf(Function); }); test('async function', async () => { From e2852df52b5e61750234f2e32c0a623c29ab0410 Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Mon, 12 Sep 2022 01:57:47 +1000 Subject: [PATCH 19/32] feat(tasks): centralising Queue and Scheduler into a single `TaskManager` * tasks can be cancelled at any stage: scheduled, queued or active * `TaskData` is suitable to be encoded into JSON and back * Graceful shutdown of `TaskManager` * `TaskHandler` gets `TaskInfo` as second parameter after the `ContextTimed` --- src/tasks/Queue.ts | 795 ------------------- src/tasks/Scheduler.ts | 448 ----------- src/tasks/Task.ts | 101 --- src/tasks/TaskEvent.ts | 33 + src/tasks/TaskManager.ts | 1251 ++++++++++++++++++++++++++++++ src/tasks/errors.ts | 124 +-- src/tasks/index.ts | 2 +- src/tasks/types.ts | 158 ++-- src/tasks/utils.ts | 139 ++-- src/utils/Plug.ts | 36 - src/utils/debug.ts | 29 + src/utils/index.ts | 1 - src/utils/utils.ts | 33 + tests/tasks/Scheduler.test.ts | 1 + tests/tasks/TaskManager.test.ts | 1266 +++++++++++++++++++++++++++++++ tests/utils/Plug.test.ts | 19 - 16 files changed, 2861 insertions(+), 1575 deletions(-) delete mode 100644 src/tasks/Queue.ts delete mode 100644 src/tasks/Scheduler.ts delete mode 100644 src/tasks/Task.ts create mode 100644 src/tasks/TaskEvent.ts create mode 100644 src/tasks/TaskManager.ts delete mode 100644 src/utils/Plug.ts create mode 100644 src/utils/debug.ts create mode 100644 tests/tasks/TaskManager.test.ts delete mode 100644 tests/utils/Plug.test.ts diff --git a/src/tasks/Queue.ts b/src/tasks/Queue.ts deleted file mode 100644 index 07bdfc07d..000000000 --- a/src/tasks/Queue.ts +++ /dev/null @@ -1,795 +0,0 @@ -import type { DB, LevelPath, KeyPath } from '@matrixai/db'; -import type { - Task, - TaskData, - TaskHandlerId, - TaskHandler, - TaskTimestamp, - TaskParameters, - TaskIdEncoded, - TaskId, - TaskPath, -} from './types'; -import type KeyManager from '../keys/KeyManager'; -import type { DBTransaction } from '@matrixai/db'; -import Logger from '@matrixai/logger'; -import { - CreateDestroyStartStop, - ready, -} from '@matrixai/async-init/dist/CreateDestroyStartStop'; -import { IdInternal } from '@matrixai/id'; -import { RWLockReader } from '@matrixai/async-locks'; -import { extractTs } from '@matrixai/id/dist/IdSortable'; -import * as tasksErrors from './errors'; -import * as tasksUtils from './utils'; -import { Plug } from '../utils/index'; - -class TaskEvent extends Event { - detail?: any; - constructor(type: string, options?: CustomEventInit) { - super(type, options); - this.detail = options?.detail; - } -} - -interface Queue extends CreateDestroyStartStop {} -@CreateDestroyStartStop( - new tasksErrors.ErrorQueueRunning(), - new tasksErrors.ErrorQueueDestroyed(), -) -class Queue { - public static async createQueue({ - db, - keyManager, - handlers = {}, - delay = false, - concurrencyLimit = Number.POSITIVE_INFINITY, - logger = new Logger(this.name), - fresh = false, - }: { - db: DB; - keyManager: KeyManager; - handlers?: Record; - delay?: boolean; - concurrencyLimit?: number; - logger?: Logger; - fresh?: boolean; - }) { - logger.info(`Creating ${this.name}`); - const queue = new this({ db, keyManager, concurrencyLimit, logger }); - await queue.start({ handlers, delay, fresh }); - logger.info(`Created ${this.name}`); - return queue; - } - - // Concurrency variables - public concurrencyLimit: number; - protected concurrencyCount: number = 0; - protected concurrencyPlug: Plug = new Plug(); - protected activeTasksPlug: Plug = new Plug(); - - protected logger: Logger; - protected db: DB; - protected queueDbPath: LevelPath = [this.constructor.name]; - /** - * Tasks collection - * `tasks/{TaskId} -> {json(Task)}` - */ - public readonly queueTasksDbPath: LevelPath = [...this.queueDbPath, 'tasks']; - public readonly queueStartTimeDbPath: LevelPath = [ - ...this.queueDbPath, - 'startTime', - ]; - /** - * This is used to track pending tasks in order of start time - */ - protected queueDbTimestampPath: LevelPath = [ - ...this.queueDbPath, - 'timestamp', - ]; - // FIXME: remove this path, data is part of the task data record - protected queueDbMetadataPath: LevelPath = [...this.queueDbPath, 'metadata']; - /** - * Tracks actively running tasks - */ - protected queueDbActivePath: LevelPath = [...this.queueDbPath, 'active']; - /** - * Tasks by Path - * `groups/...taskPath: LevelPath -> {raw(TaskId)}` - */ - public readonly queuePathDbPath: LevelPath = [...this.queueDbPath, 'groups']; - /** - * Last Task Id - */ - public readonly queueLastTaskIdPath: KeyPath = [ - ...this.queueDbPath, - 'lastTaskId', - ]; - - // /** - // * Listeners for task execution - // * When a task is executed, these listeners are synchronously executed - // * The listeners are intended for resolving or rejecting task promises - // */ - // protected listeners: Map> = new Map(); - - // variables to consuming tasks - protected activeTaskLoop: Promise | null = null; - protected taskLoopPlug: Plug = new Plug(); - protected taskLoopEnding: boolean; - // FIXME: might not be needed - protected cleanUpLock: RWLockReader = new RWLockReader(); - - protected handlers: Map = new Map(); - protected taskPromises: Map> = new Map(); - protected taskEvents: EventTarget = new EventTarget(); - protected keyManager: KeyManager; - protected generateTaskId: () => TaskId; - - public constructor({ - db, - keyManager, - concurrencyLimit, - logger, - }: { - db: DB; - keyManager: KeyManager; - concurrencyLimit: number; - logger: Logger; - }) { - this.logger = logger; - this.concurrencyLimit = concurrencyLimit; - this.db = db; - this.keyManager = keyManager; - } - - public async start({ - handlers = {}, - delay = false, - fresh = false, - }: { - handlers?: Record; - delay?: boolean; - fresh?: boolean; - } = {}): Promise { - this.logger.info(`Starting ${this.constructor.name}`); - if (fresh) { - this.handlers.clear(); - await this.db.clear(this.queueDbPath); - } - const lastTaskId = await this.getLastTaskId(); - this.generateTaskId = tasksUtils.createTaskIdGenerator( - this.keyManager.getNodeId(), - lastTaskId, - ); - for (const taskHandlerId in handlers) { - this.handlers.set( - taskHandlerId as TaskHandlerId, - handlers[taskHandlerId], - ); - } - if (!delay) await this.startTasks(); - this.logger.info(`Started ${this.constructor.name}`); - } - - public async stop(): Promise { - this.logger.info(`Stopping ${this.constructor.name}`); - await this.stopTasks(); - this.logger.info(`Stopped ${this.constructor.name}`); - } - - public async destroy() { - this.logger.info(`Destroying ${this.constructor.name}`); - this.handlers.clear(); - await this.db.clear(this.queueDbPath); - this.logger.info(`Destroyed ${this.constructor.name}`); - } - - // Promises are "connected" to events - // - // when tasks are "dispatched" to the queue - // they are actually put into a persistent system - // then we proceed to execution - // - // a task here is a function - // this is already managed by the Scheduler - // along with the actual function itself? - // we also have a priority - // - // t is a task - // but it's actually just a function - // and in this case - // note that we are "passing" in the parameters at this point - // but it is any function - // () => taskHandler(parameters) - // - // it returns a "task" - // that should be used like a "lazy" promise - // the actual task function depends on the situation - // don't we need to know actual metadata - // wait a MINUTE - // if we are "persisting" it - // do we persist it here? - - /** - * Pushes tasks into the persistent database - */ - @ready(new tasksErrors.ErrorQueueNotRunning()) - public async pushTask( - taskId: TaskId, - taskTimestampKey: Buffer, - tran?: DBTransaction, - ): Promise { - if (tran == null) { - return this.db.withTransactionF((tran) => - this.pushTask(taskId, taskTimestampKey, tran), - ); - } - - this.logger.info('adding task'); - await tran.lock([ - [...this.queueDbTimestampPath, 'loopSerialisation'].join(''), - 'read', - ]); - await tran.put( - [...this.queueStartTimeDbPath, taskId.toBuffer()], - taskTimestampKey, - true, - ); - await tran.put( - [...this.queueDbTimestampPath, taskTimestampKey], - taskId.toBuffer(), - true, - ); - await tran.put( - [...this.queueDbMetadataPath, taskId.toBuffer()], - taskTimestampKey, - true, - ); - tran.queueSuccess(async () => await this.taskLoopPlug.unplug()); - } - - /** - * Removes a task from the persistent database - */ - // @ready(new tasksErrors.ErrorQueueNotRunning(), false, ['stopping', 'starting']) - public async removeTask(taskId: TaskId, tran?: DBTransaction) { - if (tran == null) { - return this.db.withTransactionF((tran) => this.removeTask(taskId, tran)); - } - - this.logger.info('removing task'); - await tran.lock([ - [...this.queueDbTimestampPath, 'loopSerialisation'].join(''), - 'read', - ]); - const timestampBuffer = await tran.get( - [...this.queueDbMetadataPath, taskId.toBuffer()], - true, - ); - // Noop - if (timestampBuffer == null) return; - // Removing records - await tran.del([...this.queueDbTimestampPath, timestampBuffer]); - await tran.del([...this.queueDbMetadataPath, taskId.toBuffer()]); - await tran.del([...this.queueDbActivePath, taskId.toBuffer()]); - } - - /** - * This will get the next task based on priority - */ - protected async getNextTask( - tran?: DBTransaction, - ): Promise { - if (tran == null) { - return this.db.withTransactionF((tran) => this.getNextTask(tran)); - } - - await tran.lock([ - [...this.queueDbTimestampPath, 'loopSerialisation'].join(''), - 'write', - ]); - // Read out the database until we read a task not already executing - let taskId: TaskId | undefined; - for await (const [, taskIdBuffer] of tran.iterator( - this.queueDbTimestampPath, - )) { - taskId = IdInternal.fromBuffer(taskIdBuffer); - const exists = await tran.get( - [...this.queueDbActivePath, taskId.toBuffer()], - true, - ); - // Looking for an inactive task - if (exists == null) break; - taskId = undefined; - } - if (taskId == null) return; - await tran.put( - [...this.queueDbActivePath, taskId.toBuffer()], - Buffer.alloc(0, 0), - true, - ); - return taskId; - } - - @ready(new tasksErrors.ErrorQueueNotRunning(), false, ['starting']) - public async startTasks() { - // Nop if running - if (this.activeTaskLoop != null) return; - - this.activeTaskLoop = this.initTaskLoop(); - // Unplug if tasks exist to be consumed - for await (const _ of this.db.iterator(this.queueDbTimestampPath, { - limit: 1, - })) { - // Unplug if tasks exist - await this.taskLoopPlug.unplug(); - } - } - - @ready(new tasksErrors.ErrorQueueNotRunning(), false, ['stopping']) - public async stopTasks() { - this.taskLoopEnding = true; - await this.taskLoopPlug.unplug(); - await this.concurrencyPlug.unplug(); - await this.activeTaskLoop; - this.activeTaskLoop = null; - // FIXME: likely not needed, remove - await this.cleanUpLock.waitForUnlock(); - } - - protected async initTaskLoop() { - this.logger.info('initializing task loop'); - this.taskLoopEnding = false; - await this.taskLoopPlug.plug(); - const pace = async () => { - if (this.taskLoopEnding) return false; - await this.taskLoopPlug.waitForUnplug(); - await this.concurrencyPlug.waitForUnplug(); - return !this.taskLoopEnding; - }; - while (await pace()) { - // Check for task - const nextTaskId = await this.getNextTask(); - if (nextTaskId == null) { - this.logger.info('no task found, waiting'); - await this.taskLoopPlug.plug(); - continue; - } - - // Do the task with concurrency here. - // We need to call whatever dispatches tasks here - // and hook lifecycle to the promise. - // call scheduler. handleTask? - const taskIdEncoded = tasksUtils.encodeTaskId(nextTaskId); - await this.concurrencyIncrement(); - const prom = this.handleTask(nextTaskId); - this.logger.info(`started task ${taskIdEncoded}`); - - const [cleanupRelease] = await this.cleanUpLock.read()(); - const onFinally = async () => { - await this.concurrencyDecrement(); - await cleanupRelease(); - }; - - void prom.then( - async () => { - await this.removeTask(nextTaskId); - // TODO: emit an event for completed task - await onFinally(); - }, - async () => { - // FIXME: should only remove failed tasks but not cancelled - await this.removeTask(nextTaskId); - // TODO: emit an event for a failed or cancelled task - await onFinally(); - }, - ); - } - await this.activeTasksPlug.waitForUnplug(); - this.logger.info('dispatching ending'); - } - - // Concurrency limiting methods - /** - * Awaits an open slot in the concurrency. - * Must be paired with `concurrencyDecrement` when operation is done. - */ - - /** - * Increment and concurrencyPlug if full - */ - protected async concurrencyIncrement() { - if (this.concurrencyCount < this.concurrencyLimit) { - this.concurrencyCount += 1; - await this.activeTasksPlug.plug(); - if (this.concurrencyCount >= this.concurrencyLimit) { - await this.concurrencyPlug.plug(); - } - } - } - - /** - * Decrement and unplugs, resolves concurrencyActivePromise if empty - */ - protected async concurrencyDecrement() { - this.concurrencyCount -= 1; - if (this.concurrencyCount < this.concurrencyLimit) { - await this.concurrencyPlug.unplug(); - } - if (this.concurrencyCount === 0) { - await this.activeTasksPlug.unplug(); - } - } - - /** - * Will resolve when the concurrency counter reaches 0 - */ - public async allActiveTasksSettled() { - await this.activeTasksPlug.waitForUnplug(); - } - - /** - * IF a handler does not exist - * if the task is executed - * then an exception is thrown - * if listener exists, the exception is passed into this listener function - * if it doesn't exist, then it's just a reference exception in general, this can be logged - * There's nothing else to do - */ - // @ready(new tasksErrors.ErrorSchedulerNotRunning()) - // protected registerListener( - // taskId: TaskId, - // taskListener: TaskListener - // ): void { - // const taskIdString = taskId.toString() as TaskIdString; - // const taskListeners = this.listeners.get(taskIdString); - // if (taskListeners != null) { - // taskListeners.push(taskListener); - // } else { - // this.listeners.set(taskIdString, [taskListener]); - // } - // } - - // @ready(new tasksErrors.ErrorSchedulerNotRunning()) - // protected deregisterListener( - // taskId: TaskId, - // taskListener: TaskListener - // ): void { - // const taskIdString = taskId.toString() as TaskIdString; - // const taskListeners = this.listeners.get(taskIdString); - // if (taskListeners == null || taskListeners.length < 1) return; - // const index = taskListeners.indexOf(taskListener); - // if (index !== -1) { - // taskListeners.splice(index, 1); - // } - // } - - protected async handleTask(taskId: TaskId) { - // Get the task information and use the relevant handler - // throw and error if the task does not exist - // throw an error if the handler does not exist. - - return await this.db.withTransactionF(async (tran) => { - // Getting task information - const taskData = await tran.get([ - ...this.queueTasksDbPath, - taskId.toBuffer(), - ]); - if (taskData == null) throw Error('TEMP task not found'); - // Getting handler - const handler = this.getHandler(taskData.handlerId); - if (handler == null) throw Error('TEMP handler not found'); - - const prom = handler(...taskData.parameters); - - // Add the promise to the map and hook any lifecycle stuff - const taskIdEncoded = tasksUtils.encodeTaskId(taskId); - return prom - .finally(async () => { - // Cleaning up is a separate transaction - await this.db.withTransactionF(async (tran) => { - const taskTimestampKeybuffer = await tran.get( - [...this.queueStartTimeDbPath, taskId.toBuffer()], - true, - ); - await tran.del([...this.queueTasksDbPath, taskId.toBuffer()]); - await tran.del([...this.queueStartTimeDbPath, taskId.toBuffer()]); - if (taskData.path != null) { - await tran.del([ - ...this.queuePathDbPath, - ...taskData.path, - taskTimestampKeybuffer!, - ]); - } - }); - }) - .then( - (result) => { - this.taskEvents.dispatchEvent( - new TaskEvent(taskIdEncoded, { detail: [undefined, result] }), - ); - return result; - }, - (reason) => { - this.taskEvents.dispatchEvent( - new TaskEvent(taskIdEncoded, { detail: [reason] }), - ); - throw reason; - }, - ); - }); - } - - public getHandler(handlerId: TaskHandlerId): TaskHandler | undefined { - return this.handlers.get(handlerId); - } - - public getHandlers(): Record { - return Object.fromEntries(this.handlers); - } - - /** - * Registers a handler for tasks with the same `TaskHandlerId` - * If tasks are dispatched without their respective handler, - * the scheduler will throw `tasksErrors.ErrorSchedulerHandlerMissing` - */ - public registerHandler(handlerId: TaskHandlerId, handler: TaskHandler) { - this.handlers.set(handlerId, handler); - } - - /** - * Deregisters a handler - */ - public deregisterHandler(handlerId: TaskHandlerId) { - this.handlers.delete(handlerId); - } - - @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public getTaskP(taskId: TaskId, tran?: DBTransaction): Promise { - const taskIdEncoded = tasksUtils.encodeTaskId(taskId); - // This will return a task promise if it already exists - const existingTaskPromise = this.taskPromises.get(taskIdEncoded); - if (existingTaskPromise != null) return existingTaskPromise; - - // If the task exist then it will create the task promise and return that - const newTaskPromise = new Promise((resolve, reject) => { - const resultListener = (event: TaskEvent) => { - const [e, result] = event.detail; - if (e != null) reject(e); - else resolve(result); - }; - this.taskEvents.addEventListener(taskIdEncoded, resultListener, { - once: true, - }); - // If not task promise exists then with will check if the task exists - void (tran ?? this.db) - .get([...this.queueTasksDbPath, taskId.toBuffer()], true) - .then( - (taskData) => { - if (taskData == null) { - this.taskEvents.removeEventListener( - taskIdEncoded, - resultListener, - ); - reject(Error('TEMP task not found')); - } - }, - (reason) => reject(reason), - ); - }).finally(() => { - this.taskPromises.delete(taskIdEncoded); - }); - this.taskPromises.set(taskIdEncoded, newTaskPromise); - return newTaskPromise; - } - - @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public async getTask( - taskId: TaskId, - lazy: boolean = false, - tran?: DBTransaction, - ): Promise { - if (tran == null) { - return this.db.withTransactionF((tran) => - this.getTask(taskId, lazy, tran), - ); - } - const taskData = await tran.get([ - ...this.queueTasksDbPath, - taskId.toBuffer(), - ]); - if (taskData == null) throw Error('TMP task not found'); - const taskStartTime = await tran.get([ - ...this.queueStartTimeDbPath, - taskId.toBuffer(), - ]); - let promise: () => Promise; - if (lazy) { - promise = () => this.getTaskP(taskId); - } else { - const prom = this.getTaskP(taskId, tran); - promise = () => prom; - } - return { - id: taskId, - handlerId: taskData.handlerId, - parameters: taskData.parameters, - timestamp: taskData.timestamp, - startTime: taskStartTime, - path: taskData.path, - priority: taskData.priority, - promise, - }; - } - - /** - * Gets all scheduled tasks. - * Tasks are sorted by the `TaskId` - */ - @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public async *getTasks( - order: 'asc' | 'desc' = 'asc', - lazy: boolean = false, - tran?: DBTransaction, - ): AsyncGenerator { - if (tran == null) { - return yield* this.db.withTransactionG((tran) => - this.getTasks(order, lazy, tran), - ); - } - - for await (const [taskIdPath, taskData] of tran.iterator( - this.queueTasksDbPath, - { valueAsBuffer: false, reverse: order !== 'asc' }, - )) { - const taskId = IdInternal.fromBuffer(taskIdPath[0] as Buffer); - const taskStartTime = await tran.get([ - ...this.queueStartTimeDbPath, - ...taskIdPath, - ]); - let promise: () => Promise; - if (lazy) { - promise = () => this.getTaskP(taskId); - } else { - const prom = this.getTaskP(taskId, tran); - promise = () => prom; - } - yield { - id: taskId, - handlerId: taskData.handlerId, - parameters: taskData.parameters, - timestamp: taskData.timestamp, - startTime: taskStartTime, - path: taskData.path, - priority: taskData.priority, - promise, - }; - } - } - - @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public async *getTasksByPath( - path: TaskPath, - lazy: boolean = false, - tran?: DBTransaction, - ): AsyncGenerator { - if (tran == null) { - return yield* this.db.withTransactionG((tran) => - this.getTasksByPath(path, lazy, tran), - ); - } - - for await (const [, taskIdBuffer] of tran.iterator([ - ...this.queuePathDbPath, - ...path, - ])) { - const taskId = IdInternal.fromBuffer(taskIdBuffer); - yield this.getTask(taskId, lazy, tran); - } - } - - @ready(new tasksErrors.ErrorSchedulerNotRunning(), false, ['starting']) - public async getLastTaskId( - tran?: DBTransaction, - ): Promise { - const lastTaskIdBuffer = await (tran ?? this.db).get( - this.queueLastTaskIdPath, - true, - ); - if (lastTaskIdBuffer == null) return; - return IdInternal.fromBuffer(lastTaskIdBuffer); - } - - public async createTask( - handlerId: TaskHandlerId, - parameters: TaskParameters = [], - priority: number = 0, - path?: TaskPath, - lazy: boolean = false, - tran?: DBTransaction, - ): Promise { - if (tran == null) { - return this.db.withTransactionF((tran) => - this.createTask(handlerId, parameters, priority, path, lazy, tran), - ); - } - - // This does a combination of things - // 1. create save the new task within the DB - // 2. if timer exist and new delay is longer then just return the task - // 3. else cancel the timer and create a new one with the delay - const taskId = this.generateTaskId(); - // Timestamp extracted from `IdSortable` is a floating point in seconds - // with subsecond fractionals, multiply it by 1000 gives us milliseconds - const taskTimestamp = Math.trunc(extractTs(taskId) * 1000) as TaskTimestamp; - const taskPriority = tasksUtils.toPriority(priority); - const taskData: TaskData = { - handlerId, - parameters, - timestamp: taskTimestamp, - path: path, - priority: taskPriority, - }; - const taskIdBuffer = taskId.toBuffer(); - // Save the task - await tran.put([...this.queueTasksDbPath, taskIdBuffer], taskData); - // Save the last task ID - await tran.put(this.queueLastTaskIdPath, taskIdBuffer, true); - - // Adding to group - if (path != null) { - await tran.put( - [...this.queuePathDbPath, ...path, taskIdBuffer], - taskIdBuffer, - true, - ); - } - let promise: () => Promise; - if (lazy) { - promise = () => this.getTaskP(taskId); - } else { - const prom = this.getTaskP(taskId, tran); - promise = () => prom; - } - return { - id: taskId, - handlerId, - parameters, - path, - priority: taskPriority, - timestamp: taskTimestamp, - startTime: undefined, - promise, - }; - } -} - -export default Queue; - -// Epic queue -// need to do a couple things: -// 1. integrate fast-check -// 2. integrate span checks -// 3. might also consider span logs? -// 4. open tracing observability -// 5. structured logging -// 6. async hooks to get traced promises to understand the situation -// 7. do we also get fantasy land promises? and async cancellable stuff? -// 8. task abstractions? -// need to use the db for this -// 9. priority structure -// 10. timers -// abort controller - -// kinetic data structure -// the priority grows as a function of time -// order by priority <- this thing has a static value -// in a key value DB, you can maintain sorted index of values -// IDs can be lexicographically sortable - -// this is a persistent queue -// of tasks that should be EXECUTED right now -// the scheduler is a persistent scheduler of scheduled tasks -// tasks get pushed from the scheduler into the queue -// the queue connects to the WorkerManager diff --git a/src/tasks/Scheduler.ts b/src/tasks/Scheduler.ts deleted file mode 100644 index eb39c993a..000000000 --- a/src/tasks/Scheduler.ts +++ /dev/null @@ -1,448 +0,0 @@ -import type { DB, LevelPath } from '@matrixai/db'; -import type { TaskIdString } from './types'; -import type KeyManager from '../keys/KeyManager'; -import type Queue from './Queue'; -import type { DBTransaction } from '@matrixai/db'; -import type { - Task, - TaskDelay, - TaskHandlerId, - TaskId, - TaskParameters, - TaskPath, -} from './types'; -import Logger, { LogLevel } from '@matrixai/logger'; -import { IdInternal } from '@matrixai/id'; -import { - CreateDestroyStartStop, - ready, -} from '@matrixai/async-init/dist/CreateDestroyStartStop'; -import lexi from 'lexicographic-integer'; -import * as tasksUtils from './utils'; -import * as tasksErrors from './errors'; -import { Plug } from '../utils/index'; - -interface Scheduler extends CreateDestroyStartStop {} -@CreateDestroyStartStop( - new tasksErrors.ErrorSchedulerRunning(), - new tasksErrors.ErrorSchedulerDestroyed(), -) -class Scheduler { - /** - * Create the scheduler, which will create its own Queue - * This will automatically start the scheduler - * If the scheduler needs to be started after the fact - * Make sure to construct it, and then call `start` manually - */ - public static async createScheduler({ - db, - queue, - logger = new Logger(this.name), - delay = false, - fresh = false, - }: { - db: DB; - queue: Queue; - logger?: Logger; - delay?: boolean; - fresh?: boolean; - }): Promise { - logger.info(`Creating ${this.name}`); - const scheduler = new this({ db, queue, logger }); - await scheduler.start({ delay, fresh }); - logger.info(`Created ${this.name}`); - return scheduler; - } - - protected logger: Logger; - protected db: DB; - protected keyManager: KeyManager; - protected queue: Queue; - // TODO: remove this? - protected promises: Map> = new Map(); - - // TODO: swap this out for the timer system later - - protected dispatchTimer?: ReturnType; - protected dispatchTimerTimestamp: number = Number.POSITIVE_INFINITY; - protected pendingDispatch: Promise | null = null; - protected dispatchPlug: Plug = new Plug(); - protected dispatchEnding: boolean = false; - - protected schedulerDbPath: LevelPath = [this.constructor.name]; - - /** - * Tasks scheduled by time - * `time/{lexi(TaskTimestamp + TaskDelay)} -> {raw(TaskId)}` - */ - protected schedulerTimeDbPath: LevelPath = [...this.schedulerDbPath, 'time']; - - // /** - // * Tasks queued for execution - // * `pending/{lexi(TaskPriority)}/{lexi(TaskTimestamp + TaskDelay)} -> {raw(TaskId)}` - // */ - // protected schedulerPendingDbPath: LevelPath = [ - // ...this.schedulerDbPath, - // 'pending', - // ]; - - // /** - // * Task handlers - // * `handlers/{TaskHandlerId}/{TaskId} -> {raw(TaskId)}` - // */ - // protected schedulerHandlersDbPath: LevelPath = [ - // ...this.schedulerDbPath, - // 'handlers', - // ]; - - public constructor({ - db, - queue, - logger, - }: { - db: DB; - queue: Queue; - logger: Logger; - }) { - this.logger = logger; - this.db = db; - this.queue = queue; - } - - public get isDispatching(): boolean { - return this.dispatchTimer != null; - } - - public async start({ - delay = false, - fresh = false, - }: { - delay?: boolean; - fresh?: boolean; - } = {}): Promise { - this.logger.setLevel(LogLevel.INFO); - this.logger.setLevel(LogLevel.INFO); - this.logger.info(`Starting ${this.constructor.name}`); - if (fresh) { - await this.db.clear(this.schedulerDbPath); - } - // Don't start dispatching if we still want to register handlers - if (!delay) { - await this.startDispatching(); - } - this.logger.info(`Started ${this.constructor.name}`); - } - - /** - * Stop the scheduler - * This does not clear the handlers nor promises - * This maintains any registered handlers and awaiting promises - */ - public async stop(): Promise { - this.logger.info(`Stopping ${this.constructor.name}`); - await this.stopDispatching(); - this.logger.info(`Stopped ${this.constructor.name}`); - } - - /** - * Destroys the scheduler - * This must first clear all handlers - * Then it needs to cancel all promises - * Finally destroys all underlying state - */ - public async destroy() { - this.logger.info(`Destroying ${this.constructor.name}`); - await this.db.clear(this.schedulerDbPath); - this.logger.info(`Destroyed ${this.constructor.name}`); - } - - protected updateTimer(startTime: number) { - if (startTime >= this.dispatchTimerTimestamp) return; - const delay = Math.max(startTime - tasksUtils.getPerformanceTime(), 0); - clearTimeout(this.dispatchTimer); - this.dispatchTimer = setTimeout(async () => { - // This.logger.info('consuming pending tasks'); - await this.dispatchPlug.unplug(); - this.dispatchTimerTimestamp = Number.POSITIVE_INFINITY; - }, delay); - this.dispatchTimerTimestamp = startTime; - this.logger.info(`Timer was updated to ${delay} to end at ${startTime}`); - } - - /** - * Starts the dispatching of tasks - */ - @ready(new tasksErrors.ErrorSchedulerNotRunning(), false, ['starting']) - public async startDispatching(): Promise { - // Starting queue - await this.queue.startTasks(); - // If already started, do nothing - if (this.pendingDispatch == null) { - this.pendingDispatch = this.dispatchTaskLoop(); - } - } - - @ready(new tasksErrors.ErrorSchedulerNotRunning(), false, ['stopping']) - public async stopDispatching(): Promise { - const stopQueueP = this.queue.stopTasks(); - clearTimeout(this.dispatchTimer); - delete this.dispatchTimer; - this.dispatchEnding = true; - await this.dispatchPlug.unplug(); - await this.pendingDispatch; - this.pendingDispatch = null; - await stopQueueP; - } - - protected async dispatchTaskLoop(): Promise { - // This will pop tasks from the queue and put the where they need to go - this.logger.info('dispatching set up'); - this.dispatchEnding = false; - this.dispatchTimerTimestamp = Number.POSITIVE_INFINITY; - while (true) { - if (this.dispatchEnding) break; - // Setting up and waiting for plug - this.logger.info('dispatch waiting'); - await this.dispatchPlug.plug(); - // Get the next time to delay for - await this.db.withTransactionF(async (tran) => { - for await (const [keyPath] of tran.iterator(this.schedulerTimeDbPath, { - limit: 1, - })) { - const [taskTimestampKeyBuffer] = tasksUtils.splitTaskTimestampKey( - keyPath[0] as Buffer, - ); - const time = lexi.unpack(Array.from(taskTimestampKeyBuffer)); - this.updateTimer(time); - } - }); - await this.dispatchPlug.waitForUnplug(); - if (this.dispatchEnding) break; - this.logger.info('dispatch continuing'); - const time = tasksUtils.getPerformanceTime(); - // Peek ahead by 100 ms - const targetTimestamp = Buffer.from(lexi.pack(time + 100)); - await this.db.withTransactionF(async (tran) => { - for await (const [keyPath, taskIdBuffer] of tran.iterator( - this.schedulerTimeDbPath, - { - lte: targetTimestamp, - }, - )) { - const taskTimestampKeyBuffer = keyPath[0] as Buffer; - // Dispatch the task now and remove it from the scheduler - this.logger.info('dispatching task'); - await tran.del([...this.schedulerTimeDbPath, taskTimestampKeyBuffer]); - const taskId = IdInternal.fromBuffer(taskIdBuffer); - await this.queue.pushTask(taskId, taskTimestampKeyBuffer, tran); - } - }); - } - this.logger.info('dispatching ending'); - } - - // /** - // * Gets a task abstraction - // */ - // @ready(new tasksErrors.ErrorSchedulerNotRunning()) - // public async getTask(id: TaskId, tran?: DBTransaction) { - // const taskData = await (tran ?? this.db).get([...this.queueTasksDbPath, id.toBuffer()]); - // if (taskData == null) { - // return; - // } - // const { p: taskP, resolveP, rejectP } = utils.promise(); - // - // // can we standardise on the unified listener - // // that is 1 listener for every task is created automatically - // // if 1000 tasks are inserted into the DB - // // 1000 listeners are created automatically? - // - // // we can either... - // // A standardise on the listener - // // B standardise on the promise - // - // // if the creation of the promise is lazy - // // then one can standardise on the promise - // // the idea being if the promise exists, just return the promise - // // if it doesn't exist, then first check if the task id still exists - // // if so, create a promise out of that lazily - // // now you need an object map locking to prevent race conditions on promise creation - // // then there's only ever 1 promise for a given task - // // any other cases, they always give back the same promise - // - // - // const listener = (taskError, taskResult) => { - // if (taskError != null) { - // rejectP(taskError); - // } else { - // resolveP(taskResult); - // } - // this.deregisterListener(id, listener); - // }; - // this.registerListener(id, listener); - // return taskP; - // } - - /* - Const task = await scheduleTask(...); - await task; // <- any - - const task = scheduleTask(...); - await task; // <- Promise - - - const task = scheduleTask(...); - await task; // <- Task (you are actually waiting for both scheduling + task execution) - - const task = scheduleTask(..., lazy=true); - await task; // <- Task you are only awaiting the scheduling - await task.task; - - const task = scheduleTask(delay=10hrs, lazy=True); - - waited 68 hrs - - await task; <- there's no information about the task - ErrorTasksTaskMissing - - - const task = scheduleTask(delay=10hrs, lazy=True); - - waited 5 hrs - - await task; - it can register an event handler for this task - - for loop: - scheduleTask(delay=10hrs); - - - const task = await scheduler.scheduleTask(lazy=false); - await task.promise; - - const task = await scheduler.getTask(lazy=false); // this is natu - await task.promise; - - */ - - @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public async scheduleTask( - handlerId: TaskHandlerId, - parameters: TaskParameters = [], - delay: TaskDelay = 0, - priority: number = 0, - path?: TaskPath, - lazy: boolean = false, - tran?: DBTransaction, - ): Promise { - if (tran == null) { - return this.db.withTransactionF((tran) => - this.scheduleTask( - handlerId, - parameters, - delay, - priority, - path, - lazy, - tran, - ), - ); - } - - // This does a combination of things - // 1. create save the new task within the DB - // 2. if timer exist and new delay is longer then just return the task - // 3. else cancel the timer and create a new one with the delay - - const task = await this.queue.createTask( - handlerId, - parameters, - priority, - path, - lazy, - tran, - ); - const taskIdBuffer = task.id.toBuffer(); - const startTime = task.timestamp + delay; - const taskTimestampKeyBuffer = tasksUtils.makeTaskTimestampKey( - startTime, - task.id, - ); - await tran.put( - [...this.queue.queueStartTimeDbPath, taskIdBuffer], - startTime, - ); - await tran.put( - [...this.queue.queueStartTimeDbPath, taskIdBuffer], - taskTimestampKeyBuffer, - true, - ); - await tran.put( - [...this.schedulerTimeDbPath, taskTimestampKeyBuffer], - taskIdBuffer, - true, - ); - - // Only update timer if transaction succeeds - tran.queueSuccess(() => { - this.updateTimer(startTime); - this.logger.info( - `Task ${tasksUtils.encodeTaskId( - task.id, - )} was scheduled for ${startTime}`, - ); - }); - - return task; - } - - @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public async getTask( - taskId: TaskId, - lazy: boolean = false, - tran?: DBTransaction, - ): Promise { - if (tran == null) { - return this.db.withTransactionF((tran) => - this.getTask(taskId, lazy, tran), - ); - } - - // Wrapping `queue.getTask`, may want to filter for only scheduled tasks - return this.queue.getTask(taskId, lazy, tran); - } - - /** - * Gets all scheduled tasks. - * Tasks are sorted by the `TaskId` - */ - @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public async *getTasks( - order: 'asc' | 'desc' = 'asc', - lazy: boolean = false, - tran?: DBTransaction, - ): AsyncGenerator { - if (tran == null) { - return yield* this.db.withTransactionG((tran) => - this.getTasks(order, lazy, tran), - ); - } - - return yield* this.queue.getTasks(order, lazy, tran); - } - - @ready(new tasksErrors.ErrorSchedulerNotRunning()) - public async *getTasksByPath( - path: TaskPath, - lazy: boolean = false, - tran?: DBTransaction, - ): AsyncGenerator { - if (tran == null) { - return yield* this.db.withTransactionG((tran) => - this.getTasksByPath(path, lazy, tran), - ); - } - - return yield* this.queue.getTasksByPath(path, lazy, tran); - } -} - -export default Scheduler; diff --git a/src/tasks/Task.ts b/src/tasks/Task.ts deleted file mode 100644 index e88702847..000000000 --- a/src/tasks/Task.ts +++ /dev/null @@ -1,101 +0,0 @@ -import type { - TaskId, - TaskData, - TaskHandlerId, - TaskTimestamp, - TaskPriority, - TaskParameters, - TaskPath, -} from './types'; -import type { DeepReadonly } from '../types'; -import type Queue from './Queue'; - -// FIXME: this file isn't needed anymore? -class Task { - public readonly id: TaskId; - public readonly handlerId: TaskHandlerId; - public readonly parameters: DeepReadonly; - public readonly timestamp: TaskTimestamp; - // Public readonly delay: TaskDelay; - public readonly path: TaskPath | undefined; - public readonly priority: TaskPriority; - - protected taskPromise: Promise | null; - protected queue: Queue; - - constructor( - queue: Queue, - id: TaskId, - handlerId: TaskHandlerId, - parameters: TaskParameters, - timestamp: TaskTimestamp, - // Delay: TaskDelay, - path: TaskPath | undefined, - priority: TaskPriority, - taskPromise: Promise | null, - ) { - // I'm not sure about the queue - // but if this is the reference here - // then we need to add the event handler into the queue to wait for this - // this.queue = queue; - - this.id = id; - this.handlerId = handlerId; - this.parameters = parameters; - this.timestamp = timestamp; - // This.delay = delay; - this.path = path; - this.priority = priority; - this.queue = queue; - this.taskPromise = taskPromise; - } - - public toJSON(): TaskData & { id: TaskId } { - return { - id: this.id, - handlerId: this.handlerId, - // TODO: change this to `structuredClone` when available - parameters: JSON.parse(JSON.stringify(this.parameters)), - timestamp: this.timestamp, - // Delay: this.delay, - path: this.path, - priority: this.priority, - }; - } - - get promise() { - if (this.taskPromise != null) return this.taskPromise; - this.taskPromise = this.queue.getTaskP(this.id); - return this.taskPromise; - } -} - -// Const t = new Task(); -// -// const p = new Promise((resolve, reject) => { -// resolve(); -// }); -// -// p.then; -// P.catch -// p.finally -// /** -// * Represents the completion of an asynchronous operation -// */ -// interface Promise { -// /** -// * Attaches callbacks for the resolution and/or rejection of the Promise. -// * @param onfulfilled The callback to execute when the Promise is resolved. -// * @param onrejected The callback to execute when the Promise is rejected. -// * @returns A Promise for the completion of which ever callback is executed. -// */ - -// /** -// * Attaches a callback for only the rejection of the Promise. -// * @param onrejected The callback to execute when the Promise is rejected. -// * @returns A Promise for the completion of the callback. -// */ -// catch(onrejected?: ((reason: any) => TResult | PromiseLike) | undefined | null): Promise; -// } - -export default Task; diff --git a/src/tasks/TaskEvent.ts b/src/tasks/TaskEvent.ts new file mode 100644 index 000000000..54439c1f9 --- /dev/null +++ b/src/tasks/TaskEvent.ts @@ -0,0 +1,33 @@ +import type { TaskIdEncoded } from './types'; + +class TaskEvent extends Event { + public detail: + | { + status: 'success'; + result: T; + } + | { + status: 'failure'; + reason: any; + }; + + constructor( + type: TaskIdEncoded, + options: EventInit & { + detail: + | { + status: 'success'; + result: T; + } + | { + status: 'failure'; + reason: any; + }; + }, + ) { + super(type, options); + this.detail = options.detail; + } +} + +export default TaskEvent; diff --git a/src/tasks/TaskManager.ts b/src/tasks/TaskManager.ts new file mode 100644 index 000000000..dd34f0949 --- /dev/null +++ b/src/tasks/TaskManager.ts @@ -0,0 +1,1251 @@ +import type { DB, DBTransaction, LevelPath, KeyPath } from '@matrixai/db'; +import type { ResourceRelease } from '@matrixai/resources'; +import type { + TaskHandlerId, + TaskHandler, + TaskId, + TaskIdEncoded, + Task, + TaskInfo, + TaskData, + TaskStatus, + TaskParameters, + TaskTimestamp, + TaskPath, +} from './types'; +import Logger from '@matrixai/logger'; +import { IdInternal } from '@matrixai/id'; +import { + CreateDestroyStartStop, + ready, +} from '@matrixai/async-init/dist/CreateDestroyStartStop'; +import { Lock } from '@matrixai/async-locks'; +import { PromiseCancellable } from '@matrixai/async-cancellable'; +import { extractTs } from '@matrixai/id/dist/IdSortable'; +import TaskEvent from './TaskEvent'; +import * as tasksErrors from './errors'; +import * as tasksUtils from './utils'; +import Timer from '../timer/Timer'; +import * as utils from '../utils'; + +const abortSchedulingLoopReason = Symbol('abort scheduling loop reason'); +const abortQueuingLoopReason = Symbol('abort queuing loop reason'); + +@CreateDestroyStartStop( + new tasksErrors.ErrorTaskManagerRunning(), + new tasksErrors.ErrorTaskManagerDestroyed(), +) +class TaskManager { + public static async createTaskManager({ + db, + handlers = {}, + lazy = false, + activeLimit = Infinity, + logger = new Logger(this.name), + fresh = false, + }: { + db: DB; + handlers?: Record; + lazy?: boolean; + activeLimit?: number; + logger?: Logger; + fresh?: boolean; + }) { + logger.info(`Creating ${this.name}`); + const tasks = new this({ + db, + activeLimit, + logger, + }); + await tasks.start({ + handlers, + lazy, + fresh, + }); + logger.info(`Created ${this.name}`); + return tasks; + } + + protected logger: Logger; + protected schedulerLogger: Logger; + protected queueLogger: Logger; + protected db: DB; + protected handlers: Map = new Map(); + protected activeLimit: number; + protected generateTaskId: () => TaskId; + protected taskPromises: Map> = + new Map(); + protected activePromises: Map> = + new Map(); + protected taskEvents: EventTarget = new EventTarget(); + protected tasksDbPath: LevelPath = [this.constructor.name]; + /** + * Tasks collection + * `Tasks/tasks/{TaskId} -> {json(TaskData)}` + */ + protected tasksTaskDbPath: LevelPath = [...this.tasksDbPath, 'task']; + /** + * Scheduled Tasks + * This is indexed by `TaskId` at the end to avoid conflicts + * `Tasks/scheduled/{lexi(TaskTimestamp + TaskDelay)}/{TaskId} -> null` + */ + protected tasksScheduledDbPath: LevelPath = [ + ...this.tasksDbPath, + 'scheduled', + ]; + /** + * Queued Tasks + * This is indexed by `TaskId` at the end to avoid conflicts + * `Tasks/queued/{lexi(TaskPriority)}/{lexi(TaskTimestamp + TaskDelay)}/{TaskId} -> null` + */ + protected tasksQueuedDbPath: LevelPath = [...this.tasksDbPath, 'queued']; + /** + * Tracks actively running tasks + * `Tasks/active/{TaskId} -> null` + */ + protected tasksActiveDbPath: LevelPath = [...this.tasksDbPath, 'active']; + /** + * Tasks indexed path + * `Tasks/path/{...TaskPath}/{TaskId} -> null` + */ + protected tasksPathDbPath: LevelPath = [...this.tasksDbPath, 'path']; + /** + * Maintain last Task ID to preserve monotonicity across process restarts + * `Tasks/lastTaskId -> {raw(TaskId)}` + */ + protected tasksLastTaskIdPath: KeyPath = [...this.tasksDbPath, 'lastTaskId']; + /** + * Asynchronous scheduling loop + * This is blocked by the `schedulingLock` + * The `null` indicates that the scheduling loop isn't running + */ + protected schedulingLoop: PromiseCancellable | null = null; + /** + * Timer used to unblock the scheduling loop + * This releases the `schedulingLock` if it is locked + * The `null` indicates there is no timer running + */ + protected schedulingTimer: Timer | null = null; + /** + * Lock controls whether to run an iteration of the scheduling loop + */ + protected schedulingLock: Lock = new Lock(); + /** + * Releases the scheduling lock + * On the first iteration of the scheduling loop + * the lock may not be acquired yet, and therefore releaser is not set + */ + protected schedulingLockReleaser?: ResourceRelease; + /** + * Asynchronous queuing loop + * This is blocked by the `queuingLock` + * The `null` indicates that the queuing loop isn't running + */ + protected queuingLoop: PromiseCancellable | null = null; + /** + * Lock controls whether to run an iteration of the queuing loop + */ + protected queuingLock: Lock = new Lock(); + /** + * Releases the queuing lock + * On the first iteration of the queuing loop + * the lock may not be acquired yet, and therefore releaser is not set + */ + protected queuingLockReleaser?: ResourceRelease; + + public get activeCount(): number { + return this.activePromises.size; + } + + public constructor({ + db, + activeLimit, + logger, + }: { + db: DB; + activeLimit: number; + logger: Logger; + }) { + this.logger = logger; + this.schedulerLogger = logger.getChild('scheduler'); + this.queueLogger = logger.getChild('queue'); + this.db = db; + this.activeLimit = activeLimit; + } + + public async start({ + handlers = {}, + lazy = false, + fresh = false, + }: { + handlers?: Record; + lazy?: boolean; + fresh?: boolean; + } = {}): Promise { + this.logger.info( + `Starting ${this.constructor.name} ${ + lazy ? 'in Lazy Mode' : 'in Eager Mode' + }`, + ); + if (fresh) { + this.handlers.clear(); + await this.db.clear(this.tasksDbPath); + } else { + await this.repairDanglingTasks(); + } + const lastTaskId = await this.getLastTaskId(); + this.generateTaskId = tasksUtils.createTaskIdGenerator(lastTaskId); + for (const taskHandlerId in handlers) { + this.handlers.set( + taskHandlerId as TaskHandlerId, + handlers[taskHandlerId], + ); + } + if (!lazy) { + await this.startProcessing(); + } + this.logger.info(`Started ${this.constructor.name}`); + } + + public async stop() { + this.logger.info(`Stopping ${this.constructor.name}`); + await this.stopProcessing(); + await this.stopTasks(); + this.logger.info(`Stopped ${this.constructor.name}`); + } + + public async destroy() { + this.logger.info(`Destroying ${this.constructor.name}`); + this.handlers.clear(); + await this.db.clear(this.tasksDbPath); + this.logger.info(`Destroyed ${this.constructor.name}`); + } + + /** + * Start scheduling and queuing loop + * This call is idempotent + * Use this when `Tasks` is started in lazy mode + */ + @ready(new tasksErrors.ErrorTaskManagerNotRunning(), false, ['starting']) + public async startProcessing(): Promise { + await Promise.all([this.startScheduling(), this.startQueueing()]); + } + + /** + * Stop the scheduling and queuing loop + * This call is idempotent + */ + @ready(new tasksErrors.ErrorTaskManagerNotRunning(), false, ['stopping']) + public async stopProcessing(): Promise { + await Promise.all([this.stopQueueing(), this.stopScheduling()]); + } + + /** + * Stop the active tasks + * This call is idempotent + */ + @ready(new tasksErrors.ErrorTaskManagerNotRunning(), false, ['stopping']) + public async stopTasks(): Promise { + for (const [, activePromise] of this.activePromises) { + activePromise.cancel(new tasksErrors.ErrorTaskStop()); + } + await Promise.allSettled(this.activePromises.values()); + } + + public getHandler(handlerId: TaskHandlerId): TaskHandler | undefined { + return this.handlers.get(handlerId); + } + + public getHandlers(): Record { + return Object.fromEntries(this.handlers); + } + + public registerHandler(handlerId: TaskHandlerId, handler: TaskHandler) { + this.handlers.set(handlerId, handler); + } + + public deregisterHandler(handlerId: TaskHandlerId) { + this.handlers.delete(handlerId); + } + + @ready(new tasksErrors.ErrorTaskManagerNotRunning(), false, ['starting']) + public async getLastTaskId( + tran?: DBTransaction, + ): Promise { + const lastTaskIdBuffer = await (tran ?? this.db).get( + this.tasksLastTaskIdPath, + true, + ); + if (lastTaskIdBuffer == null) return; + return IdInternal.fromBuffer(lastTaskIdBuffer); + } + + @ready(new tasksErrors.ErrorTaskManagerNotRunning()) + public async getTask( + taskId: TaskId, + lazy: boolean = false, + tran?: DBTransaction, + ): Promise { + if (tran == null) { + return this.db.withTransactionF((tran) => + this.getTask(taskId, lazy, tran), + ); + } + const taskIdBuffer = taskId.toBuffer(); + const taskData = await tran.get([ + ...this.tasksTaskDbPath, + taskIdBuffer, + ]); + if (taskData == null) { + return; + } + let promise: () => PromiseCancellable; + if (lazy) { + promise = () => this.getTaskPromise(taskId); + } else { + const taskPromise = this.getTaskPromise(taskId, tran); + tran.queueFailure((e) => { + taskPromise.cancel(e); + }); + promise = () => taskPromise; + } + const cancel = (reason: any) => this.cancelTask(taskId, reason); + const taskScheduleTime = taskData.timestamp + taskData.delay; + let taskStatus: TaskStatus; + if ( + (await tran.get([...this.tasksActiveDbPath, taskId.toBuffer()])) !== + undefined + ) { + taskStatus = 'active'; + } else if ( + (await tran.get([ + ...this.tasksQueuedDbPath, + utils.lexiPackBuffer(taskData.priority), + utils.lexiPackBuffer(taskScheduleTime), + taskIdBuffer, + ])) !== undefined + ) { + taskStatus = 'queued'; + } else if ( + (await tran.get([ + ...this.tasksScheduledDbPath, + utils.lexiPackBuffer(taskScheduleTime), + taskIdBuffer, + ])) !== undefined + ) { + taskStatus = 'scheduled'; + } + return { + id: taskId, + status: taskStatus!, + promise, + cancel, + handlerId: taskData.handlerId, + parameters: taskData.parameters, + delay: tasksUtils.fromDelay(taskData.delay), + deadline: tasksUtils.fromDeadline(taskData.deadline), + priority: tasksUtils.fromPriority(taskData.priority), + path: taskData.path, + created: new Date(taskData.timestamp), + scheduled: new Date(taskScheduleTime), + }; + } + + @ready(new tasksErrors.ErrorTaskManagerNotRunning()) + public async *getTasks( + order: 'asc' | 'desc' = 'asc', + lazy: boolean = false, + path?: TaskPath, + tran?: DBTransaction, + ): AsyncGenerator { + if (tran == null) { + return yield* this.db.withTransactionG((tran) => + this.getTasks(order, lazy, path, tran), + ); + } + if (path == null) { + for await (const [[taskIdBuffer]] of tran.iterator( + [...this.tasksTaskDbPath], + { values: false, reverse: order !== 'asc' }, + )) { + const taskId = IdInternal.fromBuffer(taskIdBuffer as Buffer); + const task = (await this.getTask(taskId, lazy, tran))!; + yield task; + } + } else { + for await (const [kP] of tran.iterator( + [...this.tasksPathDbPath, ...path], + { values: false, reverse: order !== 'asc' }, + )) { + const taskIdBuffer = kP[kP.length - 1] as Buffer; + const taskId = IdInternal.fromBuffer(taskIdBuffer); + const task = (await this.getTask(taskId, lazy, tran))!; + yield task; + } + } + } + + @ready(new tasksErrors.ErrorTaskManagerNotRunning()) + public getTaskPromise( + taskId: TaskId, + tran?: DBTransaction, + ): PromiseCancellable { + const taskIdEncoded = tasksUtils.encodeTaskId(taskId); + // If the task promise is already running, return the existing promise + // this is because the task promise has a singleton cleanup operation attached + let taskPromiseCancellable = this.taskPromises.get(taskIdEncoded); + if (taskPromiseCancellable != null) return taskPromiseCancellable; + const abortController = new AbortController(); + const taskPromise = new Promise((resolve, reject) => { + // Signals cancellation to the active promise + // the active promise is lazy so the task promise is also lazy + // this means cancellation does not result in eager rejection + const signalHandler = () => + this.cancelTask(taskId, abortController.signal.reason); + const taskListener = (event: TaskEvent) => { + abortController.signal.removeEventListener('abort', signalHandler); + if (event.detail.status === 'success') { + resolve(event.detail.result); + } else { + reject(event.detail.reason); + } + }; + // Event listeners are registered synchronously + // this ensures that dispatched `TaskEvent` will be received + abortController.signal.addEventListener('abort', signalHandler); + this.taskEvents.addEventListener(taskIdEncoded, taskListener, { + once: true, + }); + // The task may not actually exist anymore + // in which case, the task listener will never settle + // Here we concurrently check if the task exists + // if it doesn't, remove all listeners and reject early + void (tran ?? this.db) + .get([...this.tasksTaskDbPath, taskId.toBuffer()]) + .then( + (taskData: TaskData | undefined) => { + if (taskData == null) { + // Rollback the event listeners + this.taskEvents.removeEventListener(taskIdEncoded, taskListener); + abortController.signal.removeEventListener( + 'abort', + signalHandler, + ); + reject(new tasksErrors.ErrorTaskMissing(taskIdEncoded)); + } + }, + (reason) => { + reject(reason); + }, + ); + }).finally(() => { + this.taskPromises.delete(taskIdEncoded); + }); + taskPromiseCancellable = PromiseCancellable.from( + taskPromise, + abortController, + ); + // Empty catch handler to ignore unhandled rejections + taskPromiseCancellable.catch(() => {}); + this.taskPromises.set(taskIdEncoded, taskPromiseCancellable); + return taskPromiseCancellable; + } + + /** + * Schedules a task + * If `this.schedulingLoop` isn't running, then this will not + * attempt to reset the `this.schedulingTimer` + */ + @ready(new tasksErrors.ErrorTaskManagerNotRunning()) + public async scheduleTask( + { + handlerId, + parameters = [], + delay = 0, + deadline = Infinity, + priority = 0, + path = [], + lazy = false, + }: { + handlerId: TaskHandlerId; + parameters?: TaskParameters; + delay?: number; + deadline?: number; + priority?: number; + path?: TaskPath; + lazy?: boolean; + }, + tran?: DBTransaction, + ): Promise { + if (tran == null) { + return this.db.withTransactionF((tran) => + this.scheduleTask( + { + handlerId, + parameters, + delay, + priority, + deadline, + path, + lazy, + }, + tran, + ), + ); + } + await this.lockLastTaskId(tran); + const taskId = this.generateTaskId(); + const taskIdEncoded = tasksUtils.encodeTaskId(taskId); + this.logger.debug( + `Scheduling Task ${taskIdEncoded} with handler \`${handlerId}\``, + ); + const taskIdBuffer = taskId.toBuffer(); + // Timestamp extracted from `IdSortable` is a floating point in seconds + // with subsecond fractionals, multiply it by 1000 gives us milliseconds + const taskTimestamp = Math.trunc(extractTs(taskId) * 1000) as TaskTimestamp; + const taskPriority = tasksUtils.toPriority(priority); + const taskDelay = tasksUtils.toDelay(delay); + const taskDeadline = tasksUtils.toDeadline(deadline); + const taskScheduleTime = taskTimestamp + taskDelay; + const taskData: TaskData = { + handlerId, + parameters, + timestamp: taskTimestamp, + priority: taskPriority, + delay: taskDelay, + deadline: taskDeadline, + path, + }; + // Saving the task + await tran.put([...this.tasksTaskDbPath, taskIdBuffer], taskData); + // Saving last task ID + await tran.put(this.tasksLastTaskIdPath, taskIdBuffer, true); + // Putting task into scheduled index + await tran.put( + [ + ...this.tasksScheduledDbPath, + utils.lexiPackBuffer(taskScheduleTime), + taskIdBuffer, + ], + null, + ); + // Putting the task into the path index + await tran.put([...this.tasksPathDbPath, ...path, taskIdBuffer], null); + // Transaction success triggers timer interception + tran.queueSuccess(() => { + // If the scheduling loop is not set then the `Tasks` system was created + // in lazy mode or the scheduling loop was explicitly stopped in either + // case, we do not attempt to intercept the scheduling timer + if (this.schedulingLoop != null) { + this.triggerScheduling(taskScheduleTime); + } + }); + let promise: () => PromiseCancellable; + if (lazy) { + promise = () => this.getTaskPromise(taskId); + } else { + const taskPromise = this.getTaskPromise(taskId, tran); + tran.queueFailure((e) => { + taskPromise.cancel(e); + }); + promise = () => taskPromise; + } + const cancel = (reason: any) => this.cancelTask(taskId, reason); + this.logger.debug( + `Scheduled Task ${taskIdEncoded} with handler \`${handlerId}\``, + ); + return { + id: taskId, + status: 'scheduled', + promise, + cancel, + handlerId, + parameters, + delay: tasksUtils.fromDelay(taskDelay), + deadline: tasksUtils.fromDeadline(taskDeadline), + priority: tasksUtils.fromPriority(taskPriority), + path, + created: new Date(taskTimestamp), + scheduled: new Date(taskScheduleTime), + }; + } + + @ready(new tasksErrors.ErrorTaskManagerNotRunning()) + public async updateTask( + taskId: TaskId, + taskPatch: Partial<{ + handlerId: TaskHandlerId; + parameters: TaskParameters; + delay: number; + deadline: number; + priority: number; + path: TaskPath; + }>, + tran?: DBTransaction, + ): Promise { + if (tran == null) { + return this.db.withTransactionF((tran) => + this.updateTask(taskId, taskPatch, tran), + ); + } + // Copy the patch POJO to avoid parameter mutation + const taskDataPatch = { ...taskPatch }; + if (taskDataPatch.delay != null) { + taskDataPatch.delay = tasksUtils.toDelay(taskDataPatch.delay); + } + if (taskDataPatch.deadline != null) { + taskDataPatch.deadline = tasksUtils.toDeadline(taskDataPatch.deadline); + } + if (taskDataPatch.priority != null) { + taskDataPatch.priority = tasksUtils.toPriority(taskDataPatch.priority); + } + await this.lockTask(tran, taskId); + const taskIdBuffer = taskId.toBuffer(); + const taskIdEncoded = tasksUtils.encodeTaskId(taskId); + const taskData = await tran.get([ + ...this.tasksTaskDbPath, + taskIdBuffer, + ]); + if (taskData == null) { + throw new tasksErrors.ErrorTaskMissing(taskIdEncoded); + } + if ( + (await tran.get([ + ...this.tasksScheduledDbPath, + utils.lexiPackBuffer(taskData.timestamp + taskData.delay), + taskIdBuffer, + ])) === undefined + ) { + // Cannot update the task if the task is already running + throw new tasksErrors.ErrorTaskRunning(taskIdEncoded); + } + const taskDataNew = { + ...taskData, + ...taskDataPatch, + }; + // Save updated task + await tran.put([...this.tasksTaskDbPath, taskIdBuffer], taskDataNew); + // Update the path index + if (taskDataPatch.path != null) { + await tran.del([...this.tasksPathDbPath, ...taskData.path, taskIdBuffer]); + await tran.put( + [...this.tasksPathDbPath, ...taskDataPatch.path, taskIdBuffer], + true, + ); + } + // Update the schedule time and trigger scheduling if delay is updated + if (taskDataPatch.delay != null) { + const taskScheduleTime = taskData.timestamp + taskData.delay; + const taskScheduleTimeNew = taskData.timestamp + taskDataPatch.delay; + await tran.del([ + ...this.tasksScheduledDbPath, + utils.lexiPackBuffer(taskScheduleTime), + taskIdBuffer, + ]); + await tran.put( + [ + ...this.tasksScheduledDbPath, + utils.lexiPackBuffer(taskScheduleTimeNew), + taskIdBuffer, + ], + null, + ); + tran.queueSuccess(async () => { + if (this.schedulingLoop != null) { + this.triggerScheduling(taskScheduleTimeNew); + } + }); + } + } + + /** + * Transition tasks from `scheduled` to `queued` + */ + protected async startScheduling() { + if (this.schedulingLoop != null) return; + this.schedulerLogger.info('Starting Scheduling Loop'); + const abortController = new AbortController(); + const abortP = utils.signalPromise(abortController.signal); + // First iteration must run + if (this.schedulingLockReleaser != null) { + await this.schedulingLockReleaser(); + } + const schedulingLoop = (async () => { + try { + while (!abortController.signal.aborted) { + // Blocks the scheduling loop until lock is released + // this ensures that each iteration of the loop is only + // run when it is required + try { + await Promise.race([this.schedulingLock.waitForUnlock(), abortP]); + } catch (e) { + if (e === abortSchedulingLoopReason) { + break; + } else { + throw e; + } + } + this.schedulerLogger.debug(`Begin scheduling loop iteration`); + [this.schedulingLockReleaser] = await this.schedulingLock.lock()(); + // Peek ahead by 100 ms in-order to prefetch some tasks + const now = + Math.trunc(performance.timeOrigin + performance.now()) + 100; + await this.db.withTransactionF(async (tran) => { + // Queue up all the tasks that are scheduled to be executed before `now` + for await (const [kP] of tran.iterator(this.tasksScheduledDbPath, { + // Upper bound of `{lexi(TaskTimestamp + TaskDelay)}/{TaskId}` + // notice the usage of `''` as the upper bound of `TaskId` + lte: [utils.lexiPackBuffer(now), ''], + values: false, + })) { + if (abortController.signal.aborted) return; + const taskIdBuffer = kP[1] as Buffer; + const taskId = IdInternal.fromBuffer(taskIdBuffer); + // If the task gets cancelled here, then queuing must be a noop + await this.queueTask(taskId); + } + }); + if (abortController.signal.aborted) break; + await this.db.withTransactionF(async (tran) => { + // Get the next task to be scheduled and set the timer accordingly + let nextScheduleTime: number | undefined; + for await (const [kP] of tran.iterator(this.tasksScheduledDbPath, { + limit: 1, + values: false, + })) { + nextScheduleTime = utils.lexiUnpackBuffer(kP[0] as Buffer); + } + if (abortController.signal.aborted) return; + if (nextScheduleTime == null) { + this.logger.debug( + 'Scheduling loop iteration found no more scheduled tasks', + ); + } else { + this.triggerScheduling(nextScheduleTime); + } + this.schedulerLogger.debug('Finish scheduling loop iteration'); + }); + } + } catch (e) { + this.schedulerLogger.error(`Failed scheduling loop ${String(e)}`); + throw new tasksErrors.ErrorTaskManagerScheduler(undefined, { + cause: e, + }); + } + })(); + this.schedulingLoop = PromiseCancellable.from( + schedulingLoop, + abortController, + ); + this.schedulerLogger.info('Started Scheduling Loop'); + } + + protected async stopScheduling(): Promise { + if (this.schedulingLoop == null) return; + this.logger.info('Stopping Scheduling Loop'); + // Cancel the timer if it exists + this.schedulingTimer?.cancel(); + this.schedulingTimer = null; + // Cancel the scheduling loop + this.schedulingLoop.cancel(abortSchedulingLoopReason); + // Wait for the cancellation signal to resolve the promise + await this.schedulingLoop; + // Indicates that the loop is no longer running + this.schedulingLoop = null; + this.logger.info('Stopped Scheduling Loop'); + } + + protected async startQueueing() { + if (this.queuingLoop != null) return; + this.queueLogger.info('Starting Queueing Loop'); + const abortController = new AbortController(); + const abortP = utils.signalPromise(abortController.signal); + // First iteration must run + if (this.queuingLockReleaser != null) await this.queuingLockReleaser(); + const queuingLoop = (async () => { + try { + while (!abortController.signal.aborted) { + try { + await Promise.race([this.queuingLock.waitForUnlock(), abortP]); + } catch (e) { + if (e === abortQueuingLoopReason) { + break; + } else { + throw e; + } + } + this.queueLogger.debug(`Begin queuing loop iteration`); + [this.queuingLockReleaser] = await this.queuingLock.lock()(); + await this.db.withTransactionF(async (tran) => { + for await (const [kP] of tran.iterator(this.tasksQueuedDbPath, { + values: false, + })) { + if (abortController.signal.aborted) break; + if (this.activePromises.size >= this.activeLimit) break; + const taskId = IdInternal.fromBuffer(kP[2] as Buffer); + await this.startTask(taskId); + } + }); + this.queueLogger.debug(`Finish queuing loop iteration`); + } + } catch (e) { + this.queueLogger.error(`Failed queuing loop ${String(e)}`); + throw new tasksErrors.ErrorTaskManagerQueue(undefined, { cause: e }); + } + })(); + // Cancellation is always a resolution + // the promise must resolve, by waiting for resolution + // it's graceful termination of the loop + this.queuingLoop = PromiseCancellable.from(queuingLoop, abortController); + this.queueLogger.info('Started Queueing Loop'); + } + + protected async stopQueueing() { + if (this.queuingLoop == null) return; + this.logger.info('Stopping Queuing Loop'); + this.queuingLoop.cancel(abortQueuingLoopReason); + await this.queuingLoop; + this.queuingLoop = null; + this.logger.info('Stopped Queuing Loop'); + } + + /** + * Triggers the scheduler on a delayed basis + * If the delay is 0, the scheduler is triggered immediately + * The scheduling timer is a singleton that can be set by both + * `this.schedulingLoop` and `this.scheduleTask` + * This ensures that the timer is set to the earliest scheduled task + */ + protected triggerScheduling(scheduleTime: number) { + if (this.schedulingTimer != null) { + if (scheduleTime >= this.schedulingTimer.scheduled!.getTime()) return; + this.schedulingTimer.cancel(); + this.schedulingTimer = null; + } + const now = Math.trunc(performance.timeOrigin + performance.now()); + const delay = Math.max(scheduleTime - now, 0); + if (delay === 0) { + this.schedulerLogger.debug( + `Setting scheduling loop iteration immediately (delay: ${delay} ms)`, + ); + this.schedulingTimer = null; + if (this.schedulingLockReleaser != null) { + void this.schedulingLockReleaser(); + } + } else { + this.schedulerLogger.debug( + `Setting scheduling loop iteration for ${new Date( + scheduleTime, + ).toISOString()} (delay: ${delay} ms)`, + ); + this.schedulingTimer = new Timer(() => { + this.schedulingTimer = null; + if (this.schedulingLockReleaser != null) { + void this.schedulingLockReleaser(); + } + }, delay); + } + } + + /** + * Same idea as triggerScheduling + * But this time unlocking the queue to proceed + * If already unlocked, subsequent unlocking is idempotent + * The unlocking of the scheduling is delayed + * Whereas this unlocking is not + * Remember the queuing just keeps running until finished + */ + protected triggerQueuing() { + if (this.activePromises.size >= this.activeLimit) return; + if (this.queuingLockReleaser != null) { + void this.queuingLockReleaser(); + } + } + + /** + * Transition from scheduled to queued + * If the task is cancelled, then this does nothing + */ + protected async queueTask(taskId: TaskId): Promise { + const taskIdBuffer = taskId.toBuffer(); + const taskIdEncoded = tasksUtils.encodeTaskId(taskId); + this.schedulerLogger.debug(`Queuing Task ${taskIdEncoded}`); + await this.db.withTransactionF(async (tran) => { + // Mutually exclude `this.updateTask` and `this.gcTask` + await this.lockTask(tran, taskId); + const taskData = await tran.get([ + ...this.tasksTaskDbPath, + taskIdBuffer, + ]); + // If the task was garbage collected, due to potentially cancellation + // then we can skip the task, as it no longer exists + if (taskData == null) { + this.schedulerLogger.debug( + `Skipped Task ${taskIdEncoded} - it is cancelled`, + ); + return; + } + // Remove task from the scheduled index + await tran.del([ + ...this.tasksScheduledDbPath, + utils.lexiPackBuffer(taskData.timestamp + taskData.delay), + taskIdBuffer, + ]); + // Put task into the queue index + await tran.put( + [ + ...this.tasksQueuedDbPath, + utils.lexiPackBuffer(taskData.priority), + utils.lexiPackBuffer(taskData.timestamp + taskData.delay), + taskIdBuffer, + ], + null, + ); + tran.queueSuccess(() => { + this.triggerQueuing(); + }); + }); + this.schedulerLogger.debug(`Queued Task ${taskIdEncoded}`); + } + + /** + * Transition from queued to active + * If the task is cancelled, then this does nothing + */ + protected async startTask(taskId: TaskId): Promise { + const taskIdBuffer = taskId.toBuffer(); + const taskIdEncoded = tasksUtils.encodeTaskId(taskId); + this.queueLogger.debug(`Starting Task ${taskIdEncoded}`); + await this.db.withTransactionF(async (tran) => { + await this.lockTask(tran, taskId); + const taskData = await tran.get([ + ...this.tasksTaskDbPath, + taskIdBuffer, + ]); + // If the task was garbage collected, due to potentially cancellation + // then we can skip the task, as it no longer exists + if (taskData == null) { + this.queueLogger.debug( + `Skipped Task ${taskIdEncoded} - it is cancelled`, + ); + return; + } + const taskHandler = this.getHandler(taskData.handlerId); + if (taskHandler == null) { + this.queueLogger.error( + `Failed Task ${taskIdEncoded} - No Handler Registered`, + ); + await this.gcTask(taskId, tran); + tran.queueSuccess(() => { + // THIS only runs after the transaction is committed + // IS IT POSSIBLE + // that I HAVE REGISTERED EVENT HANDLERS is at there + // cause if so, it would then be able to + // to get an event listener registered + // only afterwards + + this.taskEvents.dispatchEvent( + new TaskEvent(taskIdEncoded, { + detail: { + status: 'failure', + reason: new tasksErrors.ErrorTaskHandlerMissing(), + }, + }), + ); + }); + return; + } + // Remove task from the queued index + await tran.del([ + ...this.tasksQueuedDbPath, + utils.lexiPackBuffer(taskData.priority), + utils.lexiPackBuffer(taskData.timestamp + taskData.delay), + taskIdBuffer, + ]); + // Put task into the active index + // this index will be used to retry tasks if they don't finish + await tran.put([...this.tasksActiveDbPath, taskIdBuffer], null); + tran.queueSuccess(() => { + const abortController = new AbortController(); + const timeoutError = new tasksErrors.ErrorTaskTimeOut(); + const timer = new Timer( + () => void abortController.abort(timeoutError), + tasksUtils.fromDeadline(taskData.deadline), + ); + const ctx = { + timer, + signal: abortController.signal, + }; + const activePromise = (async () => { + const taskLogger = this.logger.getChild(`task ${taskIdEncoded}`); + try { + let succeeded: boolean; + let taskResult: any; + let taskReason: any; + const taskInfo: TaskInfo = { + id: taskId, + handlerId: taskData.handlerId, + parameters: taskData.parameters, + delay: tasksUtils.fromDelay(taskData.delay), + priority: tasksUtils.fromPriority(taskData.priority), + deadline: tasksUtils.fromDeadline(taskData.deadline), + path: taskData.path, + created: new Date(taskData.timestamp), + scheduled: new Date(taskData.timestamp + taskData.delay), + }; + try { + taskResult = await taskHandler( + ctx, + taskInfo, + ...taskData.parameters, + ); + succeeded = true; + } catch (e) { + taskReason = e; + succeeded = false; + } + // If the reason is `tasksErrors.ErrorTaskRetry` + // the task is not finished, and should be requeued + if (taskReason instanceof tasksErrors.ErrorTaskRetry) { + try { + await this.requeueTask(taskId); + } catch (e) { + this.logger.error(`Failed Requeuing Task ${taskIdEncoded}`); + // This is an unrecoverable error + throw new tasksErrors.ErrorTaskRequeue(taskIdEncoded, { + cause: e, + }); + } + } else { + if (succeeded) { + taskLogger.debug('Succeeded'); + } else { + taskLogger.warn(`Failed - Reason: ${String(taskReason)}`); + } + // GC the task before dispatching events + try { + await this.gcTask(taskId); + } catch (e) { + this.logger.error( + `Failed Garbage Collecting Task ${taskIdEncoded}`, + ); + // This is an unrecoverable error + throw new tasksErrors.ErrorTaskGarbageCollection( + taskIdEncoded, + { cause: e }, + ); + } + if (succeeded) { + this.taskEvents.dispatchEvent( + new TaskEvent(taskIdEncoded, { + detail: { + status: 'success', + result: taskResult, + }, + }), + ); + } else { + this.taskEvents.dispatchEvent( + new TaskEvent(taskIdEncoded, { + detail: { + status: 'failure', + reason: taskReason, + }, + }), + ); + } + } + } finally { + // Task has finished, cancel the timer + timer.cancel(); + // Remove from active promises + this.activePromises.delete(taskIdEncoded); + // Slot has opened up, trigger queueing + this.triggerQueuing(); + } + })(); + // This will be a lazy `PromiseCancellable` + const activePromiseCancellable = PromiseCancellable.from( + activePromise, + abortController, + ); + this.activePromises.set(taskIdEncoded, activePromiseCancellable); + this.queueLogger.debug(`Started Task ${taskIdEncoded}`); + }); + }); + } + + /** + * This is used to garbage collect tasks that have settled + * Explicit removal of tasks can only be done through task cancellation + */ + protected async gcTask(taskId: TaskId, tran?: DBTransaction): Promise { + if (tran == null) { + return this.db.withTransactionF((tran) => this.gcTask(taskId, tran)); + } + const taskIdEncoded = tasksUtils.encodeTaskId(taskId); + const taskIdBuffer = taskId.toBuffer(); + await this.lockTask(tran, taskId); + const taskData = await tran.get([ + ...this.tasksTaskDbPath, + taskId.toBuffer(), + ]); + if (taskData == null) return; + this.logger.debug(`Garbage Collecting Task ${taskIdEncoded}`); + const taskScheduleTime = taskData.timestamp + taskData.delay; + await tran.del([ + ...this.tasksPathDbPath, + ...taskData.path, + taskId.toBuffer(), + ]); + await tran.del([...this.tasksActiveDbPath, taskId.toBuffer()]); + await tran.del([ + ...this.tasksQueuedDbPath, + utils.lexiPackBuffer(taskData.priority), + utils.lexiPackBuffer(taskScheduleTime), + taskIdBuffer, + ]); + await tran.del([ + ...this.tasksScheduledDbPath, + utils.lexiPackBuffer(taskScheduleTime), + taskIdBuffer, + ]); + await tran.del([...this.tasksTaskDbPath, taskId.toBuffer()]); + this.logger.debug(`Garbage Collected Task ${taskIdEncoded}`); + } + + protected async requeueTask( + taskId: TaskId, + tran?: DBTransaction, + ): Promise { + if (tran == null) { + return this.db.withTransactionF((tran) => this.requeueTask(taskId, tran)); + } + const taskIdBuffer = taskId.toBuffer(); + const taskIdEncoded = tasksUtils.encodeTaskId(taskId); + this.logger.debug(`Requeuing Task ${taskIdEncoded}`); + await this.lockTask(tran, taskId); + const taskData = await tran.get([ + ...this.tasksTaskDbPath, + taskIdBuffer, + ]); + if (taskData == null) { + throw new tasksErrors.ErrorTaskMissing(taskIdEncoded); + } + // Put task into the active index + // this index will be used to retry tasks if they don't finish + await tran.del([...this.tasksActiveDbPath, taskIdBuffer]); + // Put task back into the queued index + await tran.put( + [ + ...this.tasksQueuedDbPath, + utils.lexiPackBuffer(taskData.priority), + utils.lexiPackBuffer(taskData.timestamp + taskData.delay), + taskIdBuffer, + ], + null, + ); + this.logger.debug(`Requeued Task ${taskIdEncoded}`); + } + + protected async cancelTask(taskId: TaskId, cancelReason: any): Promise { + const taskIdEncoded = tasksUtils.encodeTaskId(taskId); + this.logger.debug(`Cancelling Task ${taskIdEncoded}`); + const activePromise = this.activePromises.get(taskIdEncoded); + if (activePromise != null) { + // If the active promise exists, then we only signal for cancellation + // the active promise will clean itself up when it settles + activePromise.cancel(cancelReason); + } else { + try { + await this.gcTask(taskId); + } catch (e) { + this.logger.error( + `Failed Garbage Collecting Task ${taskIdEncoded} - ${String(e)}`, + ); + // This is an unrecoverable error + throw new tasksErrors.ErrorTaskGarbageCollection(taskIdEncoded, { + cause: e, + }); + } + this.taskEvents.dispatchEvent( + new TaskEvent(taskIdEncoded, { + detail: { + status: 'failure', + reason: cancelReason, + }, + }), + ); + } + this.logger.debug(`Cancelled Task ${taskIdEncoded}`); + } + + /** + * Mutually exclude last task ID mutation + * Prevents "counter racing" for the last task ID + */ + protected async lockLastTaskId(tran: DBTransaction): Promise { + return tran.lock(this.tasksLastTaskIdPath.join('')); + } + + /** + * Mutual exclusion for task mutation + * Used to lock: + * - `this.updateTask` + * - `this.queueTask` + * - `this.startTask` + * - `this.gcTask` + * - `this.requeueTask` + */ + protected async lockTask(tran: DBTransaction, taskId: TaskId): Promise { + return tran.lock([...this.tasksDbPath, taskId.toString()].join('')); + } + + /** + * If the process was killed ungracefully then we may need to + * repair active dangling tasks by moving them back to the queued index + */ + protected async repairDanglingTasks() { + await this.db.withTransactionF(async (tran) => { + this.logger.info('Begin Tasks Repair'); + // Move tasks from active to queued + // these tasks will be retried + for await (const [kP] of tran.iterator(this.tasksActiveDbPath, { + values: false, + })) { + const taskIdBuffer = kP[0] as Buffer; + const taskId = IdInternal.fromBuffer(taskIdBuffer); + const taskIdEncoded = tasksUtils.encodeTaskId(taskId); + const taskData = await tran.get([ + ...this.tasksTaskDbPath, + taskIdBuffer, + ]); + if (taskData == null) { + // Removing dangling task from active index + // this should not happen + await tran.del([...this.tasksActiveDbPath, ...kP]); + this.logger.warn(`Removing Dangling Active Task ${taskIdEncoded}`); + } else { + // Put task back into the queue index + await tran.put( + [ + ...this.tasksQueuedDbPath, + utils.lexiPackBuffer(taskData.priority), + utils.lexiPackBuffer(taskData.timestamp + taskData.delay), + taskIdBuffer, + ], + null, + ); + // Removing task from active index + await tran.del([...this.tasksActiveDbPath, ...kP]); + this.logger.warn( + `Moving Task ${taskIdEncoded} from Active to Queued`, + ); + } + } + this.logger.info('Finish Tasks Repair'); + }); + } +} + +export default TaskManager; diff --git a/src/tasks/errors.ts b/src/tasks/errors.ts index 5f85cfc47..601eaf223 100644 --- a/src/tasks/errors.ts +++ b/src/tasks/errors.ts @@ -2,79 +2,117 @@ import { ErrorPolykey, sysexits } from '../errors'; class ErrorTasks extends ErrorPolykey {} -class ErrorScheduler extends ErrorTasks {} - -class ErrorSchedulerRunning extends ErrorScheduler { - static description = 'Scheduler is running'; +class ErrorTaskManagerRunning extends ErrorTasks { + static description = 'TaskManager is running'; exitCode = sysexits.USAGE; } -class ErrorSchedulerNotRunning extends ErrorScheduler { - static description = 'Scheduler is not running'; +class ErrorTaskManagerNotRunning extends ErrorTasks { + static description = 'TaskManager is not running'; exitCode = sysexits.USAGE; } -class ErrorSchedulerDestroyed extends ErrorScheduler { - static description = 'Scheduler is destroyed'; +class ErrorTaskManagerDestroyed extends ErrorTasks { + static description = 'TaskManager is destroyed'; exitCode = sysexits.USAGE; } -class ErrorSchedulerHandlerMissing extends ErrorScheduler { - static description = 'Scheduler task handler is not registered'; - exitCode = sysexits.USAGE; +/** + * This is an unrecoverable error + */ +class ErrorTaskManagerScheduler extends ErrorTasks { + static description = + 'TaskManager scheduling loop encountered an unrecoverable error'; + exitCode = sysexits.SOFTWARE; } -class ErrorQueue extends ErrorTasks {} +/** + * This is an unrecoverable error + */ +class ErrorTaskManagerQueue extends ErrorTasks { + static description = + 'TaskManager queuing loop encountered an unrecoverable error'; + exitCode = sysexits.SOFTWARE; +} -class ErrorQueueRunning extends ErrorQueue { - static description = 'Queue is running'; +class ErrorTask extends ErrorTasks { + static description = 'Task error'; exitCode = sysexits.USAGE; } -class ErrorQueueNotRunning extends ErrorQueue { - static description = 'Queue is not running'; - exitCode = sysexits.USAGE; +class ErrorTaskMissing extends ErrorTask { + static description = + 'Task does not (or never) existed anymore, it may have been fulfilled or cancelled'; + exitCode = sysexits.UNAVAILABLE; } -class ErrorQueueDestroyed extends ErrorQueue { - static description = 'Queue is destroyed'; - exitCode = sysexits.USAGE; +class ErrorTaskHandlerMissing extends ErrorTask { + static description = 'Task handler is not registered'; + exitCode = sysexits.UNAVAILABLE; } -class ErrorTask extends ErrorTasks { - static description = 'Task error'; +class ErrorTaskRunning extends ErrorTask { + static description = 'Task is running, it cannot be updated'; exitCode = sysexits.USAGE; } -class ErrorTaskRejected extends ErrorTask { - static description = 'Task handler threw an exception'; - exitCode = sysexits.USAGE; +/** + * This is used as a signal reason when the `TaskDeadline` is reached + */ +class ErrorTaskTimeOut extends ErrorTask { + static description = 'Task exhausted deadline'; + exitCode = sysexits.UNAVAILABLE; } -class ErrorTaskCancelled extends ErrorTask { - static description = 'Task has been cancelled'; - exitCode = sysexits.USAGE; +/** + * This is used as a signal reason when calling `TaskManager.stopTasks()` + * If the task should be retried, then the task handler should throw `ErrorTaskRetry` + */ +class ErrorTaskStop extends ErrorTask { + static description = 'TaskManager is stopping, task is being cancelled'; + exitCode = sysexits.OK; } -class ErrorTaskMissing extends ErrorTask { - static description = - 'Task does not (or never) existed anymore, it may have been fulfilled or cancelled'; - exitCode = sysexits.USAGE; +/** + * If this is thrown by the task, the task will be requeued so it can be + * retried, if the task rejects or resolves in any other way, the task + * will be considered to have completed + */ +class ErrorTaskRetry extends ErrorTask { + static description = 'Task should be retried'; + exitCode = sysexits.TEMPFAIL; +} + +/** + * This error indicates a bug + */ +class ErrorTaskRequeue extends ErrorTask { + static description = 'Task could not be requeued'; + exitCode = sysexits.SOFTWARE; +} + +/** + * This error indicates a bug + */ +class ErrorTaskGarbageCollection extends ErrorTask { + static description = 'Task could not be garbage collected'; + exitCode = sysexits.SOFTWARE; } export { ErrorTasks, - ErrorScheduler, - ErrorSchedulerRunning, - ErrorSchedulerNotRunning, - ErrorSchedulerDestroyed, - ErrorSchedulerHandlerMissing, - ErrorQueue, - ErrorQueueRunning, - ErrorQueueNotRunning, - ErrorQueueDestroyed, + ErrorTaskManagerRunning, + ErrorTaskManagerNotRunning, + ErrorTaskManagerDestroyed, + ErrorTaskManagerScheduler, + ErrorTaskManagerQueue, ErrorTask, - ErrorTaskRejected, - ErrorTaskCancelled, ErrorTaskMissing, + ErrorTaskHandlerMissing, + ErrorTaskRunning, + ErrorTaskTimeOut, + ErrorTaskStop, + ErrorTaskRetry, + ErrorTaskRequeue, + ErrorTaskGarbageCollection, }; diff --git a/src/tasks/index.ts b/src/tasks/index.ts index ae900e45b..11ffc0c80 100644 --- a/src/tasks/index.ts +++ b/src/tasks/index.ts @@ -1,4 +1,4 @@ -export { default as Scheduler } from './Scheduler'; +export { default as TaskManager } from './TaskManager'; export * as types from './types'; export * as utils from './utils'; export * as errors from './errors'; diff --git a/src/tasks/types.ts b/src/tasks/types.ts index ab64dbdd5..0789d078e 100644 --- a/src/tasks/types.ts +++ b/src/tasks/types.ts @@ -1,117 +1,121 @@ import type { Id } from '@matrixai/id'; -import type { POJO, Opaque, Callback } from '../types'; -import type { LevelPath } from '@matrixai/db'; +import type { PromiseCancellable } from '@matrixai/async-cancellable'; +import type { Opaque } from '../types'; +import type { ContextTimed } from '../contexts/types'; -type TaskId = Opaque<'TaskId', Id>; -type TaskIdString = Opaque<'TaskIdString', string>; -type TaskIdEncoded = Opaque<'TaskIdEncoded', string>; +type TaskHandlerId = Opaque<'TaskHandlerId', string>; -/** - * Timestamp unix time in milliseconds - */ -type TaskTimestamp = number; +type TaskHandler = ( + ctx: ContextTimed, + taskInfo: TaskInfo, + ...params: TaskParameters +) => PromiseLike; -/** - * Timestamp is millisecond number >= 0 - */ -type TaskDelay = number; - -type TaskParameters = Array; +type TaskId = Opaque<'TaskId', Id>; +type TaskIdEncoded = Opaque<'TaskIdEncoded', string>; /** - * Task priority is an `uint8` [0 to 255] - * Where `0` is the highest priority and `255` is the lowest priority + * Task POJO returned to the user */ -type TaskPriority = Opaque<'TaskPriority', number>; +type Task = { + id: TaskId; + status: TaskStatus; + promise: () => PromiseCancellable; + cancel: (reason: any) => void; + handlerId: TaskHandlerId; + parameters: TaskParameters; + delay: number; + priority: number; + deadline: number; + path: TaskPath; + created: Date; + scheduled: Date; +}; /** - * Task Path, a LevelPath + * Task data decoded for the task handler */ -type TaskPath = LevelPath; +type TaskInfo = Omit; /** - * Task data to be persisted + * Task data that will be encoded into JSON for persistence */ type TaskData = { handlerId: TaskHandlerId; parameters: TaskParameters; timestamp: TaskTimestamp; - // Delay: TaskDelay; - path: TaskPath | undefined; + delay: TaskDelay; + deadline: TaskDeadline; priority: TaskPriority; + path: TaskPath; }; -type Task = TaskData & { - id: TaskId; - startTime: TaskTimestamp | undefined; - promise: () => Promise | undefined; -}; +/** + * Task state machine diagram + * ┌───────────┐ + * │ │ + * ───────► Scheduled │ + * │ │ + * └─────┬─────┘ + * ┌─────▼─────┐ + * │ │ + * │ Queued │ + * │ │ + * └─────┬─────┘ + * ┌─────▼─────┐ + * │ │ + * │ Active │ + * │ │ + * └───────────┘ + */ +type TaskStatus = 'scheduled' | 'queued' | 'active'; /** - * Task information that is returned to the user + * Task parameters */ -type TaskInfo = TaskData & { - id: TaskId; -}; +type TaskParameters = Array; -type TaskHandlerId = Opaque<'TaskHandlerId', string>; +/** + * Timestamp unix time in milliseconds + */ +type TaskTimestamp = Opaque<'TaskTimestamp', number>; -// Type TaskHandler

= [], R = any> = ( -// ...params: P -// ) => Promise; +/** + * Timestamp milliseconds is a number between 0 and maximum timeout + * It is not allowed for there to be an infinite delay + */ +type TaskDelay = Opaque<'TaskDelay', number>; -type TaskHandler = (...params: Array) => Promise; +/** + * Deadline milliseconds is a number between 0 and maximum timeout + * or it can be `null` to indicate `Infinity` + */ +type TaskDeadline = Opaque<'TaskDeadline', number | null>; /** - * Task function is the result of a lambda abstraction of applying - * `TaskHandler` to its respective parameters - * This is what gets executed + * Task priority is an `uint8` [0 to 255] + * Where `0` is the highest priority and `255` is the lowest priority */ -type TaskFunction = () => Promise; +type TaskPriority = Opaque<'TaskPriority', number>; -// Type TaskListener = Callback<[taskResult: any], void>; -// Make Task something that can be awaited on -// but when you "make" a promise or reference it -// you're for a promise -// that will resolve an event occurs -// or reject when an event occurs -// and the result of the execution -// now the exeuction of the event itself is is going to return ap romise -// something must be lisetning to it -// If you have a Record -// it has to be TaskIdString -// you can store things in it -// type X = Record; -// Task is the lowest level -// TaskData is low level -// TaskInfo is high level -// TaskId -// Task <- lazy promise -// TaskData <- low level data of a task (does not include id) -// TaskInfo <- high level (includes id) -// This is a lazy promise -// it's a promise of something that may not yet immediately executed -// type TaskPromise = Promise; -// Consider these variants... (should standardise what these are to be used) -// Task -// Tasks (usually a record, sometimes an array) -// TaskData - lower level data of a task -// TaskInfo - higher level information that is inclusive of data -// type TaskData = Record; +/** + * Task Path, a LevelPath + */ +type TaskPath = Array; export type { + TaskHandlerId, + TaskHandler, TaskId, - TaskIdString, TaskIdEncoded, Task, - TaskPath, - TaskData, TaskInfo, - TaskHandlerId, - TaskHandler, - TaskPriority, - // TaskListener + TaskData, + TaskStatus, TaskParameters, TaskTimestamp, TaskDelay, + TaskDeadline, + TaskPriority, + TaskPath, }; diff --git a/src/tasks/utils.ts b/src/tasks/utils.ts index 15e8330c6..da179a0ce 100644 --- a/src/tasks/utils.ts +++ b/src/tasks/utils.ts @@ -1,7 +1,11 @@ -import type { TaskId, TaskIdEncoded, TaskPriority } from './types'; -import type { NodeId } from '../nodes/types'; +import type { + TaskId, + TaskIdEncoded, + TaskPriority, + TaskDelay, + TaskDeadline, +} from './types'; import { IdInternal, IdSortable } from '@matrixai/id'; -import lexi from 'lexicographic-integer'; /** * Generates TaskId @@ -9,58 +13,13 @@ import lexi from 'lexicographic-integer'; * They are strictly monotonic and unique with respect to the `nodeId` * When the `NodeId` changes, make sure to regenerate this generator */ -function createTaskIdGenerator(nodeId: NodeId, lastTaskId?: TaskId) { +function createTaskIdGenerator(lastTaskId?: TaskId) { const generator = new IdSortable({ lastId: lastTaskId, - nodeId, }); return () => generator.get(); } -/** - * Converts `int8` to flipped `uint8` task priority - * Clips number to between -128 to 127 inclusive - */ -function toPriority(n: number): TaskPriority { - n = Math.min(n, 127); - n = Math.max(n, -128); - n *= -1; - n -= 1; - n += 128; - return n as TaskPriority; -} - -/** - * Converts flipped `uint8` task priority to `int8` - */ -function fromPriority(p: TaskPriority): number { - let n = p - 128; - n += 1; - // Prevent returning `-0` - if (n !== 0) n *= -1; - return n; -} - -function makeTaskTimestampKey(time: number, taskId: TaskId): Buffer { - const timestampBuffer = Buffer.from(lexi.pack(time)); - return Buffer.concat([timestampBuffer, taskId.toBuffer()]); -} - -/** - * Returns [taskTimestampBuffer, taskIdBuffer] - */ -function splitTaskTimestampKey(timestampBuffer: Buffer) { - // Last 16 bytes are TaskId - const splitPoint = timestampBuffer.length - 16; - const timeBuffer = timestampBuffer.slice(0, splitPoint); - const idBuffer = timestampBuffer.slice(splitPoint); - return [timeBuffer, idBuffer]; -} - -function getPerformanceTime(): number { - return performance.timeOrigin + performance.now(); -} - /** * Encodes the TaskId as a `base32hex` string */ @@ -86,13 +45,85 @@ function decodeTaskId(taskIdEncoded: any): TaskId | undefined { return taskId; } +/** + * Encodes delay milliseconds + */ +function toDelay(delay: number): TaskDelay { + if (isNaN(delay)) { + delay = 0; + } else { + delay = Math.max(delay, 0); + delay = Math.min(delay, 2 ** 31 - 1); + } + return delay as TaskDelay; +} + +/** + * Decodes task delay + */ +function fromDelay(taskDelay: TaskDelay): number { + return taskDelay; +} + +/** + * Encodes deadline milliseconds + * If deadline is `Infinity`, it is encoded as `null` + * If deadline is `NaN, it is encoded as `0` + */ +function toDeadline(deadline: number): TaskDeadline { + let taskDeadline: number | null; + if (isNaN(deadline)) { + taskDeadline = 0; + } else { + taskDeadline = Math.max(deadline, 0); + // Infinity is converted to `null` because `Infinity` is not supported in JSON + if (!isFinite(taskDeadline)) taskDeadline = null; + } + return taskDeadline as TaskDeadline; +} + +/** + * Decodes task deadline + * If task deadline is `null`, it is decoded as `Infinity` + */ +function fromDeadline(taskDeadline: TaskDeadline): number { + if (taskDeadline == null) return Infinity; + return taskDeadline; +} + +/** + * Converts `int8` to flipped `uint8` task priority + * Clips number to between -128 to 127 inclusive + */ +function toPriority(n: number): TaskPriority { + if (isNaN(n)) n = 0; + n = Math.min(n, 127); + n = Math.max(n, -128); + n *= -1; + n -= 1; + n += 128; + return n as TaskPriority; +} + +/** + * Converts flipped `uint8` task priority to `int8` + */ +function fromPriority(p: TaskPriority): number { + let n = p - 128; + n += 1; + // Prevent returning `-0` + if (n !== 0) n *= -1; + return n; +} + export { createTaskIdGenerator, - toPriority, - fromPriority, - makeTaskTimestampKey, - splitTaskTimestampKey, - getPerformanceTime, encodeTaskId, decodeTaskId, + toDelay, + fromDelay, + toDeadline, + fromDeadline, + toPriority, + fromPriority, }; diff --git a/src/utils/Plug.ts b/src/utils/Plug.ts deleted file mode 100644 index bde43ea38..000000000 --- a/src/utils/Plug.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Lock } from '@matrixai/async-locks'; - -/** - * Abstraction for using a Lock as a plug for asynchronous pausing of loops - */ -class Plug { - protected lock: Lock = new Lock(); - protected lockReleaser: (e?: Error) => Promise = async () => {}; - - /** - * Will cause waitForUnplug to block - */ - public async plug() { - if (this.lock.isLocked()) return; - [this.lockReleaser] = await this.lock.lock(0)(); - } - /** - * Will release waitForUnplug from blocking - */ - public async unplug() { - await this.lockReleaser(); - } - - /** - * Will block if plugged - */ - public async waitForUnplug() { - await this.lock.waitForUnlock(); - } - - public isPlugged() { - return this.lock.isLocked(); - } -} - -export default Plug; diff --git a/src/utils/debug.ts b/src/utils/debug.ts new file mode 100644 index 000000000..a2c83fbef --- /dev/null +++ b/src/utils/debug.ts @@ -0,0 +1,29 @@ +function isPrintableASCII(str: string): boolean { + return /^[\x20-\x7E]*$/.test(str); +} + +/** + * Used for debugging DB dumps + */ +function inspectBufferStructure(obj: any): any { + if (obj instanceof Buffer) { + const str = obj.toString('utf8'); + if (isPrintableASCII(str)) { + return str; + } else { + return '0x' + obj.toString('hex'); + } + } else if (Array.isArray(obj)) { + return obj.map(inspectBufferStructure); + } else if (typeof obj === 'object') { + const obj_: any = {}; + for (const k in obj) { + obj_[k] = inspectBufferStructure(obj[k]); + } + return obj_; + } else { + return obj; + } +} + +export { isPrintableASCII, inspectBufferStructure }; diff --git a/src/utils/index.ts b/src/utils/index.ts index c1d5c537b..2ee8414ff 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,5 +1,4 @@ export { default as sysexits } from './sysexits'; -export { default as Plug } from './Plug'; export * from './utils'; export * from './matchers'; export * from './binary'; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 03058031e..0d5fdf553 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -7,6 +7,7 @@ import type { import os from 'os'; import process from 'process'; import path from 'path'; +import lexi from 'lexicographic-integer'; import * as utilsErrors from './errors'; const AsyncFunction = (async () => {}).constructor; @@ -195,6 +196,22 @@ function promise(): PromiseDeconstructed { }; } +/** + * Promise constructed from signal + * This rejects when the signal is aborted + */ +function signalPromise(signal: AbortSignal): Promise { + return new Promise((_, reject) => { + if (signal.aborted) { + reject(signal.reason); + return; + } + signal.addEventListener('abort', () => { + reject(signal.reason); + }); + }); +} + function timerStart(timeout: number): Timer { const timer = {} as Timer; timer.timedOut = false; @@ -355,6 +372,19 @@ function isAsyncGenerator(v: any): v is AsyncGenerator { typeof v.throw === 'function' ); } + +/** + * Encodes whole numbers (inc of 0) to lexicographic buffers + */ +function lexiPackBuffer(n: number): Buffer { + return Buffer.from(lexi.pack(n)); +} + +/** + * Decodes lexicographic buffers to whole numbers (inc of 0) + */ +function lexiUnpackBuffer(b: Buffer): number { + return lexi.unpack([...b]); } export { @@ -373,6 +403,7 @@ export { poll, promisify, promise, + signalPromise, timerStart, timerStop, arraySet, @@ -386,4 +417,6 @@ export { isPromiseLike, isGenerator, isAsyncGenerator, + lexiPackBuffer, + lexiUnpackBuffer, }; diff --git a/tests/tasks/Scheduler.test.ts b/tests/tasks/Scheduler.test.ts index 1145789b7..a9c4e704d 100644 --- a/tests/tasks/Scheduler.test.ts +++ b/tests/tasks/Scheduler.test.ts @@ -116,4 +116,5 @@ describe(Scheduler.name, () => { test.todo('tasks timestamps are unique on taskId'); test.todo('can remove scheduled tasks'); test.todo('can not remove active tasks'); + test.todo('Should clean up any inconsistent state during creation'); }); diff --git a/tests/tasks/TaskManager.test.ts b/tests/tasks/TaskManager.test.ts new file mode 100644 index 000000000..3088b25fe --- /dev/null +++ b/tests/tasks/TaskManager.test.ts @@ -0,0 +1,1266 @@ +import type { ContextTimed } from '../../dist/contexts/types'; +import type { Task, TaskHandlerId, TaskPath } from '../../src/tasks/types'; +import fs from 'fs'; +import path from 'path'; +import os from 'os'; +import { DB } from '@matrixai/db'; +import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; +import * as fc from 'fast-check'; +import { Lock } from '@matrixai/async-locks'; +import * as utils from '@/utils/index'; +import { promise, sleep, never } from '@/utils'; +import TaskManager from '@/tasks/TaskManager'; +import { Timer } from '@/timer/index'; +import * as tasksErrors from '@/tasks/errors'; + +// TODO: move to testing utils +const scheduleCall = ( + s: fc.Scheduler, + f: () => Promise, + label: string = 'scheduled call', +) => s.schedule(Promise.resolve(label)).then(() => f()); + +describe(TaskManager.name, () => { + const logger = new Logger(`${TaskManager.name} test`, LogLevel.DEBUG, [ + new StreamHandler(), + ]); + const handlerId = 'testId' as TaskHandlerId; + let dataDir: string; + let db: DB; + + beforeEach(async () => { + logger.info('SETTING UP'); + dataDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'polykey-test-'), + ); + const dbPath = path.join(dataDir, 'db'); + db = await DB.createDB({ + dbPath, + logger, + }); + logger.info('SET UP'); + }); + afterEach(async () => { + logger.info('CLEANING UP'); + await db.stop(); + await fs.promises.rm(dataDir, { recursive: true, force: true }); + logger.info('CLEANED UP'); + }); + + test('can start and stop', async () => { + const taskManager = await TaskManager.createTaskManager({ + db, + lazy: false, + logger, + }); + await taskManager.stop(); + await taskManager.start(); + await taskManager.stop(); + }); + // TODO: use timer mocking to speed up testing + test('tasks persist between Tasks object creation', async () => { + let taskManager = await TaskManager.createTaskManager({ + db, + lazy: true, + logger, + }); + const handlerId = 'asd' as TaskHandlerId; + const handler = jest.fn(); + handler.mockImplementation(async () => {}); + taskManager.registerHandler(handlerId, handler); + + await taskManager.startProcessing(); + await taskManager.scheduleTask({ + handlerId, + parameters: [1], + delay: 1000, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [2], + delay: 100, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [3], + delay: 2000, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [4], + delay: 10, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [5], + delay: 10, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [6], + delay: 10, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [7], + delay: 3000, + lazy: true, + }); + + await sleep(500); + logger.info('STOPPING'); + await taskManager.stop(); + expect(handler).toHaveBeenCalledTimes(4); + + logger.info('CREATING'); + handler.mockClear(); + taskManager = await TaskManager.createTaskManager({ + db, + lazy: true, + logger, + }); + taskManager.registerHandler(handlerId, handler); + await taskManager.startProcessing(); + await sleep(4000); + logger.info('STOPPING AGAIN'); + await taskManager.stop(); + expect(handler).toHaveBeenCalledTimes(3); + }); + // TODO: use timer mocking to speed up testing + test('tasks persist between Tasks stop and starts', async () => { + const taskManager = await TaskManager.createTaskManager({ + db, + lazy: true, + logger, + }); + const handlerId = 'asd' as TaskHandlerId; + const handler = jest.fn(); + handler.mockImplementation(async () => {}); + taskManager.registerHandler(handlerId, handler); + + await taskManager.startProcessing(); + await taskManager.scheduleTask({ + handlerId, + parameters: [1], + delay: 1000, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [2], + delay: 100, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [3], + delay: 2000, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [4], + delay: 10, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [5], + delay: 10, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [6], + delay: 10, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [7], + delay: 3000, + lazy: true, + }); + + await sleep(500); + logger.info('STOPPING'); + await taskManager.stop(); + expect(handler).toHaveBeenCalledTimes(4); + handler.mockClear(); + logger.info('STARTING'); + await taskManager.start(); + await sleep(4000); + logger.info('STOPPING AGAIN'); + await taskManager.stop(); + expect(handler).toHaveBeenCalledTimes(3); + }); + // FIXME: needs more experimenting to get this to work. + test.skip('tasks persist between Tasks stop and starts TIMER FAKING', async () => { + const taskManager = await TaskManager.createTaskManager({ + db, + lazy: true, + logger, + }); + const handlerId = 'asd' as TaskHandlerId; + const handler = jest.fn(); + handler.mockImplementation(async () => {}); + taskManager.registerHandler(handlerId, handler); + console.log('a'); + await taskManager.scheduleTask({ handlerId, parameters: [1], delay: 1000 }); + const t1 = await taskManager.scheduleTask({ + handlerId, + parameters: [1], + delay: 100, + lazy: false, + }); + await taskManager.scheduleTask({ handlerId, parameters: [1], delay: 2000 }); + await taskManager.scheduleTask({ handlerId, parameters: [1], delay: 10 }); + await taskManager.scheduleTask({ handlerId, parameters: [1], delay: 10 }); + await taskManager.scheduleTask({ handlerId, parameters: [1], delay: 10 }); + await taskManager.scheduleTask({ handlerId, parameters: [1], delay: 3000 }); + + // Setting up actions + jest.useFakeTimers(); + setTimeout(async () => { + console.log('starting processing'); + await taskManager.startProcessing(); + }, 0); + setTimeout(async () => { + console.log('stop'); + await taskManager.stop(); + }, 500); + setTimeout(async () => { + console.log('start'); + await taskManager.start(); + }, 1000); + + // Running tests here... + // after 600 ms we should stop and 4 taskManager should've run + console.log('b'); + jest.advanceTimersByTime(400); + jest.runAllTimers(); + console.log('b'); + jest.advanceTimersByTime(200); + console.log('b'); + console.log(jest.getTimerCount()); + jest.runAllTimers(); + console.log(jest.getTimerCount()); + await t1.promise(); + console.log('b'); + expect(handler).toHaveBeenCalledTimes(4); + // After another 5000ms the rest should've been called + console.log('b'); + handler.mockClear(); + console.log('b'); + jest.advanceTimersByTime(5000); + console.log('b'); + // Expect(handler).toHaveBeenCalledTimes(3); + console.log('b'); + jest.useRealTimers(); + console.log('b'); + await taskManager.stop(); + console.log('b'); + }); + // TODO: Use fastCheck here, this needs to be re-written + test('activeLimit is enforced', async () => { + // Const mockedTimers = jest.useFakeTimers(); + const taskArb = fc.record({ + delay: fc.integer({ min: 0, max: 1000 }), + // Priority: fc.integer({min: -200, max: 200}), + }); + const taskManagerArb = fc.array(taskArb, { minLength: 10, maxLength: 50 }); + await fc.assert( + fc.asyncProperty( + fc.scheduler(), + fc.scheduler(), + taskManagerArb, + async (sCall, sHandle, taskManagerDatas) => { + console.log('a'); + const taskManager = await TaskManager.createTaskManager({ + activeLimit: 0, + db, + fresh: true, + lazy: true, + logger, + }); + console.log('a'); + let handledTaskCount = 0; + const handlerId: TaskHandlerId = 'handlerId' as TaskHandlerId; + const handler = jest.fn(); + handler.mockImplementation(async () => { + // Schedule to resolve randomly + logger.info(`ACTIVE TASKS: ${taskManager.activeCount}`); + await sHandle.schedule(Promise.resolve()); + handledTaskCount += 1; + }); + taskManager.registerHandler(handlerId, handler); + console.log('a'); + await taskManager.startProcessing(); + console.log('a'); + + // Scheduling taskManager to be scheduled + const calls: Array> = []; + const pendingTasks: Array = []; + console.log('a'); + for (const taskManagerData of taskManagerDatas) { + calls.push( + scheduleCall( + sCall, + async () => { + const task = await taskManager.scheduleTask({ + delay: taskManagerData.delay, + handlerId, + lazy: false, + }); + pendingTasks.push(task); + }, + `delay: ${taskManagerData.delay}`, + ), + ); + } + + while (handledTaskCount < taskManagerDatas.length) { + await sleep(10); + logger.info(`handledTaskCount: ${handledTaskCount}`); + // Advance time and check expectations until all taskManager are complete + // mockedTimers.advanceTimersToNextTimer(); + console.log(sHandle.count(), sCall.count()); + while (sHandle.count() > 0) { + await sHandle.waitOne(); + logger.info('resolving 1 handle'); + } + // Shoot off 5 each step + if (sCall.count() > 0) { + for (let i = 0; i < 5; i++) { + await sCall.waitOne(); + } + } + } + const promises = pendingTasks.map((task) => task.promise()); + await Promise.all(calls).then( + (result) => console.log(result), + (reason) => { + console.error(reason); + throw reason; + }, + ); + await Promise.all(promises).then( + (result) => console.log(result), + (reason) => { + console.error(reason); + throw reason; + }, + ); + await taskManager.stop(); + console.log('done'); + }, + ), + { interruptAfterTimeLimit: globalThis.defaultTimeout - 2000, numRuns: 1 }, + ); + }); + // TODO: Use fastCheck for this + test('tasks are handled exactly once per task', async () => { + const handler = jest.fn(); + const pendingLock = new Lock(); + const [lockReleaser] = await pendingLock.lock()(); + const resolvedTasks = new Map(); + const totalTasks = 50; + handler.mockImplementation(async (_, number: number) => { + resolvedTasks.set(number, (resolvedTasks.get(number) ?? 0) + 1); + if (resolvedTasks.size >= totalTasks) await lockReleaser(); + }); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + logger, + }); + + await db.withTransactionF(async (tran) => { + for (let i = 0; i < totalTasks; i++) { + await taskManager.scheduleTask( + { + handlerId, + parameters: [i], + lazy: true, + }, + tran, + ); + } + }); + + await pendingLock.waitForUnlock(); + // Each task called exactly once + resolvedTasks.forEach((value) => expect(value).toEqual(1)); + + await taskManager.stop(); + expect(handler).toHaveBeenCalledTimes(totalTasks); + }); + // TODO: use fastCheck + test('awaited taskPromises resolve', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (_, fail) => { + if (!fail) throw Error('three'); + return fail; + }); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + logger, + }); + + const taskSucceed = await taskManager.scheduleTask({ + handlerId, + parameters: [true], + lazy: false, + }); + + // Promise should succeed with result + const taskSucceedP = taskSucceed!.promise(); + await expect(taskSucceedP).resolves.toBe(true); + + await taskManager.stop(); + }); + // TODO: use fastCheck + test('awaited taskPromises reject', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (_, fail) => { + if (!fail) throw Error('three'); + return fail; + }); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + logger, + }); + + const taskFail = await taskManager.scheduleTask({ + handlerId, + parameters: [false], + lazy: false, + }); + + // Promise should throw + const taskFailP = taskFail.promise(); + await expect(taskFailP).rejects.toThrow(Error); + + await taskManager.stop(); + }); + // TODO: use fastCheck + test('awaited taskPromises resolve or reject', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (_, fail) => { + if (!fail) throw Error('three'); + return fail; + }); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + logger, + }); + + const taskFail = await taskManager.scheduleTask({ + handlerId, + parameters: [false], + lazy: false, + }); + + const taskSuccess = await taskManager.scheduleTask({ + handlerId, + parameters: [true], + lazy: false, + }); + + // Promise should succeed with result + await expect(taskSuccess.promise()).resolves.toBe(true); + await expect(taskFail.promise()).rejects.toThrow(Error); + + await taskManager.stop(); + }); + test('tasks fail with no handler', async () => { + const taskManager = await TaskManager.createTaskManager({ + db, + logger, + }); + + const taskFail = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: false, + }); + + // Promise should throw + const taskFailP = taskFail.promise(); + await expect(taskFailP).rejects.toThrow( + tasksErrors.ErrorTaskHandlerMissing, + ); + + await taskManager.stop(); + }); + test('tasks fail with unregistered handler', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (_, fail) => { + if (!fail) throw Error('three'); + return fail; + }); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + logger, + }); + + const taskSucceed = await taskManager.scheduleTask({ + handlerId, + parameters: [false], + lazy: false, + }); + + // Promise should succeed + const taskSucceedP = taskSucceed.promise(); + await expect(taskSucceedP).rejects.not.toThrow( + tasksErrors.ErrorTaskHandlerMissing, + ); + + // Deregister + taskManager.deregisterHandler(handlerId); + const taskFail = await taskManager.scheduleTask({ + handlerId, + parameters: [false], + lazy: false, + }); + const taskFailP = taskFail.promise(); + await expect(taskFailP).rejects.toThrow( + tasksErrors.ErrorTaskHandlerMissing, + ); + + await taskManager.stop(); + }); + test('eager taskPromise resolves when awaited after task completion', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (_, fail) => { + if (!fail) throw Error('three'); + return fail; + }); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + lazy: true, + logger, + }); + + const taskSucceed1 = await taskManager.scheduleTask({ + handlerId, + parameters: [true], + lazy: false, + }); + await taskManager.startProcessing(); + await expect(taskSucceed1.promise()).resolves.toBe(true); + const taskSucceed2 = await taskManager.scheduleTask({ + handlerId, + parameters: [true], + lazy: false, + }); + await expect(taskSucceed2.promise()).resolves.toBe(true); + await taskManager.stop(); + }); + test('lazy taskPromise rejects when awaited after task completion', async () => { + const handler = jest.fn(); + handler.mockImplementation(async () => {}); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + lazy: true, + logger, + }); + + const taskSucceed = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: true, + }); + const taskProm = taskManager.getTaskPromise(taskSucceed.id); + await taskManager.startProcessing(); + await taskProm; + await expect(taskSucceed.promise()).rejects.toThrow(); + await taskManager.stop(); + }); + test('Task Promises should be singletons', async () => { + const taskManager = await TaskManager.createTaskManager({ + db, + lazy: true, + logger, + }); + + const task1 = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: false, + }); + const task2 = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: true, + }); + expect(task1.promise()).toBe(task1.promise()); + expect(task1.promise()).toBe(taskManager.getTaskPromise(task1.id)); + expect(taskManager.getTaskPromise(task1.id)).toBe( + taskManager.getTaskPromise(task1.id), + ); + expect(task2.promise()).toBe(task2.promise()); + expect(task2.promise()).toBe(taskManager.getTaskPromise(task2.id)); + expect(taskManager.getTaskPromise(task2.id)).toBe( + taskManager.getTaskPromise(task2.id), + ); + await taskManager.stop(); + }); + test('can cancel scheduled task, clean up and reject taskPromise', async () => { + const taskManager = await TaskManager.createTaskManager({ + db, + lazy: true, + logger, + }); + + const task1 = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: false, + }); + const task2 = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: true, + }); + + // Cancellation should reject promise + const taskPromise = task1.promise(); + taskPromise.cancel('cancelled'); + await expect(taskPromise).rejects.toBe('cancelled'); + // Should cancel without awaiting anything + task2.cancel('cancelled'); + await sleep(200); + + // Task should be cleaned up + expect(await taskManager.getTask(task1.id)).toBeUndefined(); + expect(await taskManager.getTask(task2.id)).toBeUndefined(); + + await taskManager.stop(); + }); + test('can cancel queued task, clean up and reject taskPromise', async () => { + const taskManager = await TaskManager.createTaskManager({ + db, + lazy: true, + logger, + }); + + const task1 = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: false, + }); + const task2 = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: true, + }); + // @ts-ignore: private method + await taskManager.startScheduling(); + await sleep(100); + + // Cancellation should reject promise + const taskPromise = task1.promise(); + taskPromise.cancel('cancelled'); + await expect(taskPromise).rejects.toBe('cancelled'); + task2.cancel('cancelled'); + await sleep(200); + + // Task should be cleaned up + expect(await taskManager.getTask(task1.id)).toBeUndefined(); + expect(await taskManager.getTask(task2.id)).toBeUndefined(); + + await taskManager.stop(); + }); + test('can cancel active task, clean up and reject taskPromise', async () => { + const handler = jest.fn(); + const pauseProm = promise(); + handler.mockImplementation(async (ctx: ContextTimed) => { + const abortProm = new Promise((resolve, reject) => + ctx.signal.addEventListener('abort', () => reject(ctx.signal.reason)), + ); + await Promise.race([pauseProm.p, abortProm]); + }); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + lazy: true, + logger, + }); + + const task1 = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: false, + }); + const task2 = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: true, + }); + await taskManager.startProcessing(); + await sleep(100); + + // Cancellation should reject promise + const taskPromise = task1.promise(); + taskPromise.cancel('cancelled'); + // Await taskPromise.catch(reason => console.error(reason)); + await expect(taskPromise).rejects.toBe('cancelled'); + task2.cancel('cancelled'); + await sleep(200); + + // Task should be cleaned up + expect(await taskManager.getTask(task1.id, true)).toBeUndefined(); + expect(await taskManager.getTask(task2.id, true)).toBeUndefined(); + pauseProm.resolveP(); + + await taskManager.stop(); + }); + test('incomplete active tasks cleaned up during startup', async () => { + const handler = jest.fn(); + handler.mockImplementation(async () => {}); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + lazy: true, + logger, + }); + + // Seeding data + const task = await taskManager.scheduleTask({ + handlerId, + parameters: [], + deadline: 100, + lazy: false, + }); + + // Moving task to active in database + const taskScheduleTime = task.scheduled.getTime(); + // @ts-ignore: private property + const tasksScheduledDbPath = taskManager.tasksScheduledDbPath; + // @ts-ignore: private property + const tasksActiveDbPath = taskManager.tasksActiveDbPath; + const taskIdBuffer = task.id.toBuffer(); + await db.withTransactionF(async (tran) => { + await tran.del([ + ...tasksScheduledDbPath, + utils.lexiPackBuffer(taskScheduleTime), + taskIdBuffer, + ]); + await tran.put([...tasksActiveDbPath, taskIdBuffer], null); + }); + + // Task should be active + const newTask1 = await taskManager.getTask(task.id); + expect(newTask1!.status).toBe('active'); + + // Restart to clean up + await taskManager.stop(); + await taskManager.start({ lazy: true }); + + // Task should be back to queued + const newTask2 = await taskManager.getTask(task.id, false); + expect(newTask2!.status).toBe('queued'); + await taskManager.startProcessing(); + await newTask2!.promise(); + + await taskManager.stop(); + }); + test('stopping should gracefully end active tasks', async () => { + const handler = jest.fn(); + const pauseProm = promise(); + handler.mockImplementation(async (ctx: ContextTimed) => { + const abortProm = new Promise((resolve, reject) => + ctx.signal.addEventListener('abort', () => reject(ctx.signal.reason)), + ); + await Promise.race([pauseProm.p, abortProm]); + }); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + lazy: true, + logger, + }); + + const task1 = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: false, + }); + const task2 = await taskManager.scheduleTask({ + handlerId, + parameters: [], + lazy: false, + }); + await taskManager.startProcessing(); + await sleep(100); + await taskManager.stopTasks(); + await taskManager.stop(); + + // TaskManager should still exist. + await taskManager.start({ lazy: true }); + expect(await taskManager.getTask(task1.id)).toBeDefined(); + expect(await taskManager.getTask(task2.id)).toBeDefined(); + await task1; + await task2; + + await taskManager.stop(); + }); + test('tests for taskPath', async () => { + const taskManager = await TaskManager.createTaskManager({ + db, + lazy: true, + logger, + }); + + await taskManager.scheduleTask({ + handlerId, + parameters: [1], + path: ['one'], + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [2], + path: ['two'], + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [3], + path: ['two'], + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [4], + path: ['group1', 'three'], + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [5], + path: ['group1', 'four'], + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [6], + path: ['group1', 'four'], + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [7], + path: ['group2', 'five'], + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + parameters: [8], + path: ['group2', 'six'], + lazy: true, + }); + + const listTasks = async (taskGroup: TaskPath) => { + const taskManagerList: Array = []; + for await (const task of taskManager.getTasks( + undefined, + true, + taskGroup, + )) { + taskManagerList.push(task); + } + return taskManagerList; + }; + + expect(await listTasks(['one'])).toHaveLength(1); + expect(await listTasks(['two'])).toHaveLength(2); + expect(await listTasks(['group1'])).toHaveLength(3); + expect(await listTasks(['group1', 'four'])).toHaveLength(2); + expect(await listTasks(['group2'])).toHaveLength(2); + expect(await listTasks([])).toHaveLength(8); + }); + test('getTask', async () => { + const taskManager = await TaskManager.createTaskManager({ + db, + lazy: true, + logger, + }); + + const task1 = await taskManager.scheduleTask({ + handlerId, + parameters: [1], + lazy: true, + }); + const task2 = await taskManager.scheduleTask({ + handlerId, + parameters: [2], + lazy: true, + }); + + const gotTask1 = await taskManager.getTask(task1.id, true); + expect(task1.toString()).toEqual(gotTask1?.toString()); + const gotTask2 = await taskManager.getTask(task2.id, true); + expect(task2.toString()).toEqual(gotTask2?.toString()); + }); + test('getTasks', async () => { + const taskManager = await TaskManager.createTaskManager({ + db, + lazy: true, + logger, + }); + + await taskManager.scheduleTask({ handlerId, parameters: [1], lazy: true }); + await taskManager.scheduleTask({ handlerId, parameters: [2], lazy: true }); + await taskManager.scheduleTask({ handlerId, parameters: [3], lazy: true }); + await taskManager.scheduleTask({ handlerId, parameters: [4], lazy: true }); + + const taskList: Array = []; + for await (const task of taskManager.getTasks()) { + taskList.push(task); + } + + expect(taskList.length).toBe(4); + }); + test('updating tasks while scheduled', async () => { + const handlerId1 = 'handler1' as TaskHandlerId; + const handlerId2 = 'handler2' as TaskHandlerId; + const handler1 = jest.fn(); + const handler2 = jest.fn(); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId1]: handler1, [handlerId2]: handler2 }, + lazy: true, + logger, + }); + + const task1 = await taskManager.scheduleTask({ + handlerId: handlerId1, + delay: 100000, + parameters: [], + lazy: false, + }); + await taskManager.updateTask(task1.id, { + handlerId: handlerId2, + delay: 0, + parameters: [1], + priority: 100, + deadline: 100, + path: ['newPath'], + }); + + // Task should be updated + const oldTask = await taskManager.getTask(task1.id); + if (oldTask == null) never(); + expect(oldTask.id.equals(task1.id)).toBeTrue(); + expect(oldTask.handlerId).toEqual(handlerId2); + expect(oldTask.delay).toBe(0); + expect(oldTask.parameters).toEqual([1]); + expect(oldTask.priority).toEqual(100); + expect(oldTask.deadline).toEqual(100); + expect(oldTask.path).toEqual(['newPath']); + + // Path should've been updated + let task_: Task | undefined; + for await (const task of taskManager.getTasks(undefined, true, [ + 'newPath', + ])) { + task_ = task; + expect(task.id.equals(task1.id)).toBeTrue(); + } + expect(task_).toBeDefined(); + + await taskManager.stop(); + }); + test('updating tasks while queued or active should fail', async () => { + const handler = jest.fn(); + handler.mockImplementation(async (_, value) => value); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + lazy: true, + logger, + }); + // @ts-ignore: private method, only schedule tasks + await taskManager.startScheduling(); + + logger.info('Scheduling task'); + const task1 = await taskManager.scheduleTask({ + handlerId, + delay: 0, + parameters: [], + lazy: false, + }); + + await sleep(100); + + logger.info('Updating task'); + await expect( + taskManager.updateTask(task1.id, { + delay: 1000, + parameters: [1], + }), + ).rejects.toThrow(tasksErrors.ErrorTaskRunning); + + // Task has not been updated + const oldTask = await taskManager.getTask(task1.id); + if (oldTask == null) never(); + expect(oldTask.delay).toBe(0); + expect(oldTask.parameters).toEqual([]); + + await taskManager.stop(); + }); + test('updating tasks delay should update schedule timer', async () => { + const handlerId1 = 'handler1' as TaskHandlerId; + const handlerId2 = 'handler2' as TaskHandlerId; + const handler1 = jest.fn(); + const handler2 = jest.fn(); + handler1.mockImplementation(async (_, value) => value); + handler2.mockImplementation(async (_, value) => value); + + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId1]: handler1, [handlerId2]: handler2 }, + lazy: true, + logger, + }); + + const task1 = await taskManager.scheduleTask({ + handlerId: handlerId1, + delay: 100000, + parameters: [], + lazy: false, + }); + const task2 = await taskManager.scheduleTask({ + handlerId: handlerId1, + delay: 100000, + parameters: [], + lazy: false, + }); + + await taskManager.updateTask(task1.id, { + delay: 0, + parameters: [1], + }); + + // Task should be updated + const newTask = await taskManager.getTask(task1.id); + if (newTask == null) never(); + expect(newTask.delay).toBe(0); + expect(newTask.parameters).toEqual([1]); + + // Task should resolve with new parameter + await taskManager.startProcessing(); + await expect(task1.promise()).resolves.toBe(1); + + await sleep(100); + expect(handler1).toHaveBeenCalledTimes(1); + + // Updating task should update existing timer + await taskManager.updateTask(task2.id, { + delay: 0, + parameters: [1], + handlerId: handlerId2, + }); + await expect(task2.promise()).resolves.toBe(1); + expect(handler1).toHaveBeenCalledTimes(1); + expect(handler2).toHaveBeenCalledTimes(1); + + await taskManager.stop(); + }); + test('task should run after scheduled delay', async () => { + const handler = jest.fn(); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + lazy: true, + logger, + }); + + // Edge case delays + // same as 0 delay + await taskManager.scheduleTask({ + handlerId, + delay: NaN, + lazy: true, + }); + // Same as max delay + await taskManager.scheduleTask({ + handlerId, + delay: Infinity, + lazy: true, + }); + + // Normal delays + await taskManager.scheduleTask({ + handlerId, + delay: 500, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + delay: 1000, + lazy: true, + }); + await taskManager.scheduleTask({ + handlerId, + delay: 1500, + lazy: true, + }); + + expect(handler).toHaveBeenCalledTimes(0); + await taskManager.startProcessing(); + await sleep(250); + expect(handler).toHaveBeenCalledTimes(1); + await sleep(500); + expect(handler).toHaveBeenCalledTimes(2); + await sleep(500); + expect(handler).toHaveBeenCalledTimes(3); + await sleep(500); + expect(handler).toHaveBeenCalledTimes(4); + + await taskManager.stop(); + }); + test('queued tasks should be started in priority order', async () => { + const handler = jest.fn(); + const pendingProm = promise(); + const totalTasks = 31; + const completedTaskOrder: Array = []; + handler.mockImplementation(async (_, priority) => { + completedTaskOrder.push(priority); + if (completedTaskOrder.length >= totalTasks) pendingProm.resolveP(); + }); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + lazy: true, + logger, + }); + const expectedTaskOrder: Array = []; + for (let i = 0; i < totalTasks; i += 1) { + const priority = 150 - i * 10; + expectedTaskOrder.push(priority); + await taskManager.scheduleTask({ + handlerId, + parameters: [priority], + priority, + lazy: true, + }); + } + + // @ts-ignore: start scheduling first + await taskManager.startScheduling(); + await sleep(500); + // @ts-ignore: Then queueing + await taskManager.startQueueing(); + // Wait for all tasks to complete + await pendingProm.p; + expect(completedTaskOrder).toEqual(expectedTaskOrder); + + await taskManager.stop(); + }); + test('task exceeding deadline should abort and clean up', async () => { + const handler = jest.fn(); + const pauseProm = promise(); + handler.mockImplementation(async (ctx: ContextTimed) => { + const abortProm = new Promise((resolve, reject) => + ctx.signal.addEventListener('abort', () => reject(ctx.signal.reason)), + ); + await Promise.race([pauseProm.p, abortProm]); + }); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId]: handler }, + lazy: true, + logger, + }); + + const task = await taskManager.scheduleTask({ + handlerId, + parameters: [], + deadline: 100, + lazy: false, + }); + await taskManager.startProcessing(); + + // Cancellation should reject promise + const taskPromise = task.promise(); + // FIXME: check for deadline timeout error + await expect(taskPromise).rejects.toThrow(tasksErrors.ErrorTaskTimeOut); + + // Task should be cleaned up + const oldTask = await taskManager.getTask(task.id); + expect(oldTask).toBeUndefined(); + pauseProm.resolveP(); + + await taskManager.stop(); + }); + test.todo('scheduled task times should not conflict'); + // TODO: this should move the clock backwards with mocking + test.todo('taskIds are monotonic'); + // TODO: needs fast check + test.todo('general concurrent API usage to test robustness'); +}); + +test('test', async () => { + jest.useFakeTimers(); + new Timer(() => console.log('test'), 100000); + console.log('a'); + jest.advanceTimersByTime(100000); + console.log('a'); + jest.useRealTimers(); +}); + +test('arb', async () => { + const taskArb = fc.record({ + handlerId: fc.constant('handlerId' as TaskHandlerId), + delay: fc.integer({ min: 10, max: 1000 }), + parameters: fc.constant([]), + priority: fc.integer({ min: -200, max: 200 }), + }); + + const scheduleCommandArb = taskArb.map((taskSpec) => async (context) => { + await context.taskManager.scheduleTask({ + ...taskSpec, + lazy: false, + }); + }); + + const sleepCommandArb = fc + .integer({ min: 10, max: 1000 }) + .map((value) => async (context) => { + console.log('sleeping', value); + await sleep(value); + }); + + const commandsArb = fc.array( + fc.oneof( + { arbitrary: scheduleCommandArb, weight: 1 }, + { arbitrary: sleepCommandArb, weight: 1 }, + ), + { maxLength: 10, minLength: 10 }, + ); + + await fc.assert( + fc.asyncProperty(commandsArb, async (commands) => { + const context = { taskManager: {} }; + for (const command of commands) { + await command(context); + } + }), + { numRuns: 2 }, + ); +}); diff --git a/tests/utils/Plug.test.ts b/tests/utils/Plug.test.ts deleted file mode 100644 index a1effeefd..000000000 --- a/tests/utils/Plug.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import Plug from '@/utils/Plug'; - -describe(Plug.name, () => { - test('can plug and unplug', async () => { - const plug = new Plug(); - - // Calls are idempotent - await plug.plug(); - await plug.plug(); - await plug.plug(); - expect(plug.isPlugged()).toBeTrue(); - - // Calls are idempotent - await plug.unplug(); - await plug.unplug(); - await plug.unplug(); - expect(plug.isPlugged()).toBeFalse(); - }); -}); From 19fbf141f0f12f73576bf2eb32a555cf5430f9da Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Mon, 12 Sep 2022 02:03:56 +1000 Subject: [PATCH 20/32] style: updated eslint dependencies --- package-lock.json | 162 +++++++++++++++++++++++----------------------- package.json | 4 +- 2 files changed, 84 insertions(+), 82 deletions(-) diff --git a/package-lock.json b/package-lock.json index f9aa13d13..1a0325b7a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -61,8 +61,8 @@ "@types/prompts": "^2.0.13", "@types/readable-stream": "^2.3.11", "@types/uuid": "^8.3.0", - "@typescript-eslint/eslint-plugin": "^5.23.0", - "@typescript-eslint/parser": "^5.23.0", + "@typescript-eslint/eslint-plugin": "^5.36.2", + "@typescript-eslint/parser": "^5.36.2", "babel-jest": "^28.1.3", "benny": "^3.7.1", "common-tags": "^1.8.2", @@ -3105,14 +3105,14 @@ "dev": true }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.28.0.tgz", - "integrity": "sha512-DXVU6Cg29H2M6EybqSg2A+x8DgO9TCUBRp4QEXQHJceLS7ogVDP0g3Lkg/SZCqcvkAP/RruuQqK0gdlkgmhSUA==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.36.2.tgz", + "integrity": "sha512-OwwR8LRwSnI98tdc2z7mJYgY60gf7I9ZfGjN5EjCwwns9bdTuQfAXcsjSB2wSQ/TVNYSGKf4kzVXbNGaZvwiXw==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "5.28.0", - "@typescript-eslint/type-utils": "5.28.0", - "@typescript-eslint/utils": "5.28.0", + "@typescript-eslint/scope-manager": "5.36.2", + "@typescript-eslint/type-utils": "5.36.2", + "@typescript-eslint/utils": "5.36.2", "debug": "^4.3.4", "functional-red-black-tree": "^1.0.1", "ignore": "^5.2.0", @@ -3153,14 +3153,14 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.28.0.tgz", - "integrity": "sha512-ekqoNRNK1lAcKhZESN/PdpVsWbP9jtiNqzFWkp/yAUdZvJalw2heCYuqRmM5eUJSIYEkgq5sGOjq+ZqsLMjtRA==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.36.2.tgz", + "integrity": "sha512-qS/Kb0yzy8sR0idFspI9Z6+t7mqk/oRjnAYfewG+VN73opAUvmYL3oPIMmgOX6CnQS6gmVIXGshlb5RY/R22pA==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "5.28.0", - "@typescript-eslint/types": "5.28.0", - "@typescript-eslint/typescript-estree": "5.28.0", + "@typescript-eslint/scope-manager": "5.36.2", + "@typescript-eslint/types": "5.36.2", + "@typescript-eslint/typescript-estree": "5.36.2", "debug": "^4.3.4" }, "engines": { @@ -3180,13 +3180,13 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.28.0.tgz", - "integrity": "sha512-LeBLTqF/he1Z+boRhSqnso6YrzcKMTQ8bO/YKEe+6+O/JGof9M0g3IJlIsqfrK/6K03MlFIlycbf1uQR1IjE+w==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.36.2.tgz", + "integrity": "sha512-cNNP51L8SkIFSfce8B1NSUBTJTu2Ts4nWeWbFrdaqjmn9yKrAaJUBHkyTZc0cL06OFHpb+JZq5AUHROS398Orw==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.28.0", - "@typescript-eslint/visitor-keys": "5.28.0" + "@typescript-eslint/types": "5.36.2", + "@typescript-eslint/visitor-keys": "5.36.2" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -3197,12 +3197,13 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.28.0.tgz", - "integrity": "sha512-SyKjKh4CXPglueyC6ceAFytjYWMoPHMswPQae236zqe1YbhvCVQyIawesYywGiu98L9DwrxsBN69vGIVxJ4mQQ==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.36.2.tgz", + "integrity": "sha512-rPQtS5rfijUWLouhy6UmyNquKDPhQjKsaKH0WnY6hl/07lasj8gPaH2UD8xWkePn6SC+jW2i9c2DZVDnL+Dokw==", "dev": true, "dependencies": { - "@typescript-eslint/utils": "5.28.0", + "@typescript-eslint/typescript-estree": "5.36.2", + "@typescript-eslint/utils": "5.36.2", "debug": "^4.3.4", "tsutils": "^3.21.0" }, @@ -3223,9 +3224,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.28.0.tgz", - "integrity": "sha512-2OOm8ZTOQxqkPbf+DAo8oc16sDlVR5owgJfKheBkxBKg1vAfw2JsSofH9+16VPlN9PWtv8Wzhklkqw3k/zCVxA==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.36.2.tgz", + "integrity": "sha512-9OJSvvwuF1L5eS2EQgFUbECb99F0mwq501w0H0EkYULkhFa19Qq7WFbycdw1PexAc929asupbZcgjVIe6OK/XQ==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -3236,13 +3237,13 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.28.0.tgz", - "integrity": "sha512-9GX+GfpV+F4hdTtYc6OV9ZkyYilGXPmQpm6AThInpBmKJEyRSIjORJd1G9+bknb7OTFYL+Vd4FBJAO6T78OVqA==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.36.2.tgz", + "integrity": "sha512-8fyH+RfbKc0mTspfuEjlfqA4YywcwQK2Amcf6TDOwaRLg7Vwdu4bZzyvBZp4bjt1RRjQ5MDnOZahxMrt2l5v9w==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.28.0", - "@typescript-eslint/visitor-keys": "5.28.0", + "@typescript-eslint/types": "5.36.2", + "@typescript-eslint/visitor-keys": "5.36.2", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -3278,15 +3279,15 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.28.0.tgz", - "integrity": "sha512-E60N5L0fjv7iPJV3UGc4EC+A3Lcj4jle9zzR0gW7vXhflO7/J29kwiTGITA2RlrmPokKiZbBy2DgaclCaEUs6g==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.36.2.tgz", + "integrity": "sha512-uNcopWonEITX96v9pefk9DC1bWMdkweeSsewJ6GeC7L6j2t0SJywisgkr9wUTtXk90fi2Eljj90HSHm3OGdGRg==", "dev": true, "dependencies": { "@types/json-schema": "^7.0.9", - "@typescript-eslint/scope-manager": "5.28.0", - "@typescript-eslint/types": "5.28.0", - "@typescript-eslint/typescript-estree": "5.28.0", + "@typescript-eslint/scope-manager": "5.36.2", + "@typescript-eslint/types": "5.36.2", + "@typescript-eslint/typescript-estree": "5.36.2", "eslint-scope": "^5.1.1", "eslint-utils": "^3.0.0" }, @@ -3302,12 +3303,12 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.28.0.tgz", - "integrity": "sha512-BtfP1vCor8cWacovzzPFOoeW4kBQxzmhxGoOpt0v1SFvG+nJ0cWaVdJk7cky1ArTcFHHKNIxyo2LLr3oNkSuXA==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.36.2.tgz", + "integrity": "sha512-BtRvSR6dEdrNt7Net2/XDjbYKU5Ml6GqJgVfXT0CxTCJlnIqK7rAGreuWKMT2t8cFUT2Msv5oxw0GMRD7T5J7A==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.28.0", + "@typescript-eslint/types": "5.36.2", "eslint-visitor-keys": "^3.3.0" }, "engines": { @@ -13832,14 +13833,14 @@ "dev": true }, "@typescript-eslint/eslint-plugin": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.28.0.tgz", - "integrity": "sha512-DXVU6Cg29H2M6EybqSg2A+x8DgO9TCUBRp4QEXQHJceLS7ogVDP0g3Lkg/SZCqcvkAP/RruuQqK0gdlkgmhSUA==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.36.2.tgz", + "integrity": "sha512-OwwR8LRwSnI98tdc2z7mJYgY60gf7I9ZfGjN5EjCwwns9bdTuQfAXcsjSB2wSQ/TVNYSGKf4kzVXbNGaZvwiXw==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "5.28.0", - "@typescript-eslint/type-utils": "5.28.0", - "@typescript-eslint/utils": "5.28.0", + "@typescript-eslint/scope-manager": "5.36.2", + "@typescript-eslint/type-utils": "5.36.2", + "@typescript-eslint/utils": "5.36.2", "debug": "^4.3.4", "functional-red-black-tree": "^1.0.1", "ignore": "^5.2.0", @@ -13860,52 +13861,53 @@ } }, "@typescript-eslint/parser": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.28.0.tgz", - "integrity": "sha512-ekqoNRNK1lAcKhZESN/PdpVsWbP9jtiNqzFWkp/yAUdZvJalw2heCYuqRmM5eUJSIYEkgq5sGOjq+ZqsLMjtRA==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.36.2.tgz", + "integrity": "sha512-qS/Kb0yzy8sR0idFspI9Z6+t7mqk/oRjnAYfewG+VN73opAUvmYL3oPIMmgOX6CnQS6gmVIXGshlb5RY/R22pA==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "5.28.0", - "@typescript-eslint/types": "5.28.0", - "@typescript-eslint/typescript-estree": "5.28.0", + "@typescript-eslint/scope-manager": "5.36.2", + "@typescript-eslint/types": "5.36.2", + "@typescript-eslint/typescript-estree": "5.36.2", "debug": "^4.3.4" } }, "@typescript-eslint/scope-manager": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.28.0.tgz", - "integrity": "sha512-LeBLTqF/he1Z+boRhSqnso6YrzcKMTQ8bO/YKEe+6+O/JGof9M0g3IJlIsqfrK/6K03MlFIlycbf1uQR1IjE+w==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.36.2.tgz", + "integrity": "sha512-cNNP51L8SkIFSfce8B1NSUBTJTu2Ts4nWeWbFrdaqjmn9yKrAaJUBHkyTZc0cL06OFHpb+JZq5AUHROS398Orw==", "dev": true, "requires": { - "@typescript-eslint/types": "5.28.0", - "@typescript-eslint/visitor-keys": "5.28.0" + "@typescript-eslint/types": "5.36.2", + "@typescript-eslint/visitor-keys": "5.36.2" } }, "@typescript-eslint/type-utils": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.28.0.tgz", - "integrity": "sha512-SyKjKh4CXPglueyC6ceAFytjYWMoPHMswPQae236zqe1YbhvCVQyIawesYywGiu98L9DwrxsBN69vGIVxJ4mQQ==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.36.2.tgz", + "integrity": "sha512-rPQtS5rfijUWLouhy6UmyNquKDPhQjKsaKH0WnY6hl/07lasj8gPaH2UD8xWkePn6SC+jW2i9c2DZVDnL+Dokw==", "dev": true, "requires": { - "@typescript-eslint/utils": "5.28.0", + "@typescript-eslint/typescript-estree": "5.36.2", + "@typescript-eslint/utils": "5.36.2", "debug": "^4.3.4", "tsutils": "^3.21.0" } }, "@typescript-eslint/types": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.28.0.tgz", - "integrity": "sha512-2OOm8ZTOQxqkPbf+DAo8oc16sDlVR5owgJfKheBkxBKg1vAfw2JsSofH9+16VPlN9PWtv8Wzhklkqw3k/zCVxA==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.36.2.tgz", + "integrity": "sha512-9OJSvvwuF1L5eS2EQgFUbECb99F0mwq501w0H0EkYULkhFa19Qq7WFbycdw1PexAc929asupbZcgjVIe6OK/XQ==", "dev": true }, "@typescript-eslint/typescript-estree": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.28.0.tgz", - "integrity": "sha512-9GX+GfpV+F4hdTtYc6OV9ZkyYilGXPmQpm6AThInpBmKJEyRSIjORJd1G9+bknb7OTFYL+Vd4FBJAO6T78OVqA==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.36.2.tgz", + "integrity": "sha512-8fyH+RfbKc0mTspfuEjlfqA4YywcwQK2Amcf6TDOwaRLg7Vwdu4bZzyvBZp4bjt1RRjQ5MDnOZahxMrt2l5v9w==", "dev": true, "requires": { - "@typescript-eslint/types": "5.28.0", - "@typescript-eslint/visitor-keys": "5.28.0", + "@typescript-eslint/types": "5.36.2", + "@typescript-eslint/visitor-keys": "5.36.2", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -13925,26 +13927,26 @@ } }, "@typescript-eslint/utils": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.28.0.tgz", - "integrity": "sha512-E60N5L0fjv7iPJV3UGc4EC+A3Lcj4jle9zzR0gW7vXhflO7/J29kwiTGITA2RlrmPokKiZbBy2DgaclCaEUs6g==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.36.2.tgz", + "integrity": "sha512-uNcopWonEITX96v9pefk9DC1bWMdkweeSsewJ6GeC7L6j2t0SJywisgkr9wUTtXk90fi2Eljj90HSHm3OGdGRg==", "dev": true, "requires": { "@types/json-schema": "^7.0.9", - "@typescript-eslint/scope-manager": "5.28.0", - "@typescript-eslint/types": "5.28.0", - "@typescript-eslint/typescript-estree": "5.28.0", + "@typescript-eslint/scope-manager": "5.36.2", + "@typescript-eslint/types": "5.36.2", + "@typescript-eslint/typescript-estree": "5.36.2", "eslint-scope": "^5.1.1", "eslint-utils": "^3.0.0" } }, "@typescript-eslint/visitor-keys": { - "version": "5.28.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.28.0.tgz", - "integrity": "sha512-BtfP1vCor8cWacovzzPFOoeW4kBQxzmhxGoOpt0v1SFvG+nJ0cWaVdJk7cky1ArTcFHHKNIxyo2LLr3oNkSuXA==", + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.36.2.tgz", + "integrity": "sha512-BtRvSR6dEdrNt7Net2/XDjbYKU5Ml6GqJgVfXT0CxTCJlnIqK7rAGreuWKMT2t8cFUT2Msv5oxw0GMRD7T5J7A==", "dev": true, "requires": { - "@typescript-eslint/types": "5.28.0", + "@typescript-eslint/types": "5.36.2", "eslint-visitor-keys": "^3.3.0" } }, diff --git a/package.json b/package.json index ce5da85c3..54b14cbca 100644 --- a/package.json +++ b/package.json @@ -125,8 +125,8 @@ "@types/prompts": "^2.0.13", "@types/readable-stream": "^2.3.11", "@types/uuid": "^8.3.0", - "@typescript-eslint/eslint-plugin": "^5.23.0", - "@typescript-eslint/parser": "^5.23.0", + "@typescript-eslint/eslint-plugin": "^5.36.2", + "@typescript-eslint/parser": "^5.36.2", "babel-jest": "^28.1.3", "benny": "^3.7.1", "common-tags": "^1.8.2", From f62035b8394ad49ed7c1b81517b6c81b093db1bc Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Mon, 12 Sep 2022 02:04:26 +1000 Subject: [PATCH 21/32] style: allow throwing literals due to abort signal reasons --- .eslintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.eslintrc b/.eslintrc index ed7535105..44a8d5ac5 100644 --- a/.eslintrc +++ b/.eslintrc @@ -115,7 +115,7 @@ "@typescript-eslint/consistent-type-imports": ["error"], "@typescript-eslint/consistent-type-exports": ["error"], "no-throw-literal": "off", - "@typescript-eslint/no-throw-literal": ["error"], + "@typescript-eslint/no-throw-literal": "off", "@typescript-eslint/no-floating-promises": ["error", { "ignoreVoid": true, "ignoreIIFE": true From 9a0424de2e92a75e63e1c7ace6906881b0943451 Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Mon, 12 Sep 2022 02:05:17 +1000 Subject: [PATCH 22/32] style: linting timer and contexts --- src/contexts/decorators/context.ts | 2 +- src/contexts/decorators/timedCancellable.ts | 7 +-- src/contexts/functions/cancellable.ts | 14 ++--- src/contexts/functions/timed.ts | 50 +++++----------- src/contexts/functions/timedCancellable.ts | 4 +- src/timer/Timer.ts | 4 +- tests/contexts/decorators/cancellable.test.ts | 58 ++++++++++--------- tests/contexts/decorators/timed.test.ts | 22 ++++--- tests/contexts/functions/cancellable.test.ts | 18 +++--- tests/contexts/functions/timed.test.ts | 46 ++++++++------- 10 files changed, 104 insertions(+), 121 deletions(-) diff --git a/src/contexts/decorators/context.ts b/src/contexts/decorators/context.ts index 1b6df8a0f..fe4b0ae21 100644 --- a/src/contexts/decorators/context.ts +++ b/src/contexts/decorators/context.ts @@ -4,7 +4,7 @@ import * as contextsUtils from '../utils'; * Context parameter decorator * It is only allowed to be used once */ -function context(target: Object, key: string | symbol, index: number) { +function context(target: any, key: string | symbol, index: number) { const targetName = target['name'] ?? target.constructor.name; const method = target[key]; if (contextsUtils.contexts.has(method)) { diff --git a/src/contexts/decorators/timedCancellable.ts b/src/contexts/decorators/timedCancellable.ts index 995482b27..f86949629 100644 --- a/src/contexts/decorators/timedCancellable.ts +++ b/src/contexts/decorators/timedCancellable.ts @@ -1,5 +1,4 @@ - -// equivalent to timed(cancellable()) +// Equivalent to timed(cancellable()) // timeout is always lazy // it's only if you call cancel // PLUS this only works with PromiseLike @@ -11,8 +10,6 @@ function timedCancellable( lazy: boolean = false, delay: number = Infinity, errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedTimeOut, -) { - -} +) {} export default timedCancellable; diff --git a/src/contexts/functions/cancellable.ts b/src/contexts/functions/cancellable.ts index 5194832c0..e564d1e1a 100644 --- a/src/contexts/functions/cancellable.ts +++ b/src/contexts/functions/cancellable.ts @@ -1,18 +1,16 @@ -import type { ContextCancellable } from "../types"; +import type { ContextCancellable } from '../types'; import { PromiseCancellable } from '@matrixai/async-cancellable'; type ContextRemaining = Omit; -type ContextAndParameters> = - keyof ContextRemaining extends never +type ContextAndParameters< + C, + P extends Array, +> = keyof ContextRemaining extends never ? [Partial?, ...P] : [Partial & ContextRemaining, ...P]; -function cancellable< - C extends ContextCancellable, - P extends Array, - R ->( +function cancellable, R>( f: (ctx: C, ...params: P) => PromiseLike, lazy: boolean = false, ): (...params: ContextAndParameters) => PromiseCancellable { diff --git a/src/contexts/functions/timed.ts b/src/contexts/functions/timed.ts index 5c60c6b69..5b885f447 100644 --- a/src/contexts/functions/timed.ts +++ b/src/contexts/functions/timed.ts @@ -18,10 +18,7 @@ function setupContext( return () => { timer.cancel(); }; - } else if ( - ctx.timer === undefined && - ctx.signal instanceof AbortSignal - ) { + } else if (ctx.timer === undefined && ctx.signal instanceof AbortSignal) { const abortController = new AbortController(); const e = new errorTimeoutConstructor(); const timer = new Timer(() => void abortController.abort(e), delay); @@ -85,8 +82,10 @@ function setupContext( type ContextRemaining = Omit; -type ContextAndParameters> = - keyof ContextRemaining extends never +type ContextAndParameters< + C, + P extends Array, +> = keyof ContextRemaining extends never ? [Partial?, ...P] : [Partial & ContextRemaining, ...P]; @@ -94,32 +93,21 @@ type ContextAndParameters> = * Timed HOF * This overloaded signature is external signature */ -function timed< - C extends ContextTimed, - P extends Array, - R ->( +function timed, R>( f: (ctx: C, ...params: P) => R, delay?: number, errorTimeoutConstructor?: new () => Error, -): ( ...params: ContextAndParameters) => R; -function timed< - C extends ContextTimed, - P extends Array ->( +): (...params: ContextAndParameters) => R; +function timed>( f: (ctx: C, ...params: P) => any, delay: number = Infinity, errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedTimeOut, -): ( ...params: ContextAndParameters) => any { +): (...params: ContextAndParameters) => any { if (f instanceof utils.AsyncFunction) { return async (...params) => { const ctx = params[0] ?? {}; const args = params.slice(1) as P; - const teardownContext = setupContext( - delay, - errorTimeoutConstructor, - ctx, - ); + const teardownContext = setupContext(delay, errorTimeoutConstructor, ctx); try { return await f(ctx as C, ...args); } finally { @@ -130,11 +118,7 @@ function timed< return function* (...params) { const ctx = params[0] ?? {}; const args = params.slice(1) as P; - const teardownContext = setupContext( - delay, - errorTimeoutConstructor, - ctx, - ); + const teardownContext = setupContext(delay, errorTimeoutConstructor, ctx); try { return yield* f(ctx as C, ...args); } finally { @@ -145,11 +129,7 @@ function timed< return async function* (...params) { const ctx = params[0] ?? {}; const args = params.slice(1) as P; - const teardownContext = setupContext( - delay, - errorTimeoutConstructor, - ctx, - ); + const teardownContext = setupContext(delay, errorTimeoutConstructor, ctx); try { return yield* f(ctx as C, ...args); } finally { @@ -160,11 +140,7 @@ function timed< return (...params) => { const ctx = params[0] ?? {}; const args = params.slice(1) as P; - const teardownContext = setupContext( - delay, - errorTimeoutConstructor, - ctx, - ); + const teardownContext = setupContext(delay, errorTimeoutConstructor, ctx); const result = f(ctx as C, ...args); if (utils.isPromiseLike(result)) { return result.then( diff --git a/src/contexts/functions/timedCancellable.ts b/src/contexts/functions/timedCancellable.ts index 4f54f8c8b..3f8ff65ac 100644 --- a/src/contexts/functions/timedCancellable.ts +++ b/src/contexts/functions/timedCancellable.ts @@ -1,5 +1,3 @@ -function timedCancellable() { - -} +function timedCancellable() {} export default timedCancellable; diff --git a/src/timer/Timer.ts b/src/timer/Timer.ts index ad14b316a..fd56c9c23 100644 --- a/src/timer/Timer.ts +++ b/src/timer/Timer.ts @@ -121,7 +121,7 @@ class Timer if (isFinite(delay)) { // Clip to delay <= 2147483647 (maximum timeout) // but only if delay is finite - delay = Math.min(delay, 2**31 - 1); + delay = Math.min(delay, 2 ** 31 - 1); } } this.handler = handler; @@ -154,7 +154,7 @@ class Timer } else { // Infinite interval, make sure you are cancelling the `Timer` // otherwise you will keep the process alive - this.timeoutRef = setInterval(() => {}, 2**31 - 1); + this.timeoutRef = setInterval(() => {}, 2 ** 31 - 1); this.timestamp = new Date(performance.timeOrigin + performance.now()); } } diff --git a/tests/contexts/decorators/cancellable.test.ts b/tests/contexts/decorators/cancellable.test.ts index 348fb8547..d9969fb25 100644 --- a/tests/contexts/decorators/cancellable.test.ts +++ b/tests/contexts/decorators/cancellable.test.ts @@ -173,24 +173,26 @@ describe('context/decorators/cancellable', () => { f(ctx?: Partial): PromiseCancellable; @cancellable() f(@context ctx: ContextCancellable): PromiseCancellable { - const pC = new PromiseCancellable((resolve, reject, signal) => { - if (signal.aborted) { - reject('eager 2:' + signal.reason); - } else { - signal.onabort = () => { - reject('lazy 2:' + signal.reason); - }; - } - sleep(10).then(() => { - resolve('hello world'); - }); - }); + const pC = new PromiseCancellable( + (resolve, reject, signal) => { + if (signal.aborted) { + reject('eager 2:' + signal.reason); + } else { + signal.onabort = () => { + reject('lazy 2:' + signal.reason); + }; + } + void sleep(10).then(() => { + resolve('hello world'); + }); + }, + ); if (ctx.signal.aborted) { pC.cancel('eager 1:' + ctx.signal.reason); } else { ctx.signal.onabort = () => { pC.cancel('lazy 1:' + ctx.signal.reason); - } + }; } return pC; } @@ -211,24 +213,26 @@ describe('context/decorators/cancellable', () => { f(ctx?: Partial): PromiseCancellable; @cancellable(true) f(@context ctx: ContextCancellable): PromiseCancellable { - const pC = new PromiseCancellable((resolve, reject, signal) => { - if (signal.aborted) { - reject('eager 2:' + signal.reason); - } else { - signal.onabort = () => { - reject('lazy 2:' + signal.reason); - }; - } - sleep(10).then(() => { - resolve('hello world'); - }); - }); + const pC = new PromiseCancellable( + (resolve, reject, signal) => { + if (signal.aborted) { + reject('eager 2:' + signal.reason); + } else { + signal.onabort = () => { + reject('lazy 2:' + signal.reason); + }; + } + void sleep(10).then(() => { + resolve('hello world'); + }); + }, + ); if (ctx.signal.aborted) { pC.cancel('eager 1:' + ctx.signal.reason); } else { ctx.signal.onabort = () => { pC.cancel('lazy 1:' + ctx.signal.reason); - } + }; } return pC; } @@ -360,7 +364,7 @@ describe('context/decorators/cancellable', () => { class C { f(ctx?: Partial): PromiseCancellable; @cancellable() - async f(@context ctx: ContextCancellable): Promise { + async f(@context _ctx: ContextCancellable): Promise { return 'hello world'; } } diff --git a/tests/contexts/decorators/timed.test.ts b/tests/contexts/decorators/timed.test.ts index aee7af5a5..08e2b0993 100644 --- a/tests/contexts/decorators/timed.test.ts +++ b/tests/contexts/decorators/timed.test.ts @@ -80,7 +80,7 @@ describe('context/decorators/timed', () => { expect(ctx.signal).toBeInstanceOf(AbortSignal); expect(ctx.timer).toBeInstanceOf(Timer); if (check != null) check(ctx.timer); - return [1,2,3,4]; + return [1, 2, 3, 4]; } functionPromise( @@ -183,18 +183,22 @@ describe('context/decorators/timed', () => { test('functionValue', () => { expect(x.functionValue()).toBe('hello world'); expect(x.functionValue({})).toBe('hello world'); - expect(x.functionValue({ timer: new Timer({ delay: 100 }) }, (t) => { - expect(t.delay).toBe(100); - })).toBe('hello world'); + expect( + x.functionValue({ timer: new Timer({ delay: 100 }) }, (t) => { + expect(t.delay).toBe(100); + }), + ).toBe('hello world'); expect(x.functionValue).toBeInstanceOf(Function); expect(x.functionValue.name).toBe('functionValue'); }); test('functionValueArray', () => { - expect(x.functionValueArray()).toStrictEqual([1,2,3,4]); - expect(x.functionValueArray({})).toStrictEqual([1,2,3,4]); - expect(x.functionValueArray({ timer: new Timer({ delay: 100 }) }, (t) => { - expect(t.delay).toBe(100); - })).toStrictEqual([1,2,3,4]); + expect(x.functionValueArray()).toStrictEqual([1, 2, 3, 4]); + expect(x.functionValueArray({})).toStrictEqual([1, 2, 3, 4]); + expect( + x.functionValueArray({ timer: new Timer({ delay: 100 }) }, (t) => { + expect(t.delay).toBe(100); + }), + ).toStrictEqual([1, 2, 3, 4]); expect(x.functionValueArray).toBeInstanceOf(Function); expect(x.functionValueArray.name).toBe('functionValueArray'); }); diff --git a/tests/contexts/functions/cancellable.test.ts b/tests/contexts/functions/cancellable.test.ts index 06bad3e39..8a0992e98 100644 --- a/tests/contexts/functions/cancellable.test.ts +++ b/tests/contexts/functions/cancellable.test.ts @@ -41,7 +41,7 @@ describe('context/functions/cancellable', () => { await expect(pC).rejects.toBeUndefined(); }); test('async function cancel - lazy', async () => { - const f = async(ctx: ContextCancellable): Promise => { + const f = async (ctx: ContextCancellable): Promise => { expect(ctx.signal.aborted).toBe(false); while (true) { if (ctx.signal.aborted) break; @@ -96,7 +96,7 @@ describe('context/functions/cancellable', () => { reject('lazy 2:' + signal.reason); }; } - sleep(10).then(() => { + void sleep(10).then(() => { resolve('hello world'); }); }); @@ -105,7 +105,7 @@ describe('context/functions/cancellable', () => { } else { ctx.signal.onabort = () => { pC.cancel('lazy 1:' + ctx.signal.reason); - } + }; } return pC; }; @@ -130,7 +130,7 @@ describe('context/functions/cancellable', () => { reject('lazy 2:' + signal.reason); }; } - sleep(10).then(() => { + void sleep(10).then(() => { resolve('hello world'); }); }); @@ -139,7 +139,7 @@ describe('context/functions/cancellable', () => { } else { ctx.signal.onabort = () => { pC.cancel('lazy 1:' + ctx.signal.reason); - } + }; } return pC; }; @@ -198,7 +198,7 @@ describe('context/functions/cancellable', () => { expect(signal!.aborted).toBe(true); }); test('nested cancellable - lazy then lazy', async () => { - const f = async(ctx: ContextCancellable): Promise => { + const f = async (ctx: ContextCancellable): Promise => { expect(ctx.signal.aborted).toBe(false); while (true) { if (ctx.signal.aborted) { @@ -214,7 +214,7 @@ describe('context/functions/cancellable', () => { await expect(pC).rejects.toBe('throw:cancel reason'); }); test('nested cancellable - lazy then eager', async () => { - const f = async(ctx: ContextCancellable): Promise => { + const f = async (ctx: ContextCancellable): Promise => { expect(ctx.signal.aborted).toBe(false); while (true) { if (ctx.signal.aborted) { @@ -230,7 +230,7 @@ describe('context/functions/cancellable', () => { await expect(pC).rejects.toBe('cancel reason'); }); test('nested cancellable - eager then lazy', async () => { - const f = async(ctx: ContextCancellable): Promise => { + const f = async (ctx: ContextCancellable): Promise => { expect(ctx.signal.aborted).toBe(false); while (true) { if (ctx.signal.aborted) { @@ -246,7 +246,7 @@ describe('context/functions/cancellable', () => { await expect(pC).rejects.toBe('cancel reason'); }); test('signal event listeners are removed', async () => { - const f = async (ctx: ContextCancellable): Promise => { + const f = async (_ctx: ContextCancellable): Promise => { return 'hello world'; }; const abortController = new AbortController(); diff --git a/tests/contexts/functions/timed.test.ts b/tests/contexts/functions/timed.test.ts index d9a4d0bac..cfd19fb54 100644 --- a/tests/contexts/functions/timed.test.ts +++ b/tests/contexts/functions/timed.test.ts @@ -6,7 +6,7 @@ import { AsyncFunction, GeneratorFunction, AsyncGeneratorFunction, - sleep + sleep, } from '@/utils'; describe('context/functions/timed', () => { @@ -24,9 +24,11 @@ describe('context/functions/timed', () => { const fTimed = timed(f); expect(fTimed(undefined)).toBe('hello world'); expect(fTimed({})).toBe('hello world'); - expect(fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { - expect(t.delay).toBe(50); - })).toBe('hello world'); + expect( + fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + }), + ).toBe('hello world'); expect(fTimed).toBeInstanceOf(Function); }); test('function value array', () => { @@ -37,14 +39,16 @@ describe('context/functions/timed', () => { expect(ctx.timer).toBeInstanceOf(Timer); expect(ctx.signal).toBeInstanceOf(AbortSignal); if (check != null) check(ctx.timer); - return [1,2,3,4]; + return [1, 2, 3, 4]; }; const fTimed = timed(f); - expect(fTimed(undefined)).toStrictEqual([1,2,3,4]); - expect(fTimed({})).toStrictEqual([1,2,3,4]); - expect(fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { - expect(t.delay).toBe(50); - })).toStrictEqual([1,2,3,4]); + expect(fTimed(undefined)).toStrictEqual([1, 2, 3, 4]); + expect(fTimed({})).toStrictEqual([1, 2, 3, 4]); + expect( + fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + }), + ).toStrictEqual([1, 2, 3, 4]); expect(fTimed).toBeInstanceOf(Function); }); test('function promise', async () => { @@ -60,9 +64,11 @@ describe('context/functions/timed', () => { const fTimed = timed(f); expect(await fTimed(undefined)).toBeUndefined(); expect(await fTimed({})).toBeUndefined(); - expect(await fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { - expect(t.delay).toBe(50); - })).toBeUndefined(); + expect( + await fTimed({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + }), + ).toBeUndefined(); expect(fTimed).toBeInstanceOf(Function); }); test('async function', async () => { @@ -181,7 +187,7 @@ describe('context/functions/timed', () => { contextsErrors.ErrorContextsTimedTimeOut, ); return 'hello world'; - } + }; const fTimed = timed(f, 50); await expect(fTimed()).resolves.toBe('hello world'); }); @@ -224,7 +230,7 @@ describe('context/functions/timed', () => { }); }; const fTimed = timed(f, 50); - // const c = new C(); + // Const c = new C(); await expect(fTimed()).resolves.toBe('hello world'); }); test('promise function expiry and late rejection', async () => { @@ -281,7 +287,7 @@ describe('context/functions/timed', () => { expect(timeout).toBeUndefined(); }); test('async generator expiry', async () => { - const f = async function *(ctx: ContextTimed): AsyncGenerator { + const f = async function* (ctx: ContextTimed): AsyncGenerator { while (true) { if (ctx.signal.aborted) { throw ctx.signal.reason; @@ -361,7 +367,7 @@ describe('context/functions/timed', () => { expect(ctx.timer.delay).toBe(50); expect(ctx.signal.aborted).toBe(false); return 'g'; - } + }; const gTimed = timed(g, 25); const f = async (ctx: ContextTimed): Promise => { expect(ctx.timer).toBeInstanceOf(Timer); @@ -389,7 +395,7 @@ describe('context/functions/timed', () => { expect(ctx.timer.delay).toBe(25); expect(ctx.signal.aborted).toBe(false); return 'g'; - } + }; const gTimed = timed(g, 25); const f = async (ctx: ContextTimed): Promise => { expect(ctx.timer).toBeInstanceOf(Timer); @@ -415,7 +421,7 @@ describe('context/functions/timed', () => { expect(ctx.timer.delay).toBe(25); expect(ctx.signal.aborted).toBe(false); return 'g'; - } + }; const gTimed = timed(g, 25); const f = async (ctx: ContextTimed): Promise => { expect(ctx.timer).toBeInstanceOf(Timer); @@ -467,7 +473,7 @@ describe('context/functions/timed', () => { // it may reject after some time await hTimed(ctx); return 'hello world'; - } + }; const fTimed = timed(f, 25); await expect(fTimed()).rejects.toThrow( contextsErrors.ErrorContextsTimedTimeOut, From fb28a532711d122314591a553351010e8721a78c Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Mon, 12 Sep 2022 14:04:02 +1000 Subject: [PATCH 23/32] tests: fixing concurrency limit test --- src/tasks/TaskManager.ts | 2 +- tests/tasks/TaskManager.test.ts | 202 ++++++++++++++------------------ 2 files changed, 89 insertions(+), 115 deletions(-) diff --git a/src/tasks/TaskManager.ts b/src/tasks/TaskManager.ts index dd34f0949..f43e59ec5 100644 --- a/src/tasks/TaskManager.ts +++ b/src/tasks/TaskManager.ts @@ -170,7 +170,7 @@ class TaskManager { this.schedulerLogger = logger.getChild('scheduler'); this.queueLogger = logger.getChild('queue'); this.db = db; - this.activeLimit = activeLimit; + this.activeLimit = Math.max(1, activeLimit); } public async start({ diff --git a/tests/tasks/TaskManager.test.ts b/tests/tasks/TaskManager.test.ts index 3088b25fe..a441ef4a7 100644 --- a/tests/tasks/TaskManager.test.ts +++ b/tests/tasks/TaskManager.test.ts @@ -1,5 +1,6 @@ import type { ContextTimed } from '../../dist/contexts/types'; import type { Task, TaskHandlerId, TaskPath } from '../../src/tasks/types'; +import type { PromiseCancellable } from '@matrixai/async-cancellable'; import fs from 'fs'; import path from 'path'; import os from 'os'; @@ -10,7 +11,6 @@ import { Lock } from '@matrixai/async-locks'; import * as utils from '@/utils/index'; import { promise, sleep, never } from '@/utils'; import TaskManager from '@/tasks/TaskManager'; -import { Timer } from '@/timer/index'; import * as tasksErrors from '@/tasks/errors'; // TODO: move to testing utils @@ -269,99 +269,82 @@ describe(TaskManager.name, () => { }); // TODO: Use fastCheck here, this needs to be re-written test('activeLimit is enforced', async () => { - // Const mockedTimers = jest.useFakeTimers(); - const taskArb = fc.record({ - delay: fc.integer({ min: 0, max: 1000 }), - // Priority: fc.integer({min: -200, max: 200}), - }); - const taskManagerArb = fc.array(taskArb, { minLength: 10, maxLength: 50 }); - await fc.assert( - fc.asyncProperty( - fc.scheduler(), - fc.scheduler(), - taskManagerArb, - async (sCall, sHandle, taskManagerDatas) => { - console.log('a'); - const taskManager = await TaskManager.createTaskManager({ - activeLimit: 0, - db, - fresh: true, - lazy: true, - logger, - }); - console.log('a'); - let handledTaskCount = 0; - const handlerId: TaskHandlerId = 'handlerId' as TaskHandlerId; - const handler = jest.fn(); - handler.mockImplementation(async () => { - // Schedule to resolve randomly - logger.info(`ACTIVE TASKS: ${taskManager.activeCount}`); - await sHandle.schedule(Promise.resolve()); - handledTaskCount += 1; - }); - taskManager.registerHandler(handlerId, handler); - console.log('a'); - await taskManager.startProcessing(); - console.log('a'); - - // Scheduling taskManager to be scheduled - const calls: Array> = []; - const pendingTasks: Array = []; - console.log('a'); - for (const taskManagerData of taskManagerDatas) { - calls.push( - scheduleCall( - sCall, - async () => { - const task = await taskManager.scheduleTask({ - delay: taskManagerData.delay, - handlerId, - lazy: false, - }); - pendingTasks.push(task); - }, - `delay: ${taskManagerData.delay}`, - ), - ); - } - - while (handledTaskCount < taskManagerDatas.length) { - await sleep(10); - logger.info(`handledTaskCount: ${handledTaskCount}`); - // Advance time and check expectations until all taskManager are complete - // mockedTimers.advanceTimersToNextTimer(); - console.log(sHandle.count(), sCall.count()); - while (sHandle.count() > 0) { - await sHandle.waitOne(); - logger.info('resolving 1 handle'); - } - // Shoot off 5 each step - if (sCall.count() > 0) { - for (let i = 0; i < 5; i++) { - await sCall.waitOne(); - } - } - } - const promises = pendingTasks.map((task) => task.promise()); - await Promise.all(calls).then( - (result) => console.log(result), - (reason) => { - console.error(reason); - throw reason; - }, - ); - await Promise.all(promises).then( - (result) => console.log(result), - (reason) => { - console.error(reason); - throw reason; - }, - ); - await taskManager.stop(); - console.log('done'); - }, + const activeLimit = 5; + + const taskArb = fc + .record({ + handlerId: fc.constant(handlerId), + delay: fc.integer({ min: 10, max: 1000 }), + parameters: fc.constant([]), + priority: fc.integer({ min: -200, max: 200 }), + }) + .noShrink(); + + const scheduleCommandArb = taskArb.map( + (taskSpec) => async (context: { taskManager: TaskManager }) => { + return await context.taskManager.scheduleTask({ + ...taskSpec, + lazy: false, + }); + }, + ); + + const sleepCommandArb = fc + .integer({ min: 10, max: 100 }) + .noShrink() + .map((value) => async (_context) => { + logger.info(`sleeping ${value}`); + await sleep(value); + }); + + const commandsArb = fc.array( + fc.oneof( + { arbitrary: scheduleCommandArb, weight: 2 }, + { arbitrary: sleepCommandArb, weight: 1 }, ), - { interruptAfterTimeLimit: globalThis.defaultTimeout - 2000, numRuns: 1 }, + { maxLength: 50, minLength: 50 }, + ); + + await fc.assert( + fc.asyncProperty(commandsArb, async (commands) => { + const taskManager = await TaskManager.createTaskManager({ + activeLimit, + db, + fresh: true, + logger, + }); + const handler = jest.fn(); + handler.mockImplementation(async () => { + await sleep(200); + }); + await taskManager.registerHandler(handlerId, handler); + await taskManager.startProcessing(); + const context = { taskManager }; + + // Scheduling taskManager to be scheduled + const pendingTasks: Array> = []; + for (const command of commands) { + expect(taskManager.activeCount).toBeLessThanOrEqual(activeLimit); + const task = await command(context); + if (task != null) pendingTasks.push(task.promise()); + } + + let completed = false; + const waitForcompletionProm = (async () => { + await Promise.all(pendingTasks); + completed = true; + })(); + + // Check for active tasks while tasks are still running + while (!completed) { + expect(taskManager.activeCount).toBeLessThanOrEqual(activeLimit); + logger.info(`Active tasks: ${taskManager.activeCount}`); + await Promise.race([sleep(100), waitForcompletionProm]); + } + + await taskManager.stop(); + }), + { interruptAfterTimeLimit: globalThis.defaultTimeout - 2000, numRuns: 3 }, ); }); // TODO: Use fastCheck for this @@ -371,7 +354,7 @@ describe(TaskManager.name, () => { const [lockReleaser] = await pendingLock.lock()(); const resolvedTasks = new Map(); const totalTasks = 50; - handler.mockImplementation(async (_, number: number) => { + handler.mockImplementation(async (_ctx, _taskInfo, number: number) => { resolvedTasks.set(number, (resolvedTasks.get(number) ?? 0) + 1); if (resolvedTasks.size >= totalTasks) await lockReleaser(); }); @@ -404,7 +387,7 @@ describe(TaskManager.name, () => { // TODO: use fastCheck test('awaited taskPromises resolve', async () => { const handler = jest.fn(); - handler.mockImplementation(async (_, fail) => { + handler.mockImplementation(async (_ctx, _taskInfo, fail) => { if (!fail) throw Error('three'); return fail; }); @@ -429,7 +412,7 @@ describe(TaskManager.name, () => { // TODO: use fastCheck test('awaited taskPromises reject', async () => { const handler = jest.fn(); - handler.mockImplementation(async (_, fail) => { + handler.mockImplementation(async (_ctx, _taskInfo, fail) => { if (!fail) throw Error('three'); return fail; }); @@ -454,7 +437,7 @@ describe(TaskManager.name, () => { // TODO: use fastCheck test('awaited taskPromises resolve or reject', async () => { const handler = jest.fn(); - handler.mockImplementation(async (_, fail) => { + handler.mockImplementation(async (_ctx, _taskInfo, fail) => { if (!fail) throw Error('three'); return fail; }); @@ -504,7 +487,7 @@ describe(TaskManager.name, () => { }); test('tasks fail with unregistered handler', async () => { const handler = jest.fn(); - handler.mockImplementation(async (_, fail) => { + handler.mockImplementation(async (_ctx, _taskInfo, fail) => { if (!fail) throw Error('three'); return fail; }); @@ -542,7 +525,7 @@ describe(TaskManager.name, () => { }); test('eager taskPromise resolves when awaited after task completion', async () => { const handler = jest.fn(); - handler.mockImplementation(async (_, fail) => { + handler.mockImplementation(async (_ctx, _taskInfo, fail) => { if (!fail) throw Error('three'); return fail; }); @@ -987,7 +970,7 @@ describe(TaskManager.name, () => { }); test('updating tasks while queued or active should fail', async () => { const handler = jest.fn(); - handler.mockImplementation(async (_, value) => value); + handler.mockImplementation(async (_ctx, _taskInfo, value) => value); const taskManager = await TaskManager.createTaskManager({ db, handlers: { [handlerId]: handler }, @@ -1028,8 +1011,8 @@ describe(TaskManager.name, () => { const handlerId2 = 'handler2' as TaskHandlerId; const handler1 = jest.fn(); const handler2 = jest.fn(); - handler1.mockImplementation(async (_, value) => value); - handler2.mockImplementation(async (_, value) => value); + handler1.mockImplementation(async (_ctx, _taskInfo, value) => value); + handler2.mockImplementation(async (_ctx, _taskInfo, value) => value); const taskManager = await TaskManager.createTaskManager({ db, @@ -1139,7 +1122,7 @@ describe(TaskManager.name, () => { const pendingProm = promise(); const totalTasks = 31; const completedTaskOrder: Array = []; - handler.mockImplementation(async (_, priority) => { + handler.mockImplementation(async (_ctx, _taskInfo, priority) => { completedTaskOrder.push(priority); if (completedTaskOrder.length >= totalTasks) pendingProm.resolveP(); }); @@ -1215,15 +1198,6 @@ describe(TaskManager.name, () => { test.todo('general concurrent API usage to test robustness'); }); -test('test', async () => { - jest.useFakeTimers(); - new Timer(() => console.log('test'), 100000); - console.log('a'); - jest.advanceTimersByTime(100000); - console.log('a'); - jest.useRealTimers(); -}); - test('arb', async () => { const taskArb = fc.record({ handlerId: fc.constant('handlerId' as TaskHandlerId), @@ -1241,8 +1215,8 @@ test('arb', async () => { const sleepCommandArb = fc .integer({ min: 10, max: 1000 }) - .map((value) => async (context) => { - console.log('sleeping', value); + .map((value) => async (_context) => { + // console.log('sleeping', value); await sleep(value); }); From 6a63ac5b867d4656366cb88b32526882126bd553 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Mon, 12 Sep 2022 14:11:09 +1000 Subject: [PATCH 24/32] style: linting `Tasks` tests --- tests/tasks/TaskManager.test.ts | 76 ++++----------------------------- tests/utils/utils.ts | 11 +++++ 2 files changed, 20 insertions(+), 67 deletions(-) diff --git a/tests/tasks/TaskManager.test.ts b/tests/tasks/TaskManager.test.ts index a441ef4a7..79e0a40ea 100644 --- a/tests/tasks/TaskManager.test.ts +++ b/tests/tasks/TaskManager.test.ts @@ -13,13 +13,6 @@ import { promise, sleep, never } from '@/utils'; import TaskManager from '@/tasks/TaskManager'; import * as tasksErrors from '@/tasks/errors'; -// TODO: move to testing utils -const scheduleCall = ( - s: fc.Scheduler, - f: () => Promise, - label: string = 'scheduled call', -) => s.schedule(Promise.resolve(label)).then(() => f()); - describe(TaskManager.name, () => { const logger = new Logger(`${TaskManager.name} test`, LogLevel.DEBUG, [ new StreamHandler(), @@ -211,7 +204,7 @@ describe(TaskManager.name, () => { const handler = jest.fn(); handler.mockImplementation(async () => {}); taskManager.registerHandler(handlerId, handler); - console.log('a'); + // Console.log('a'); await taskManager.scheduleTask({ handlerId, parameters: [1], delay: 1000 }); const t1 = await taskManager.scheduleTask({ handlerId, @@ -228,44 +221,34 @@ describe(TaskManager.name, () => { // Setting up actions jest.useFakeTimers(); setTimeout(async () => { - console.log('starting processing'); + // Console.log('starting processing'); await taskManager.startProcessing(); }, 0); setTimeout(async () => { - console.log('stop'); + // Console.log('stop'); await taskManager.stop(); }, 500); setTimeout(async () => { - console.log('start'); + // Console.log('start'); await taskManager.start(); }, 1000); // Running tests here... // after 600 ms we should stop and 4 taskManager should've run - console.log('b'); jest.advanceTimersByTime(400); jest.runAllTimers(); - console.log('b'); jest.advanceTimersByTime(200); - console.log('b'); - console.log(jest.getTimerCount()); + // Console.log(jest.getTimerCount()); jest.runAllTimers(); - console.log(jest.getTimerCount()); + // Console.log(jest.getTimerCount()); await t1.promise(); - console.log('b'); expect(handler).toHaveBeenCalledTimes(4); // After another 5000ms the rest should've been called - console.log('b'); handler.mockClear(); - console.log('b'); jest.advanceTimersByTime(5000); - console.log('b'); // Expect(handler).toHaveBeenCalledTimes(3); - console.log('b'); jest.useRealTimers(); - console.log('b'); await taskManager.stop(); - console.log('b'); }); // TODO: Use fastCheck here, this needs to be re-written test('activeLimit is enforced', async () => { @@ -317,7 +300,7 @@ describe(TaskManager.name, () => { handler.mockImplementation(async () => { await sleep(200); }); - await taskManager.registerHandler(handlerId, handler); + taskManager.registerHandler(handlerId, handler); await taskManager.startProcessing(); const context = { taskManager }; @@ -796,8 +779,8 @@ describe(TaskManager.name, () => { await taskManager.start({ lazy: true }); expect(await taskManager.getTask(task1.id)).toBeDefined(); expect(await taskManager.getTask(task2.id)).toBeDefined(); - await task1; - await task2; + await task1.promise(); + await task2.promise(); await taskManager.stop(); }); @@ -1197,44 +1180,3 @@ describe(TaskManager.name, () => { // TODO: needs fast check test.todo('general concurrent API usage to test robustness'); }); - -test('arb', async () => { - const taskArb = fc.record({ - handlerId: fc.constant('handlerId' as TaskHandlerId), - delay: fc.integer({ min: 10, max: 1000 }), - parameters: fc.constant([]), - priority: fc.integer({ min: -200, max: 200 }), - }); - - const scheduleCommandArb = taskArb.map((taskSpec) => async (context) => { - await context.taskManager.scheduleTask({ - ...taskSpec, - lazy: false, - }); - }); - - const sleepCommandArb = fc - .integer({ min: 10, max: 1000 }) - .map((value) => async (_context) => { - // console.log('sleeping', value); - await sleep(value); - }); - - const commandsArb = fc.array( - fc.oneof( - { arbitrary: scheduleCommandArb, weight: 1 }, - { arbitrary: sleepCommandArb, weight: 1 }, - ), - { maxLength: 10, minLength: 10 }, - ); - - await fc.assert( - fc.asyncProperty(commandsArb, async (commands) => { - const context = { taskManager: {} }; - for (const command of commands) { - await command(context); - } - }), - { numRuns: 2 }, - ); -}); diff --git a/tests/utils/utils.ts b/tests/utils/utils.ts index 96a831828..b2fa14e2b 100644 --- a/tests/utils/utils.ts +++ b/tests/utils/utils.ts @@ -2,6 +2,7 @@ import type { NodeId } from '@/nodes/types'; import type { PrivateKeyPem } from '@/keys/types'; import type { StatusLive } from '@/status/types'; import type Logger from '@matrixai/logger'; +import type * as fc from 'fast-check'; import path from 'path'; import fs from 'fs'; import readline from 'readline'; @@ -157,6 +158,15 @@ function describeIf(condition: boolean) { return condition ? describe : describe.skip; } +/** + * Used with fast-check to schedule calling of a function + */ +const scheduleCall = ( + s: fc.Scheduler, + f: () => Promise, + label: string = 'scheduled call', +) => s.schedule(Promise.resolve(label)).then(() => f()); + export { setupGlobalKeypair, setupTestAgent, @@ -164,4 +174,5 @@ export { expectRemoteError, testIf, describeIf, + scheduleCall, }; From 03c973f9e2da271aa91603d2af6677c6c0883a8a Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Mon, 12 Sep 2022 16:11:39 +1000 Subject: [PATCH 25/32] tests: removing old `Queue` and `Scheduler` tests --- tests/tasks/Queue.test.ts | 415 ---------------------------------- tests/tasks/Scheduler.test.ts | 120 ---------- 2 files changed, 535 deletions(-) delete mode 100644 tests/tasks/Queue.test.ts delete mode 100644 tests/tasks/Scheduler.test.ts diff --git a/tests/tasks/Queue.test.ts b/tests/tasks/Queue.test.ts deleted file mode 100644 index 65f54648a..000000000 --- a/tests/tasks/Queue.test.ts +++ /dev/null @@ -1,415 +0,0 @@ -import type { TaskHandlerId, TaskId } from '../../src/tasks/types'; -import type { TaskPath, Task } from '../../src/tasks/types'; -import os from 'os'; -import path from 'path'; -import fs from 'fs'; -import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; -import { DB } from '@matrixai/db'; -import { sleep } from '@matrixai/async-locks/dist/utils'; -import { IdInternal } from '@matrixai/id'; -import { promise } from 'encryptedfs/dist/utils'; -import Scheduler from '@/tasks/Scheduler'; -import Queue from '@/tasks/Queue'; -import * as keysUtils from '@/keys/utils'; -import * as tasksUtils from '@/tasks/utils'; -import KeyManager from '@/keys/KeyManager'; -import { globalRootKeyPems } from '../fixtures/globalRootKeyPems'; - -describe(Queue.name, () => { - const password = 'password'; - const logger = new Logger(`${Scheduler.name} test`, LogLevel.INFO, [ - new StreamHandler(), - ]); - let dbKey: Buffer; - let dbPath: string; - let db: DB; - let keyManager: KeyManager; - const handlerId = 'testId' as TaskHandlerId; - - const pushTask = async ( - queue: Queue, - handlerId, - params: Array, - lazy = true, - ) => { - const task = await queue.createTask( - handlerId, - params, - undefined, - undefined, - lazy, - ); - const timestampBuffer = tasksUtils.makeTaskTimestampKey( - task.timestamp, - task.id, - ); - await queue.pushTask(task.id, timestampBuffer); - return task; - }; - - beforeAll(async () => { - dataDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), 'polykey-test-'), - ); - const keysPath = `${dataDir}/keys`; - keyManager = await KeyManager.createKeyManager({ - password, - keysPath, - logger, - privateKeyPemOverride: globalRootKeyPems[0], - }); - dbKey = await keysUtils.generateKey(); - dbPath = `${dataDir}/db`; - }); - beforeEach(async () => { - db = await DB.createDB({ - dbPath, - logger, - crypto: { - key: dbKey, - ops: { - encrypt: keysUtils.encryptWithKey, - decrypt: keysUtils.decryptWithKey, - }, - }, - }); - }); - afterEach(async () => { - await db.stop(); - await db.destroy(); - }); - - test('can start and stop', async () => { - const queue = await Queue.createQueue({ - db, - keyManager, - concurrencyLimit: 2, - logger, - }); - await queue.stop(); - await queue.start(); - await queue.stop(); - }); - test('can consume tasks', async () => { - const handler = jest.fn(); - handler.mockImplementation(async () => {}); - const queue = await Queue.createQueue({ - db, - keyManager, - handlers: { [handlerId]: handler }, - concurrencyLimit: 2, - logger, - }); - await queue.startTasks(); - await pushTask(queue, handlerId, [0]); - await pushTask(queue, handlerId, [1]); - await queue.allActiveTasksSettled(); - await queue.stop(); - expect(handler).toHaveBeenCalled(); - }); - test('tasks persist', async () => { - const handler = jest.fn(); - handler.mockImplementation(async () => sleep(0)); - let queue = await Queue.createQueue({ - db, - keyManager, - delay: true, - concurrencyLimit: 2, - logger, - }); - - await pushTask(queue, handlerId, [0]); - await pushTask(queue, handlerId, [1]); - await pushTask(queue, handlerId, [2]); - await queue.stop(); - - queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - concurrencyLimit: 2, - logger, - }); - // Time for tasks to start processing - await sleep(100); - await queue.allActiveTasksSettled(); - await queue.stop(); - expect(handler).toHaveBeenCalled(); - }); - test('concurrency is enforced', async () => { - const handler = jest.fn(); - const prom = promise(); - handler.mockImplementation(async () => { - await prom.p; - }); - const queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - concurrencyLimit: 2, - logger, - }); - - await queue.startTasks(); - await pushTask(queue, handlerId, [0]); - await pushTask(queue, handlerId, [1]); - await pushTask(queue, handlerId, [2]); - await pushTask(queue, handlerId, [3]); - await sleep(200); - expect(handler).toHaveBeenCalledTimes(2); - prom.resolveP(); - await sleep(200); - await queue.allActiveTasksSettled(); - await queue.stop(); - expect(handler).toHaveBeenCalledTimes(4); - }); - test('called exactly 4 times', async () => { - const handler = jest.fn(); - handler.mockImplementation(async () => {}); - const queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - logger, - }); - - await queue.startTasks(); - await pushTask(queue, handlerId, [0]); - await pushTask(queue, handlerId, [1]); - await pushTask(queue, handlerId, [2]); - await pushTask(queue, handlerId, [3]); - await sleep(100); - await queue.stop(); - expect(handler).toHaveBeenCalledTimes(4); - }); - test('tasks can have an optional group', async () => { - const handler = jest.fn(); - handler.mockImplementation(async (nextTaskId) => { - // Await sleep(1000); - logger.info(`task complete ${tasksUtils.encodeTaskId(nextTaskId)}`); - }); - const queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - delay: true, - concurrencyLimit: 2, - logger, - }); - - await queue.createTask(handlerId, [1], undefined, ['one'], true); - await queue.createTask(handlerId, [2], undefined, ['two'], true); - await queue.createTask(handlerId, [3], undefined, ['two'], true); - await queue.createTask( - handlerId, - [4], - undefined, - ['group1', 'three'], - true, - ); - await queue.createTask(handlerId, [5], undefined, ['group1', 'four'], true); - await queue.createTask(handlerId, [6], undefined, ['group1', 'four'], true); - await queue.createTask(handlerId, [7], undefined, ['group2', 'five'], true); - await queue.createTask(handlerId, [8], undefined, ['group2', 'six'], true); - - const listTasks = async (taskGroup: TaskPath) => { - const tasks: Array = []; - for await (const task of queue.getTasksByPath(taskGroup)) { - tasks.push(task); - } - return tasks; - }; - - expect(await listTasks(['one'])).toHaveLength(1); - expect(await listTasks(['two'])).toHaveLength(2); - expect(await listTasks(['group1'])).toHaveLength(3); - expect(await listTasks(['group1', 'four'])).toHaveLength(2); - expect(await listTasks(['group2'])).toHaveLength(2); - expect(await listTasks([])).toHaveLength(8); - - await queue.stop(); - }); - test('completed tasks emit events', async () => { - const handler = jest.fn(); - handler.mockImplementation(async () => { - return 'completed'; - }); - const queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - concurrencyLimit: 2, - logger, - }); - - await pushTask(queue, handlerId, [0]); - await pushTask(queue, handlerId, [1]); - await pushTask(queue, handlerId, [2]); - await pushTask(queue, handlerId, [4]); - await queue.startTasks(); - await sleep(200); - await queue.allActiveTasksSettled(); - await queue.stop(); - expect(handler).toHaveBeenCalledTimes(4); - }); - test('can await a task promise resolve', async () => { - const handler = jest.fn(); - handler.mockImplementation(async (fail) => { - if (!fail) throw Error('three'); - return fail; - }); - const queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - concurrencyLimit: 2, - logger, - }); - - const taskSucceed = await pushTask(queue, handlerId, [true], false); - - // Promise should succeed with result - const taskSucceedP = taskSucceed!.promise(); - await expect(taskSucceedP).resolves.toBe(true); - - await queue.stop(); - }); - test('can await a task promise reject', async () => { - const handler = jest.fn(); - handler.mockImplementation(async (fail) => { - if (!fail) throw Error('three'); - return fail; - }); - const queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - concurrencyLimit: 2, - logger, - }); - - const taskFail = await pushTask(queue, handlerId, [false], false); - // Promise should fail - const taskFailP = taskFail!.promise(); - await expect(taskFailP).rejects.toBeInstanceOf(Error); - - await queue.stop(); - }); - test('getting multiple promises for a task should be the same promise', async () => { - const handler = jest.fn(); - handler.mockImplementation(async (fail) => { - if (!fail) throw Error('three'); - return fail; - }); - const queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - delay: true, - concurrencyLimit: 2, - logger, - }); - - const taskSucceed = await pushTask(queue, handlerId, [true], false); - // If we get a 2nd task promise, it should be the same promise - const prom1 = queue.getTaskP(taskSucceed.id); - const prom2 = queue.getTaskP(taskSucceed.id); - expect(prom1).toBe(prom2); - expect(prom1).toBe(taskSucceed!.promise()); - - await queue.stop(); - }); - test('task promise for invalid task should throw', async () => { - const handler = jest.fn(); - handler.mockImplementation(async (fail) => { - if (!fail) throw Error('three'); - return fail; - }); - const queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - delay: true, - concurrencyLimit: 2, - logger, - }); - - // Getting task promise should not throw - const invalidTask = queue.getTaskP( - IdInternal.fromBuffer(Buffer.alloc(16, 0)), - ); - // Task promise will throw an error if task not found - await expect(invalidTask).rejects.toThrow(); - - await queue.stop(); - }); - test('lazy task promise for completed task should throw', async () => { - const handler = jest.fn(); - handler.mockImplementation(async (fail) => { - if (!fail) throw Error('three'); - return fail; - }); - const queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - delay: true, - concurrencyLimit: 2, - logger, - }); - - const taskSucceed = await pushTask(queue, handlerId, [true], true); - const prom = queue.getTaskP(taskSucceed.id); - await queue.startTasks(); - await prom; - // Finished tasks should throw - await expect(taskSucceed?.promise()).rejects.toThrow(); - - await queue.stop(); - }); - test('eager task promise for completed task should resolve', async () => { - const handler = jest.fn(); - handler.mockImplementation(async (fail) => { - if (!fail) throw Error('three'); - return fail; - }); - const queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - delay: true, - concurrencyLimit: 2, - logger, - }); - - await queue.startTasks(); - const taskSucceed = await pushTask(queue, handlerId, [true], false); - await expect(taskSucceed?.promise()).resolves.toBe(true); - - await queue.stop(); - }); - - test('template', async () => { - const handler = jest.fn(); - handler.mockImplementation(async (nextTaskId) => { - // Await sleep(1000); - logger.info(`task complete ${tasksUtils.encodeTaskId(nextTaskId)}`); - }); - const queue = await Queue.createQueue({ - db, - handlers: { [handlerId]: handler }, - keyManager, - concurrencyLimit: 2, - logger, - }); - - await pushTask(queue, handlerId, [0]); - await pushTask(queue, handlerId, [1]); - await pushTask(queue, handlerId, [2]); - - await queue.startTasks(); - await sleep(100); - await queue.stop(); - expect(handler).toHaveBeenCalledTimes(3); - }); -}); diff --git a/tests/tasks/Scheduler.test.ts b/tests/tasks/Scheduler.test.ts deleted file mode 100644 index a9c4e704d..000000000 --- a/tests/tasks/Scheduler.test.ts +++ /dev/null @@ -1,120 +0,0 @@ -import type { TaskHandlerId } from '../../src/tasks/types'; -import os from 'os'; -import path from 'path'; -import fs from 'fs'; -import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; -import { DB } from '@matrixai/db'; -import { sleep } from '@matrixai/async-locks/dist/utils'; -import KeyManager from '@/keys/KeyManager'; -import Scheduler from '@/tasks/Scheduler'; -import * as keysUtils from '@/keys/utils'; -import Queue from '@/tasks/Queue'; -import { globalRootKeyPems } from '../fixtures/globalRootKeyPems'; - -describe(Scheduler.name, () => { - const password = 'password'; - const logger = new Logger(`${Scheduler.name} test`, LogLevel.INFO, [ - new StreamHandler(), - ]); - let keyManager: KeyManager; - let dbKey: Buffer; - let dbPath: string; - let db: DB; - beforeAll(async () => { - dataDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), 'polykey-test-'), - ); - const keysPath = `${dataDir}/keys`; - keyManager = await KeyManager.createKeyManager({ - password, - keysPath, - logger, - privateKeyPemOverride: globalRootKeyPems[0], - }); - dbKey = await keysUtils.generateKey(); - dbPath = `${dataDir}/db`; - }); - beforeEach(async () => { - db = await DB.createDB({ - dbPath, - logger, - crypto: { - key: dbKey, - ops: { - encrypt: keysUtils.encryptWithKey, - decrypt: keysUtils.decryptWithKey, - }, - }, - }); - }); - afterEach(async () => { - await db.stop(); - await db.destroy(); - }); - test('can add tasks with scheduled delay', async () => { - const queue = await Queue.createQueue({ - db, - keyManager, - logger, - }); - const scheduler = await Scheduler.createScheduler({ - db, - queue, - logger, - }); - const taskHandler = 'asd' as TaskHandlerId; - const handler = jest.fn(); - handler.mockImplementation(async () => sleep(100)); - queue.registerHandler(taskHandler, handler); - - await scheduler.scheduleTask(taskHandler, [1], 1000); - await scheduler.scheduleTask(taskHandler, [2], 100); - await scheduler.scheduleTask(taskHandler, [3], 2000); - await scheduler.scheduleTask(taskHandler, [4], 10); - await scheduler.scheduleTask(taskHandler, [5], 10); - await scheduler.scheduleTask(taskHandler, [6], 10); - await scheduler.scheduleTask(taskHandler, [7], 3000); - await sleep(4000); - await scheduler.stop(); - expect(handler).toHaveBeenCalledTimes(7); - }); - test('scheduled tasks persist', async () => { - const queue = await Queue.createQueue({ - db, - keyManager, - logger, - }); - const scheduler = await Scheduler.createScheduler({ - db, - queue, - logger, - }); - const taskHandler = 'asd' as TaskHandlerId; - const handler = jest.fn(); - handler.mockImplementation(async () => sleep(100)); - queue.registerHandler(taskHandler, handler); - - await scheduler.start(); - await scheduler.scheduleTask(taskHandler, [1], 1000); - await scheduler.scheduleTask(taskHandler, [2], 100); - await scheduler.scheduleTask(taskHandler, [3], 2000); - await scheduler.scheduleTask(taskHandler, [4], 10); - await scheduler.scheduleTask(taskHandler, [5], 10); - await scheduler.scheduleTask(taskHandler, [6], 10); - await scheduler.scheduleTask(taskHandler, [7], 3000); - await sleep(500); - await scheduler.stop(); - - logger.info('intermission!!!!'); - - await scheduler.start(); - await sleep(4000); - await scheduler.stop(); - expect(handler).toHaveBeenCalledTimes(7); - }); - test.todo('Scheculed tasks get moved to queue after delay'); - test.todo('tasks timestamps are unique on taskId'); - test.todo('can remove scheduled tasks'); - test.todo('can not remove active tasks'); - test.todo('Should clean up any inconsistent state during creation'); -}); From d3ec10a2beb48cb21e12d06df2bfb0760287a358 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Mon, 12 Sep 2022 15:28:11 +1000 Subject: [PATCH 26/32] tests: expanding encode/decode tests --- tests/tasks/TaskManager.test.ts | 68 +++++++++++++++++++++++++++++++ tests/tasks/utils.test.ts | 71 ++++++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 1 deletion(-) diff --git a/tests/tasks/TaskManager.test.ts b/tests/tasks/TaskManager.test.ts index 79e0a40ea..7894f21b0 100644 --- a/tests/tasks/TaskManager.test.ts +++ b/tests/tasks/TaskManager.test.ts @@ -784,6 +784,74 @@ describe(TaskManager.name, () => { await taskManager.stop(); }); + test('stopped tasks should run again if allowed', async () => { + const pauseProm = promise(); + const handlerId1 = 'handler1' as TaskHandlerId; + const handler1 = jest.fn(); + handler1.mockImplementation(async (ctx: ContextTimed) => { + const abortProm = new Promise((resolve, reject) => + ctx.signal.addEventListener('abort', () => reject(ctx.signal.reason)), + ); + await Promise.race([pauseProm.p, abortProm]); + }); + const handlerId2 = 'handler2' as TaskHandlerId; + const handler2 = jest.fn(); + handler2.mockImplementation(async (ctx: ContextTimed) => { + const abortProm = new Promise((resolve, reject) => + ctx.signal.addEventListener('abort', () => + reject(Error('different error')), + ), + ); + await Promise.race([pauseProm.p, abortProm]); + }); + const taskManager = await TaskManager.createTaskManager({ + db, + handlers: { [handlerId1]: handler1, [handlerId2]: handler2 }, + lazy: true, + logger, + }); + + const task1 = await taskManager.scheduleTask({ + handlerId: handlerId1, + parameters: [], + lazy: false, + }); + const task2 = await taskManager.scheduleTask({ + handlerId: handlerId2, + parameters: [], + lazy: false, + }); + await taskManager.startProcessing(); + await sleep(100); + await taskManager.stopTasks(); + await taskManager.stop(); + + // Tasks were run + expect(handler1).toHaveBeenCalled(); + expect(handler2).toHaveBeenCalled(); + handler1.mockClear(); + handler2.mockClear(); + + // Tasks should complete + await expect(task1.promise()).rejects.toThrow(); + await expect(task2.promise()).rejects.toThrow(); + + await taskManager.start({ lazy: true }); + const task1New = await taskManager.getTask(task1.id, false); + const task2New = await taskManager.getTask(task2.id, false); + await taskManager.startProcessing(); + // Task1 should still exist + expect(task1New).toBeDefined(); + // Task2 should've been removed + expect(task2New).toBeUndefined(); + await expect(task1New?.promise()).resolves.toBeUndefined(); + + // Tasks were run + expect(handler1).toHaveBeenCalled(); + expect(handler2).not.toHaveBeenCalled(); + + await taskManager.stop(); + }); test('tests for taskPath', async () => { const taskManager = await TaskManager.createTaskManager({ db, diff --git a/tests/tasks/utils.test.ts b/tests/tasks/utils.test.ts index 9bf3e1cab..179cf91f5 100644 --- a/tests/tasks/utils.test.ts +++ b/tests/tasks/utils.test.ts @@ -1,4 +1,10 @@ -import type { TaskPriority } from '@/tasks/types'; +import type { + TaskPriority, + TaskDeadline, + TaskDelay, + TaskId, +} from '@/tasks/types'; +import { IdInternal } from '@matrixai/id'; import * as tasksUtils from '@/tasks/utils'; describe('tasks/utils', () => { @@ -26,4 +32,67 @@ describe('tasks/utils', () => { expect(tasksUtils.fromPriority(254 as TaskPriority)).toBe(-127); expect(tasksUtils.fromPriority(255 as TaskPriority)).toBe(-128); }); + test('toDeadline', async () => { + expect(tasksUtils.toDeadline(NaN)).toBe(0); + expect(tasksUtils.toDeadline(0)).toBe(0); + expect(tasksUtils.toDeadline(100)).toBe(100); + expect(tasksUtils.toDeadline(1000)).toBe(1000); + expect(tasksUtils.toDeadline(Infinity)).toBe(null); + }); + test('fromDeadline', async () => { + expect(tasksUtils.fromDeadline(0 as TaskDeadline)).toBe(0); + expect(tasksUtils.fromDeadline(100 as TaskDeadline)).toBe(100); + expect(tasksUtils.fromDeadline(1000 as TaskDeadline)).toBe(1000); + // @ts-ignore: typescript complains about null here + expect(tasksUtils.fromDeadline(null as TaskDeadline)).toBe(Infinity); + }); + test('toDelay', async () => { + expect(tasksUtils.toDelay(NaN)).toBe(0); + expect(tasksUtils.toDelay(0)).toBe(0); + expect(tasksUtils.toDelay(100)).toBe(100); + expect(tasksUtils.toDelay(1000)).toBe(1000); + expect(tasksUtils.toDelay(2 ** 31 - 1)).toBe(2 ** 31 - 1); + expect(tasksUtils.toDelay(2 ** 31 + 100)).toBe(2 ** 31 - 1); + expect(tasksUtils.toDelay(Infinity)).toBe(2 ** 31 - 1); + }); + test('fromDelay', async () => { + expect(tasksUtils.fromDelay((2 ** 31 - 1) as TaskDelay)).toBe(2 ** 31 - 1); + expect(tasksUtils.fromDelay((2 ** 31 + 100) as TaskDelay)).toBe( + 2 ** 31 + 100, + ); + expect(tasksUtils.fromDelay(1000 as TaskDelay)).toBe(1000); + expect(tasksUtils.fromDelay(100 as TaskDelay)).toBe(100); + expect(tasksUtils.fromDelay(0 as TaskDelay)).toBe(0); + }); + test('encodeTaskId', async () => { + const taskId1 = IdInternal.fromBuffer(Buffer.alloc(16, 0)); + const taskId2 = IdInternal.fromBuffer(Buffer.alloc(16, 100)); + const taskId3 = IdInternal.fromBuffer(Buffer.alloc(16, 255)); + + expect(tasksUtils.encodeTaskId(taskId1)).toBe( + 'v00000000000000000000000000', + ); + expect(tasksUtils.encodeTaskId(taskId2)).toBe( + 'vchi68p34chi68p34chi68p34cg', + ); + expect(tasksUtils.encodeTaskId(taskId3)).toBe( + 'vvvvvvvvvvvvvvvvvvvvvvvvvvs', + ); + }); + test('decodeTaskId', async () => { + const taskId1 = IdInternal.fromBuffer(Buffer.alloc(16, 0)); + const taskId2 = IdInternal.fromBuffer(Buffer.alloc(16, 100)); + const taskId3 = IdInternal.fromBuffer(Buffer.alloc(16, 255)); + + expect( + tasksUtils.decodeTaskId('v00000000000000000000000000')?.equals(taskId1), + ).toBe(true); + expect( + tasksUtils.decodeTaskId('vchi68p34chi68p34chi68p34cg')?.equals(taskId2), + ).toBe(true); + expect( + tasksUtils.decodeTaskId('vvvvvvvvvvvvvvvvvvvvvvvvvvs')?.equals(taskId3), + ).toBe(true); + }); + test; }); From 2d577377e869d3366749853b6e646627781aed22 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Mon, 12 Sep 2022 16:08:09 +1000 Subject: [PATCH 27/32] fix: set `TaskManager` test logger level to `WARN` --- tests/tasks/TaskManager.test.ts | 40 +++++++++++++++++---------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/tasks/TaskManager.test.ts b/tests/tasks/TaskManager.test.ts index 7894f21b0..2a836b8fc 100644 --- a/tests/tasks/TaskManager.test.ts +++ b/tests/tasks/TaskManager.test.ts @@ -14,7 +14,7 @@ import TaskManager from '@/tasks/TaskManager'; import * as tasksErrors from '@/tasks/errors'; describe(TaskManager.name, () => { - const logger = new Logger(`${TaskManager.name} test`, LogLevel.DEBUG, [ + const logger = new Logger(`${TaskManager.name} test`, LogLevel.WARN, [ new StreamHandler(), ]); const handlerId = 'testId' as TaskHandlerId; @@ -250,7 +250,6 @@ describe(TaskManager.name, () => { jest.useRealTimers(); await taskManager.stop(); }); - // TODO: Use fastCheck here, this needs to be re-written test('activeLimit is enforced', async () => { const activeLimit = 5; @@ -749,7 +748,13 @@ describe(TaskManager.name, () => { const pauseProm = promise(); handler.mockImplementation(async (ctx: ContextTimed) => { const abortProm = new Promise((resolve, reject) => - ctx.signal.addEventListener('abort', () => reject(ctx.signal.reason)), + ctx.signal.addEventListener('abort', () => + reject( + new tasksErrors.ErrorTaskRetry(undefined, { + cause: ctx.signal.reason, + }), + ), + ), ); await Promise.race([pauseProm.p, abortProm]); }); @@ -763,24 +768,21 @@ describe(TaskManager.name, () => { const task1 = await taskManager.scheduleTask({ handlerId, parameters: [], - lazy: false, + lazy: true, }); const task2 = await taskManager.scheduleTask({ handlerId, parameters: [], - lazy: false, + lazy: true, }); await taskManager.startProcessing(); await sleep(100); - await taskManager.stopTasks(); await taskManager.stop(); // TaskManager should still exist. await taskManager.start({ lazy: true }); expect(await taskManager.getTask(task1.id)).toBeDefined(); expect(await taskManager.getTask(task2.id)).toBeDefined(); - await task1.promise(); - await task2.promise(); await taskManager.stop(); }); @@ -790,7 +792,13 @@ describe(TaskManager.name, () => { const handler1 = jest.fn(); handler1.mockImplementation(async (ctx: ContextTimed) => { const abortProm = new Promise((resolve, reject) => - ctx.signal.addEventListener('abort', () => reject(ctx.signal.reason)), + ctx.signal.addEventListener('abort', () => + reject( + new tasksErrors.ErrorTaskRetry(undefined, { + cause: ctx.signal.reason, + }), + ), + ), ); await Promise.race([pauseProm.p, abortProm]); }); @@ -798,9 +806,7 @@ describe(TaskManager.name, () => { const handler2 = jest.fn(); handler2.mockImplementation(async (ctx: ContextTimed) => { const abortProm = new Promise((resolve, reject) => - ctx.signal.addEventListener('abort', () => - reject(Error('different error')), - ), + ctx.signal.addEventListener('abort', () => reject(ctx.signal.reason)), ); await Promise.race([pauseProm.p, abortProm]); }); @@ -814,16 +820,15 @@ describe(TaskManager.name, () => { const task1 = await taskManager.scheduleTask({ handlerId: handlerId1, parameters: [], - lazy: false, + lazy: true, }); const task2 = await taskManager.scheduleTask({ handlerId: handlerId2, parameters: [], - lazy: false, + lazy: true, }); await taskManager.startProcessing(); await sleep(100); - await taskManager.stopTasks(); await taskManager.stop(); // Tasks were run @@ -832,10 +837,6 @@ describe(TaskManager.name, () => { handler1.mockClear(); handler2.mockClear(); - // Tasks should complete - await expect(task1.promise()).rejects.toThrow(); - await expect(task2.promise()).rejects.toThrow(); - await taskManager.start({ lazy: true }); const task1New = await taskManager.getTask(task1.id, false); const task2New = await taskManager.getTask(task2.id, false); @@ -844,6 +845,7 @@ describe(TaskManager.name, () => { expect(task1New).toBeDefined(); // Task2 should've been removed expect(task2New).toBeUndefined(); + pauseProm.resolveP(); await expect(task1New?.promise()).resolves.toBeUndefined(); // Tasks were run From 37733c4d0d079947d27ad275114f740901c3063a Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Mon, 12 Sep 2022 16:44:01 +1000 Subject: [PATCH 28/32] build(timer): removing `timer` domain and adding `@matrixai/timer` dependency --- package-lock.json | 17 ++ package.json | 1 + src/contexts/decorators/timed.ts | 2 +- src/contexts/functions/timed.ts | 2 +- src/contexts/types.ts | 2 +- src/tasks/TaskManager.ts | 2 +- src/timer/Timer.ts | 277 ------------------------ src/timer/index.ts | 1 - tests/contexts/decorators/timed.test.ts | 2 +- tests/contexts/functions/timed.test.ts | 2 +- tests/timer/Timer.test.ts | 227 ------------------- 11 files changed, 24 insertions(+), 511 deletions(-) delete mode 100644 src/timer/Timer.ts delete mode 100644 src/timer/index.ts delete mode 100644 tests/timer/Timer.test.ts diff --git a/package-lock.json b/package-lock.json index 1a0325b7a..c17e588f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@matrixai/id": "^3.3.3", "@matrixai/logger": "^3.0.0", "@matrixai/resources": "^1.1.4", + "@matrixai/timer": "^1.0.0", "@matrixai/workers": "^1.3.6", "ajv": "^7.0.4", "bip39": "^3.0.3", @@ -2695,6 +2696,14 @@ "resolved": "https://registry.npmjs.org/@matrixai/resources/-/resources-1.1.4.tgz", "integrity": "sha512-YZSMtklbXah0+SxcKOVEm0ONQdWhlJecQ1COx6hg9Dl80WOybZjZ9A+N+OZfvWk9y25NuoIPzOsjhr8G1aTnIg==" }, + "node_modules/@matrixai/timer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@matrixai/timer/-/timer-1.0.0.tgz", + "integrity": "sha512-ZcsgIW+gMfoU206aryeDFPymSz/FVCY4w6Klw0CCQxSRpa20bdzFJ9UdCMJZzHiEBD1TSAdc2wPTqeXq5OUlPw==", + "dependencies": { + "@matrixai/async-cancellable": "^1.0.2" + } + }, "node_modules/@matrixai/workers": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/@matrixai/workers/-/workers-1.3.6.tgz", @@ -13461,6 +13470,14 @@ "resolved": "https://registry.npmjs.org/@matrixai/resources/-/resources-1.1.4.tgz", "integrity": "sha512-YZSMtklbXah0+SxcKOVEm0ONQdWhlJecQ1COx6hg9Dl80WOybZjZ9A+N+OZfvWk9y25NuoIPzOsjhr8G1aTnIg==" }, + "@matrixai/timer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@matrixai/timer/-/timer-1.0.0.tgz", + "integrity": "sha512-ZcsgIW+gMfoU206aryeDFPymSz/FVCY4w6Klw0CCQxSRpa20bdzFJ9UdCMJZzHiEBD1TSAdc2wPTqeXq5OUlPw==", + "requires": { + "@matrixai/async-cancellable": "^1.0.2" + } + }, "@matrixai/workers": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/@matrixai/workers/-/workers-1.3.6.tgz", diff --git a/package.json b/package.json index 54b14cbca..b003138d9 100644 --- a/package.json +++ b/package.json @@ -87,6 +87,7 @@ "@matrixai/logger": "^3.0.0", "@matrixai/resources": "^1.1.4", "@matrixai/workers": "^1.3.6", + "@matrixai/timer": "^1.0.0", "ajv": "^7.0.4", "bip39": "^3.0.3", "canonicalize": "^1.0.5", diff --git a/src/contexts/decorators/timed.ts b/src/contexts/decorators/timed.ts index 875aa1363..d54e946e3 100644 --- a/src/contexts/decorators/timed.ts +++ b/src/contexts/decorators/timed.ts @@ -1,7 +1,7 @@ import type { ContextTimed } from '../types'; +import { Timer } from '@matrixai/timer'; import * as contextsUtils from '../utils'; import * as contextsErrors from '../errors'; -import Timer from '../../timer/Timer'; import * as utils from '../../utils'; /** diff --git a/src/contexts/functions/timed.ts b/src/contexts/functions/timed.ts index 5b885f447..1f33a0c4f 100644 --- a/src/contexts/functions/timed.ts +++ b/src/contexts/functions/timed.ts @@ -1,6 +1,6 @@ import type { ContextTimed } from '../types'; +import { Timer } from '@matrixai/timer'; import * as contextsErrors from '../errors'; -import Timer from '../../timer/Timer'; import * as utils from '../../utils'; function setupContext( diff --git a/src/contexts/types.ts b/src/contexts/types.ts index 6160ef3da..047368657 100644 --- a/src/contexts/types.ts +++ b/src/contexts/types.ts @@ -1,4 +1,4 @@ -import type Timer from '../timer/Timer'; +import type { Timer } from '@matrixai/timer'; type ContextCancellable = { signal: AbortSignal; diff --git a/src/tasks/TaskManager.ts b/src/tasks/TaskManager.ts index f43e59ec5..6dc221def 100644 --- a/src/tasks/TaskManager.ts +++ b/src/tasks/TaskManager.ts @@ -22,10 +22,10 @@ import { import { Lock } from '@matrixai/async-locks'; import { PromiseCancellable } from '@matrixai/async-cancellable'; import { extractTs } from '@matrixai/id/dist/IdSortable'; +import { Timer } from '@matrixai/timer'; import TaskEvent from './TaskEvent'; import * as tasksErrors from './errors'; import * as tasksUtils from './utils'; -import Timer from '../timer/Timer'; import * as utils from '../utils'; const abortSchedulingLoopReason = Symbol('abort scheduling loop reason'); diff --git a/src/timer/Timer.ts b/src/timer/Timer.ts deleted file mode 100644 index fd56c9c23..000000000 --- a/src/timer/Timer.ts +++ /dev/null @@ -1,277 +0,0 @@ -import type { PromiseCancellableController } from '@matrixai/async-cancellable'; -import { performance } from 'perf_hooks'; -import { PromiseCancellable } from '@matrixai/async-cancellable'; - -/** - * Unlike `setTimeout` or `setInterval`, - * this will not keep the NodeJS event loop alive - */ -class Timer - implements Pick, keyof PromiseCancellable> -{ - /** - * Delay in milliseconds - * This may be `Infinity` - */ - public readonly delay: number; - - /** - * If it is lazy, the timer will not eagerly reject - * on cancellation if the handler has started executing - */ - public readonly lazy: boolean; - - /** - * Timestamp when this is constructed - * Guaranteed to be weakly monotonic within the process lifetime - * Compare this with `performance.now()` not `Date.now()` - */ - public readonly timestamp: Date; - - /** - * Timestamp when this is scheduled to finish and execute the handler - * Guaranteed to be weakly monotonic within the process lifetime - * Compare this with `performance.now()` not `Date.now()` - */ - public readonly scheduled?: Date; - - /** - * Handler to be executed - */ - protected handler?: (signal: AbortSignal) => T | PromiseLike; - - /** - * Deconstructed promise - */ - protected p: PromiseCancellable; - - /** - * Resolve deconstructed promise - */ - protected resolveP: (value?: T) => void; - - /** - * Reject deconstructed promise - */ - protected rejectP: (reason?: any) => void; - - /** - * Abort controller allows immediate cancellation - */ - protected abortController: AbortController; - - /** - * Internal timeout reference - */ - protected timeoutRef?: ReturnType; - - /** - * The status indicates when we have started settling or settled - */ - protected _status: 'settling' | 'settled' | null = null; - - /** - * Construct a Timer - * By default `lazy` is false, which means it will eagerly reject - * the timer, even if the handler has already started executing - * If `lazy` is true, this will make the timer wait for the handler - * to finish executing - * Note that passing a custom controller does not stop the default behaviour - */ - constructor( - handler?: (signal: AbortSignal) => T | PromiseLike, - delay?: number, - lazy?: boolean, - controller?: PromiseCancellableController, - ); - constructor(opts?: { - handler?: (signal: AbortSignal) => T | PromiseLike; - delay?: number; - lazy?: boolean; - controller?: PromiseCancellableController; - }); - constructor( - handlerOrOpts?: - | ((signal: AbortSignal) => T | PromiseLike) - | { - handler?: (signal: AbortSignal) => T | PromiseLike; - delay?: number; - lazy?: boolean; - controller?: PromiseCancellableController; - }, - delay: number = 0, - lazy: boolean = false, - controller?: PromiseCancellableController, - ) { - let handler: ((signal: AbortSignal) => T | PromiseLike) | undefined; - if (typeof handlerOrOpts === 'function') { - handler = handlerOrOpts; - } else if (typeof handlerOrOpts === 'object' && handlerOrOpts !== null) { - handler = handlerOrOpts.handler; - delay = handlerOrOpts.delay ?? delay; - lazy = handlerOrOpts.lazy ?? lazy; - controller = handlerOrOpts.controller ?? controller; - } - // Coerce NaN to minimal delay of 0 - if (isNaN(delay)) { - delay = 0; - } else { - // Clip to delay >= 0 - delay = Math.max(delay, 0); - if (isFinite(delay)) { - // Clip to delay <= 2147483647 (maximum timeout) - // but only if delay is finite - delay = Math.min(delay, 2 ** 31 - 1); - } - } - this.handler = handler; - this.delay = delay; - this.lazy = lazy; - let abortController: AbortController; - if (typeof controller === 'function') { - abortController = new AbortController(); - controller(abortController.signal); - } else if (controller != null) { - abortController = controller; - } else { - abortController = new AbortController(); - abortController.signal.addEventListener( - 'abort', - () => void this.reject(abortController.signal.reason), - ); - } - this.p = new PromiseCancellable((resolve, reject) => { - this.resolveP = resolve.bind(this.p); - this.rejectP = reject.bind(this.p); - }, abortController); - this.abortController = abortController; - // If the delay is Infinity, this promise will never resolve - // it may still reject however - if (isFinite(delay)) { - this.timeoutRef = setTimeout(() => void this.fulfill(), delay); - this.timestamp = new Date(performance.timeOrigin + performance.now()); - this.scheduled = new Date(this.timestamp.getTime() + delay); - } else { - // Infinite interval, make sure you are cancelling the `Timer` - // otherwise you will keep the process alive - this.timeoutRef = setInterval(() => {}, 2 ** 31 - 1); - this.timestamp = new Date(performance.timeOrigin + performance.now()); - } - } - - public get [Symbol.toStringTag](): string { - return this.constructor.name; - } - - public get status(): 'settling' | 'settled' | null { - return this._status; - } - - /** - * Gets the remaining time in milliseconds - * This will return `Infinity` if `delay` is `Infinity` - * This will return `0` if status is `settling` or `settled` - */ - public getTimeout(): number { - if (this._status !== null) return 0; - if (this.scheduled == null) return Infinity; - return Math.max( - Math.trunc( - this.scheduled.getTime() - (performance.timeOrigin + performance.now()), - ), - 0, - ); - } - - /** - * To remaining time as a string - * This may return `'Infinity'` if `this.delay` is `Infinity` - * This will return `'0'` if status is `settling` or `settled` - */ - public toString(): string { - return this.getTimeout().toString(); - } - - /** - * To remaining time as a number - * This may return `Infinity` if `this.delay` is `Infinity` - * This will return `0` if status is `settling` or `settled` - */ - public valueOf(): number { - return this.getTimeout(); - } - - /** - * Cancels the timer - * Unlike `PromiseCancellable`, canceling the timer will not result - * in an unhandled promise rejection, all promise rejections are ignored - */ - public cancel(reason?: any): void { - void this.p.catch(() => {}); - this.p.cancel(reason); - } - - public then( - onFulfilled?: - | ((value: T, signal: AbortSignal) => TResult1 | PromiseLike) - | undefined - | null, - onRejected?: - | ((reason: any, signal: AbortSignal) => TResult2 | PromiseLike) - | undefined - | null, - controller?: PromiseCancellableController, - ): PromiseCancellable { - return this.p.then(onFulfilled, onRejected, controller); - } - - public catch( - onRejected?: - | ((reason: any, signal: AbortSignal) => TResult | PromiseLike) - | undefined - | null, - controller?: PromiseCancellableController, - ): PromiseCancellable { - return this.p.catch(onRejected, controller); - } - - public finally( - onFinally?: ((signal: AbortSignal) => void) | undefined | null, - controller?: PromiseCancellableController, - ): PromiseCancellable { - return this.p.finally(onFinally, controller); - } - - protected async fulfill(): Promise { - this._status = 'settling'; - clearTimeout(this.timeoutRef); - delete this.timeoutRef; - if (this.handler != null) { - try { - const result = await this.handler(this.abortController.signal); - this.resolveP(result); - } catch (e) { - this.rejectP(e); - } - } else { - this.resolveP(); - } - this._status = 'settled'; - } - - protected async reject(reason?: any): Promise { - if ( - (this.lazy && (this._status == null || this._status === 'settling')) || - this._status === 'settled' - ) { - return; - } - this._status = 'settling'; - clearTimeout(this.timeoutRef); - delete this.timeoutRef; - this.rejectP(reason); - this._status = 'settled'; - } -} - -export default Timer; diff --git a/src/timer/index.ts b/src/timer/index.ts deleted file mode 100644 index ed32c1af2..000000000 --- a/src/timer/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default as Timer } from './Timer'; diff --git a/tests/contexts/decorators/timed.test.ts b/tests/contexts/decorators/timed.test.ts index 08e2b0993..d0088ce6f 100644 --- a/tests/contexts/decorators/timed.test.ts +++ b/tests/contexts/decorators/timed.test.ts @@ -1,8 +1,8 @@ import type { ContextTimed } from '@/contexts/types'; +import { Timer } from '@matrixai/timer'; import context from '@/contexts/decorators/context'; import timed from '@/contexts/decorators/timed'; import * as contextsErrors from '@/contexts/errors'; -import Timer from '@/timer/Timer'; import { AsyncFunction, GeneratorFunction, diff --git a/tests/contexts/functions/timed.test.ts b/tests/contexts/functions/timed.test.ts index cfd19fb54..5444ac4fd 100644 --- a/tests/contexts/functions/timed.test.ts +++ b/tests/contexts/functions/timed.test.ts @@ -1,7 +1,7 @@ import type { ContextTimed } from '@/contexts/types'; +import { Timer } from '@matrixai/timer'; import timed from '@/contexts/functions/timed'; import * as contextsErrors from '@/contexts/errors'; -import Timer from '@/timer/Timer'; import { AsyncFunction, GeneratorFunction, diff --git a/tests/timer/Timer.test.ts b/tests/timer/Timer.test.ts deleted file mode 100644 index 9b43cdd32..000000000 --- a/tests/timer/Timer.test.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { performance } from 'perf_hooks'; -import { Timer } from '@/timer'; -import { sleep } from '@/utils'; - -describe(Timer.name, () => { - test('timer is thenable and awaitable', async () => { - const t1 = new Timer(); - expect(await t1).toBeUndefined(); - expect(t1.status).toBe('settled'); - const t2 = new Timer(); - await expect(t2).resolves.toBeUndefined(); - expect(t2.status).toBe('settled'); - }); - test('timer delays', async () => { - const t1 = new Timer({ delay: 20, handler: () => 1 }); - const t2 = new Timer(() => 2, 10); - const result = await Promise.any([t1, t2]); - expect(result).toBe(2); - }); - test('timer handlers', async () => { - const t1 = new Timer(() => 123); - expect(await t1).toBe(123); - expect(t1.status).toBe('settled'); - const t2 = new Timer({ delay: 100, handler: () => '123' }); - expect(await t2).toBe('123'); - expect(t2.status).toBe('settled'); - }); - test('timer timestamps', async () => { - const start = new Date(performance.timeOrigin + performance.now()); - await sleep(10); - const t = new Timer({ delay: 100 }); - expect(t.status).toBeNull(); - expect(t.timestamp).toBeAfter(start); - expect(t.scheduled).toBeAfter(start); - expect(t.scheduled).toBeAfterOrEqualTo(t.timestamp); - const delta = t.scheduled!.getTime() - t.timestamp.getTime(); - expect(t.getTimeout()).toBeLessThanOrEqual(delta); - }); - test('timer primitive string and number', () => { - const t1 = new Timer(); - expect(t1.valueOf()).toBe(0); - expect(+t1).toBe(0); - expect(t1.toString()).toBe('0'); - expect(`${t1}`).toBe('0'); - const t2 = new Timer({ delay: 100 }); - expect(t2.valueOf()).toBePositive(); - expect(+t2).toBePositive(); - expect(t2.toString()).toMatch(/\d+/); - expect(`${t2}`).toMatch(/\d+/); - }); - test('timer with infinite delay', async () => { - const t1 = new Timer({ delay: Infinity }); - expect(t1.delay).toBe(Infinity); - expect(t1.scheduled).toBeUndefined(); - expect(t1.getTimeout()).toBe(Infinity); - expect(t1.valueOf()).toBe(Infinity); - expect(+t1).toBe(Infinity); - expect(t1.toString()).toBe('Infinity'); - expect(`${t1}`).toBe('Infinity'); - t1.cancel(new Error('Oh No')); - await expect(t1).rejects.toThrow('Oh No'); - }); - test('custom signal handler ignores default rejection', async () => { - const onabort = jest.fn(); - const t = new Timer( - () => 1, - 50, - false, - (signal) => { - signal.onabort = onabort; - }, - ); - t.cancel('abort'); - await expect(t).resolves.toBe(1); - expect(onabort).toBeCalled(); - }); - test('custom abort controller ignores default rejection', async () => { - const onabort = jest.fn(); - const abortController = new AbortController(); - abortController.signal.onabort = onabort; - const t = new Timer(() => 1, 50, false, abortController); - t.cancel('abort'); - await expect(t).resolves.toBe(1); - expect(onabort).toBeCalled(); - }); - describe('timer cancellation', () => { - test('cancellation rejects the timer with the reason', async () => { - const t1 = new Timer(undefined, 100); - t1.cancel(); - await expect(t1).rejects.toBeUndefined(); - expect(t1.status).toBe('settled'); - const t2 = new Timer({ delay: 100 }); - const results = await Promise.all([ - (async () => { - try { - await t2; - } catch (e) { - return e; - } - })(), - (async () => { - t2.cancel('Surprise!'); - })(), - ]); - expect(results[0]).toBe('Surprise!'); - expect(t2.status).toBe('settled'); - }); - test('non-lazy cancellation is early/eager rejection', async () => { - let resolveHandlerCalledP; - const handlerCalledP = new Promise((resolve) => { - resolveHandlerCalledP = resolve; - }); - let p; - const handler = jest.fn().mockImplementation((signal: AbortSignal) => { - resolveHandlerCalledP(); - p = new Promise((resolve, reject) => { - if (signal.aborted) { - reject('handler abort start'); - return; - } - const timeout = setTimeout(() => resolve('handler result'), 100); - signal.addEventListener( - 'abort', - () => { - clearTimeout(timeout); - reject('handler abort during'); - }, - { once: true }, - ); - }); - return p; - }); - // Non-lazy means that it will do an early rejection - const t = new Timer({ - handler, - delay: 100, - lazy: false, - }); - await handlerCalledP; - expect(handler).toBeCalledWith(expect.any(AbortSignal)); - t.cancel('timer abort'); - await expect(t).rejects.toBe('timer abort'); - await expect(p).rejects.toBe('handler abort during'); - }); - test('lazy cancellation', async () => { - let resolveHandlerCalledP; - const handlerCalledP = new Promise((resolve) => { - resolveHandlerCalledP = resolve; - }); - let p; - const handler = jest.fn().mockImplementation((signal: AbortSignal) => { - resolveHandlerCalledP(); - p = new Promise((resolve, reject) => { - if (signal.aborted) { - reject('handler abort start'); - return; - } - const timeout = setTimeout(() => resolve('handler result'), 100); - signal.addEventListener( - 'abort', - () => { - clearTimeout(timeout); - reject('handler abort during'); - }, - { once: true }, - ); - }); - return p; - }); - // Lazy means that it will not do an early rejection - const t = new Timer({ - handler, - delay: 100, - lazy: true, - }); - await handlerCalledP; - expect(handler).toBeCalledWith(expect.any(AbortSignal)); - t.cancel('timer abort'); - await expect(t).rejects.toBe('handler abort during'); - await expect(p).rejects.toBe('handler abort during'); - }); - test('cancellation should not have an unhandled promise rejection', async () => { - const timer = new Timer(); - timer.cancel('reason'); - }); - test('multiple cancellations should have an unhandled promise rejection', async () => { - const timer = new Timer(); - timer.cancel('reason 1'); - timer.cancel('reason 2'); - }); - test('only the first reason is used in multiple cancellations', async () => { - const timer = new Timer(); - timer.cancel('reason 1'); - timer.cancel('reason 2'); - await expect(timer).rejects.toBe('reason 1'); - }); - test('lazy cancellation allows resolution if signal is ignored', async () => { - const timer = new Timer({ - handler: (signal) => { - expect(signal.aborted).toBe(true); - return new Promise((resolve) => { - setTimeout(() => { - resolve('result'); - }, 50); - }); - }, - lazy: true, - }); - timer.cancel('reason'); - expect(await timer).toBe('result'); - }); - test('lazy cancellation allows rejection if signal is ignored', async () => { - const timer = new Timer({ - handler: () => { - return new Promise((resolve, reject) => { - setTimeout(() => { - reject('error'); - }, 50); - }); - }, - lazy: true, - }); - timer.cancel('reason'); - await expect(timer).rejects.toBe('error'); - }); - }); -}); From d20fb5042a04708ed5f9f8d2084cbae7bbaabe9c Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Mon, 12 Sep 2022 18:35:46 +1000 Subject: [PATCH 29/32] fix(contexts): when timed decorator inherits timer and signal, it should do nothing There are 3 properties for the `timed` wrapper: A. If timer times out, signal is aborted B. If signal is aborted, timer is cancelled C. If timer is owned by the wrapper, then it must be cancelled when the target finishes There are 4 cases where the wrapper is used and where the properties are applied: 1. Nothing is inherited - A B C 2. Signal is inherited - A B C 3. Timer is inherited - A 4. Both signal and timer are inherited - A* B and C are only applied to case 1 and 2, because that's when the `Timer` is owned by the wrapper. *Case 4 is a special case, because the timer and signal are inherited, so it is assumed that the handlers are already setup betwen the timer and signal. --- src/contexts/decorators/timed.ts | 44 ++++++++++++++++++------- src/contexts/functions/timed.ts | 44 ++++++++++++++++++------- tests/contexts/decorators/timed.test.ts | 12 ++++++- tests/contexts/functions/timed.test.ts | 12 ++++++- 4 files changed, 86 insertions(+), 26 deletions(-) diff --git a/src/contexts/decorators/timed.ts b/src/contexts/decorators/timed.ts index d54e946e3..9da23cd1b 100644 --- a/src/contexts/decorators/timed.ts +++ b/src/contexts/decorators/timed.ts @@ -42,14 +42,38 @@ function setupContext( `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`signal\` property is not an instance of \`AbortSignal\``, ); } - // Mutating the `context` parameter + // There are 3 properties of timer and signal: + // + // A. If timer times out, signal is aborted + // B. If signal is aborted, timer is cancelled + // C. If timer is owned by the wrapper, then it must be cancelled when the target finishes + // + // There are 4 cases where the wrapper is used: + // + // 1. Nothing is inherited - A B C + // 2. Signal is inherited - A B C + // 3. Timer is inherited - A + // 4. Both signal and timer are inherited - A* + // + // Property B and C only applies to case 1 and 2 because the timer is owned + // by the wrapper and it is not inherited, if it is inherited, the caller may + // need to reuse the timer. + // In situation 4, there's a caveat for property A: it is assumed that the + // caller has already setup the property A relationship, therefore this + // wrapper will not re-setup this property A relationship. if (context.timer === undefined && context.signal === undefined) { const abortController = new AbortController(); const e = new errorTimeoutConstructor(); + // Property A const timer = new Timer(() => void abortController.abort(e), delay); + abortController.signal.addEventListener('abort', () => { + // Property B + timer.cancel(); + }); context.signal = abortController.signal; context.timer = timer; return () => { + // Property C timer.cancel(); }; } else if ( @@ -58,14 +82,17 @@ function setupContext( ) { const abortController = new AbortController(); const e = new errorTimeoutConstructor(); + // Property A const timer = new Timer(() => void abortController.abort(e), delay); const signalUpstream = context.signal; const signalHandler = () => { + // Property B timer.cancel(); abortController.abort(signalUpstream.reason); }; // If already aborted, abort target and cancel the timer if (signalUpstream.aborted) { + // Property B timer.cancel(); abortController.abort(signalUpstream.reason); } else { @@ -76,6 +103,7 @@ function setupContext( context.timer = timer; return () => { signalUpstream.removeEventListener('abort', signalHandler); + // Property C timer.cancel(); }; } else if (context.timer instanceof Timer && context.signal === undefined) { @@ -88,6 +116,7 @@ function setupContext( // If the timer is aborted after it resolves // then don't bother aborting the target function if (!finished && !s.aborted) { + // Property A abortController.abort(e); } return r; @@ -103,17 +132,8 @@ function setupContext( } else { // In this case, `context.timer` and `context.signal` are both instances of // `Timer` and `AbortSignal` respectively - const signalHandler = () => { - context.timer!.cancel(); - }; - if (context.signal!.aborted) { - context.timer!.cancel(); - } else { - context.signal!.addEventListener('abort', signalHandler); - } - return () => { - context.signal!.removeEventListener('abort', signalHandler); - }; + // It is assumed that both the timer and signal are already hooked up to each other + return () => {}; } } diff --git a/src/contexts/functions/timed.ts b/src/contexts/functions/timed.ts index 1f33a0c4f..0afb9a430 100644 --- a/src/contexts/functions/timed.ts +++ b/src/contexts/functions/timed.ts @@ -8,27 +8,54 @@ function setupContext( errorTimeoutConstructor: new () => Error, ctx: Partial, ): () => void { - // Mutating the `context` parameter + // There are 3 properties of timer and signal: + // + // A. If timer times out, signal is aborted + // B. If signal is aborted, timer is cancelled + // C. If timer is owned by the wrapper, then it must be cancelled when the target finishes + // + // There are 4 cases where the wrapper is used: + // + // 1. Nothing is inherited - A B C + // 2. Signal is inherited - A B C + // 3. Timer is inherited - A + // 4. Both signal and timer are inherited - A* + // + // Property B and C only applies to case 1 and 2 because the timer is owned + // by the wrapper and it is not inherited, if it is inherited, the caller may + // need to reuse the timer. + // In situation 4, there's a caveat for property A: it is assumed that the + // caller has already setup the property A relationship, therefore this + // wrapper will not re-setup this property A relationship. if (ctx.timer === undefined && ctx.signal === undefined) { const abortController = new AbortController(); const e = new errorTimeoutConstructor(); + // Property A const timer = new Timer(() => void abortController.abort(e), delay); + abortController.signal.addEventListener('abort', () => { + // Property B + timer.cancel(); + }); ctx.signal = abortController.signal; ctx.timer = timer; return () => { + // Property C timer.cancel(); }; } else if (ctx.timer === undefined && ctx.signal instanceof AbortSignal) { const abortController = new AbortController(); const e = new errorTimeoutConstructor(); + // Property A const timer = new Timer(() => void abortController.abort(e), delay); const signalUpstream = ctx.signal; const signalHandler = () => { + // Property B timer.cancel(); abortController.abort(signalUpstream.reason); }; // If already aborted, abort target and cancel the timer if (signalUpstream.aborted) { + // Property B timer.cancel(); abortController.abort(signalUpstream.reason); } else { @@ -39,6 +66,7 @@ function setupContext( ctx.timer = timer; return () => { signalUpstream.removeEventListener('abort', signalHandler); + // Property C timer.cancel(); }; } else if (ctx.timer instanceof Timer && ctx.signal === undefined) { @@ -51,6 +79,7 @@ function setupContext( // If the timer is aborted after it resolves // then don't bother aborting the target function if (!finished && !s.aborted) { + // Property A abortController.abort(e); } return r; @@ -66,17 +95,8 @@ function setupContext( } else { // In this case, `ctx.timer` and `ctx.signal` are both instances of // `Timer` and `AbortSignal` respectively - const signalHandler = () => { - ctx!.timer!.cancel(); - }; - if (ctx.signal!.aborted) { - ctx.timer!.cancel(); - } else { - ctx.signal!.addEventListener('abort', signalHandler); - } - return () => { - ctx!.signal!.removeEventListener('abort', signalHandler); - }; + // It is assumed that both the timer and signal are already hooked up to each other + return () => {}; } } diff --git a/tests/contexts/decorators/timed.test.ts b/tests/contexts/decorators/timed.test.ts index d0088ce6f..48b7dd56a 100644 --- a/tests/contexts/decorators/timed.test.ts +++ b/tests/contexts/decorators/timed.test.ts @@ -734,8 +734,18 @@ describe('context/decorators/timed', () => { await expect(p).rejects.toBe('reason during'); }); test('explicit signal signal abortion with passed in timer - during', async () => { - const timer = new Timer({ delay: 100 }); + // By passing in the timer and signal explicitly + // it is expected that the timer and signal handling is already setup const abortController = new AbortController(); + const timer = new Timer({ + handler: () => { + abortController.abort(new contextsErrors.ErrorContextsTimedTimeOut); + }, + delay: 100 + }); + abortController.signal.addEventListener('abort', () => { + timer.cancel(); + }); const p = c.f({ timer, signal: abortController.signal }); abortController.abort('abort reason'); expect(ctx_!.timer.status).toBe('settled'); diff --git a/tests/contexts/functions/timed.test.ts b/tests/contexts/functions/timed.test.ts index 5444ac4fd..36a8808ea 100644 --- a/tests/contexts/functions/timed.test.ts +++ b/tests/contexts/functions/timed.test.ts @@ -542,8 +542,18 @@ describe('context/functions/timed', () => { await expect(p).rejects.toBe('reason during'); }); test('explicit signal signal abortion with passed in timer - during', async () => { - const timer = new Timer({ delay: 100 }); + // By passing in the timer and signal explicitly + // it is expected that the timer and signal handling is already setup const abortController = new AbortController(); + const timer = new Timer({ + handler: () => { + abortController.abort(new contextsErrors.ErrorContextsTimedTimeOut); + }, + delay: 100 + }); + abortController.signal.addEventListener('abort', () => { + timer.cancel(); + }); const p = fTimed({ timer, signal: abortController.signal }); abortController.abort('abort reason'); expect(ctx_!.timer.status).toBe('settled'); From eb4e287fa0442d3f3292f5cd1059275151f5e1b9 Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Mon, 12 Sep 2022 20:35:49 +1000 Subject: [PATCH 30/32] feat(contexts): introducing `timedCancellable` decorator and HOF and factored out common functionality in contexts domain --- src/contexts/decorators/cancellable.ts | 86 +- src/contexts/decorators/timed.ts | 214 +---- src/contexts/decorators/timedCancellable.ts | 56 +- src/contexts/functions/cancellable.ts | 107 ++- src/contexts/functions/timed.ts | 46 +- src/contexts/functions/timedCancellable.ts | 170 +++- src/contexts/utils.ts | 62 +- tests/contexts/decorators/cancellable.test.ts | 2 + tests/contexts/decorators/timed.test.ts | 4 +- .../decorators/timedCancellable.test.ts | 872 ++++++++++++++++++ tests/contexts/functions/timed.test.ts | 4 +- .../functions/timedCancellable.test.ts | 674 ++++++++++++++ 12 files changed, 1983 insertions(+), 314 deletions(-) create mode 100644 tests/contexts/decorators/timedCancellable.test.ts create mode 100644 tests/contexts/functions/timedCancellable.test.ts diff --git a/src/contexts/decorators/cancellable.ts b/src/contexts/decorators/cancellable.ts index ae4301256..c76ce8b20 100644 --- a/src/contexts/decorators/cancellable.ts +++ b/src/contexts/decorators/cancellable.ts @@ -1,5 +1,5 @@ import type { ContextCancellable } from '../types'; -import { PromiseCancellable } from '@matrixai/async-cancellable'; +import { setupCancellable } from '../functions/cancellable'; import * as contextsUtils from '../utils'; function cancellable(lazy: boolean = false) { @@ -20,79 +20,21 @@ function cancellable(lazy: boolean = false) { `\`${targetName}.${key.toString()}\` is not a function`, ); } - const contextIndex = contextsUtils.contexts.get(target[key]); - if (contextIndex == null) { - throw new TypeError( - `\`${targetName}.${key.toString()}\` does not have a \`@context\` parameter decorator`, - ); - } - descriptor['value'] = function (...params) { - let context: Partial = params[contextIndex]; - if (context === undefined) { - context = {}; - params[contextIndex] = context; + const contextIndex = contextsUtils.getContextIndex(target, key, targetName); + descriptor['value'] = function (...args) { + let ctx: Partial = args[contextIndex]; + if (ctx === undefined) { + ctx = {}; + args[contextIndex] = ctx; } // Runtime type check on the context parameter - if (typeof context !== 'object' || context === null) { - throw new TypeError( - `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter is not a context object`, - ); - } - if ( - context.signal !== undefined && - !(context.signal instanceof AbortSignal) - ) { - throw new TypeError( - `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`signal\` property is not an instance of \`AbortSignal\``, - ); - } - // Mutating the `context` parameter - if (context.signal === undefined) { - const abortController = new AbortController(); - context.signal = abortController.signal; - const result = f.apply(this, params); - return new PromiseCancellable((resolve, reject, signal) => { - if (!lazy) { - signal.addEventListener('abort', () => { - reject(signal.reason); - }); - } - void result.then(resolve, reject); - }, abortController); - } else { - // In this case, `context.signal` is set - // and we chain the upsteam signal to the downstream signal - const abortController = new AbortController(); - const signalUpstream = context.signal; - const signalHandler = () => { - abortController.abort(signalUpstream.reason); - }; - if (signalUpstream.aborted) { - abortController.abort(signalUpstream.reason); - } else { - signalUpstream.addEventListener('abort', signalHandler); - } - // Overwrite the signal property with this context's `AbortController.signal` - context.signal = abortController.signal; - const result = f.apply(this, params); - // The `abortController` must be shared in the `finally` clause - // to link up final promise's cancellation with the target - // function's signal - return new PromiseCancellable((resolve, reject, signal) => { - if (!lazy) { - if (signal.aborted) { - reject(signal.reason); - } else { - signal.addEventListener('abort', () => { - reject(signal.reason); - }); - } - } - void result.then(resolve, reject); - }, abortController).finally(() => { - signalUpstream.removeEventListener('abort', signalHandler); - }, abortController); - } + contextsUtils.checkContextCancellable(ctx, key, targetName); + return setupCancellable( + (_, ...args) => f.apply(this, args), + lazy, + ctx, + args, + ); }; // Preserve the name Object.defineProperty(descriptor['value'], 'name', { diff --git a/src/contexts/decorators/timed.ts b/src/contexts/decorators/timed.ts index 9da23cd1b..08345f0a6 100644 --- a/src/contexts/decorators/timed.ts +++ b/src/contexts/decorators/timed.ts @@ -1,142 +1,9 @@ import type { ContextTimed } from '../types'; -import { Timer } from '@matrixai/timer'; +import { setupTimedContext } from '../functions/timed'; import * as contextsUtils from '../utils'; import * as contextsErrors from '../errors'; import * as utils from '../../utils'; -/** - * This sets up the context - * This will mutate the `params` parameter - * It returns a teardown function to be called - * when the target function is finished - */ -function setupContext( - delay: number, - errorTimeoutConstructor: new () => Error, - targetName: string, - key: string | symbol, - contextIndex: number, - params: Array, -): () => void { - let context: Partial = params[contextIndex]; - if (context === undefined) { - context = {}; - params[contextIndex] = context; - } - // Runtime type check on the context parameter - if (typeof context !== 'object' || context === null) { - throw new TypeError( - `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter is not a context object`, - ); - } - if (context.timer !== undefined && !(context.timer instanceof Timer)) { - throw new TypeError( - `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`timer\` property is not an instance of \`Timer\``, - ); - } - if ( - context.signal !== undefined && - !(context.signal instanceof AbortSignal) - ) { - throw new TypeError( - `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`signal\` property is not an instance of \`AbortSignal\``, - ); - } - // There are 3 properties of timer and signal: - // - // A. If timer times out, signal is aborted - // B. If signal is aborted, timer is cancelled - // C. If timer is owned by the wrapper, then it must be cancelled when the target finishes - // - // There are 4 cases where the wrapper is used: - // - // 1. Nothing is inherited - A B C - // 2. Signal is inherited - A B C - // 3. Timer is inherited - A - // 4. Both signal and timer are inherited - A* - // - // Property B and C only applies to case 1 and 2 because the timer is owned - // by the wrapper and it is not inherited, if it is inherited, the caller may - // need to reuse the timer. - // In situation 4, there's a caveat for property A: it is assumed that the - // caller has already setup the property A relationship, therefore this - // wrapper will not re-setup this property A relationship. - if (context.timer === undefined && context.signal === undefined) { - const abortController = new AbortController(); - const e = new errorTimeoutConstructor(); - // Property A - const timer = new Timer(() => void abortController.abort(e), delay); - abortController.signal.addEventListener('abort', () => { - // Property B - timer.cancel(); - }); - context.signal = abortController.signal; - context.timer = timer; - return () => { - // Property C - timer.cancel(); - }; - } else if ( - context.timer === undefined && - context.signal instanceof AbortSignal - ) { - const abortController = new AbortController(); - const e = new errorTimeoutConstructor(); - // Property A - const timer = new Timer(() => void abortController.abort(e), delay); - const signalUpstream = context.signal; - const signalHandler = () => { - // Property B - timer.cancel(); - abortController.abort(signalUpstream.reason); - }; - // If already aborted, abort target and cancel the timer - if (signalUpstream.aborted) { - // Property B - timer.cancel(); - abortController.abort(signalUpstream.reason); - } else { - signalUpstream.addEventListener('abort', signalHandler); - } - // Overwrite the signal property with this context's `AbortController.signal` - context.signal = abortController.signal; - context.timer = timer; - return () => { - signalUpstream.removeEventListener('abort', signalHandler); - // Property C - timer.cancel(); - }; - } else if (context.timer instanceof Timer && context.signal === undefined) { - const abortController = new AbortController(); - const e = new errorTimeoutConstructor(); - let finished = false; - // If the timer resolves, then abort the target function - void context.timer.then( - (r: any, s: AbortSignal) => { - // If the timer is aborted after it resolves - // then don't bother aborting the target function - if (!finished && !s.aborted) { - // Property A - abortController.abort(e); - } - return r; - }, - () => { - // Ignore any upstream cancellation - }, - ); - context.signal = abortController.signal; - return () => { - finished = true; - }; - } else { - // In this case, `context.timer` and `context.signal` are both instances of - // `Timer` and `AbortSignal` respectively - // It is assumed that both the timer and signal are already hooked up to each other - return () => {}; - } -} - /** * Timed method decorator */ @@ -158,71 +25,82 @@ function timed( `\`${targetName}.${key.toString()}\` is not a function`, ); } - const contextIndex = contextsUtils.contexts.get(target[key]); - if (contextIndex == null) { - throw new TypeError( - `\`${targetName}.${key.toString()}\` does not have a \`@context\` parameter decorator`, - ); - } + const contextIndex = contextsUtils.getContextIndex(target, key, targetName); if (f instanceof utils.AsyncFunction) { - descriptor['value'] = async function (...params) { - const teardownContext = setupContext( + descriptor['value'] = async function (...args) { + let ctx: Partial = args[contextIndex]; + if (ctx === undefined) { + ctx = {}; + args[contextIndex] = ctx; + } + // Runtime type check on the context parameter + contextsUtils.checkContextTimed(ctx, key, targetName); + const teardownContext = setupTimedContext( delay, errorTimeoutConstructor, - targetName, - key, - contextIndex, - params, + ctx, ); try { - return await f.apply(this, params); + return await f.apply(this, args); } finally { teardownContext(); } }; } else if (f instanceof utils.GeneratorFunction) { - descriptor['value'] = function* (...params) { - const teardownContext = setupContext( + descriptor['value'] = function* (...args) { + let ctx: Partial = args[contextIndex]; + if (ctx === undefined) { + ctx = {}; + args[contextIndex] = ctx; + } + // Runtime type check on the context parameter + contextsUtils.checkContextTimed(ctx, key, targetName); + const teardownContext = setupTimedContext( delay, errorTimeoutConstructor, - targetName, - key, - contextIndex, - params, + ctx, ); try { - return yield* f.apply(this, params); + return yield* f.apply(this, args); } finally { teardownContext(); } }; } else if (f instanceof utils.AsyncGeneratorFunction) { - descriptor['value'] = async function* (...params) { - const teardownContext = setupContext( + descriptor['value'] = async function* (...args) { + let ctx: Partial = args[contextIndex]; + if (ctx === undefined) { + ctx = {}; + args[contextIndex] = ctx; + } + // Runtime type check on the context parameter + contextsUtils.checkContextTimed(ctx, key, targetName); + const teardownContext = setupTimedContext( delay, errorTimeoutConstructor, - targetName, - key, - contextIndex, - params, + ctx, ); try { - return yield* f.apply(this, params); + return yield* f.apply(this, args); } finally { teardownContext(); } }; } else { - descriptor['value'] = function (...params) { - const teardownContext = setupContext( + descriptor['value'] = function (...args) { + let ctx: Partial = args[contextIndex]; + if (ctx === undefined) { + ctx = {}; + args[contextIndex] = ctx; + } + // Runtime type check on the context parameter + contextsUtils.checkContextTimed(ctx, key, targetName); + const teardownContext = setupTimedContext( delay, errorTimeoutConstructor, - targetName, - key, - contextIndex, - params, + ctx, ); - const result = f.apply(this, params); + const result = f.apply(this, args); if (utils.isPromiseLike(result)) { return result.then( (r) => { diff --git a/src/contexts/decorators/timedCancellable.ts b/src/contexts/decorators/timedCancellable.ts index f86949629..46c7196fa 100644 --- a/src/contexts/decorators/timedCancellable.ts +++ b/src/contexts/decorators/timedCancellable.ts @@ -1,15 +1,55 @@ -// Equivalent to timed(cancellable()) -// timeout is always lazy -// it's only if you call cancel -// PLUS this only works with PromiseLike -// the timed just wraps that together -// and the result is a bit more efficient -// to avoid having to chain the signals up too much +import type { ContextTimed } from '../types'; +import { setupTimedCancellable } from '../functions/timedCancellable'; +import * as contextsUtils from '../utils'; +import * as contextsErrors from '../errors'; function timedCancellable( lazy: boolean = false, delay: number = Infinity, errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedTimeOut, -) {} +) { + return < + T extends TypedPropertyDescriptor< + (...params: Array) => PromiseLike + >, + >( + target: any, + key: string | symbol, + descriptor: T, + ) => { + // Target is instance prototype for instance methods + // or the class prototype for static methods + const targetName: string = target['name'] ?? target.constructor.name; + const f = descriptor['value']; + if (typeof f !== 'function') { + throw new TypeError( + `\`${targetName}.${key.toString()}\` is not a function`, + ); + } + const contextIndex = contextsUtils.getContextIndex(target, key, targetName); + descriptor['value'] = function (...args) { + let ctx: Partial = args[contextIndex]; + if (ctx === undefined) { + ctx = {}; + args[contextIndex] = ctx; + } + // Runtime type check on the context parameter + contextsUtils.checkContextTimed(ctx, key, targetName); + return setupTimedCancellable( + (_, ...args) => f.apply(this, args), + lazy, + delay, + errorTimeoutConstructor, + ctx, + args, + ); + }; + // Preserve the name + Object.defineProperty(descriptor['value'], 'name', { + value: typeof key === 'symbol' ? `[${key.description}]` : key, + }); + return descriptor; + }; +} export default timedCancellable; diff --git a/src/contexts/functions/cancellable.ts b/src/contexts/functions/cancellable.ts index e564d1e1a..77fd8e898 100644 --- a/src/contexts/functions/cancellable.ts +++ b/src/contexts/functions/cancellable.ts @@ -10,6 +10,64 @@ type ContextAndParameters< ? [Partial?, ...P] : [Partial & ContextRemaining, ...P]; +function setupCancellable< + C extends ContextCancellable, + P extends Array, + R, +>( + f: (ctx: C, ...params: P) => PromiseLike, + lazy: boolean, + ctx: Partial, + args: P, +): PromiseCancellable { + if (ctx.signal === undefined) { + const abortController = new AbortController(); + ctx.signal = abortController.signal; + const result = f(ctx as C, ...args); + return new PromiseCancellable((resolve, reject, signal) => { + if (!lazy) { + signal.addEventListener('abort', () => { + reject(signal.reason); + }); + } + void result.then(resolve, reject); + }, abortController); + } else { + // In this case, `context.signal` is set + // and we chain the upsteam signal to the downstream signal + const abortController = new AbortController(); + const signalUpstream = ctx.signal; + const signalHandler = () => { + abortController.abort(signalUpstream.reason); + }; + if (signalUpstream.aborted) { + abortController.abort(signalUpstream.reason); + } else { + signalUpstream.addEventListener('abort', signalHandler); + } + // Overwrite the signal property with this context's `AbortController.signal` + ctx.signal = abortController.signal; + const result = f(ctx as C, ...args); + // The `abortController` must be shared in the `finally` clause + // to link up final promise's cancellation with the target + // function's signal + return new PromiseCancellable((resolve, reject, signal) => { + if (!lazy) { + if (signal.aborted) { + reject(signal.reason); + } else { + signal.addEventListener('abort', () => { + reject(signal.reason); + }); + } + } + void result.then(resolve, reject); + }, abortController).finally(() => { + signalUpstream.removeEventListener('abort', signalHandler); + }, abortController); + } +} + function cancellable, R>( f: (ctx: C, ...params: P) => PromiseLike, lazy: boolean = false, @@ -17,53 +75,10 @@ function cancellable, R>( return (...params) => { const ctx = params[0] ?? {}; const args = params.slice(1) as P; - if (ctx.signal === undefined) { - const abortController = new AbortController(); - ctx.signal = abortController.signal; - const result = f(ctx as C, ...args); - return new PromiseCancellable((resolve, reject, signal) => { - if (!lazy) { - signal.addEventListener('abort', () => { - reject(signal.reason); - }); - } - void result.then(resolve, reject); - }, abortController); - } else { - // In this case, `context.signal` is set - // and we chain the upsteam signal to the downstream signal - const abortController = new AbortController(); - const signalUpstream = ctx.signal; - const signalHandler = () => { - abortController.abort(signalUpstream.reason); - }; - if (signalUpstream.aborted) { - abortController.abort(signalUpstream.reason); - } else { - signalUpstream.addEventListener('abort', signalHandler); - } - // Overwrite the signal property with this context's `AbortController.signal` - ctx.signal = abortController.signal; - const result = f(ctx as C, ...args); - // The `abortController` must be shared in the `finally` clause - // to link up final promise's cancellation with the target - // function's signal - return new PromiseCancellable((resolve, reject, signal) => { - if (!lazy) { - if (signal.aborted) { - reject(signal.reason); - } else { - signal.addEventListener('abort', () => { - reject(signal.reason); - }); - } - } - void result.then(resolve, reject); - }, abortController).finally(() => { - signalUpstream.removeEventListener('abort', signalHandler); - }, abortController); - } + return setupCancellable(f, lazy, ctx, args); }; } export default cancellable; + +export { setupCancellable }; diff --git a/src/contexts/functions/timed.ts b/src/contexts/functions/timed.ts index 0afb9a430..3c4e621c6 100644 --- a/src/contexts/functions/timed.ts +++ b/src/contexts/functions/timed.ts @@ -3,7 +3,16 @@ import { Timer } from '@matrixai/timer'; import * as contextsErrors from '../errors'; import * as utils from '../../utils'; -function setupContext( +type ContextRemaining = Omit; + +type ContextAndParameters< + C, + P extends Array, +> = keyof ContextRemaining extends never + ? [Partial?, ...P] + : [Partial & ContextRemaining, ...P]; + +function setupTimedContext( delay: number, errorTimeoutConstructor: new () => Error, ctx: Partial, @@ -100,15 +109,6 @@ function setupContext( } } -type ContextRemaining = Omit; - -type ContextAndParameters< - C, - P extends Array, -> = keyof ContextRemaining extends never - ? [Partial?, ...P] - : [Partial & ContextRemaining, ...P]; - /** * Timed HOF * This overloaded signature is external signature @@ -127,7 +127,11 @@ function timed>( return async (...params) => { const ctx = params[0] ?? {}; const args = params.slice(1) as P; - const teardownContext = setupContext(delay, errorTimeoutConstructor, ctx); + const teardownContext = setupTimedContext( + delay, + errorTimeoutConstructor, + ctx, + ); try { return await f(ctx as C, ...args); } finally { @@ -138,7 +142,11 @@ function timed>( return function* (...params) { const ctx = params[0] ?? {}; const args = params.slice(1) as P; - const teardownContext = setupContext(delay, errorTimeoutConstructor, ctx); + const teardownContext = setupTimedContext( + delay, + errorTimeoutConstructor, + ctx, + ); try { return yield* f(ctx as C, ...args); } finally { @@ -149,7 +157,11 @@ function timed>( return async function* (...params) { const ctx = params[0] ?? {}; const args = params.slice(1) as P; - const teardownContext = setupContext(delay, errorTimeoutConstructor, ctx); + const teardownContext = setupTimedContext( + delay, + errorTimeoutConstructor, + ctx, + ); try { return yield* f(ctx as C, ...args); } finally { @@ -160,7 +172,11 @@ function timed>( return (...params) => { const ctx = params[0] ?? {}; const args = params.slice(1) as P; - const teardownContext = setupContext(delay, errorTimeoutConstructor, ctx); + const teardownContext = setupTimedContext( + delay, + errorTimeoutConstructor, + ctx, + ); const result = f(ctx as C, ...args); if (utils.isPromiseLike(result)) { return result.then( @@ -198,3 +214,5 @@ function timed>( } export default timed; + +export { setupTimedContext }; diff --git a/src/contexts/functions/timedCancellable.ts b/src/contexts/functions/timedCancellable.ts index 3f8ff65ac..332302358 100644 --- a/src/contexts/functions/timedCancellable.ts +++ b/src/contexts/functions/timedCancellable.ts @@ -1,3 +1,171 @@ -function timedCancellable() {} +import type { ContextTimed } from '../types'; +import { PromiseCancellable } from '@matrixai/async-cancellable'; +import { Timer } from '@matrixai/timer'; +import * as contextsErrors from '../errors'; + +type ContextRemaining = Omit; + +type ContextAndParameters< + C, + P extends Array, +> = keyof ContextRemaining extends never + ? [Partial?, ...P] + : [Partial & ContextRemaining, ...P]; + +function setupTimedCancellable, R>( + f: (ctx: C, ...params: P) => PromiseLike, + lazy: boolean, + delay: number, + errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedTimeOut, + ctx: Partial, + args: P, +): PromiseCancellable { + // There are 3 properties of timer and signal: + // + // A. If timer times out, signal is aborted + // B. If signal is aborted, timer is cancelled + // C. If timer is owned by the wrapper, then it must be cancelled when the target finishes + // + // There are 4 cases where the wrapper is used: + // + // 1. Nothing is inherited - A B C + // 2. Signal is inherited - A B C + // 3. Timer is inherited - A + // 4. Both signal and timer are inherited - A* + // + // Property B and C only applies to case 1 and 2 because the timer is owned + // by the wrapper and it is not inherited, if it is inherited, the caller may + // need to reuse the timer. + // In situation 4, there's a caveat for property A: it is assumed that the + // caller has already setup the property A relationship, therefore this + // wrapper will not re-setup this property A relationship. + let abortController: AbortController; + let teardownContext: () => void; + if (ctx.timer === undefined && ctx.signal === undefined) { + abortController = new AbortController(); + const e = new errorTimeoutConstructor(); + // Property A + const timer = new Timer(() => void abortController.abort(e), delay); + abortController.signal.addEventListener('abort', () => { + // Property B + timer.cancel(); + }); + ctx.signal = abortController.signal; + ctx.timer = timer; + teardownContext = () => { + // Property C + timer.cancel(); + }; + } else if (ctx.timer === undefined && ctx.signal instanceof AbortSignal) { + abortController = new AbortController(); + const e = new errorTimeoutConstructor(); + // Property A + const timer = new Timer(() => void abortController.abort(e), delay); + const signalUpstream = ctx.signal; + const signalHandler = () => { + // Property B + timer.cancel(); + abortController.abort(signalUpstream.reason); + }; + // If already aborted, abort target and cancel the timer + if (signalUpstream.aborted) { + // Property B + timer.cancel(); + abortController.abort(signalUpstream.reason); + } else { + signalUpstream.addEventListener('abort', signalHandler); + } + // Overwrite the signal property with this ctx's `AbortController.signal` + ctx.signal = abortController.signal; + ctx.timer = timer; + teardownContext = () => { + signalUpstream.removeEventListener('abort', signalHandler); + // Property C + timer.cancel(); + }; + } else if (ctx.timer instanceof Timer && ctx.signal === undefined) { + abortController = new AbortController(); + const e = new errorTimeoutConstructor(); + let finished = false; + // If the timer resolves, then abort the target function + void ctx.timer.then( + (r: any, s: AbortSignal) => { + // If the timer is aborted after it resolves + // then don't bother aborting the target function + if (!finished && !s.aborted) { + // Property A + abortController.abort(e); + } + return r; + }, + () => { + // Ignore any upstream cancellation + }, + ); + ctx.signal = abortController.signal; + teardownContext = () => { + finished = true; + }; + } else { + // In this case, `context.timer` and `context.signal` are both instances of + // `Timer` and `AbortSignal` respectively + // It is assumed that both the timer and signal are already hooked up to each other + abortController = new AbortController(); + const signalUpstream = ctx.signal!; + const signalHandler = () => { + abortController.abort(signalUpstream.reason); + }; + if (signalUpstream.aborted) { + abortController.abort(signalUpstream.reason); + } else { + signalUpstream.addEventListener('abort', signalHandler); + } + // Overwrite the signal property with this context's `AbortController.signal` + ctx.signal = abortController.signal; + teardownContext = () => { + signalUpstream.removeEventListener('abort', signalHandler); + }; + } + const result = f(ctx as C, ...args); + // The `abortController` must be shared in the `finally` clause + // to link up final promise's cancellation with the target + // function's signal + return new PromiseCancellable((resolve, reject, signal) => { + if (!lazy) { + if (signal.aborted) { + reject(signal.reason); + } else { + signal.addEventListener('abort', () => { + reject(signal.reason); + }); + } + } + void result.then(resolve, reject); + }, abortController).finally(() => { + teardownContext(); + }, abortController); +} + +function timedCancellable, R>( + f: (ctx: C, ...params: P) => PromiseLike, + lazy: boolean = false, + delay: number = Infinity, + errorTimeoutConstructor: new () => Error = contextsErrors.ErrorContextsTimedTimeOut, +): (...params: ContextAndParameters) => PromiseCancellable { + return (...params) => { + const ctx = params[0] ?? {}; + const args = params.slice(1) as P; + return setupTimedCancellable( + f, + lazy, + delay, + errorTimeoutConstructor, + ctx, + args, + ); + }; +} export default timedCancellable; + +export { setupTimedCancellable }; diff --git a/src/contexts/utils.ts b/src/contexts/utils.ts index d4f675f9c..6a9ba00c1 100644 --- a/src/contexts/utils.ts +++ b/src/contexts/utils.ts @@ -1,3 +1,63 @@ +import { Timer } from '@matrixai/timer'; + const contexts = new WeakMap(); -export { contexts }; +function getContextIndex( + target: any, + key: string | symbol, + targetName: string, +): number { + const contextIndex = contexts.get(target[key]); + if (contextIndex == null) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` does not have a \`@context\` parameter decorator`, + ); + } + return contextIndex; +} + +function checkContextCancellable( + ctx: any, + key: string | symbol, + targetName: string, +): void { + if (typeof ctx !== 'object' || ctx === null) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter is not a context object`, + ); + } + if (ctx.signal !== undefined && !(ctx.signal instanceof AbortSignal)) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`signal\` property is not an instance of \`AbortSignal\``, + ); + } +} + +function checkContextTimed( + ctx: any, + key: string | symbol, + targetName: string, +): void { + if (typeof ctx !== 'object' || ctx === null) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter is not a context object`, + ); + } + if (ctx.signal !== undefined && !(ctx.signal instanceof AbortSignal)) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`signal\` property is not an instance of \`AbortSignal\``, + ); + } + if (ctx.timer !== undefined && !(ctx.timer instanceof Timer)) { + throw new TypeError( + `\`${targetName}.${key.toString()}\` decorated \`@context\` parameter's \`timer\` property is not an instance of \`Timer\``, + ); + } +} + +export { + contexts, + getContextIndex, + checkContextCancellable, + checkContextTimed, +}; diff --git a/tests/contexts/decorators/cancellable.test.ts b/tests/contexts/decorators/cancellable.test.ts index d9969fb25..f1b08298f 100644 --- a/tests/contexts/decorators/cancellable.test.ts +++ b/tests/contexts/decorators/cancellable.test.ts @@ -75,6 +75,7 @@ describe('context/decorators/cancellable', () => { test('asyncFunction', async () => { const pC = x.asyncFunction(); expect(pC).toBeInstanceOf(PromiseCancellable); + await pC; await x.asyncFunction({}); await x.asyncFunction({ signal: new AbortController().signal }); expect(x.asyncFunction).toBeInstanceOf(Function); @@ -84,6 +85,7 @@ describe('context/decorators/cancellable', () => { test('symbolFunction', async () => { const pC = x[symbolFunction](); expect(pC).toBeInstanceOf(PromiseCancellable); + await pC; await x[symbolFunction]({}); await x[symbolFunction]({ signal: new AbortController().signal }); expect(x[symbolFunction]).toBeInstanceOf(Function); diff --git a/tests/contexts/decorators/timed.test.ts b/tests/contexts/decorators/timed.test.ts index 48b7dd56a..b5d0ce0b7 100644 --- a/tests/contexts/decorators/timed.test.ts +++ b/tests/contexts/decorators/timed.test.ts @@ -739,9 +739,9 @@ describe('context/decorators/timed', () => { const abortController = new AbortController(); const timer = new Timer({ handler: () => { - abortController.abort(new contextsErrors.ErrorContextsTimedTimeOut); + abortController.abort(new contextsErrors.ErrorContextsTimedTimeOut()); }, - delay: 100 + delay: 100, }); abortController.signal.addEventListener('abort', () => { timer.cancel(); diff --git a/tests/contexts/decorators/timedCancellable.test.ts b/tests/contexts/decorators/timedCancellable.test.ts new file mode 100644 index 000000000..d32dfdcbe --- /dev/null +++ b/tests/contexts/decorators/timedCancellable.test.ts @@ -0,0 +1,872 @@ +import type { ContextTimed } from '@/contexts/types'; +import { Timer } from '@matrixai/timer'; +import { PromiseCancellable } from '@matrixai/async-cancellable'; +import context from '@/contexts/decorators/context'; +import timedCancellable from '@/contexts/decorators/timedCancellable'; +import * as contextsErrors from '@/contexts/errors'; +import { AsyncFunction, sleep, promise } from '@/utils'; + +describe('context/decorators/timedCancellable', () => { + describe('timedCancellable decorator runtime validation', () => { + test('timedCancellable decorator requires context decorator', async () => { + expect(() => { + class C { + @timedCancellable() + async f(_ctx: ContextTimed): Promise { + return 'hello world'; + } + } + return C; + }).toThrow(TypeError); + }); + test('cancellable decorator fails on invalid context', async () => { + await expect(async () => { + class C { + @timedCancellable() + async f(@context _ctx: ContextTimed): Promise { + return 'hello world'; + } + } + const c = new C(); + // @ts-ignore invalid context signal + await c.f({ signal: 'lol' }); + }).rejects.toThrow(TypeError); + }); + }); + describe('timedCancellable decorator syntax', () => { + // Decorators cannot change type signatures + // use overloading to change required context parameter to optional context parameter + const symbolFunction = Symbol('sym'); + class X { + functionPromise( + ctx?: Partial, + check?: (t: Timer) => any, + ): PromiseCancellable; + @timedCancellable(false, 1000) + functionPromise( + @context ctx: ContextTimed, + check?: (t: Timer) => any, + ): Promise { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(ctx.timer).toBeInstanceOf(Timer); + if (check != null) check(ctx.timer); + return new Promise((resolve) => void resolve()); + } + + asyncFunction( + ctx?: Partial, + check?: (t: Timer) => any, + ): PromiseCancellable; + @timedCancellable(true, Infinity) + async asyncFunction( + @context ctx: ContextTimed, + check?: (t: Timer) => any, + ): Promise { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(ctx.timer).toBeInstanceOf(Timer); + if (check != null) check(ctx.timer); + } + + [symbolFunction]( + ctx?: Partial, + check?: (t: Timer) => any, + ): PromiseCancellable; + @timedCancellable() + [symbolFunction]( + @context ctx: ContextTimed, + check?: (t: Timer) => any, + ): Promise { + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(ctx.timer).toBeInstanceOf(Timer); + if (check != null) check(ctx.timer); + return new Promise((resolve) => void resolve()); + } + } + const x = new X(); + test('functionPromise', async () => { + const pC = x.functionPromise(); + expect(pC).toBeInstanceOf(PromiseCancellable); + await pC; + await x.functionPromise({}); + await x.functionPromise({ timer: new Timer({ delay: 100 }) }, (t) => { + expect(t.delay).toBe(100); + }); + expect(x.functionPromise).toBeInstanceOf(Function); + expect(x.functionPromise.name).toBe('functionPromise'); + }); + test('asyncFunction', async () => { + const pC = x.asyncFunction(); + expect(pC).toBeInstanceOf(PromiseCancellable); + await pC; + await x.asyncFunction({}); + await x.asyncFunction({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + }); + expect(x.functionPromise).toBeInstanceOf(Function); + // Returning `PromiseCancellable` means it cannot be an async function + expect(x.asyncFunction).not.toBeInstanceOf(AsyncFunction); + expect(x.asyncFunction.name).toBe('asyncFunction'); + }); + test('symbolFunction', async () => { + const pC = x[symbolFunction](); + expect(pC).toBeInstanceOf(PromiseCancellable); + await pC; + await x[symbolFunction]({}); + await x[symbolFunction]({ timer: new Timer({ delay: 250 }) }, (t) => { + expect(t.delay).toBe(250); + }); + expect(x[symbolFunction]).toBeInstanceOf(Function); + expect(x[symbolFunction].name).toBe('[sym]'); + }); + }); + describe('timedCancellable decorator expiry', () => { + test('async function expiry - eager', async () => { + const { p: finishedP, resolveP: resolveFinishedP } = promise(); + class C { + /** + * Async function + */ + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(false, 50) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf( + contextsErrors.ErrorContextsTimedTimeOut, + ); + resolveFinishedP(); + return 'hello world'; + } + } + const c = new C(); + await expect(c.f()).rejects.toThrow( + contextsErrors.ErrorContextsTimedTimeOut, + ); + // Eager rejection allows the promise finish its side effects + await expect(finishedP).resolves.toBeUndefined(); + }); + test('async function expiry - lazy', async () => { + class C { + /** + * Async function + */ + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true, 50) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf( + contextsErrors.ErrorContextsTimedTimeOut, + ); + return 'hello world'; + } + } + const c = new C(); + await expect(c.f()).resolves.toBe('hello world'); + }); + test('async function expiry with custom error - eager', async () => { + class ErrorCustom extends Error {} + class C { + /** + * Async function + */ + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(false, 50, ErrorCustom) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf(ErrorCustom); + throw ctx.signal.reason; + } + } + const c = new C(); + await expect(c.f()).rejects.toBeInstanceOf(ErrorCustom); + }); + test('async function expiry with custom error - lazy', async () => { + class ErrorCustom extends Error {} + class C { + /** + * Async function + */ + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true, 50, ErrorCustom) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf(ErrorCustom); + throw ctx.signal.reason; + } + } + const c = new C(); + await expect(c.f()).rejects.toBeInstanceOf(ErrorCustom); + }); + test('promise function expiry - lazy', async () => { + class C { + /** + * Regular function returning promise + */ + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true, 50) + f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + return sleep(15) + .then(() => { + expect(ctx.signal.aborted).toBe(false); + }) + .then(() => sleep(40)) + .then(() => { + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf( + contextsErrors.ErrorContextsTimedTimeOut, + ); + }) + .then(() => { + return 'hello world'; + }); + } + } + const c = new C(); + await expect(c.f()).resolves.toBe('hello world'); + }); + test('promise function expiry and late rejection - lazy', async () => { + let timeout: ReturnType | undefined; + class C { + /** + * Regular function that actually rejects + * when the signal is aborted + */ + f(ctx?: Partial): Promise; + @timedCancellable(true, 50) + f(@context ctx: ContextTimed): Promise { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + } + timeout = setTimeout(() => { + resolve('hello world'); + }, 50000); + ctx.signal.onabort = () => { + clearTimeout(timeout); + timeout = undefined; + reject(ctx.signal.reason); + }; + }); + } + } + const c = new C(); + await expect(c.f()).rejects.toBeInstanceOf( + contextsErrors.ErrorContextsTimedTimeOut, + ); + expect(timeout).toBeUndefined(); + }); + test('promise function expiry and early rejection - lazy', async () => { + let timeout: ReturnType | undefined; + class C { + /** + * Regular function that actually rejects immediately + */ + f(ctx?: Partial): Promise; + @timedCancellable(true, 0) + f(@context ctx: ContextTimed): Promise { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + } + timeout = setTimeout(() => { + resolve('hello world'); + }, 50000); + ctx.signal.onabort = () => { + clearTimeout(timeout); + timeout = undefined; + reject(ctx.signal.reason); + }; + }); + } + } + const c = new C(); + await expect(c.f()).rejects.toBeInstanceOf( + contextsErrors.ErrorContextsTimedTimeOut, + ); + expect(timeout).toBeUndefined(); + }); + }); + describe('timedCancellable decorator cancellation', () => { + test('async function cancel - eager', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable() + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel(); + await expect(pC).rejects.toBeUndefined(); + }); + test('async function cancel - lazy', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel(); + await expect(pC).resolves.toBe('hello world'); + }); + test('async function cancel with custom error and eager rejection', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable() + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('async function cancel with custom error and lazy rejection', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw ctx.signal.reason; + } + await sleep(1); + } + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('promise timedCancellable function - eager rejection', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable() + f(@context ctx: ContextTimed): PromiseCancellable { + const pC = new PromiseCancellable( + (resolve, reject, signal) => { + if (signal.aborted) { + reject('eager 2:' + signal.reason); + } else { + signal.onabort = () => { + reject('lazy 2:' + signal.reason); + }; + } + void sleep(10).then(() => { + resolve('hello world'); + }); + }, + ); + if (ctx.signal.aborted) { + pC.cancel('eager 1:' + ctx.signal.reason); + } else { + ctx.signal.onabort = () => { + pC.cancel('lazy 1:' + ctx.signal.reason); + }; + } + return pC; + } + } + const c = new C(); + // Signal is aborted afterwards + const pC1 = c.f(); + pC1.cancel('cancel reason'); + await expect(pC1).rejects.toBe('cancel reason'); + // Signal is already aborted + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC2 = c.f({ signal: abortController.signal }); + await expect(pC2).rejects.toBe('cancel reason'); + }); + test('promise timedCancellable function - lazy rejection', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true) + f(@context ctx: ContextTimed): PromiseCancellable { + const pC = new PromiseCancellable( + (resolve, reject, signal) => { + if (signal.aborted) { + reject('eager 2:' + signal.reason); + } else { + signal.onabort = () => { + reject('lazy 2:' + signal.reason); + }; + } + void sleep(10).then(() => { + resolve('hello world'); + }); + }, + ); + if (ctx.signal.aborted) { + pC.cancel('eager 1:' + ctx.signal.reason); + } else { + ctx.signal.onabort = () => { + pC.cancel('lazy 1:' + ctx.signal.reason); + }; + } + return pC; + } + } + const c = new C(); + // Signal is aborted afterwards + const pC1 = c.f(); + pC1.cancel('cancel reason'); + await expect(pC1).rejects.toBe('lazy 2:lazy 1:cancel reason'); + // Signal is already aborted + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC2 = c.f({ signal: abortController.signal }); + await expect(pC2).rejects.toBe('lazy 2:eager 1:cancel reason'); + }); + }); + describe('timedCancellable decorator propagation', () => { + test('propagate timer and signal', async () => { + let timer: Timer; + let signal: AbortSignal; + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true, 50) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await this.g(ctx); + } + + g(ctx?: Partial): PromiseCancellable; + @timedCancellable(true, 25) + async g(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + // Timer will be propagated + expect(timer).toBe(ctx.timer); + // Signal will be chained + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(50); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + } + } + const c = new C(); + await expect(c.f()).resolves.toBe('g'); + }); + test('propagate timer only', async () => { + let timer: Timer; + let signal: AbortSignal; + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true, 50) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await this.g({ timer: ctx.timer }); + } + + g(ctx?: Partial): PromiseCancellable; + @timedCancellable(true, 25) + async g(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(timer).toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(50); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + } + } + const c = new C(); + await expect(c.f()).resolves.toBe('g'); + }); + test('propagate signal only', async () => { + let timer: Timer; + let signal: AbortSignal; + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true, 50) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + if (!signal.aborted) { + expect(timer.getTimeout()).toBeGreaterThan(0); + } else { + expect(timer.getTimeout()).toBe(0); + } + return await this.g({ signal: ctx.signal }); + } + + g(ctx?: Partial): PromiseCancellable; + @timedCancellable(true, 25) + g(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + // Even though signal is propagated + // because the timer isn't, the signal here is chained + expect(timer).not.toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + if (!signal.aborted) { + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(25); + } else { + expect(timer.getTimeout()).toBe(0); + } + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject('early:' + ctx.signal.reason); + } else { + const timeout = setTimeout(() => { + resolve('g'); + }, 10); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject('during:' + ctx.signal.reason); + }); + } + }); + } + } + const c = new C(); + const pC1 = c.f(); + await expect(pC1).resolves.toBe('g'); + expect(signal!.aborted).toBe(false); + const pC2 = c.f(); + pC2.cancel('cancel reason'); + await expect(pC2).rejects.toBe('during:cancel reason'); + expect(signal!.aborted).toBe(true); + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC3 = c.f({ signal: abortController.signal }); + await expect(pC3).rejects.toBe('early:cancel reason'); + expect(signal!.aborted).toBe(true); + }); + test('propagate nothing', async () => { + let timer: Timer; + let signal: AbortSignal; + class C { + f(ctx?: Partial): Promise; + @timedCancellable(true, 50) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await this.g(); + } + + g(ctx?: Partial): Promise; + @timedCancellable(true, 25) + async g(@context ctx: ContextTimed): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(timer).not.toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(25); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + } + } + const c = new C(); + await expect(c.f()).resolves.toBe('g'); + }); + test('propagated expiry', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true, 25) + async f(@context ctx: ContextTimed): Promise { + // The `g` will use up all the remaining time + const counter = await this.g(ctx.timer.getTimeout()); + expect(counter).toBeGreaterThan(0); + // The `h` will reject eventually + // it may reject immediately + // it may reject after some time + await this.h(ctx); + return 'hello world'; + } + + async g(timeout: number): Promise { + const start = performance.now(); + let counter = 0; + while (true) { + if (performance.now() - start > timeout) { + break; + } + await sleep(1); + counter++; + } + return counter; + } + + h(ctx?: Partial): PromiseCancellable; + @timedCancellable(true, 25) + async h(@context ctx: ContextTimed): Promise { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + return; + } + const timeout = setTimeout(() => { + resolve('hello world'); + }, 25); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject(ctx.signal.reason); + }); + }); + } + } + const c = new C(); + await expect(c.f()).rejects.toThrow( + contextsErrors.ErrorContextsTimedTimeOut, + ); + }); + test('nested cancellable - lazy then lazy', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true) + @timedCancellable(true) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('throw:cancel reason'); + }); + test('nested cancellable - lazy then eager', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(true) + @timedCancellable(false) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('nested cancellable - eager then lazy', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable(false) + @timedCancellable(true) + async f(@context ctx: ContextTimed): Promise { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + } + } + const c = new C(); + const pC = c.f(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('signal event listeners are removed', async () => { + class C { + f(ctx?: Partial): PromiseCancellable; + @timedCancellable() + async f(@context _ctx: ContextTimed): Promise { + return 'hello world'; + } + } + const abortController = new AbortController(); + let listenerCount = 0; + const signal = new Proxy(abortController.signal, { + get(target, prop, receiver) { + if (prop === 'addEventListener') { + return function addEventListener(...args) { + listenerCount++; + return target[prop].apply(this, args); + }; + } else if (prop === 'removeEventListener') { + return function addEventListener(...args) { + listenerCount--; + return target[prop].apply(this, args); + }; + } else { + return Reflect.get(target, prop, receiver); + } + }, + }); + const c = new C(); + await c.f({ signal }); + await c.f({ signal }); + const pC = c.f({ signal }); + pC.cancel(); + await expect(pC).rejects.toBe(undefined); + expect(listenerCount).toBe(0); + }); + }); + describe('timedCancellable decorator explicit timer cancellation or signal abortion', () => { + // If the timer is cancelled + // there will be no timeout error + let ctx_: ContextTimed | undefined; + class C { + f(ctx?: Partial): Promise; + @timedCancellable(true, 50) + f(@context ctx: ContextTimed): Promise { + ctx_ = ctx; + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason + ' begin'); + return; + } + const timeout = setTimeout(() => { + resolve('hello world'); + }, 25); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject(ctx.signal.reason + ' during'); + }); + }); + } + } + const c = new C(); + beforeEach(() => { + ctx_ = undefined; + }); + test('explicit timer cancellation - begin', async () => { + const timer = new Timer({ delay: 100 }); + timer.cancel('reason'); + const p = c.f({ timer }); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit timer cancellation - during', async () => { + const timer = new Timer({ delay: 100 }); + const p = c.f({ timer }); + timer.cancel('reason'); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit timer cancellation - during after sleep', async () => { + const timer = new Timer({ delay: 20 }); + const p = c.f({ timer }); + await sleep(1); + timer.cancel('reason'); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit signal abortion - begin', async () => { + const abortController = new AbortController(); + abortController.abort('reason'); + const p = c.f({ signal: abortController.signal }); + expect(ctx_!.timer.status).toBe('settled'); + await expect(p).rejects.toBe('reason begin'); + }); + test('explicit signal abortion - during', async () => { + const abortController = new AbortController(); + const p = c.f({ signal: abortController.signal }); + abortController.abort('reason'); + // Timer is also cancelled immediately + expect(ctx_!.timer.status).toBe('settled'); + await expect(p).rejects.toBe('reason during'); + }); + test('explicit signal signal abortion with passed in timer - during', async () => { + // By passing in the timer and signal explicitly + // it is expected that the timer and signal handling is already setup + const abortController = new AbortController(); + const timer = new Timer({ + handler: () => { + abortController.abort(new contextsErrors.ErrorContextsTimedTimeOut()); + }, + delay: 100, + }); + abortController.signal.addEventListener('abort', () => { + timer.cancel(); + }); + const p = c.f({ timer, signal: abortController.signal }); + abortController.abort('abort reason'); + expect(ctx_!.timer.status).toBe('settled'); + expect(timer.status).toBe('settled'); + expect(ctx_!.signal.aborted).toBe(true); + await expect(p).rejects.toBe('abort reason during'); + }); + test('explicit timer cancellation and signal abortion - begin', async () => { + const timer = new Timer({ delay: 100 }); + timer.cancel('timer reason'); + const abortController = new AbortController(); + abortController.abort('abort reason'); + const p = c.f({ timer, signal: abortController.signal }); + expect(ctx_!.timer.status).toBe('settled'); + expect(ctx_!.signal.aborted).toBe(true); + await expect(p).rejects.toBe('abort reason begin'); + }); + }); +}); diff --git a/tests/contexts/functions/timed.test.ts b/tests/contexts/functions/timed.test.ts index 36a8808ea..2cacc61bb 100644 --- a/tests/contexts/functions/timed.test.ts +++ b/tests/contexts/functions/timed.test.ts @@ -547,9 +547,9 @@ describe('context/functions/timed', () => { const abortController = new AbortController(); const timer = new Timer({ handler: () => { - abortController.abort(new contextsErrors.ErrorContextsTimedTimeOut); + abortController.abort(new contextsErrors.ErrorContextsTimedTimeOut()); }, - delay: 100 + delay: 100, }); abortController.signal.addEventListener('abort', () => { timer.cancel(); diff --git a/tests/contexts/functions/timedCancellable.test.ts b/tests/contexts/functions/timedCancellable.test.ts new file mode 100644 index 000000000..579a0195e --- /dev/null +++ b/tests/contexts/functions/timedCancellable.test.ts @@ -0,0 +1,674 @@ +import type { ContextTimed } from '@/contexts/types'; +import { Timer } from '@matrixai/timer'; +import { PromiseCancellable } from '@matrixai/async-cancellable'; +import timedCancellable from '@/contexts/functions/timedCancellable'; +import * as contextsErrors from '@/contexts/errors'; +import { AsyncFunction, sleep, promise } from '@/utils'; + +describe('context/functions/timedCancellable', () => { + describe('timedCancellable syntax', () => { + test('function promise', async () => { + const f = function ( + ctx: ContextTimed, + check?: (t: Timer) => any, + ): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + if (check != null) check(ctx.timer); + return new Promise((resolve) => void resolve()); + }; + const fTimedCancellable = timedCancellable(f, true); + const pC = fTimedCancellable(undefined); + expect(pC).toBeInstanceOf(PromiseCancellable); + await pC; + expect(await fTimedCancellable({})).toBeUndefined(); + expect( + await fTimedCancellable({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + }), + ).toBeUndefined(); + expect(fTimedCancellable).toBeInstanceOf(Function); + }); + test('async function', async () => { + const f = async function ( + ctx: ContextTimed, + check?: (t: Timer) => any, + ): Promise { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + if (check != null) check(ctx.timer); + return; + }; + const fTimedCancellable = timedCancellable(f, true); + const pC = fTimedCancellable(undefined); + expect(pC).toBeInstanceOf(PromiseCancellable); + await pC; + await fTimedCancellable({}); + await fTimedCancellable({ timer: new Timer({ delay: 50 }) }, (t) => { + expect(t.delay).toBe(50); + }); + expect(fTimedCancellable).not.toBeInstanceOf(AsyncFunction); + }); + }); + describe('timedCancellable expiry', () => { + test('async function expiry - eager', async () => { + const { p: finishedP, resolveP: resolveFinishedP } = promise(); + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf( + contextsErrors.ErrorContextsTimedTimeOut, + ); + resolveFinishedP(); + return 'hello world'; + }; + const fTimedCancellable = timedCancellable(f, false, 50); + await expect(fTimedCancellable()).rejects.toThrow( + contextsErrors.ErrorContextsTimedTimeOut, + ); + // Eager rejection allows the promise finish its side effects + await expect(finishedP).resolves.toBeUndefined(); + }); + test('async function expiry - lazy', async () => { + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf( + contextsErrors.ErrorContextsTimedTimeOut, + ); + return 'hello world'; + }; + const fTimedCancellable = timedCancellable(f, true, 50); + await expect(fTimedCancellable()).resolves.toBe('hello world'); + }); + test('async function expiry with custom error - eager', async () => { + class ErrorCustom extends Error {} + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf(ErrorCustom); + throw ctx.signal.reason; + }; + const fTimedCancellable = timedCancellable(f, false, 50, ErrorCustom); + await expect(fTimedCancellable()).rejects.toBeInstanceOf(ErrorCustom); + }); + test('async function expiry with custom error - lazy', async () => { + class ErrorCustom extends Error {} + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + await sleep(15); + expect(ctx.signal.aborted).toBe(false); + await sleep(40); + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf(ErrorCustom); + throw ctx.signal.reason; + }; + const fTimedCancellable = timedCancellable(f, true, 50, ErrorCustom); + await expect(fTimedCancellable()).rejects.toBeInstanceOf(ErrorCustom); + }); + test('promise function expiry - lazy', async () => { + const f = (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + return sleep(15) + .then(() => { + expect(ctx.signal.aborted).toBe(false); + }) + .then(() => sleep(40)) + .then(() => { + expect(ctx.signal.aborted).toBe(true); + expect(ctx.signal.reason).toBeInstanceOf( + contextsErrors.ErrorContextsTimedTimeOut, + ); + }) + .then(() => { + return 'hello world'; + }); + }; + const fTimedCancellable = timedCancellable(f, true, 50); + await expect(fTimedCancellable()).resolves.toBe('hello world'); + }); + test('promise function expiry and late rejection - lazy', async () => { + let timeout: ReturnType | undefined; + const f = (ctx: ContextTimed): Promise => { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + } + timeout = setTimeout(() => { + resolve('hello world'); + }, 50000); + ctx.signal.onabort = () => { + clearTimeout(timeout); + timeout = undefined; + reject(ctx.signal.reason); + }; + }); + }; + const fTimedCancellable = timedCancellable(f, true, 50); + await expect(fTimedCancellable()).rejects.toBeInstanceOf( + contextsErrors.ErrorContextsTimedTimeOut, + ); + expect(timeout).toBeUndefined(); + }); + test('promise function expiry and early rejection - lazy', async () => { + let timeout: ReturnType | undefined; + const f = (ctx: ContextTimed): Promise => { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + } + timeout = setTimeout(() => { + resolve('hello world'); + }, 50000); + ctx.signal.onabort = () => { + clearTimeout(timeout); + timeout = undefined; + reject(ctx.signal.reason); + }; + }); + }; + const fTimedCancellable = timedCancellable(f, true, 0); + await expect(fTimedCancellable()).rejects.toBeInstanceOf( + contextsErrors.ErrorContextsTimedTimeOut, + ); + expect(timeout).toBeUndefined(); + }); + }); + describe('timedCancellable cancellation', () => { + test('async function cancel - eager', async () => { + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + }; + const fTimedCancellable = timedCancellable(f); + const pC = fTimedCancellable(); + await sleep(1); + pC.cancel(); + await expect(pC).rejects.toBeUndefined(); + }); + test('async function cancel - lazy', async () => { + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + }; + const fTimedCancellable = timedCancellable(f, true); + const pC = fTimedCancellable(); + await sleep(1); + pC.cancel(); + await expect(pC).resolves.toBe('hello world'); + }); + test('async function cancel with custom error and eager rejection', async () => { + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) break; + await sleep(1); + } + return 'hello world'; + }; + const fTimedCancellable = timedCancellable(f); + const pC = fTimedCancellable(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('async function cancel with custom error and lazy rejection', async () => { + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw ctx.signal.reason; + } + await sleep(1); + } + }; + const fTimedCancellable = timedCancellable(f, true); + const pC = fTimedCancellable(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('promise timedCancellable function - eager rejection', async () => { + const f = (ctx: ContextTimed): PromiseCancellable => { + const pC = new PromiseCancellable((resolve, reject, signal) => { + if (signal.aborted) { + reject('eager 2:' + signal.reason); + } else { + signal.onabort = () => { + reject('lazy 2:' + signal.reason); + }; + } + void sleep(10).then(() => { + resolve('hello world'); + }); + }); + if (ctx.signal.aborted) { + pC.cancel('eager 1:' + ctx.signal.reason); + } else { + ctx.signal.onabort = () => { + pC.cancel('lazy 1:' + ctx.signal.reason); + }; + } + return pC; + }; + const fTimedCancellable = timedCancellable(f); + // Signal is aborted afterwards + const pC1 = fTimedCancellable(); + pC1.cancel('cancel reason'); + await expect(pC1).rejects.toBe('cancel reason'); + // Signal is already aborted + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC2 = fTimedCancellable({ signal: abortController.signal }); + await expect(pC2).rejects.toBe('cancel reason'); + }); + test('promise timedCancellable function - lazy rejection', async () => { + const f = (ctx: ContextTimed): PromiseCancellable => { + const pC = new PromiseCancellable((resolve, reject, signal) => { + if (signal.aborted) { + reject('eager 2:' + signal.reason); + } else { + signal.onabort = () => { + reject('lazy 2:' + signal.reason); + }; + } + void sleep(10).then(() => { + resolve('hello world'); + }); + }); + if (ctx.signal.aborted) { + pC.cancel('eager 1:' + ctx.signal.reason); + } else { + ctx.signal.onabort = () => { + pC.cancel('lazy 1:' + ctx.signal.reason); + }; + } + return pC; + }; + const fTimedCancellable = timedCancellable(f, true); + // Signal is aborted afterwards + const pC1 = fTimedCancellable(); + pC1.cancel('cancel reason'); + await expect(pC1).rejects.toBe('lazy 2:lazy 1:cancel reason'); + // Signal is already aborted + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC2 = fTimedCancellable({ signal: abortController.signal }); + await expect(pC2).rejects.toBe('lazy 2:eager 1:cancel reason'); + }); + }); + describe('timedCancellable propagation', () => { + test('propagate timer and signal', async () => { + let timer: Timer; + let signal: AbortSignal; + const g = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + // Timer will be propagated + expect(timer).toBe(ctx.timer); + // Signal will be chained + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(50); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + }; + const gTimedCancellable = timedCancellable(g, true, 25); + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await gTimedCancellable(ctx); + }; + const fTimedCancellable = timedCancellable(f, true, 50); + await expect(fTimedCancellable()).resolves.toBe('g'); + }); + test('propagate timer only', async () => { + let timer: Timer; + let signal: AbortSignal; + const g = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(timer).toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(50); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + }; + const gTimedCancellable = timedCancellable(g, true, 25); + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await gTimedCancellable({ timer: ctx.timer }); + }; + const fTimedCancellable = timedCancellable(f, true, 50); + await expect(fTimedCancellable()).resolves.toBe('g'); + }); + test('propagate signal only', async () => { + let timer: Timer; + let signal: AbortSignal; + const g = (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + // Even though signal is propagated + // because the timer isn't, the signal here is chained + expect(timer).not.toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + if (!signal.aborted) { + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(25); + } else { + expect(timer.getTimeout()).toBe(0); + } + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject('early:' + ctx.signal.reason); + } else { + const timeout = setTimeout(() => { + resolve('g'); + }, 10); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject('during:' + ctx.signal.reason); + }); + } + }); + }; + const gTimedCancellable = timedCancellable(g, true, 25); + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + if (!signal.aborted) { + expect(timer.getTimeout()).toBeGreaterThan(0); + } else { + expect(timer.getTimeout()).toBe(0); + } + return await gTimedCancellable({ signal: ctx.signal }); + }; + const fTimedCancellable = timedCancellable(f, true, 50); + const pC1 = fTimedCancellable(); + await expect(pC1).resolves.toBe('g'); + expect(signal!.aborted).toBe(false); + const pC2 = fTimedCancellable(); + pC2.cancel('cancel reason'); + await expect(pC2).rejects.toBe('during:cancel reason'); + expect(signal!.aborted).toBe(true); + const abortController = new AbortController(); + abortController.abort('cancel reason'); + const pC3 = fTimedCancellable({ signal: abortController.signal }); + await expect(pC3).rejects.toBe('early:cancel reason'); + expect(signal!.aborted).toBe(true); + }); + test('propagate nothing', async () => { + let timer: Timer; + let signal: AbortSignal; + const g = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + expect(timer).not.toBe(ctx.timer); + expect(signal).not.toBe(ctx.signal); + expect(ctx.timer.getTimeout()).toBeGreaterThan(0); + expect(ctx.timer.delay).toBe(25); + expect(ctx.signal.aborted).toBe(false); + return 'g'; + }; + const gTimedCancellable = timedCancellable(g, true, 25); + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.timer).toBeInstanceOf(Timer); + expect(ctx.signal).toBeInstanceOf(AbortSignal); + timer = ctx.timer; + signal = ctx.signal; + expect(timer.getTimeout()).toBeGreaterThan(0); + expect(signal.aborted).toBe(false); + return await gTimedCancellable(); + }; + const fTimedCancellable = timedCancellable(f, true, 50); + await expect(fTimedCancellable()).resolves.toBe('g'); + }); + test('propagated expiry', async () => { + const g = async (timeout: number): Promise => { + const start = performance.now(); + let counter = 0; + while (true) { + if (performance.now() - start > timeout) { + break; + } + await sleep(1); + counter++; + } + return counter; + }; + const h = async (ctx: ContextTimed): Promise => { + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason); + return; + } + const timeout = setTimeout(() => { + resolve('hello world'); + }, 25); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject(ctx.signal.reason); + }); + }); + }; + const hTimedCancellable = timedCancellable(h, true, 25); + const f = async (ctx: ContextTimed): Promise => { + // The `g` will use up all the remaining time + const counter = await g(ctx.timer.getTimeout()); + expect(counter).toBeGreaterThan(0); + // The `h` will reject eventually + // it may reject immediately + // it may reject after some time + await hTimedCancellable(ctx); + return 'hello world'; + }; + const fTimedCancellable = timedCancellable(f, true, 25); + await expect(fTimedCancellable()).rejects.toThrow( + contextsErrors.ErrorContextsTimedTimeOut, + ); + }); + test('nested cancellable - lazy then lazy', async () => { + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + }; + const fTimedCancellable = timedCancellable( + timedCancellable(f, true), + true, + ); + const pC = fTimedCancellable(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('throw:cancel reason'); + }); + test('nested cancellable - lazy then eager', async () => { + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + }; + const fCancellable = timedCancellable(timedCancellable(f, true), false); + const pC = fCancellable(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('nested cancellable - eager then lazy', async () => { + const f = async (ctx: ContextTimed): Promise => { + expect(ctx.signal.aborted).toBe(false); + while (true) { + if (ctx.signal.aborted) { + throw 'throw:' + ctx.signal.reason; + } + await sleep(1); + } + }; + const fCancellable = timedCancellable(timedCancellable(f, false), true); + const pC = fCancellable(); + await sleep(1); + pC.cancel('cancel reason'); + await expect(pC).rejects.toBe('cancel reason'); + }); + test('signal event listeners are removed', async () => { + const f = async (_ctx: ContextTimed): Promise => { + return 'hello world'; + }; + const abortController = new AbortController(); + let listenerCount = 0; + const signal = new Proxy(abortController.signal, { + get(target, prop, receiver) { + if (prop === 'addEventListener') { + return function addEventListener(...args) { + listenerCount++; + return target[prop].apply(this, args); + }; + } else if (prop === 'removeEventListener') { + return function addEventListener(...args) { + listenerCount--; + return target[prop].apply(this, args); + }; + } else { + return Reflect.get(target, prop, receiver); + } + }, + }); + const fTimedCancellable = timedCancellable(f); + await fTimedCancellable({ signal }); + await fTimedCancellable({ signal }); + const pC = fTimedCancellable({ signal }); + pC.cancel(); + await expect(pC).rejects.toBe(undefined); + expect(listenerCount).toBe(0); + }); + }); + describe('timedCancellable explicit timer cancellation or signal abortion', () => { + // If the timer is cancelled + // there will be no timeout error + let ctx_: ContextTimed | undefined; + const f = (ctx: ContextTimed): Promise => { + ctx_ = ctx; + return new Promise((resolve, reject) => { + if (ctx.signal.aborted) { + reject(ctx.signal.reason + ' begin'); + return; + } + const timeout = setTimeout(() => { + resolve('hello world'); + }, 25); + ctx.signal.addEventListener('abort', () => { + clearTimeout(timeout); + reject(ctx.signal.reason + ' during'); + }); + }); + }; + const fTimedCancellable = timedCancellable(f, true, 50); + beforeEach(() => { + ctx_ = undefined; + }); + test('explicit timer cancellation - begin', async () => { + const timer = new Timer({ delay: 100 }); + timer.cancel('reason'); + const p = fTimedCancellable({ timer }); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit timer cancellation - during', async () => { + const timer = new Timer({ delay: 100 }); + const p = fTimedCancellable({ timer }); + timer.cancel('reason'); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit timer cancellation - during after sleep', async () => { + const timer = new Timer({ delay: 20 }); + const p = fTimedCancellable({ timer }); + await sleep(1); + timer.cancel('reason'); + await expect(p).resolves.toBe('hello world'); + expect(ctx_!.signal.aborted).toBe(false); + }); + test('explicit signal abortion - begin', async () => { + const abortController = new AbortController(); + abortController.abort('reason'); + const p = fTimedCancellable({ signal: abortController.signal }); + expect(ctx_!.timer.status).toBe('settled'); + await expect(p).rejects.toBe('reason begin'); + }); + test('explicit signal abortion - during', async () => { + const abortController = new AbortController(); + const p = fTimedCancellable({ signal: abortController.signal }); + abortController.abort('reason'); + // Timer is also cancelled immediately + expect(ctx_!.timer.status).toBe('settled'); + await expect(p).rejects.toBe('reason during'); + }); + test('explicit signal signal abortion with passed in timer - during', async () => { + // By passing in the timer and signal explicitly + // it is expected that the timer and signal handling is already setup + const abortController = new AbortController(); + const timer = new Timer({ + handler: () => { + abortController.abort(new contextsErrors.ErrorContextsTimedTimeOut()); + }, + delay: 100, + }); + abortController.signal.addEventListener('abort', () => { + timer.cancel(); + }); + const p = fTimedCancellable({ timer, signal: abortController.signal }); + abortController.abort('abort reason'); + expect(ctx_!.timer.status).toBe('settled'); + expect(timer.status).toBe('settled'); + expect(ctx_!.signal.aborted).toBe(true); + await expect(p).rejects.toBe('abort reason during'); + }); + test('explicit timer cancellation and signal abortion - begin', async () => { + const timer = new Timer({ delay: 100 }); + timer.cancel('timer reason'); + const abortController = new AbortController(); + abortController.abort('abort reason'); + const p = fTimedCancellable({ timer, signal: abortController.signal }); + expect(ctx_!.timer.status).toBe('settled'); + expect(ctx_!.signal.aborted).toBe(true); + await expect(p).rejects.toBe('abort reason begin'); + }); + }); +}); From cc9920efb487e0656f2642f93937554da189c002 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Tue, 13 Sep 2022 10:48:38 +1000 Subject: [PATCH 31/32] tests: cleaing up `TaskManager.test.ts` --- tests/tasks/TaskManager.test.ts | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/tests/tasks/TaskManager.test.ts b/tests/tasks/TaskManager.test.ts index 2a836b8fc..57d50ce34 100644 --- a/tests/tasks/TaskManager.test.ts +++ b/tests/tasks/TaskManager.test.ts @@ -1,17 +1,17 @@ -import type { ContextTimed } from '../../dist/contexts/types'; -import type { Task, TaskHandlerId, TaskPath } from '../../src/tasks/types'; import type { PromiseCancellable } from '@matrixai/async-cancellable'; +import type { ContextTimed } from '@/contexts/types'; +import type { Task, TaskHandlerId, TaskPath } from '@/tasks/types'; import fs from 'fs'; import path from 'path'; import os from 'os'; import { DB } from '@matrixai/db'; import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; -import * as fc from 'fast-check'; import { Lock } from '@matrixai/async-locks'; -import * as utils from '@/utils/index'; -import { promise, sleep, never } from '@/utils'; +import * as fc from 'fast-check'; import TaskManager from '@/tasks/TaskManager'; import * as tasksErrors from '@/tasks/errors'; +import * as utils from '@/utils'; +import { promise, sleep, never } from '@/utils'; describe(TaskManager.name, () => { const logger = new Logger(`${TaskManager.name} test`, LogLevel.WARN, [ @@ -22,7 +22,6 @@ describe(TaskManager.name, () => { let db: DB; beforeEach(async () => { - logger.info('SETTING UP'); dataDir = await fs.promises.mkdtemp( path.join(os.tmpdir(), 'polykey-test-'), ); @@ -31,13 +30,10 @@ describe(TaskManager.name, () => { dbPath, logger, }); - logger.info('SET UP'); }); afterEach(async () => { - logger.info('CLEANING UP'); await db.stop(); await fs.promises.rm(dataDir, { recursive: true, force: true }); - logger.info('CLEANED UP'); }); test('can start and stop', async () => { @@ -107,11 +103,9 @@ describe(TaskManager.name, () => { }); await sleep(500); - logger.info('STOPPING'); await taskManager.stop(); expect(handler).toHaveBeenCalledTimes(4); - logger.info('CREATING'); handler.mockClear(); taskManager = await TaskManager.createTaskManager({ db, @@ -121,7 +115,6 @@ describe(TaskManager.name, () => { taskManager.registerHandler(handlerId, handler); await taskManager.startProcessing(); await sleep(4000); - logger.info('STOPPING AGAIN'); await taskManager.stop(); expect(handler).toHaveBeenCalledTimes(3); }); @@ -182,14 +175,11 @@ describe(TaskManager.name, () => { }); await sleep(500); - logger.info('STOPPING'); await taskManager.stop(); expect(handler).toHaveBeenCalledTimes(4); handler.mockClear(); - logger.info('STARTING'); await taskManager.start(); await sleep(4000); - logger.info('STOPPING AGAIN'); await taskManager.stop(); expect(handler).toHaveBeenCalledTimes(3); }); @@ -275,7 +265,6 @@ describe(TaskManager.name, () => { .integer({ min: 10, max: 100 }) .noShrink() .map((value) => async (_context) => { - logger.info(`sleeping ${value}`); await sleep(value); }); @@ -320,7 +309,6 @@ describe(TaskManager.name, () => { // Check for active tasks while tasks are still running while (!completed) { expect(taskManager.activeCount).toBeLessThanOrEqual(activeLimit); - logger.info(`Active tasks: ${taskManager.activeCount}`); await Promise.race([sleep(100), waitForcompletionProm]); } @@ -1033,7 +1021,6 @@ describe(TaskManager.name, () => { // @ts-ignore: private method, only schedule tasks await taskManager.startScheduling(); - logger.info('Scheduling task'); const task1 = await taskManager.scheduleTask({ handlerId, delay: 0, @@ -1043,7 +1030,6 @@ describe(TaskManager.name, () => { await sleep(100); - logger.info('Updating task'); await expect( taskManager.updateTask(task1.id, { delay: 1000, From 5647b39c43d09f3d43892528a8abcc2dacdffde9 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Tue, 13 Sep 2022 10:48:59 +1000 Subject: [PATCH 32/32] fix: fixing type bug in `Discovery` --- src/discovery/Discovery.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/discovery/Discovery.ts b/src/discovery/Discovery.ts index 37bc416f6..834b6c733 100644 --- a/src/discovery/Discovery.ts +++ b/src/discovery/Discovery.ts @@ -489,7 +489,7 @@ class Discovery { // Get our own auth identity id const authIdentityIds = await provider.getAuthIdentityIds(); // If we don't have one then we can't request data so just skip - if (authIdentityIds === [] || authIdentityIds[0] == null) { + if (authIdentityIds.length === 0 || authIdentityIds[0] == null) { return undefined; } const authIdentityId = authIdentityIds[0];