From ca93503778e71cfd0e4d24c5a7e38177e1dff194 Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko <60982217+rostislavdeepcrawl@users.noreply.github.com> Date: Tue, 15 Dec 2020 18:28:46 +0300 Subject: [PATCH] [ODIN-436] iterator/iterable (#60) --- .eslintignore | 20 +- .gitignore | 20 +- .npmignore | 12 +- .prettierignore | 14 +- docs/api/node-duckdb.connection.close.md | 2 +- docs/api/node-duckdb.connection.execute.md | 6 +- docs/api/node-duckdb.connection.md | 10 +- docs/api/node-duckdb.md | 1 - ...duckdb.resultiterator._symbol.iterator_.md | 15 + docs/api/node-duckdb.resultiterator.md | 18 +- docs/api/node-duckdb.resultiterator.next.md | 15 + docs/api/node-duckdb.resultstream.md | 19 -- src/addon/connection.ts | 59 ++-- src/addon/duckdb.ts | 264 +++++++++--------- src/addon/index.ts | 7 +- src/addon/result-iterator.ts | 121 ++++---- src/addon/result-stream.ts | 48 +--- src/index.ts | 2 +- src/tests/result-iterator-protocol.test.ts | 45 +++ src/tests/result-stream.test.ts | 25 +- 20 files changed, 371 insertions(+), 352 deletions(-) create mode 100644 docs/api/node-duckdb.resultiterator._symbol.iterator_.md create mode 100644 docs/api/node-duckdb.resultiterator.next.md delete mode 100644 docs/api/node-duckdb.resultstream.md create mode 100644 src/tests/result-iterator-protocol.test.ts diff --git a/.eslintignore b/.eslintignore index 80e9d67..dedf96f 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,10 +1,10 @@ -coverage/ -dist -node_modules/ -.eslintrc.js -jest.config.js -tsconfig.json -duckdb -addon -nodemon.json -examples +/coverage/ +/dist +/node_modules/ +/.eslintrc.js +/jest.config.js +/tsconfig.json +/duckdb +/addon +/nodemon.json +/examples diff --git a/.gitignore b/.gitignore index 1f6dd6d..1f83713 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ -node_modules -build -dist -coverage -duckdb -duckdb.tar.gz -prebuilds -temp -etc -yarn-error.log +/node_modules +/build +/dist +/coverage +/duckdb +/duckdb.tar.gz +/prebuilds +/temp +/etc +/yarn-error.log diff --git a/.npmignore b/.npmignore index 590f89c..56f91df 100644 --- a/.npmignore +++ b/.npmignore @@ -1,6 +1,6 @@ -node_modules -coverage -build -duckdb -prebuilds -examples +/node_modules +/coverage +/build +/duckdb +/prebuilds +/examples diff --git a/.prettierignore b/.prettierignore index f174405..a6e7c04 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,7 +1,7 @@ -coverage/ -node_modules/ -duckdb -addon -dist -tsconfig.json -examples +/coverage/ +/node_modules/ +/duckdb +/addon +/dist +/tsconfig.json +/examples diff --git a/docs/api/node-duckdb.connection.close.md b/docs/api/node-duckdb.connection.close.md index 48c6f31..c660723 100644 --- a/docs/api/node-duckdb.connection.close.md +++ b/docs/api/node-duckdb.connection.close.md @@ -4,7 +4,7 @@ ## Connection.close() method -Close the connection (also closes all [ResultStream](./node-duckdb.resultstream.md) or [ResultIterator](./node-duckdb.resultiterator.md) objects associated with this connection). +Close the connection (also closes all [Readable](https://nodejs.org/api/stream.html#stream_class_stream_readable) or [ResultIterator](./node-duckdb.resultiterator.md) objects associated with this connection). Signature: diff --git a/docs/api/node-duckdb.connection.execute.md b/docs/api/node-duckdb.connection.execute.md index 0548277..6737cc6 100644 --- a/docs/api/node-duckdb.connection.execute.md +++ b/docs/api/node-duckdb.connection.execute.md @@ -4,12 +4,12 @@ ## Connection.execute() method -Asynchronously executes the query and returns a node.js stream that wraps the result set. +Asynchronously executes the query and returns a [Readable stream](https://nodejs.org/api/stream.html#stream_class_stream_readable) that wraps the result set. Signature: ```typescript -execute(command: string, options?: IExecuteOptions): Promise>; +execute(command: string, options?: IExecuteOptions): Promise; ``` ## Parameters @@ -21,7 +21,7 @@ execute(command: string, options?: IExecuteOptions): Promise> Returns: -Promise<[ResultStream](./node-duckdb.resultstream.md)<T>> +Promise<Readable> ## Example diff --git a/docs/api/node-duckdb.connection.md b/docs/api/node-duckdb.connection.md index 8875da1..4bc9dfc 100644 --- a/docs/api/node-duckdb.connection.md +++ b/docs/api/node-duckdb.connection.md @@ -30,8 +30,8 @@ A single db instance can have multiple connections. Having more than one connect ## Methods -| Method | Modifiers | Description | -| -------------------------------------------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [close()](./node-duckdb.connection.close.md) | | Close the connection (also closes all [ResultStream](./node-duckdb.resultstream.md) or [ResultIterator](./node-duckdb.resultiterator.md) objects associated with this connection). | -| [execute(command, options)](./node-duckdb.connection.execute.md) | | Asynchronously executes the query and returns a node.js stream that wraps the result set. | -| [executeIterator(command, options)](./node-duckdb.connection.executeiterator.md) | | Asynchronously executes the query and returns an iterator that points to the first result in the result set. | +| Method | Modifiers | Description | +| -------------------------------------------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [close()](./node-duckdb.connection.close.md) | | Close the connection (also closes all [Readable](https://nodejs.org/api/stream.html#stream_class_stream_readable) or [ResultIterator](./node-duckdb.resultiterator.md) objects associated with this connection). | +| [execute(command, options)](./node-duckdb.connection.execute.md) | | Asynchronously executes the query and returns a [Readable stream](https://nodejs.org/api/stream.html#stream_class_stream_readable) that wraps the result set. | +| [executeIterator(command, options)](./node-duckdb.connection.executeiterator.md) | | Asynchronously executes the query and returns an iterator that points to the first result in the result set. | diff --git a/docs/api/node-duckdb.md b/docs/api/node-duckdb.md index 73d44fc..f058086 100644 --- a/docs/api/node-duckdb.md +++ b/docs/api/node-duckdb.md @@ -52,7 +52,6 @@ For more examples see [here](https://github.com/deepcrawl/node-duckdb/tree/featu | [Connection](./node-duckdb.connection.md) | Represents a DuckDB connection. | | [DuckDB](./node-duckdb.duckdb.md) | The DuckDB class represents a DuckDB database instance. | | [ResultIterator](./node-duckdb.resultiterator.md) | ResultIterator represents the result set of a DuckDB query. Instances of this class are returned by the [Connection.executeIterator](./node-duckdb.connection.executeiterator.md). | -| [ResultStream](./node-duckdb.resultstream.md) | This is a Readable stream that wrapps the ResultIterator. Instances of this class are returned by [Connection.execute](./node-duckdb.connection.execute.md). | ## Enumerations diff --git a/docs/api/node-duckdb.resultiterator._symbol.iterator_.md b/docs/api/node-duckdb.resultiterator._symbol.iterator_.md new file mode 100644 index 0000000..12d8cbb --- /dev/null +++ b/docs/api/node-duckdb.resultiterator._symbol.iterator_.md @@ -0,0 +1,15 @@ + + +[Node-DuckDB API](./node-duckdb.md) > [ResultIterator](./node-duckdb.resultiterator.md) > [\[Symbol.iterator\]](./node-duckdb.resultiterator._symbol.iterator_.md) + +## ResultIterator.\[Symbol.iterator\]() method + +Signature: + +```typescript +[Symbol.iterator](): this; +``` + +Returns: + +this diff --git a/docs/api/node-duckdb.resultiterator.md b/docs/api/node-duckdb.resultiterator.md index 1511883..e3c4af0 100644 --- a/docs/api/node-duckdb.resultiterator.md +++ b/docs/api/node-duckdb.resultiterator.md @@ -9,9 +9,11 @@ ResultIterator represents the result set of a DuckDB query. Instances of this cl Signature: ```typescript -export declare class ResultIterator +export declare class ResultIterator implements IterableIterator ``` +Implements: IterableIterator<T> + ## Remarks The constructor for this class is marked as internal. Third-party code should not call the constructor directly or create subclasses that extend the `ResultIterator` class. @@ -25,9 +27,11 @@ The constructor for this class is marked as internal. Third-party code should no ## Methods -| Method | Modifiers | Description | -| -------------------------------------------------------------- | --------- | ------------------------------- | -| [close()](./node-duckdb.resultiterator.close.md) | | Close the ResultIterator | -| [describe()](./node-duckdb.resultiterator.describe.md) | | Describe the result set schema. | -| [fetchAllRows()](./node-duckdb.resultiterator.fetchallrows.md) | | Fetch all rows | -| [fetchRow()](./node-duckdb.resultiterator.fetchrow.md) | | Fetch the next row | +| Method | Modifiers | Description | +| -------------------------------------------------------------------------- | --------- | ------------------------------- | +| [\[Symbol.iterator\]()](./node-duckdb.resultiterator._symbol.iterator_.md) | | | +| [close()](./node-duckdb.resultiterator.close.md) | | Close the ResultIterator | +| [describe()](./node-duckdb.resultiterator.describe.md) | | Describe the result set schema. | +| [fetchAllRows()](./node-duckdb.resultiterator.fetchallrows.md) | | Fetch all rows | +| [fetchRow()](./node-duckdb.resultiterator.fetchrow.md) | | Fetch the next row | +| [next()](./node-duckdb.resultiterator.next.md) | | | diff --git a/docs/api/node-duckdb.resultiterator.next.md b/docs/api/node-duckdb.resultiterator.next.md new file mode 100644 index 0000000..61a6891 --- /dev/null +++ b/docs/api/node-duckdb.resultiterator.next.md @@ -0,0 +1,15 @@ + + +[Node-DuckDB API](./node-duckdb.md) > [ResultIterator](./node-duckdb.resultiterator.md) > [next](./node-duckdb.resultiterator.next.md) + +## ResultIterator.next() method + +Signature: + +```typescript +next(): IteratorResult; +``` + +Returns: + +IteratorResult<T> diff --git a/docs/api/node-duckdb.resultstream.md b/docs/api/node-duckdb.resultstream.md deleted file mode 100644 index d206280..0000000 --- a/docs/api/node-duckdb.resultstream.md +++ /dev/null @@ -1,19 +0,0 @@ - - -[Node-DuckDB API](./node-duckdb.md) > [ResultStream](./node-duckdb.resultstream.md) - -## ResultStream class - -This is a Readable stream that wrapps the ResultIterator. Instances of this class are returned by [Connection.execute](./node-duckdb.connection.execute.md). - -Signature: - -```typescript -export declare class ResultStream extends Readable -``` - -Extends: Readable - -## Remarks - -The constructor for this class is marked as internal. Third-party code should not call the constructor directly or create subclasses that extend the `ResultStream` class. diff --git a/src/addon/connection.ts b/src/addon/connection.ts index cdff6ce..8cbbd14 100644 --- a/src/addon/connection.ts +++ b/src/addon/connection.ts @@ -1,40 +1,43 @@ -import { ConnectionBinding } from "../addon-bindings"; -import { ResultStream } from "./result-stream"; -import {DuckDB} from "./duckdb"; -import { ResultIterator } from "./result-iterator"; +import { Readable } from "stream"; + +import { ConnectionBinding } from "@addon-bindings"; import { IExecuteOptions } from "@addon-types"; +import { DuckDB } from "./duckdb"; +import { ResultIterator } from "./result-iterator"; +import { getResultStream } from "./result-stream"; + /** * Represents a DuckDB connection. - * + * * @remarks * A single db instance can have multiple connections. Having more than one connection instance is required when executing concurrent queries. - * + * * @public */ export class Connection { /** - * Connection constructor. - * @param duckdb - {@link DuckDB | DuckDB} instance to connect to. - * - * - * @example - * Initializing a connection: - * ```ts - * import { DuckDB } from "node-duckdb"; - * const db = new DuckDB(); - * const connection = new Connection(db); - * ``` - * - * @public - */ + * Connection constructor. + * @param duckdb - {@link DuckDB | DuckDB} instance to connect to. + * + * + * @example + * Initializing a connection: + * ```ts + * import { DuckDB } from "node-duckdb"; + * const db = new DuckDB(); + * const connection = new Connection(db); + * ``` + * + * @public + */ constructor(private duckdb: DuckDB) {} private connectionBinding = new ConnectionBinding(this.duckdb.db); /** - * Asynchronously executes the query and returns a node.js stream that wraps the result set. + * Asynchronously executes the query and returns a {@link https://nodejs.org/api/stream.html#stream_class_stream_readable | Readable stream} that wraps the result set. * @param command - SQL command to execute * @param options - optional options object of type {@link IExecuteOptions | IExecuteOptions} - * + * * @example * Streaming results of a DuckDB query into a CSV file: * ```ts @@ -64,15 +67,15 @@ export class Connection { * outputToFileAsCsv(); * ``` */ - public async execute(command: string, options?: IExecuteOptions): Promise> { + public async execute(command: string, options?: IExecuteOptions): Promise { const resultIteratorBinding = await this.connectionBinding.execute(command, options); - return new ResultStream(new ResultIterator(resultIteratorBinding)); + return getResultStream(new ResultIterator(resultIteratorBinding)); } /** * Asynchronously executes the query and returns an iterator that points to the first result in the result set. * @param command - SQL command to execute * @param options - optional options object of type {@link IExecuteOptions | IExecuteOptions} - * + * * @example * Printing rows: * ```ts @@ -83,7 +86,7 @@ export class Connection { * await connection.executeIterator("CREATE TABLE people(id INTEGER, name VARCHAR);"); * await connection.executeIterator("INSERT INTO people VALUES (1, 'Mark'), (2, 'Hannes'), (3, 'Bob');"); * const result = await connection.executeIterator("SELECT * FROM people;"); - * // print the first row + * // print the first row * console.log(result.fetchRow()); * // print the rest of the rows * console.log(result.fetchAllRows()); @@ -94,7 +97,7 @@ export class Connection { * } * queryDatabaseWithIterator(); * ``` - * + * * @example * Providing generics type: * ```ts @@ -108,7 +111,7 @@ export class Connection { return new ResultIterator(await this.connectionBinding.execute(command, options)); } /** - * Close the connection (also closes all {@link ResultStream | ResultStream} or {@link ResultIterator | ResultIterator} objects associated with this connection). + * Close the connection (also closes all {@link https://nodejs.org/api/stream.html#stream_class_stream_readable | Readable} or {@link ResultIterator | ResultIterator} objects associated with this connection). * @remarks * Even though GC will automatically destroy the Connection object at some point, DuckDB data is stored in the native address space, not the V8 heap, meaning you can easily have a Node.js process taking gigabytes of memory (more than the default heap size for Node.js) with V8 not triggering GC. So, definitely think about manually calling `close()`. */ diff --git a/src/addon/duckdb.ts b/src/addon/duckdb.ts index 6f099b2..82fbd39 100644 --- a/src/addon/duckdb.ts +++ b/src/addon/duckdb.ts @@ -1,141 +1,141 @@ -import { IDuckDBConfig } from "@addon-types"; -import { DuckDBBinding, DuckDBClass } from "../addon-bindings"; import { promises as fs } from "fs"; import { join } from "path"; -import { AccessMode, OrderType, OrderByNullType } from "@addon-types"; + +import { DuckDBBinding, DuckDBClass } from "@addon-bindings"; +import { IDuckDBConfig, AccessMode, OrderType, OrderByNullType } from "@addon-types"; /** * The DuckDB class represents a DuckDB database instance. * @public */ export class DuckDB { - /** - * Returns the current version of the node-duckdb package (from package.json) - * - * @remarks - * Useful for logging/debugging - * - * @public - */ - public static async getBindingsVersion(): Promise { - return JSON.parse(await fs.readFile(join(__dirname, "../../package.json"), {encoding: "utf-8"})).version; - } - private duckdb: DuckDBClass; - /** - * Represents a native instance of DuckDB. - * @param config - optional configuration object of type {@link IDuckDBConfig | IDuckDBConfig}. - * - * - * @example - * Initializing a duckdb database in memory: - * ```ts - * import { DuckDB } from "node-duckdb"; - * const db = new DuckDB(); - * ``` - * - * @example - * Initializing a duckdb database from file: - * ```ts - * import { DuckDB } from "node-duckdb"; - * const db = new DuckDB({ path: join(__dirname, "./mydb") }); - * ``` - * - * @example - * Initializing a duckdb database from file and setting some additional options: - * ```ts - * import { DuckDB, OrderType } from "node-duckdb"; - * const db = new DuckDB({ path: join(__dirname, "./mydb"), options: { defaultOrderType: OrderType.Descending, temporaryDirectory: false } }); - * ``` - * - * @public - */ - constructor(config: IDuckDBConfig = {}) { - this.duckdb = new DuckDBBinding(config); - } - /** - * Closes the underlying duckdb database, frees associated memory and renders it unusuable. - * @remarks - * Even though GC will automatically destroy the Database object at some point, DuckDB data is stored in the native address space, not the V8 heap, meaning you can easily have a Node.js process taking gigabytes of memory (more than the default heap size for Node.js) with V8 not triggering GC. So, definitely think about manually calling `close()`. - * @public - */ - public close(): void { - return this.duckdb.close(); - } - /** - * Returns underlying binding instance. - * @internal - */ - public get db() { - return this.duckdb; - } - /** - * Returns true if the underlying database has been closed, false otherwise. - * @public - */ - public get isClosed(): boolean { - return this.duckdb.isClosed; - } - /** - * Returns the {@link AccessMode | access mode} used by the database. - * @public - */ - public get accessMode() { - return this.duckdb.accessMode; - } - /** - * Returns the checkpoint write ahead log size used by the database. - * @public - */ - public get checkPointWALSize() { - return this.duckdb.checkPointWALSize; - } - /** - * Returns the maximum memory limit for the database. - * @public - */ - public get maximumMemory() { - return this.duckdb.maximumMemory; - } - /** - * Returns true if the database uses a temporary directory for storing data that does not fit into memory, false otherwise. - * @public - */ - public get useTemporaryDirectory() { - return this.duckdb.useTemporaryDirectory; - } - /** - * Returns the temporary directory location for the database. - * @public - */ - public get temporaryDirectory() { - return this.duckdb.temporaryDirectory; - } - /** - * Returns the collation used by the database. - * @public - */ - public get collation() { - return this.duckdb.collation; - } - /** - * Returns the default {@link OrderType | sort order}. - * @public - */ - public get defaultOrderType() { - return this.duckdb.defaultOrderType; - } - /** - * Returns the default {@link OrderByNullType | sort order for null values}. - * @public - */ - public get defaultNullOrder() { - return this.duckdb.defaultNullOrder; - } - /** - * Returns true of copying is enabled, false otherwise. - * @public - */ - public get enableCopy() { - return this.duckdb.enableCopy; - } + /** + * Returns the current version of the node-duckdb package (from package.json) + * + * @remarks + * Useful for logging/debugging + * + * @public + */ + public static async getBindingsVersion(): Promise { + return JSON.parse(await fs.readFile(join(__dirname, "../../package.json"), { encoding: "utf-8" })).version; + } + private duckdb: DuckDBClass; + /** + * Represents a native instance of DuckDB. + * @param config - optional configuration object of type {@link IDuckDBConfig | IDuckDBConfig}. + * + * + * @example + * Initializing a duckdb database in memory: + * ```ts + * import { DuckDB } from "node-duckdb"; + * const db = new DuckDB(); + * ``` + * + * @example + * Initializing a duckdb database from file: + * ```ts + * import { DuckDB } from "node-duckdb"; + * const db = new DuckDB({ path: join(__dirname, "./mydb") }); + * ``` + * + * @example + * Initializing a duckdb database from file and setting some additional options: + * ```ts + * import { DuckDB, OrderType } from "node-duckdb"; + * const db = new DuckDB({ path: join(__dirname, "./mydb"), options: { defaultOrderType: OrderType.Descending, temporaryDirectory: false } }); + * ``` + * + * @public + */ + constructor(config: IDuckDBConfig = {}) { + this.duckdb = new DuckDBBinding(config); + } + /** + * Closes the underlying duckdb database, frees associated memory and renders it unusuable. + * @remarks + * Even though GC will automatically destroy the Database object at some point, DuckDB data is stored in the native address space, not the V8 heap, meaning you can easily have a Node.js process taking gigabytes of memory (more than the default heap size for Node.js) with V8 not triggering GC. So, definitely think about manually calling `close()`. + * @public + */ + public close(): void { + return this.duckdb.close(); + } + /** + * Returns underlying binding instance. + * @internal + */ + public get db(): DuckDBClass { + return this.duckdb; + } + /** + * Returns true if the underlying database has been closed, false otherwise. + * @public + */ + public get isClosed(): boolean { + return this.duckdb.isClosed; + } + /** + * Returns the {@link AccessMode | access mode} used by the database. + * @public + */ + public get accessMode(): AccessMode { + return this.duckdb.accessMode; + } + /** + * Returns the checkpoint write ahead log size used by the database. + * @public + */ + public get checkPointWALSize(): number { + return this.duckdb.checkPointWALSize; + } + /** + * Returns the maximum memory limit for the database. + * @public + */ + public get maximumMemory(): number { + return this.duckdb.maximumMemory; + } + /** + * Returns true if the database uses a temporary directory for storing data that does not fit into memory, false otherwise. + * @public + */ + public get useTemporaryDirectory(): boolean { + return this.duckdb.useTemporaryDirectory; + } + /** + * Returns the temporary directory location for the database. + * @public + */ + public get temporaryDirectory(): string { + return this.duckdb.temporaryDirectory; + } + /** + * Returns the collation used by the database. + * @public + */ + public get collation(): string { + return this.duckdb.collation; + } + /** + * Returns the default {@link OrderType | sort order}. + * @public + */ + public get defaultOrderType(): OrderType { + return this.duckdb.defaultOrderType; + } + /** + * Returns the default {@link OrderByNullType | sort order for null values}. + * @public + */ + public get defaultNullOrder(): OrderByNullType { + return this.duckdb.defaultNullOrder; + } + /** + * Returns true of copying is enabled, false otherwise. + * @public + */ + public get enableCopy(): boolean { + return this.duckdb.enableCopy; + } } diff --git a/src/addon/index.ts b/src/addon/index.ts index c041760..00f79b5 100644 --- a/src/addon/index.ts +++ b/src/addon/index.ts @@ -1,4 +1,3 @@ -export {DuckDB} from "./duckdb"; -export {ResultIterator} from "./result-iterator"; -export {ResultStream} from "./result-stream"; -export {Connection} from "./connection"; +export { DuckDB } from "./duckdb"; +export { ResultIterator } from "./result-iterator"; +export { Connection } from "./connection"; diff --git a/src/addon/result-iterator.ts b/src/addon/result-iterator.ts index bb018cb..c40424a 100644 --- a/src/addon/result-iterator.ts +++ b/src/addon/result-iterator.ts @@ -1,63 +1,74 @@ +import { ResultIteratorClass } from "@addon-bindings"; import { ResultType } from "@addon-types"; -import { ResultIteratorClass } from "../addon-bindings"; /** * ResultIterator represents the result set of a DuckDB query. Instances of this class are returned by the {@link Connection.executeIterator | Connection.executeIterator}. - * + * * @public */ -export class ResultIterator { - /** - * - * @internal - */ - constructor(private resultInterator: ResultIteratorClass) {} - /** - * Fetch the next row - * - * @remarks - * First call returns the first row, when no more rows left `null` is returned. - */ - public fetchRow(): T { - return this.resultInterator.fetchRow(); - } - /** - * Fetch all rows - * - * @remarks - * Note, this may produce a `heap out of bounds` error in case when there is too much data. Either use the {@link ResultIterator.fetchRow | fetchRow} or the {@link Connection.execute | Connection.execute} method when there is a lot of data. - */ - public fetchAllRows(): T[] { - const allRows: T[] = []; - for (let element = this.fetchRow(); element !== null; element = this.fetchRow()) { - allRows.push(element); - } - return allRows; - } - /** - * Describe the result set schema. - */ - public describe(): string[][] { - return this.resultInterator.describe(); - } - /** - * Close the ResultIterator - * @remarks - * {@link Connection.close | Connection.close} automatically closes all associated ResultIterators. - */ - public close(): void { - return this.resultInterator.close(); - } - /** - * Get the {@link ResultType | ResultType} of the ResultIterator. This is specified by the {@link IExecuteOptions.forceMaterialized | options} argument on {@link Connection.executeIterator | executeIterator}. - */ - public get type(): ResultType { - return this.resultInterator.type; - } - /** - * Returns true if ResultIterator is closed, false otherwise. - */ - public get isClosed(): boolean { - return this.resultInterator.isClosed; +export class ResultIterator implements IterableIterator { + /** + * + * @internal + */ + constructor(private resultInterator: ResultIteratorClass) {} + /** + * Fetch the next row + * + * @remarks + * First call returns the first row, when no more rows left `null` is returned. + */ + public fetchRow(): T { + return this.resultInterator.fetchRow(); + } + /** + * Fetch all rows + * + * @remarks + * Note, this may produce a `heap out of bounds` error in case when there is too much data. Either use the {@link ResultIterator.fetchRow | fetchRow} or the {@link Connection.execute | Connection.execute} method when there is a lot of data. + */ + public fetchAllRows(): T[] { + return [...this]; + } + /** + * Describe the result set schema. + */ + public describe(): string[][] { + return this.resultInterator.describe(); + } + /** + * Close the ResultIterator + * @remarks + * {@link Connection.close | Connection.close} automatically closes all associated ResultIterators. + */ + public close(): void { + return this.resultInterator.close(); + } + /** + * Get the {@link ResultType | ResultType} of the ResultIterator. This is specified by the {@link IExecuteOptions.forceMaterialized | options} argument on {@link Connection.executeIterator | executeIterator}. + */ + public get type(): ResultType { + return this.resultInterator.type; + } + /** + * Returns true if ResultIterator is closed, false otherwise. + */ + public get isClosed(): boolean { + return this.resultInterator.isClosed; + } + + public next(): IteratorResult { + const row = this.fetchRow(); + if (row === null) { + // TS interface is incorrect: when `done` is true, there should be no `value` + return >{ done: true }; } + return { + value: row, + done: false, + }; + } + public [Symbol.iterator](): this { + return this; + } } diff --git a/src/addon/result-stream.ts b/src/addon/result-stream.ts index 41e6994..8225001 100644 --- a/src/addon/result-stream.ts +++ b/src/addon/result-stream.ts @@ -1,47 +1,11 @@ import { Readable } from "stream"; import { ResultIterator } from "./result-iterator"; -/** - * This is a Readable stream that wrapps the ResultIterator. Instances of this class are returned by {@link Connection.execute | Connection.execute}. - * @public - */ -export class ResultStream extends Readable { - /** - * @internal - */ - constructor(private resultIterator: ResultIterator) { - super({ objectMode: true }); - } - /** - * @internal - */ - public _read(): void { - try { - const element = this.resultIterator.fetchRow(); - this.push(element); - if (element === null) { - this.close(); - } - } catch (e) { - this.destroy(e); - } - } - /** - * - * @internal - */ - public _destroy(error: Error | null, callback: (error?: Error | null) => void): void { - this.close(); - callback(error); - } - - private close(): void { - this.resultIterator.close(); - if (!this.resultIterator.isClosed) { - this.emit("error", new Error("Close wasn't successful")); - return; - } - this.emit("close"); - } +export function getResultStream(iterator: ResultIterator): Readable { + return Readable.from(iterator, { + destroy() { + iterator.close(); + }, + }); } diff --git a/src/index.ts b/src/index.ts index 8951d0c..b452ccb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -39,5 +39,5 @@ * ``` * For more examples see {@link https://github.com/deepcrawl/node-duckdb/tree/feature/ODIN-423-welcome-page/examples | here}. */ -export { DuckDB, Connection, ResultIterator, ResultStream } from "./addon"; +export { DuckDB, Connection, ResultIterator } from "./addon"; export * from "./addon-types"; diff --git a/src/tests/result-iterator-protocol.test.ts b/src/tests/result-iterator-protocol.test.ts new file mode 100644 index 0000000..a8d4372 --- /dev/null +++ b/src/tests/result-iterator-protocol.test.ts @@ -0,0 +1,45 @@ +import { Connection, DuckDB } from "@addon"; +import { RowResultFormat } from "@addon-types"; + +describe("Result iterator as well formed JS iterator/iterable", () => { + let db: DuckDB; + let connection: Connection; + beforeEach(() => { + db = new DuckDB(); + connection = new Connection(db); + }); + + afterEach(() => { + connection.close(); + db.close(); + }); + + it("supports for/each", async () => { + const results = []; + const result = await connection.executeIterator( + "SELECT * FROM read_csv_auto('src/tests/test-fixtures/web_page.csv')", + { rowResultFormat: RowResultFormat.Array }, + ); + // eslint-disable-next-line no-loops/no-loops + for (const row of result) { + results.push(row); + } + expect(results[0]).toEqual([ + 1, + "AAAAAAAABAAAAAAA", + 873244800000, + null, + 2450810, + 2452620, + "Y", + 98539, + "http://www.foo.com", + "welcome", + 2531, + 8, + 3, + 4, + ]); + expect(results.length).toBe(60); + }); +}); diff --git a/src/tests/result-stream.test.ts b/src/tests/result-stream.test.ts index 72481c4..4e7b09c 100644 --- a/src/tests/result-stream.test.ts +++ b/src/tests/result-stream.test.ts @@ -1,11 +1,13 @@ -import { Connection, DuckDB, ResultStream } from "@addon"; +import { Readable } from "stream"; + +import { Connection, DuckDB } from "@addon"; import { IExecuteOptions, RowResultFormat } from "@addon-types"; const query = "SELECT * FROM read_csv_auto('src/tests/test-fixtures/web_page.csv')"; const executeOptions: IExecuteOptions = { rowResultFormat: RowResultFormat.Array }; -function readStream(rs: ResultStream): Promise { +function readStream(rs: Readable): Promise { return new Promise((resolve, reject) => { const elements: T[] = []; rs.on("data", (el: any) => elements.push(el)); @@ -71,25 +73,6 @@ describe("Result stream", () => { expect(hasClosedFired).toBe(true); }); - it("closes resource when all data has been read", async () => { - const rs = await connection.execute(query, executeOptions); - let hasClosedFired = false; - rs.on("close", () => (hasClosedFired = true)); - const elements = await readStream(rs); - expect(elements.length).toBe(60); - expect(hasClosedFired).toBe(true); - }); - - it("closes resource on manual destroy", async () => { - const rs1 = await connection.execute(query, executeOptions); - let hasClosedFired = false; - rs1.on("close", () => (hasClosedFired = true)); - void readStream(rs1); - expect(hasClosedFired).toBe(false); - rs1.destroy(); - expect(hasClosedFired).toBe(true); - }); - it("is able to read from two streams on separate connections to one database while interleaving", async () => { const connection1 = new Connection(db); const connection2 = new Connection(db);