diff --git a/clients/typescript/src/client/conversions/converter.ts b/clients/typescript/src/client/conversions/converter.ts index 92add21828..2197210c55 100644 --- a/clients/typescript/src/client/conversions/converter.ts +++ b/clients/typescript/src/client/conversions/converter.ts @@ -1,4 +1,3 @@ -import { Row } from '../../util/types' import { AnyTableSchema } from '../model' import { PgType } from './types' @@ -14,16 +13,19 @@ export interface Converter { * @param row The row to encode * @param tableSchema The schema of the table for this row. */ - encodeRow(row: Row, tableSchema: AnyTableSchema): Record + encodeRow = Record>( + row: Record, + tableSchema: Pick + ): T /** * Encodes the provided rows for storing in the database. * @param rows The rows to encode * @param tableSchema The schema of the table for these rows. */ - encodeRows( - rows: Array, - tableSchema: AnyTableSchema - ): Array> + encodeRows = Record>( + rows: Array>, + tableSchema: Pick + ): Array /** * Decodes the provided value from the database. * @param v The value to decode. @@ -36,8 +38,8 @@ export interface Converter { * @param tableSchema The schema of the table for this row. */ decodeRow = Record>( - row: Row, - tableSchema: AnyTableSchema + row: Record, + tableSchema: Pick ): T /** * Decodes the provided rows from the database. @@ -45,8 +47,8 @@ export interface Converter { * @param tableSchema The schema of the table for these rows. */ decodeRows = Record>( - rows: Array, - tableSchema: AnyTableSchema + rows: Array>, + tableSchema: Pick ): Array } @@ -62,25 +64,32 @@ export function isDataObject(v: unknown): boolean { return v instanceof Date || typeof v === 'bigint' || ArrayBuffer.isView(v) } -export function mapRow = Record>( - row: Row, - tableSchema: AnyTableSchema, +export function mapRow< + T extends Record = Record +>( + row: Record, + tableSchema: Pick, f: (v: any, pgType: PgType) => any ): T { - const decodedRow = {} as T + const mappedRow = {} as T for (const [key, value] of Object.entries(row)) { const pgType = tableSchema.fields[key] - const decodedValue = f(value, pgType) - decodedRow[key as keyof T] = decodedValue + const mappedValue = + pgType === undefined + ? value // it's an unknown column, leave it as is + : f(value, pgType) + mappedRow[key as keyof T] = mappedValue } - return decodedRow + return mappedRow } -export function mapRows = Record>( - rows: Array, - tableSchema: AnyTableSchema, +export function mapRows< + T extends Record = Record +>( + rows: Array>, + tableSchema: Pick, f: (v: any, pgType: PgType) => any ): T[] { return rows.map((row) => mapRow(row, tableSchema, f)) diff --git a/clients/typescript/src/client/conversions/postgres.ts b/clients/typescript/src/client/conversions/postgres.ts index 1798e0f7ae..9d15a89458 100644 --- a/clients/typescript/src/client/conversions/postgres.ts +++ b/clients/typescript/src/client/conversions/postgres.ts @@ -4,7 +4,6 @@ import { deserialiseDate, serialiseDate } from './datatypes/date' import { isJsonNull } from './datatypes/json' import { PgBasicType, PgDateType, PgType } from './types' import { AnyTableSchema } from '../model/schema' -import { Row } from '../../util/types' /** * This module takes care of converting TypeScript values to a Postgres storeable value and back. @@ -99,17 +98,21 @@ export function fromPostgres(v: any, pgType: PgType): any { export const postgresConverter: Converter = { encode: toPostgres, - encodeRow: (row: Row, tableSchema: AnyTableSchema) => - mapRow(row, tableSchema, toPostgres), - encodeRows: (rows: Array, tableSchema: AnyTableSchema) => - mapRows(rows, tableSchema, toPostgres), + encodeRow: = Record>( + row: Record, + tableSchema: AnyTableSchema + ) => mapRow(row, tableSchema, toPostgres), + encodeRows: = Record>( + rows: Array>, + tableSchema: AnyTableSchema + ) => mapRows(rows, tableSchema, toPostgres), decode: fromPostgres, decodeRow: = Record>( - row: Row, + row: Record, tableSchema: AnyTableSchema ) => mapRow(row, tableSchema, fromPostgres), decodeRows: = Record>( - rows: Array, + rows: Array>, tableSchema: AnyTableSchema ) => mapRows(rows, tableSchema, fromPostgres), } diff --git a/clients/typescript/src/client/conversions/sqlite.ts b/clients/typescript/src/client/conversions/sqlite.ts index 62a6d1d074..5f2bbc7608 100644 --- a/clients/typescript/src/client/conversions/sqlite.ts +++ b/clients/typescript/src/client/conversions/sqlite.ts @@ -6,7 +6,6 @@ import { deserialiseDate, serialiseDate } from './datatypes/date' import { deserialiseJSON, serialiseJSON } from './datatypes/json' import { PgBasicType, PgDateType, PgType, isPgDateType } from './types' import { AnyTableSchema } from '../model/schema' -import { Row } from '../../util/types' /** * This module takes care of converting TypeScript values for Postgres-specific types to a SQLite storeable value and back. @@ -95,17 +94,21 @@ export function fromSqlite(v: any, pgType: PgType): any { export const sqliteConverter: Converter = { encode: toSqlite, - encodeRow: (row: Row, tableSchema: AnyTableSchema) => - mapRow(row, tableSchema, toSqlite), - encodeRows: (rows: Array, tableSchema: AnyTableSchema) => - mapRows(rows, tableSchema, toSqlite), + encodeRow: = Record>( + row: Record, + tableSchema: AnyTableSchema + ) => mapRow(row, tableSchema, toSqlite), + encodeRows: = Record>( + rows: Array>, + tableSchema: AnyTableSchema + ) => mapRows(rows, tableSchema, toSqlite), decode: fromSqlite, decodeRow: = Record>( - row: Row, + row: Record, tableSchema: AnyTableSchema ) => mapRow(row, tableSchema, fromSqlite), - decodeRows: = Record>( - rows: Array, + decodeRows: = Record>( + rows: Array>, tableSchema: AnyTableSchema ) => mapRows(rows, tableSchema, fromSqlite), } diff --git a/clients/typescript/src/client/execution/nonTransactionalDB.ts b/clients/typescript/src/client/execution/nonTransactionalDB.ts index 93c7897c35..6a202aa3dc 100644 --- a/clients/typescript/src/client/execution/nonTransactionalDB.ts +++ b/clients/typescript/src/client/execution/nonTransactionalDB.ts @@ -3,7 +3,6 @@ import { QueryBuilder } from 'squel' import { DB } from './db' import * as z from 'zod' import { Row, Statement } from '../../util' -import { Transformation, transformFields } from '../conversions/input' import { Fields } from '../model/schema' import { Converter } from '../conversions/converter' @@ -59,12 +58,9 @@ export class NonTransactionalDB implements DB { // convert SQLite/PG values back to JS values // and then parse the transformed object // with the Zod schema to validate it - const transformedRow = transformFields( - row, - this._fields, - this._converter, - Transformation.Decode - ) + const transformedRow = this._converter.decodeRow(row, { + fields: this._fields, + }) return schema.parse(transformedRow) }) successCallback(this, objects) diff --git a/clients/typescript/src/client/execution/transactionalDB.ts b/clients/typescript/src/client/execution/transactionalDB.ts index 1dc97a757f..b3170b7372 100644 --- a/clients/typescript/src/client/execution/transactionalDB.ts +++ b/clients/typescript/src/client/execution/transactionalDB.ts @@ -4,7 +4,6 @@ import { DB } from './db' import * as z from 'zod' import { Row, Statement } from '../../util' import { Fields } from '../model/schema' -import { Transformation, transformFields } from '../conversions/input' import { Converter } from '../conversions/converter' export class TransactionalDB implements DB { @@ -52,12 +51,9 @@ export class TransactionalDB implements DB { // convert SQLite/PG values back to JS values // and then parse the transformed object // with the Zod schema to validate it - const transformedRow = transformFields( - row, - this._fields, - this._converter, - Transformation.Decode - ) + const transformedRow = this._converter.decodeRow(row, { + fields: this._fields, + }) return schema.parse(transformedRow) }) successCallback( diff --git a/clients/typescript/src/client/model/client.ts b/clients/typescript/src/client/model/client.ts index eb58533dd8..13b52f5e31 100644 --- a/clients/typescript/src/client/model/client.ts +++ b/clients/typescript/src/client/model/client.ts @@ -153,9 +153,7 @@ export class ElectricClient< } } - setReplicationTransform< - T extends Record = Record - >( + setReplicationTransform( qualifiedTableName: QualifiedTablename, i: ReplicatedRowTransformer ): void { diff --git a/clients/typescript/src/client/model/transforms.ts b/clients/typescript/src/client/model/transforms.ts index 6c6deb2d0f..35d85297b4 100644 --- a/clients/typescript/src/client/model/transforms.ts +++ b/clients/typescript/src/client/model/transforms.ts @@ -3,9 +3,9 @@ import { QualifiedTablename, ReplicatedRowTransformer, DbRecord as DataRecord, + Row, } from '../../util' import { Converter } from '../conversions/converter' -import { Transformation, transformFields } from '../conversions/input' import { validate, validateRecordTransformation, @@ -20,7 +20,7 @@ export interface IReplicationTransformManager { ): void clearTableTransform(tableName: QualifiedTablename): void - transformTableRecord>( + transformTableRecord( record: DataRecord, transformRow: (row: T) => T, fields: Fields, @@ -45,7 +45,7 @@ export class ReplicationTransformManager this.satellite.clearReplicationTransform(tableName) } - transformTableRecord>( + transformTableRecord( record: DataRecord, transformRow: (row: T) => T, fields: Fields, @@ -74,7 +74,7 @@ export class ReplicationTransformManager * @param immutableFields - fields that cannot be modified by {@link transformRow} * @return the transformed raw record */ -export function transformTableRecord>( +export function transformTableRecord( record: DataRecord, transformRow: (row: T) => T, fields: Fields, @@ -83,12 +83,7 @@ export function transformTableRecord>( immutableFields: string[] ): DataRecord { // parse raw record according to specified fields - const parsedRow = transformFields( - record, - fields, - converter, - Transformation.Decode - ) as T + const parsedRow = converter.decodeRow(record, { fields }) // apply specified transformation const transformedParsedRow = transformRow(parsedRow as Readonly) @@ -100,12 +95,12 @@ export function transformTableRecord>( schema !== undefined ? validate(transformedParsedRow, schema) : transformedParsedRow - const transformedRecord = transformFields( + const transformedRecord = converter.encodeRow( validatedTransformedParsedRow, - fields, - converter, - Transformation.Encode - ) as DataRecord + { + fields, + } + ) // check if any of the immutable fields were modified const validatedTransformedRecord = validateRecordTransformation( @@ -117,9 +112,7 @@ export function transformTableRecord>( return validatedTransformedRecord } -export function setReplicationTransform< - T extends Record = Record ->( +export function setReplicationTransform( dbDescription: DbSchema, replicationTransformManager: IReplicationTransformManager, qualifiedTableName: QualifiedTablename, diff --git a/clients/typescript/src/satellite/client.ts b/clients/typescript/src/satellite/client.ts index da4286badb..125ae1f434 100644 --- a/clients/typescript/src/satellite/client.ts +++ b/clients/typescript/src/satellite/client.ts @@ -76,6 +76,7 @@ import { ReplicatedRowTransformer, DataGone, GoneBatchCallback, + SqlValue, } from '../util/types' import { base64, @@ -1547,7 +1548,7 @@ function deserializeColumnData( // All values serialized as textual representation function serializeColumnData( - columnValue: boolean | string | number | object, + columnValue: SqlValue, columnType: PgType, encoder: TypeEncoder ): Uint8Array { diff --git a/clients/typescript/src/satellite/oplog.ts b/clients/typescript/src/satellite/oplog.ts index bc748c02d4..a2795fef23 100644 --- a/clients/typescript/src/satellite/oplog.ts +++ b/clients/typescript/src/satellite/oplog.ts @@ -398,7 +398,7 @@ export function extractPK(c: DataChange) { .reduce((primaryKeyRec, col) => { primaryKeyRec[col.name] = columnValues[col.name]! return primaryKeyRec - }, {} as Record) + }, {} as Row) ) } @@ -520,11 +520,7 @@ export const opLogEntryToChange = ( * @param primaryKeyObj object representing all columns of a primary key * @returns a stringified JSON with stable sorting on column names */ -export const primaryKeyToStr = < - T extends Record ->( - primaryKeyObj: T -): string => { +export const primaryKeyToStr = (primaryKeyObj: T): string => { // Sort the keys then insert them in order in a fresh object // cf. https://stackoverflow.com/questions/5467129/sort-javascript-object-by-key diff --git a/clients/typescript/src/util/types.ts b/clients/typescript/src/util/types.ts index c2c7b86963..b7ee713771 100644 --- a/clients/typescript/src/util/types.ts +++ b/clients/typescript/src/util/types.ts @@ -19,7 +19,14 @@ export type Query = string export type Row = { [key: string]: SqlValue } export type RowCallback = (row: Row) => void export type RowId = number -export type SqlValue = string | number | null | Uint8Array | bigint +export type SqlValue = + | boolean + | string + | number + | Uint8Array + | undefined + | null + | bigint export type StatementId = string export type Tablename = string export type VoidOrPromise = void | Promise @@ -188,7 +195,7 @@ export function isDataChange(change: Change): change is DataChange { } export type DbRecord = { - [key: string]: boolean | string | number | Uint8Array | undefined | null + [key: string]: SqlValue } export type Replication = { diff --git a/e2e/satellite_client/src/client.ts b/e2e/satellite_client/src/client.ts index 86e991f685..0e9f8dafff 100644 --- a/e2e/satellite_client/src/client.ts +++ b/e2e/satellite_client/src/client.ts @@ -792,7 +792,7 @@ const write_json_dal = async (electric: Electric, id: string, jsb: any) => { } const write_json_raw = async (electric: Electric, id: string, jsb: any) => { - const r = converter.encode({ id, jsb}, schema.tables.jsons) + const r = converter.encodeRow({ id, jsb}, schema.tables.jsons) const [ row ] = await electric.adapter.query({ sql: `INSERT INTO jsons (id, jsb) VALUES (${builder.makePositionalParam(1)}, ${builder.makePositionalParam(2)}) RETURNING *;`, args: [r.id, r.jsb], @@ -899,7 +899,7 @@ const write_blob_raw = async ( id: string, blob: Uint8Array | null ) => { - const r = converter.encode({ id, blob }, schema.tables.blobs) + const r = converter.encodeRow({ id, blob }, schema.tables.blobs) const [ row ] = await electric.adapter.query({ sql: `INSERT INTO blobs (id, blob) VALUES (${builder.makePositionalParam(1)}, ${builder.makePositionalParam(2)}) RETURNING *;`, args: [r.id, r.blob],