Replace occurences of transformFields by converter.encodeRow/decodeRow

clients/typescript/src/client/conversions/converter.ts

@@ -1,4 +1,3 @@
-import { Row } from '../../util/types'
 import { AnyTableSchema } from '../model'
 import { PgType } from './types'
 
@@ -14,16 +13,19 @@ export interface Converter {
    * @param row The row to encode
    * @param tableSchema The schema of the table for this row.
    */
-  encodeRow(row: Row, tableSchema: AnyTableSchema): Record<string, any>
+  encodeRow<T extends Record<string, any> = Record<string, any>>(
+    row: Record<string, unknown>,
+    tableSchema: Pick<AnyTableSchema, 'fields'>
+  ): T
   /**
    * Encodes the provided rows for storing in the database.
    * @param rows The rows to encode
    * @param tableSchema The schema of the table for these rows.
    */
-  encodeRows(
-    rows: Array<Row>,
-    tableSchema: AnyTableSchema
-  ): Array<Record<string, any>>
+  encodeRows<T extends Record<string, any> = Record<string, any>>(
+    rows: Array<Record<string, unknown>>,
+    tableSchema: Pick<AnyTableSchema, 'fields'>
+  ): Array<T>
   /**
    * Decodes the provided value from the database.
    * @param v The value to decode.
@@ -36,17 +38,17 @@ export interface Converter {
    * @param tableSchema The schema of the table for this row.
    */
   decodeRow<T extends Record<string, any> = Record<string, any>>(
-    row: Row,
-    tableSchema: AnyTableSchema
+    row: Record<string, unknown>,
+    tableSchema: Pick<AnyTableSchema, 'fields'>
   ): T
   /**
    * Decodes the provided rows from the database.
    * @param rows The rows to decode
    * @param tableSchema The schema of the table for these rows.
    */
   decodeRows<T extends Record<string, any> = Record<string, any>>(
-    rows: Array<Row>,
-    tableSchema: AnyTableSchema
+    rows: Array<Record<string, unknown>>,
+    tableSchema: Pick<AnyTableSchema, 'fields'>
   ): Array<T>
 }
 
@@ -62,25 +64,32 @@ export function isDataObject(v: unknown): boolean {
   return v instanceof Date || typeof v === 'bigint' || ArrayBuffer.isView(v)
 }
 
-export function mapRow<T extends Record<string, any> = Record<string, any>>(
-  row: Row,
-  tableSchema: AnyTableSchema,
+export function mapRow<
+  T extends Record<string, unknown> = Record<string, unknown>
+>(
+  row: Record<string, unknown>,
+  tableSchema: Pick<AnyTableSchema, 'fields'>,
   f: (v: any, pgType: PgType) => any
 ): T {
-  const decodedRow = {} as T
+  const mappedRow = {} as T
 
   for (const [key, value] of Object.entries(row)) {
     const pgType = tableSchema.fields[key]
-    const decodedValue = f(value, pgType)
-    decodedRow[key as keyof T] = decodedValue
+    const mappedValue =
+      pgType === undefined
+        ? value // it's an unknown column, leave it as is
+        : f(value, pgType)
+    mappedRow[key as keyof T] = mappedValue
   }
 
-  return decodedRow
+  return mappedRow
 }
 
-export function mapRows<T extends Record<string, any> = Record<string, any>>(
-  rows: Array<Row>,
-  tableSchema: AnyTableSchema,
+export function mapRows<
+  T extends Record<string, unknown> = Record<string, unknown>
+>(
+  rows: Array<Record<string, unknown>>,
+  tableSchema: Pick<AnyTableSchema, 'fields'>,
   f: (v: any, pgType: PgType) => any
 ): T[] {
   return rows.map((row) => mapRow<T>(row, tableSchema, f))

clients/typescript/src/client/conversions/postgres.ts

@@ -4,7 +4,6 @@ import { deserialiseDate, serialiseDate } from './datatypes/date'
 import { isJsonNull } from './datatypes/json'
 import { PgBasicType, PgDateType, PgType } from './types'
 import { AnyTableSchema } from '../model/schema'
-import { Row } from '../../util/types'
 
 /**
  * This module takes care of converting TypeScript values to a Postgres storeable value and back.
@@ -99,17 +98,21 @@ export function fromPostgres(v: any, pgType: PgType): any {
 
 export const postgresConverter: Converter = {
   encode: toPostgres,
-  encodeRow: (row: Row, tableSchema: AnyTableSchema) =>
-    mapRow(row, tableSchema, toPostgres),
-  encodeRows: (rows: Array<Row>, tableSchema: AnyTableSchema) =>
-    mapRows(rows, tableSchema, toPostgres),
+  encodeRow: <T extends Record<string, unknown> = Record<string, unknown>>(
+    row: Record<string, unknown>,
+    tableSchema: AnyTableSchema
+  ) => mapRow<T>(row, tableSchema, toPostgres),
+  encodeRows: <T extends Record<string, unknown> = Record<string, unknown>>(
+    rows: Array<Record<string, unknown>>,
+    tableSchema: AnyTableSchema
+  ) => mapRows<T>(rows, tableSchema, toPostgres),
   decode: fromPostgres,
   decodeRow: <T extends Record<string, any> = Record<string, any>>(
-    row: Row,
+    row: Record<string, unknown>,
     tableSchema: AnyTableSchema
   ) => mapRow<T>(row, tableSchema, fromPostgres),
   decodeRows: <T extends Record<string, any> = Record<string, any>>(
-    rows: Array<Row>,
+    rows: Array<Record<string, unknown>>,
     tableSchema: AnyTableSchema
   ) => mapRows<T>(rows, tableSchema, fromPostgres),
 }

clients/typescript/src/client/conversions/sqlite.ts

@@ -6,7 +6,6 @@ import { deserialiseDate, serialiseDate } from './datatypes/date'
 import { deserialiseJSON, serialiseJSON } from './datatypes/json'
 import { PgBasicType, PgDateType, PgType, isPgDateType } from './types'
 import { AnyTableSchema } from '../model/schema'
-import { Row } from '../../util/types'
 
 /**
  * This module takes care of converting TypeScript values for Postgres-specific types to a SQLite storeable value and back.
@@ -95,17 +94,21 @@ export function fromSqlite(v: any, pgType: PgType): any {
 
 export const sqliteConverter: Converter = {
   encode: toSqlite,
-  encodeRow: (row: Row, tableSchema: AnyTableSchema) =>
-    mapRow(row, tableSchema, toSqlite),
-  encodeRows: (rows: Array<Row>, tableSchema: AnyTableSchema) =>
-    mapRows(rows, tableSchema, toSqlite),
+  encodeRow: <T extends Record<string, unknown> = Record<string, unknown>>(
+    row: Record<string, unknown>,
+    tableSchema: AnyTableSchema
+  ) => mapRow<T>(row, tableSchema, toSqlite),
+  encodeRows: <T extends Record<string, unknown> = Record<string, unknown>>(
+    rows: Array<Record<string, unknown>>,
+    tableSchema: AnyTableSchema
+  ) => mapRows<T>(rows, tableSchema, toSqlite),
   decode: fromSqlite,
   decodeRow: <T extends Record<string, any> = Record<string, any>>(
-    row: Row,
+    row: Record<string, unknown>,
     tableSchema: AnyTableSchema
   ) => mapRow<T>(row, tableSchema, fromSqlite),
-  decodeRows: <T extends Record<string, any> = Record<string, any>>(
-    rows: Array<Row>,
+  decodeRows: <T extends Record<string, unknown> = Record<string, unknown>>(
+    rows: Array<Record<string, unknown>>,
     tableSchema: AnyTableSchema
   ) => mapRows<T>(rows, tableSchema, fromSqlite),
 }

clients/typescript/src/client/execution/nonTransactionalDB.ts

@@ -3,7 +3,6 @@ import { QueryBuilder } from 'squel'
 import { DB } from './db'
 import * as z from 'zod'
 import { Row, Statement } from '../../util'
-import { Transformation, transformFields } from '../conversions/input'
 import { Fields } from '../model/schema'
 import { Converter } from '../conversions/converter'
 
@@ -59,12 +58,9 @@ export class NonTransactionalDB implements DB {
             // convert SQLite/PG values back to JS values
             // and then parse the transformed object
             // with the Zod schema to validate it
-            const transformedRow = transformFields(
-              row,
-              this._fields,
-              this._converter,
-              Transformation.Decode
-            )
+            const transformedRow = this._converter.decodeRow(row, {
+              fields: this._fields,
+            })
             return schema.parse(transformedRow)
           })
           successCallback(this, objects)

clients/typescript/src/client/execution/transactionalDB.ts

@@ -4,7 +4,6 @@ import { DB } from './db'
 import * as z from 'zod'
 import { Row, Statement } from '../../util'
 import { Fields } from '../model/schema'
-import { Transformation, transformFields } from '../conversions/input'
 import { Converter } from '../conversions/converter'
 
 export class TransactionalDB implements DB {
@@ -52,12 +51,9 @@ export class TransactionalDB implements DB {
             // convert SQLite/PG values back to JS values
             // and then parse the transformed object
             // with the Zod schema to validate it
-            const transformedRow = transformFields(
-              row,
-              this._fields,
-              this._converter,
-              Transformation.Decode
-            )
+            const transformedRow = this._converter.decodeRow(row, {
+              fields: this._fields,
+            })
             return schema.parse(transformedRow)
           })
           successCallback(

clients/typescript/src/client/model/client.ts

@@ -153,9 +153,7 @@ export class ElectricClient<
     }
   }
 
-  setReplicationTransform<
-    T extends Record<string, unknown> = Record<string, unknown>
-  >(
+  setReplicationTransform<T extends Row = Row>(
     qualifiedTableName: QualifiedTablename,
     i: ReplicatedRowTransformer<T>
   ): void {

clients/typescript/src/client/model/transforms.ts

@@ -3,9 +3,9 @@ import {
   QualifiedTablename,
   ReplicatedRowTransformer,
   DbRecord as DataRecord,
+  Row,
 } from '../../util'
 import { Converter } from '../conversions/converter'
-import { Transformation, transformFields } from '../conversions/input'
 import {
   validate,
   validateRecordTransformation,
@@ -20,7 +20,7 @@ export interface IReplicationTransformManager {
   ): void
   clearTableTransform(tableName: QualifiedTablename): void
 
-  transformTableRecord<T extends Record<string, unknown>>(
+  transformTableRecord<T extends Row>(
     record: DataRecord,
     transformRow: (row: T) => T,
     fields: Fields,
@@ -45,7 +45,7 @@ export class ReplicationTransformManager
     this.satellite.clearReplicationTransform(tableName)
   }
 
-  transformTableRecord<T extends Record<string, unknown>>(
+  transformTableRecord<T extends Row>(
     record: DataRecord,
     transformRow: (row: T) => T,
     fields: Fields,
@@ -74,7 +74,7 @@ export class ReplicationTransformManager
  * @param immutableFields - fields that cannot be modified by {@link transformRow}
  * @return the transformed raw record
  */
-export function transformTableRecord<T extends Record<string, unknown>>(
+export function transformTableRecord<T extends Row>(
   record: DataRecord,
   transformRow: (row: T) => T,
   fields: Fields,
@@ -83,12 +83,7 @@ export function transformTableRecord<T extends Record<string, unknown>>(
   immutableFields: string[]
 ): DataRecord {
   // parse raw record according to specified fields
-  const parsedRow = transformFields(
-    record,
-    fields,
-    converter,
-    Transformation.Decode
-  ) as T
+  const parsedRow = converter.decodeRow(record, { fields })
 
   // apply specified transformation
   const transformedParsedRow = transformRow(parsedRow as Readonly<T>)
@@ -100,12 +95,12 @@ export function transformTableRecord<T extends Record<string, unknown>>(
     schema !== undefined
       ? validate(transformedParsedRow, schema)
       : transformedParsedRow
-  const transformedRecord = transformFields(
+  const transformedRecord = converter.encodeRow<DataRecord>(
     validatedTransformedParsedRow,
-    fields,
-    converter,
-    Transformation.Encode
-  ) as DataRecord
+    {
+      fields,
+    }
+  )
 
   // check if any of the immutable fields were modified
   const validatedTransformedRecord = validateRecordTransformation(
@@ -117,9 +112,7 @@ export function transformTableRecord<T extends Record<string, unknown>>(
   return validatedTransformedRecord
 }
 
-export function setReplicationTransform<
-  T extends Record<string, unknown> = Record<string, unknown>
->(
+export function setReplicationTransform<T extends Row = Row>(
   dbDescription: DbSchema<TableSchemas>,
   replicationTransformManager: IReplicationTransformManager,
   qualifiedTableName: QualifiedTablename,

clients/typescript/src/satellite/client.ts

@@ -76,6 +76,7 @@ import {
   ReplicatedRowTransformer,
   DataGone,
   GoneBatchCallback,
+  SqlValue,
 } from '../util/types'
 import {
   base64,
@@ -1547,7 +1548,7 @@ function deserializeColumnData(
 
 // All values serialized as textual representation
 function serializeColumnData(
-  columnValue: boolean | string | number | object,
+  columnValue: SqlValue,
   columnType: PgType,
   encoder: TypeEncoder
 ): Uint8Array {

clients/typescript/src/satellite/oplog.ts

@@ -398,7 +398,7 @@ export function extractPK(c: DataChange) {
       .reduce((primaryKeyRec, col) => {
         primaryKeyRec[col.name] = columnValues[col.name]!
         return primaryKeyRec
-      }, {} as Record<string, boolean | string | number | Uint8Array>)
+      }, {} as Row)
   )
 }
 
@@ -520,11 +520,7 @@ export const opLogEntryToChange = (
  * @param primaryKeyObj object representing all columns of a primary key
  * @returns a stringified JSON with stable sorting on column names
  */
-export const primaryKeyToStr = <
-  T extends Record<string, boolean | string | number | Uint8Array>
->(
-  primaryKeyObj: T
-): string => {
+export const primaryKeyToStr = <T extends Row>(primaryKeyObj: T): string => {
   // Sort the keys then insert them in order in a fresh object
   // cf. https://stackoverflow.com/questions/5467129/sort-javascript-object-by-key
 

clients/typescript/src/util/types.ts

@@ -19,7 +19,14 @@ export type Query = string
 export type Row = { [key: string]: SqlValue }
 export type RowCallback = (row: Row) => void
 export type RowId = number
-export type SqlValue = string | number | null | Uint8Array | bigint
+export type SqlValue =
+  | boolean
+  | string
+  | number
+  | Uint8Array
+  | undefined
+  | null
+  | bigint
 export type StatementId = string
 export type Tablename = string
 export type VoidOrPromise = void | Promise<void>
@@ -188,7 +195,7 @@ export function isDataChange(change: Change): change is DataChange {
 }
 
 export type DbRecord = {
-  [key: string]: boolean | string | number | Uint8Array | undefined | null
+  [key: string]: SqlValue
 }
 
 export type Replication<TransactionType> = {

e2e/satellite_client/src/client.ts

@@ -792,7 +792,7 @@ const write_json_dal = async (electric: Electric, id: string, jsb: any) => {
 }
 
 const write_json_raw = async (electric: Electric, id: string, jsb: any) => {
-  const r = converter.encode({ id, jsb}, schema.tables.jsons)
+  const r = converter.encodeRow({ id, jsb}, schema.tables.jsons)
   const [ row ] = await electric.adapter.query({
     sql: `INSERT INTO jsons (id, jsb) VALUES (${builder.makePositionalParam(1)}, ${builder.makePositionalParam(2)}) RETURNING *;`,
     args: [r.id, r.jsb],
@@ -899,7 +899,7 @@ const write_blob_raw = async (
   id: string,
   blob: Uint8Array | null
 ) => {
-  const r = converter.encode({ id, blob }, schema.tables.blobs)
+  const r = converter.encodeRow({ id, blob }, schema.tables.blobs)
   const [ row ] = await electric.adapter.query({
     sql: `INSERT INTO blobs (id, blob) VALUES (${builder.makePositionalParam(1)}, ${builder.makePositionalParam(2)}) RETURNING *;`,
     args: [r.id, r.blob],