From 58c22fd41e536c039b020f36d9fe3a419d812d3d Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 17:44:29 +0800 Subject: [PATCH 01/80] airtable to pattern --- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 +++ .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 14 ++++-- apps/app-config/package.json | 2 +- apps/app-config/providers.ts | 2 - .../AirtableClient.ts | 0 integrations/integration-airtable/def.ts | 26 ++++++++++ .../index.ts | 3 +- .../package.json | 2 +- .../server.ts} | 26 ++++------ packages/airbyte/Dockerfile | 2 +- packages/cdk-core/makeSyncProvider.ts | 2 + pnpm-lock.yaml | 50 ++++++++++--------- 14 files changed, 90 insertions(+), 51 deletions(-) rename integrations/{core-integration-airtable => integration-airtable}/AirtableClient.ts (100%) create mode 100644 integrations/integration-airtable/def.ts rename integrations/{core-integration-airtable => integration-airtable}/index.ts (79%) rename integrations/{core-integration-airtable => integration-airtable}/package.json (86%) rename integrations/{core-integration-airtable/airtableProvider.ts => integration-airtable/server.ts} (67%) diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index 3055cf52..acddf1c2 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -1,4 +1,5 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand +import {default as integrationAirtable} from '@usevenice/integration-airtable/def' import {default as integrationBrex} from '@usevenice/integration-brex/def' import {default as integrationHeron} from '@usevenice/integration-heron/def' import {default as integrationMercury} from '@usevenice/integration-mercury/def' @@ -10,6 +11,7 @@ import {default as integrationTeller} from '@usevenice/integration-teller/def' import {default as integrationYodlee} from '@usevenice/integration-yodlee/def' export const defIntegrations = { + airtable: integrationAirtable, brex: integrationBrex, heron: integrationHeron, mercury: integrationMercury, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index bd8f1489..9b7eaab9 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -1,4 +1,6 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand +import {default as integrationAirtable_def} from '@usevenice/integration-airtable/def' +import {default as integrationAirtable_server} from '@usevenice/integration-airtable/server' import {default as integrationBrex_def} from '@usevenice/integration-brex/def' import {default as integrationBrex_server} from '@usevenice/integration-brex/server' import {default as integrationHeron_def} from '@usevenice/integration-heron/def' @@ -22,6 +24,11 @@ import {default as integrationYodlee_client} from '@usevenice/integration-yodlee import {default as integrationYodlee_def} from '@usevenice/integration-yodlee/def' import {default as integrationYodlee_server} from '@usevenice/integration-yodlee/server' +const integrationAirtable = { + ...integrationAirtable_def, + ...integrationAirtable_server, +} + const integrationBrex = { ...integrationBrex_def, ...integrationBrex_server, @@ -72,6 +79,7 @@ const integrationYodlee = { } export const mergedIntegrations = { + airtable: integrationAirtable, brex: integrationBrex, heron: integrationHeron, mercury: integrationMercury, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index def4ee46..4210b66d 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -1,4 +1,5 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand +import {default as integrationAirtable} from '@usevenice/integration-airtable/server' import {default as integrationBrex} from '@usevenice/integration-brex/server' import {default as integrationHeron} from '@usevenice/integration-heron/server' import {default as integrationMerge} from '@usevenice/integration-merge/server' @@ -9,6 +10,7 @@ import {default as integrationTeller} from '@usevenice/integration-teller/server import {default as integrationYodlee} from '@usevenice/integration-yodlee/server' export const serverIntegrations = { + airtable: integrationAirtable, brex: integrationBrex, heron: integrationHeron, merge: integrationMerge, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index 0199fbf4..16744993 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -1,11 +1,6 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand module.exports = [ - { - dirName: 'core-integration-airtable', - varName: 'coreIntegrationAirtable', - imports: {}, - }, { dirName: 'core-integration-firebase', varName: 'coreIntegrationFirebase', @@ -32,6 +27,15 @@ module.exports = [ varName: 'coreIntegrationWebhook', imports: {}, }, + { + name: 'airtable', + dirName: 'integration-airtable', + varName: 'integrationAirtable', + imports: { + def: '@usevenice/integration-airtable/def', + server: '@usevenice/integration-airtable/server', + }, + }, { dirName: 'integration-alphavantage', varName: 'integrationAlphavantage', diff --git a/apps/app-config/package.json b/apps/app-config/package.json index 003ba45a..ee834252 100644 --- a/apps/app-config/package.json +++ b/apps/app-config/package.json @@ -12,7 +12,7 @@ "@t3-oss/env-nextjs": "0.3.1", "@usevenice/cdk-core": "workspace:*", "@usevenice/cdk-ledger": "workspace:*", - "@usevenice/core-integration-airtable": "workspace:*", + "@usevenice/integration-airtable": "workspace:*", "@usevenice/core-integration-firebase": "workspace:*", "@usevenice/core-integration-fs": "workspace:*", "@usevenice/core-integration-mongodb": "workspace:*", diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index cdebec37..12f85f42 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -1,7 +1,6 @@ // @deprecated. Soon to be fully replaced by integrations.merge.ts import {debugProvider} from '@usevenice/cdk-core' -import {airtableProvider} from '@usevenice/core-integration-airtable' import {firebaseProvider} from '@usevenice/core-integration-firebase' import {fsProvider} from '@usevenice/core-integration-fs' import {mongodbProvider} from '@usevenice/core-integration-mongodb' @@ -40,7 +39,6 @@ export const PROVIDERS = [ firebaseProvider, mongodbProvider, corePostgresProvider, - airtableProvider, webhookProvider, beancountProvider, spreadsheetProvider, diff --git a/integrations/core-integration-airtable/AirtableClient.ts b/integrations/integration-airtable/AirtableClient.ts similarity index 100% rename from integrations/core-integration-airtable/AirtableClient.ts rename to integrations/integration-airtable/AirtableClient.ts diff --git a/integrations/integration-airtable/def.ts b/integrations/integration-airtable/def.ts new file mode 100644 index 00000000..ac5715e4 --- /dev/null +++ b/integrations/integration-airtable/def.ts @@ -0,0 +1,26 @@ +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import type {EntityPayloadWithExternal} from '@usevenice/cdk-ledger' +import {z, zCast} from '@usevenice/util' + +import {zAirtableResourceSettings} from './AirtableClient' + +export const airtableSchemas = { + name: z.literal('airtable'), + resourceSettings: zAirtableResourceSettings, + destinationInputEntity: zCast(), +} satisfies IntegrationSchemas + +export const helpers = intHelpers(airtableSchemas) + +export const airtableDef = { + metadata: { + layer: 'core', + categories: ['database'], + logoUrl: '/_assets/logo-airtable.svg', + }, + name: 'airtable', + def: airtableSchemas, +} satisfies IntegrationDef + +export default airtableDef diff --git a/integrations/core-integration-airtable/index.ts b/integrations/integration-airtable/index.ts similarity index 79% rename from integrations/core-integration-airtable/index.ts rename to integrations/integration-airtable/index.ts index e98daf45..cc02ff01 100644 --- a/integrations/core-integration-airtable/index.ts +++ b/integrations/integration-airtable/index.ts @@ -1,4 +1,5 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' +export * from './server' export * from './AirtableClient' -export * from './airtableProvider' // codegen:end diff --git a/integrations/core-integration-airtable/package.json b/integrations/integration-airtable/package.json similarity index 86% rename from integrations/core-integration-airtable/package.json rename to integrations/integration-airtable/package.json index 5c9f4b03..0bed2792 100644 --- a/integrations/core-integration-airtable/package.json +++ b/integrations/integration-airtable/package.json @@ -1,5 +1,5 @@ { - "name": "@usevenice/core-integration-airtable", + "name": "@usevenice/integration-airtable", "version": "0.0.0", "private": true, "sideEffects": [], diff --git a/integrations/core-integration-airtable/airtableProvider.ts b/integrations/integration-airtable/server.ts similarity index 67% rename from integrations/core-integration-airtable/airtableProvider.ts rename to integrations/integration-airtable/server.ts index 0123b133..d2a0476b 100644 --- a/integrations/core-integration-airtable/airtableProvider.ts +++ b/integrations/integration-airtable/server.ts @@ -1,22 +1,12 @@ -import {handlersLink, makeSyncProvider} from '@usevenice/cdk-core' -import type {EntityPayloadWithExternal} from '@usevenice/cdk-ledger' +import type {IntegrationServer} from '@usevenice/cdk-core' +import {handlersLink} from '@usevenice/cdk-core' import type {Standard} from '@usevenice/standard' -import {fromCompletion, z, zCast} from '@usevenice/util' +import {fromCompletion} from '@usevenice/util' -import {makeAirtableClient, zAirtableResourceSettings} from './AirtableClient' +import {makeAirtableClient} from './AirtableClient' +import type {airtableSchemas} from './def' -const def = makeSyncProvider.def({ - ...makeSyncProvider.def.defaults, - name: z.literal('airtable'), - resourceSettings: zAirtableResourceSettings, - destinationInputEntity: zCast(), -}) - -export const airtableProvider = makeSyncProvider({ - ...makeSyncProvider.defaults, - metadata: {categories: ['database'], logoUrl: '/_assets/logo-airtable.svg'}, - def, - // metadata: {logoUrl: '/_assets/logo-airtable.png'}, +export const airtableServer = { destinationSync: ({settings}) => { const airtable = makeAirtableClient(settings) airtable.initBase() @@ -59,4 +49,6 @@ export const airtableProvider = makeSyncProvider({ }, }) }, -}) +} satisfies IntegrationServer + +export default airtableServer diff --git a/packages/airbyte/Dockerfile b/packages/airbyte/Dockerfile index c6208e56..5179417f 100644 --- a/packages/airbyte/Dockerfile +++ b/packages/airbyte/Dockerfile @@ -40,7 +40,7 @@ COPY ./integrations/core-integration-redis/package.json ./integrations/core-inte COPY ./integrations/integration-onebrick/package.json ./integrations/integration-onebrick/package.json COPY ./integrations/core-integration-postgres/package.json ./integrations/core-integration-postgres/package.json COPY ./integrations/core-integration-firebase/package.json ./integrations/core-integration-firebase/package.json -COPY ./integrations/core-integration-airtable/package.json ./integrations/core-integration-airtable/package.json +COPY ./integrations/integration-airtable/package.json ./integrations/integration-airtable/package.json COPY ./integrations/integration-plaid/package.json ./integrations/integration-plaid/package.json COPY ./integrations/integration-moota/package.json ./integrations/integration-moota/package.json COPY ./package.json ./package.json diff --git a/packages/cdk-core/makeSyncProvider.ts b/packages/cdk-core/makeSyncProvider.ts index 8b45d7ce..dea8bff7 100644 --- a/packages/cdk-core/makeSyncProvider.ts +++ b/packages/cdk-core/makeSyncProvider.ts @@ -66,6 +66,8 @@ export interface IntegrationMetadata { logoUrl?: string logoSvg?: string displayName?: string + /** @deprecated way to indicate an integration outputs raw rather than standardized data */ + layer?: 'core' | 'ledger' platforms?: Array<'cloud' | 'local'> stage?: z.infer // labels?: Array<'featured' | 'banking' | 'accounting' | 'enrichment'> diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 45b7cc62..e1a849d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,8 @@ -lockfileVersion: '6.0' +lockfileVersion: '6.1' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false neverBuiltDependencies: - libpq @@ -154,9 +158,6 @@ importers: '@usevenice/cdk-ledger': specifier: workspace:* version: link:../../packages/cdk-ledger - '@usevenice/core-integration-airtable': - specifier: workspace:* - version: link:../../integrations/core-integration-airtable '@usevenice/core-integration-firebase': specifier: workspace:* version: link:../../integrations/core-integration-firebase @@ -181,6 +182,9 @@ importers: '@usevenice/engine-frontend': specifier: workspace:* version: link:../../packages/engine-frontend + '@usevenice/integration-airtable': + specifier: workspace:* + version: link:../../integrations/integration-airtable '@usevenice/integration-alphavantage': specifier: workspace:* version: link:../../integrations/integration-alphavantage @@ -630,25 +634,6 @@ importers: specifier: '*' version: 5.74.0(esbuild@0.17.5) - integrations/core-integration-airtable: - dependencies: - '@usevenice/cdk-core': - specifier: workspace:* - version: link:../../packages/cdk-core - '@usevenice/standard': - specifier: workspace:* - version: link:../../packages/standard - '@usevenice/util': - specifier: workspace:* - version: link:../../packages/util - airtable: - specifier: 0.11.4 - version: 0.11.4 - devDependencies: - '@usevenice/cdk-ledger': - specifier: workspace:* - version: link:../../packages/cdk-ledger - integrations/core-integration-firebase: dependencies: '@firebase/app': @@ -749,6 +734,25 @@ importers: specifier: workspace:* version: link:../../packages/util + integrations/integration-airtable: + dependencies: + '@usevenice/cdk-core': + specifier: workspace:* + version: link:../../packages/cdk-core + '@usevenice/standard': + specifier: workspace:* + version: link:../../packages/standard + '@usevenice/util': + specifier: workspace:* + version: link:../../packages/util + airtable: + specifier: 0.11.4 + version: 0.11.4 + devDependencies: + '@usevenice/cdk-ledger': + specifier: workspace:* + version: link:../../packages/cdk-ledger + integrations/integration-alphavantage: dependencies: '@usevenice/cdk-core': From 60f92f29dfbf192276f2ed803fb5f0e377096246 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 17:57:06 +0800 Subject: [PATCH 02/80] Saltedge and Venmo down 17 to go --- .../integrations/integrations.def.ts | 4 + .../integrations/integrations.merged.ts | 14 +++ .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 13 ++- apps/app-config/providers.ts | 7 +- .../{saltedgeProvider.ts => def.ts} | 98 +++---------------- integrations/integration-saltedge/index.ts | 3 +- integrations/integration-saltedge/server.ts | 43 ++++++++ .../integration-venmo/VenmoProvider.ts | 96 +++--------------- integrations/integration-venmo/def.ts | 83 ++++++++++++++++ integrations/integration-venmo/index.ts | 1 + 11 files changed, 190 insertions(+), 174 deletions(-) rename integrations/integration-saltedge/{saltedgeProvider.ts => def.ts} (56%) create mode 100644 integrations/integration-saltedge/server.ts create mode 100644 integrations/integration-venmo/def.ts diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index acddf1c2..a8fc76f1 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -6,8 +6,10 @@ import {default as integrationMercury} from '@usevenice/integration-mercury/def' import {default as integrationMerge} from '@usevenice/integration-merge/def' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/def' import {default as integrationPlaid} from '@usevenice/integration-plaid/def' +import {default as integrationSaltedge} from '@usevenice/integration-saltedge/def' import {default as integrationStripe} from '@usevenice/integration-stripe/def' import {default as integrationTeller} from '@usevenice/integration-teller/def' +import {default as integrationVenmo} from '@usevenice/integration-venmo/def' import {default as integrationYodlee} from '@usevenice/integration-yodlee/def' export const defIntegrations = { @@ -18,7 +20,9 @@ export const defIntegrations = { merge: integrationMerge, onebrick: integrationOnebrick, plaid: integrationPlaid, + saltedge: integrationSaltedge, stripe: integrationStripe, teller: integrationTeller, + venmo: integrationVenmo, yodlee: integrationYodlee, } diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 9b7eaab9..5049523f 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -15,11 +15,14 @@ import {default as integrationOnebrick_server} from '@usevenice/integration-oneb import {default as integrationPlaid_client} from '@usevenice/integration-plaid/client' import {default as integrationPlaid_def} from '@usevenice/integration-plaid/def' import {default as integrationPlaid_server} from '@usevenice/integration-plaid/server' +import {default as integrationSaltedge_def} from '@usevenice/integration-saltedge/def' +import {default as integrationSaltedge_server} from '@usevenice/integration-saltedge/server' import {default as integrationStripe_def} from '@usevenice/integration-stripe/def' import {default as integrationStripe_server} from '@usevenice/integration-stripe/server' import {default as integrationTeller_client} from '@usevenice/integration-teller/client' import {default as integrationTeller_def} from '@usevenice/integration-teller/def' import {default as integrationTeller_server} from '@usevenice/integration-teller/server' +import {default as integrationVenmo_def} from '@usevenice/integration-venmo/def' import {default as integrationYodlee_client} from '@usevenice/integration-yodlee/client' import {default as integrationYodlee_def} from '@usevenice/integration-yodlee/def' import {default as integrationYodlee_server} from '@usevenice/integration-yodlee/server' @@ -61,6 +64,11 @@ const integrationPlaid = { ...integrationPlaid_server, } +const integrationSaltedge = { + ...integrationSaltedge_def, + ...integrationSaltedge_server, +} + const integrationStripe = { ...integrationStripe_def, ...integrationStripe_server, @@ -72,6 +80,10 @@ const integrationTeller = { ...integrationTeller_server, } +const integrationVenmo = { + ...integrationVenmo_def, +} + const integrationYodlee = { ...integrationYodlee_def, ...integrationYodlee_client, @@ -86,7 +98,9 @@ export const mergedIntegrations = { merge: integrationMerge, onebrick: integrationOnebrick, plaid: integrationPlaid, + saltedge: integrationSaltedge, stripe: integrationStripe, teller: integrationTeller, + venmo: integrationVenmo, yodlee: integrationYodlee, } diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index 4210b66d..a2d19dc1 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -5,6 +5,7 @@ import {default as integrationHeron} from '@usevenice/integration-heron/server' import {default as integrationMerge} from '@usevenice/integration-merge/server' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/server' import {default as integrationPlaid} from '@usevenice/integration-plaid/server' +import {default as integrationSaltedge} from '@usevenice/integration-saltedge/server' import {default as integrationStripe} from '@usevenice/integration-stripe/server' import {default as integrationTeller} from '@usevenice/integration-teller/server' import {default as integrationYodlee} from '@usevenice/integration-yodlee/server' @@ -16,6 +17,7 @@ export const serverIntegrations = { merge: integrationMerge, onebrick: integrationOnebrick, plaid: integrationPlaid, + saltedge: integrationSaltedge, stripe: integrationStripe, teller: integrationTeller, yodlee: integrationYodlee, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index 16744993..2ff597a0 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -124,9 +124,13 @@ module.exports = [ {dirName: 'integration-qbo', varName: 'integrationQbo', imports: {}}, {dirName: 'integration-ramp', varName: 'integrationRamp', imports: {}}, { + name: 'saltedge', dirName: 'integration-saltedge', varName: 'integrationSaltedge', - imports: {}, + imports: { + def: '@usevenice/integration-saltedge/def', + server: '@usevenice/integration-saltedge/server', + }, }, { dirName: 'integration-splitwise', @@ -158,7 +162,12 @@ module.exports = [ }, }, {dirName: 'integration-toggl', varName: 'integrationToggl', imports: {}}, - {dirName: 'integration-venmo', varName: 'integrationVenmo', imports: {}}, + { + name: 'venmo', + dirName: 'integration-venmo', + varName: 'integrationVenmo', + imports: {def: '@usevenice/integration-venmo/def'}, + }, {dirName: 'integration-wise', varName: 'integrationWise', imports: {}}, { name: 'yodlee', diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 12f85f42..81a70e34 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -14,11 +14,9 @@ import {plaidProvider} from '@usevenice/integration-plaid' import {postgresProvider} from '@usevenice/integration-postgres' import {QBOProvider} from '@usevenice/integration-qbo' import {rampProvider} from '@usevenice/integration-ramp' -import {saltedgeProvider} from '@usevenice/integration-saltedge' import {splitwiseProvider} from '@usevenice/integration-splitwise' import {spreadsheetProvider} from '@usevenice/integration-spreadsheet' import {togglProvider} from '@usevenice/integration-toggl' -import {venmoProvider} from '@usevenice/integration-venmo' import {wiseProvider} from '@usevenice/integration-wise' import {mergedIntegrations} from './integrations/integrations.merged' @@ -36,9 +34,11 @@ export const PROVIDERS = [ // TODO: Migrate these over to the new paradigm debugProvider, fsProvider, + postgresProvider, firebaseProvider, mongodbProvider, corePostgresProvider, + webhookProvider, beancountProvider, spreadsheetProvider, @@ -49,9 +49,6 @@ export const PROVIDERS = [ foreceiptProvider, splitwiseProvider, - postgresProvider, mootaProvider, QBOProvider, - saltedgeProvider, - venmoProvider, ] as const diff --git a/integrations/integration-saltedge/saltedgeProvider.ts b/integrations/integration-saltedge/def.ts similarity index 56% rename from integrations/integration-saltedge/saltedgeProvider.ts rename to integrations/integration-saltedge/def.ts index 6be60dcf..53bae7cc 100644 --- a/integrations/integration-saltedge/saltedgeProvider.ts +++ b/integrations/integration-saltedge/def.ts @@ -1,27 +1,13 @@ -import {makeSyncProvider} from '@usevenice/cdk-core' -import {makePostingsMap, veniceProviderBase} from '@usevenice/cdk-ledger' +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {makePostingsMap} from '@usevenice/cdk-ledger' import type {Standard} from '@usevenice/standard' import type {Brand} from '@usevenice/util' -import { - A, - DateTime, - objectFromObject, - R, - Rx, - rxjs, - startCase, - z, - zCast, -} from '@usevenice/util' +import {A, startCase, z, zCast} from '@usevenice/util' -import { - CACHED_CATEGORIES_MAP, - makeSaltedgeClient, - zConfig, -} from './saltedgeClient' +import {CACHED_CATEGORIES_MAP, zConfig} from './saltedgeClient' -const _def = makeSyncProvider.def({ - ...veniceProviderBase.def, +export const saltedgeSchemas = { name: z.literal('saltedge'), integrationConfig: zConfig, resourceSettings: zCast(), @@ -29,14 +15,7 @@ const _def = makeSyncProvider.def({ z.object({ id: z.string(), entityName: z.literal('account'), - entity: zCast< - { - _balancesMap: Record< - ISODate, - Pick - > - } & SaltEdge.Account - >(), + entity: zCast(), }), z.object({ id: z.string(), @@ -44,13 +23,15 @@ const _def = makeSyncProvider.def({ entity: zCast(), }), ]), -}) +} satisfies IntegrationSchemas -export const saltedgeProviderDef = makeSyncProvider.def.helpers(_def) +export const saltedgeHelpers = intHelpers(saltedgeSchemas) -export const saltedgeProvider = makeSyncProvider({ +export const saltedgeDef = { + name: 'saltedge', + def: saltedgeSchemas, metadata: {categories: ['banking'], logoUrl: '/_assets/logo-saltedge.png'}, - ...veniceProviderBase(saltedgeProviderDef, { + extension: { sourceMapEntity: { account: ({entity: a}, c) => ({ id: a.id, @@ -65,14 +46,6 @@ export const saltedgeProvider = makeSyncProvider({ current: A(a.balance, a.currency_code), }, defaultUnit: a.currency_code as Unit, - balancesMap: objectFromObject( - a._balancesMap ?? {}, - (_, val): Standard.Balance => ({ - autoShiftPaddingDate: true, - holdings: [A(val.balance, val.currency_code)], - disabled: true, - }), - ), type: ((): Standard.AccountType => { switch (a.nature) { case 'account': @@ -124,46 +97,7 @@ export const saltedgeProvider = makeSyncProvider({ }, }), }, - }), - - sourceSync: ({config, settings}) => { - const saltedge = makeSaltedgeClient(config) - async function* iterateEntities() { - for await (const accounts of saltedge.iterateAllAccounts({ - connection_id: settings._id, - })) { - yield accounts.map((a) => - saltedgeProviderDef._opData('account', a.id, { - ...a, - _balancesMap: { - [DateTime.utc().toISODate()]: R.pick(a, [ - 'balance', - 'currency_code', - ]), - }, - }), - ) - } - - const txnGenerator = saltedge.iterateAllTransactions({ - connection_id: settings._id, - }) // TODO: Complete txnGenerator conditials - - for await (const transactions of txnGenerator) { - yield transactions.map((t) => - saltedgeProviderDef._opData('transaction', t.id, t), - ) - } - } - - return rxjs - .from(iterateEntities()) - .pipe( - Rx.mergeMap((ops) => - rxjs.from([...ops, saltedgeProviderDef._op('commit')]), - ), - ) - - // TODO: Move handlePushData }, -}) +} satisfies IntegrationDef + +export default saltedgeDef diff --git a/integrations/integration-saltedge/index.ts b/integrations/integration-saltedge/index.ts index 6b35a504..cbf61e23 100644 --- a/integrations/integration-saltedge/index.ts +++ b/integrations/integration-saltedge/index.ts @@ -1,4 +1,5 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' export * from './saltedgeClient' -export * from './saltedgeProvider' +export * from './server' // codegen:end diff --git a/integrations/integration-saltedge/server.ts b/integrations/integration-saltedge/server.ts new file mode 100644 index 00000000..5bc0c838 --- /dev/null +++ b/integrations/integration-saltedge/server.ts @@ -0,0 +1,43 @@ +import type {IntegrationServer} from '@usevenice/cdk-core' +import {Rx, rxjs} from '@usevenice/util' + +import type {saltedgeSchemas} from './def' +import {saltedgeHelpers} from './def' +import {makeSaltedgeClient} from './saltedgeClient' + +export const saltedgeServer = { + sourceSync: ({config, settings}) => { + const saltedge = makeSaltedgeClient(config) + async function* iterateEntities() { + for await (const accounts of saltedge.iterateAllAccounts({ + connection_id: settings._id, + })) { + yield accounts.map((a) => + saltedgeHelpers._opData('account', a.id, {...a}), + ) + } + + const txnGenerator = saltedge.iterateAllTransactions({ + connection_id: settings._id, + }) // TODO: Complete txnGenerator conditials + + for await (const transactions of txnGenerator) { + yield transactions.map((t) => + saltedgeHelpers._opData('transaction', t.id, t), + ) + } + } + + return rxjs + .from(iterateEntities()) + .pipe( + Rx.mergeMap((ops) => + rxjs.from([...ops, saltedgeHelpers._op('commit')]), + ), + ) + + // TODO: Move handlePushData + }, +} satisfies IntegrationServer + +export default saltedgeServer diff --git a/integrations/integration-venmo/VenmoProvider.ts b/integrations/integration-venmo/VenmoProvider.ts index 12cdf7e2..1c447b48 100644 --- a/integrations/integration-venmo/VenmoProvider.ts +++ b/integrations/integration-venmo/VenmoProvider.ts @@ -1,93 +1,23 @@ -import {makeSyncProvider} from '@usevenice/cdk-core' -import {veniceProviderBase} from '@usevenice/cdk-ledger' -import type {Brand} from '@usevenice/util' -import {A, DateTime, Rx, rxjs, z, zCast} from '@usevenice/util' +import type {IntegrationServer} from '@usevenice/cdk-core' +import {Rx, rxjs} from '@usevenice/util' -import { - descriptionFromTransaction, - payeeFromTransaction, - postingsFromTransaction, -} from './venmo-helpers' -import {makeVenmoClient, zConfig} from './VenmoClient' +import type {venmoSchemas} from './def' +import {helpers} from './def' +import {makeVenmoClient} from './VenmoClient' -// Venmo appears to only support USD for now -const VENMO_CURR = 'USD' as Unit - -const zSettings = z.object({ - me: zCast(), - // TODO: Store venmo credentials inside VGS rather than own db - credentials: zCast(), -}) -const _def = makeSyncProvider.def({ - ...veniceProviderBase.def, - name: z.literal('venmo'), - integrationConfig: zConfig, - resourceSettings: zSettings, - sourceOutputEntity: z.discriminatedUnion('entityName', [ - z.object({ - id: z.string(), - entityName: z.literal('account'), - entity: zCast(), - }), - z.object({ - id: z.string(), - entityName: z.literal('transaction'), - entity: zCast(), - }), - ]), -}) - -export const venmoProviderDef = makeSyncProvider.def.helpers(_def) - -export const venmoProvider = makeSyncProvider({ - metadata: {categories: ['banking'], logoUrl: '/_assets/logo-venmo.png'}, - ...veniceProviderBase(venmoProviderDef, { - sourceMapEntity: { - account: ({entity: a}, _extConn) => ({ - id: a.user.id, - entityName: 'account', - entity: { - name: `Venmo (${a.user.username})`, - informationalBalances: { - current: A(Number.parseInt(a.balance, 10), VENMO_CURR), - }, - lastFour: a.user.username, - type: 'asset/digital_wallet', - }, - }), - transaction: ({entity: t}, extConn) => ({ - id: t.id, - entityName: 'transaction', - entity: { - date: DateTime.fromISO(t.datetime_created).toISODate(), - description: descriptionFromTransaction(t), - payee: payeeFromTransaction( - t, - extConn.me.user.id ?? t._currentUserId, - ), - postingsMap: postingsFromTransaction( - t, - extConn.me.user.id ?? t._currentUserId, - ), - externalStatus: t.transfer?.status as Brand, - }, - }), - }, - }), +export const venmoServer = { sourceSync: ({config, settings: {credentials}}) => { const venmo = makeVenmoClient(config, credentials) async function* iterateEntities() { const me = await venmo.getCurrentUser() - yield [me].map((a) => - venmoProviderDef._opData('account', `${a.user.id}`, a), - ) + yield [me].map((a) => helpers._opData('account', `${a.user.id}`, a)) const iterator = venmo.iterateAllTransactions({ currentUser: me, }) // TODO: Iterate venmo statements and sync to balance records for await (const transactions of iterator) { yield transactions.map((t) => - venmoProviderDef._opData('transaction', t.id, { + helpers._opData('transaction', t.id, { ...t, _currentUserId: me.user.id, }), @@ -96,10 +26,8 @@ export const venmoProvider = makeSyncProvider({ } return rxjs .from(iterateEntities()) - .pipe( - Rx.mergeMap((ops) => - rxjs.from([...ops, venmoProviderDef._op('commit')]), - ), - ) + .pipe(Rx.mergeMap((ops) => rxjs.from([...ops, helpers._op('commit')]))) }, -}) +} satisfies IntegrationServer + +export default venmoServer diff --git a/integrations/integration-venmo/def.ts b/integrations/integration-venmo/def.ts new file mode 100644 index 00000000..4676d445 --- /dev/null +++ b/integrations/integration-venmo/def.ts @@ -0,0 +1,83 @@ +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {zEntityPayload} from '@usevenice/cdk-ledger' +import type {Brand} from '@usevenice/util' +import {A, DateTime, z, zCast} from '@usevenice/util' + +import { + descriptionFromTransaction, + payeeFromTransaction, + postingsFromTransaction, +} from './venmo-helpers' +import {zConfig} from './VenmoClient' + +// Venmo appears to only support USD for now +const VENMO_CURR = 'USD' as Unit + +const zSettings = z.object({ + me: zCast(), + // TODO: Store venmo credentials inside VGS rather than own db + credentials: zCast(), +}) + +export const venmoSchemas = { + name: z.literal('venmo'), + integrationConfig: zConfig, + resourceSettings: zSettings, + sourceOutputEntity: z.discriminatedUnion('entityName', [ + z.object({ + id: z.string(), + entityName: z.literal('account'), + entity: zCast(), + }), + z.object({ + id: z.string(), + entityName: z.literal('transaction'), + entity: zCast(), + }), + ]), + destinationInputEntity: zEntityPayload, +} satisfies IntegrationSchemas + +export const helpers = intHelpers(venmoSchemas) + +export const venmoDef = { + name: 'venmo', + def: venmoSchemas, + metadata: {categories: ['banking'], logoUrl: '/_assets/logo-venmo.png'}, + extension: { + sourceMapEntity: { + account: ({entity: a}, _extConn) => ({ + id: a.user.id, + entityName: 'account', + entity: { + name: `Venmo (${a.user.username})`, + informationalBalances: { + current: A(Number.parseInt(a.balance, 10), VENMO_CURR), + }, + lastFour: a.user.username, + type: 'asset/digital_wallet', + }, + }), + transaction: ({entity: t}, extConn) => ({ + id: t.id, + entityName: 'transaction', + entity: { + date: DateTime.fromISO(t.datetime_created).toISODate(), + description: descriptionFromTransaction(t), + payee: payeeFromTransaction( + t, + extConn.me.user.id ?? t._currentUserId, + ), + postingsMap: postingsFromTransaction( + t, + extConn.me.user.id ?? t._currentUserId, + ), + externalStatus: t.transfer?.status as Brand, + }, + }), + }, + }, +} satisfies IntegrationDef + +export default venmoDef diff --git a/integrations/integration-venmo/index.ts b/integrations/integration-venmo/index.ts index 760a2f12..b8e8dbe3 100644 --- a/integrations/integration-venmo/index.ts +++ b/integrations/integration-venmo/index.ts @@ -1,4 +1,5 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' export * from './venmo-helpers' export * from './VenmoClient' export * from './VenmoProvider' From 1d8eb6b30045c685eb4b049dfb179b9e5fbb8c5f Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 18:01:05 +0800 Subject: [PATCH 03/80] Migrated QBO --- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 ++ .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 10 +- apps/app-config/providers.ts | 2 - .../{QBOProvider.ts => def.ts} | 95 +++++++------------ integrations/integration-qbo/index.ts | 2 +- integrations/integration-qbo/server.ts | 36 +++++++ 8 files changed, 93 insertions(+), 64 deletions(-) rename integrations/integration-qbo/{QBOProvider.ts => def.ts} (89%) create mode 100644 integrations/integration-qbo/server.ts diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index a8fc76f1..1a442a02 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -6,6 +6,7 @@ import {default as integrationMercury} from '@usevenice/integration-mercury/def' import {default as integrationMerge} from '@usevenice/integration-merge/def' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/def' import {default as integrationPlaid} from '@usevenice/integration-plaid/def' +import {default as integrationQbo} from '@usevenice/integration-qbo/def' import {default as integrationSaltedge} from '@usevenice/integration-saltedge/def' import {default as integrationStripe} from '@usevenice/integration-stripe/def' import {default as integrationTeller} from '@usevenice/integration-teller/def' @@ -20,6 +21,7 @@ export const defIntegrations = { merge: integrationMerge, onebrick: integrationOnebrick, plaid: integrationPlaid, + qbo: integrationQbo, saltedge: integrationSaltedge, stripe: integrationStripe, teller: integrationTeller, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 5049523f..638ff50a 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -15,6 +15,8 @@ import {default as integrationOnebrick_server} from '@usevenice/integration-oneb import {default as integrationPlaid_client} from '@usevenice/integration-plaid/client' import {default as integrationPlaid_def} from '@usevenice/integration-plaid/def' import {default as integrationPlaid_server} from '@usevenice/integration-plaid/server' +import {default as integrationQbo_def} from '@usevenice/integration-qbo/def' +import {default as integrationQbo_server} from '@usevenice/integration-qbo/server' import {default as integrationSaltedge_def} from '@usevenice/integration-saltedge/def' import {default as integrationSaltedge_server} from '@usevenice/integration-saltedge/server' import {default as integrationStripe_def} from '@usevenice/integration-stripe/def' @@ -64,6 +66,11 @@ const integrationPlaid = { ...integrationPlaid_server, } +const integrationQbo = { + ...integrationQbo_def, + ...integrationQbo_server, +} + const integrationSaltedge = { ...integrationSaltedge_def, ...integrationSaltedge_server, @@ -98,6 +105,7 @@ export const mergedIntegrations = { merge: integrationMerge, onebrick: integrationOnebrick, plaid: integrationPlaid, + qbo: integrationQbo, saltedge: integrationSaltedge, stripe: integrationStripe, teller: integrationTeller, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index a2d19dc1..fe2973bf 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -5,6 +5,7 @@ import {default as integrationHeron} from '@usevenice/integration-heron/server' import {default as integrationMerge} from '@usevenice/integration-merge/server' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/server' import {default as integrationPlaid} from '@usevenice/integration-plaid/server' +import {default as integrationQbo} from '@usevenice/integration-qbo/server' import {default as integrationSaltedge} from '@usevenice/integration-saltedge/server' import {default as integrationStripe} from '@usevenice/integration-stripe/server' import {default as integrationTeller} from '@usevenice/integration-teller/server' @@ -17,6 +18,7 @@ export const serverIntegrations = { merge: integrationMerge, onebrick: integrationOnebrick, plaid: integrationPlaid, + qbo: integrationQbo, saltedge: integrationSaltedge, stripe: integrationStripe, teller: integrationTeller, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index 2ff597a0..1d9b236e 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -121,7 +121,15 @@ module.exports = [ varName: 'integrationPostgres', imports: {}, }, - {dirName: 'integration-qbo', varName: 'integrationQbo', imports: {}}, + { + name: 'qbo', + dirName: 'integration-qbo', + varName: 'integrationQbo', + imports: { + def: '@usevenice/integration-qbo/def', + server: '@usevenice/integration-qbo/server', + }, + }, {dirName: 'integration-ramp', varName: 'integrationRamp', imports: {}}, { name: 'saltedge', diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 81a70e34..d08b130e 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -12,7 +12,6 @@ import {lunchmoneyProvider} from '@usevenice/integration-lunchmoney' import {mootaProvider} from '@usevenice/integration-moota' import {plaidProvider} from '@usevenice/integration-plaid' import {postgresProvider} from '@usevenice/integration-postgres' -import {QBOProvider} from '@usevenice/integration-qbo' import {rampProvider} from '@usevenice/integration-ramp' import {splitwiseProvider} from '@usevenice/integration-splitwise' import {spreadsheetProvider} from '@usevenice/integration-spreadsheet' @@ -50,5 +49,4 @@ export const PROVIDERS = [ splitwiseProvider, mootaProvider, - QBOProvider, ] as const diff --git a/integrations/integration-qbo/QBOProvider.ts b/integrations/integration-qbo/def.ts similarity index 89% rename from integrations/integration-qbo/QBOProvider.ts rename to integrations/integration-qbo/def.ts index ce75e31f..927266c6 100644 --- a/integrations/integration-qbo/QBOProvider.ts +++ b/integrations/integration-qbo/def.ts @@ -1,13 +1,13 @@ -import {makeSyncProvider} from '@usevenice/cdk-core' -import {makePostingsMap, veniceProviderBase} from '@usevenice/cdk-ledger' +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {makePostingsMap} from '@usevenice/cdk-ledger' import type {Standard} from '@usevenice/standard' import type {EnumOf} from '@usevenice/util' -import {A, DateTime, Rx, rxjs, z, zCast} from '@usevenice/util' +import {A, DateTime, z, zCast} from '@usevenice/util' -import {makeQBOClient, zConfig, zCreds} from './QBOClient' +import {zConfig, zCreds} from './QBOClient' -const _def = makeSyncProvider.def({ - ...veniceProviderBase.def, +export const qboSchemas = { name: z.literal('qbo'), integrationConfig: zConfig, resourceSettings: zCreds, @@ -42,38 +42,20 @@ const _def = makeSyncProvider.def({ ]), }), ]), -}) +} satisfies IntegrationSchemas -export const QBOProviderDef = makeSyncProvider.def.helpers(_def) +export const qboHelpers = intHelpers(qboSchemas) -const QBO_CLASSFICATION_TO_ACCOUNT_TYPE: Record< - QBO.Account['Classification'], - Standard.AccountType -> = { - Asset: 'asset', - Equity: 'equity', - Liability: 'liability', - Revenue: 'income', - Expense: 'expense', -} - -function mapQboAccountType(a: QBO.Account) { - // TODO: Take into account a.AccountType and a.AccountSubtype - return QBO_CLASSFICATION_TO_ACCOUNT_TYPE[a.Classification] -} -/** Prefix id with realmId to get id global within QBO provider */ -function globalId(realmId: string, entityId: string) { - return `${realmId}_${entityId}` as ExternalId -} - -export const QBOProvider = makeSyncProvider({ +export const qboDef = { + name: 'qbo', + def: qboSchemas, metadata: { displayName: 'Quickbooks Online', stage: 'beta', categories: ['accounting'], logoUrl: '/_assets/logo-qbo.svg', }, - ...veniceProviderBase(QBOProviderDef, { + extension: { sourceMapEntity: { account: ({entity: a}) => ({ id: a.Id, @@ -315,37 +297,8 @@ export const QBOProvider = makeSyncProvider({ } }, }, - }), - sourceSync: ({config, settings}) => { - const qbo = makeQBOClient(config, settings) - async function* iterateEntities() { - for await (const res of qbo.getAll('Account')) { - yield res.entities.map((a) => - QBOProviderDef._opData('account', a.Id, a), - ) - } - const updatedSince = undefined - for (const type of Object.values(TRANSACTION_TYPE_NAME)) { - for await (const res of qbo.getAll(type, {updatedSince})) { - const entities = res.entities as QBO.Transaction[] - yield entities.map((t) => - QBOProviderDef._opData('transaction', t.Id, { - type: type as 'Purchase', - entity: t as QBO.Purchase, - realmId: settings.realmId, - }), - ) - } - } - } - - return rxjs - .from(iterateEntities()) - .pipe( - Rx.mergeMap((ops) => rxjs.from([...ops, QBOProviderDef._op('commit')])), - ) }, -}) +} satisfies IntegrationDef export const TRANSACTION_TYPE_NAME: EnumOf = { Purchase: 'Purchase', @@ -354,3 +307,25 @@ export const TRANSACTION_TYPE_NAME: EnumOf = { Invoice: 'Invoice', Payment: 'Payment', } + +const QBO_CLASSFICATION_TO_ACCOUNT_TYPE: Record< + QBO.Account['Classification'], + Standard.AccountType +> = { + Asset: 'asset', + Equity: 'equity', + Liability: 'liability', + Revenue: 'income', + Expense: 'expense', +} + +function mapQboAccountType(a: QBO.Account) { + // TODO: Take into account a.AccountType and a.AccountSubtype + return QBO_CLASSFICATION_TO_ACCOUNT_TYPE[a.Classification] +} +/** Prefix id with realmId to get id global within QBO provider */ +function globalId(realmId: string, entityId: string) { + return `${realmId}_${entityId}` as ExternalId +} + +export default qboDef diff --git a/integrations/integration-qbo/index.ts b/integrations/integration-qbo/index.ts index fdf6313d..b42349ea 100644 --- a/integrations/integration-qbo/index.ts +++ b/integrations/integration-qbo/index.ts @@ -1,4 +1,4 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} export * from './QBOClient' -export * from './QBOProvider' +export * from './server' // codegen:end diff --git a/integrations/integration-qbo/server.ts b/integrations/integration-qbo/server.ts new file mode 100644 index 00000000..4d2ab632 --- /dev/null +++ b/integrations/integration-qbo/server.ts @@ -0,0 +1,36 @@ +import type {IntegrationServer} from '@usevenice/cdk-core' +import {Rx, rxjs} from '@usevenice/util' + +import type {qboSchemas} from './def' +import {qboHelpers, TRANSACTION_TYPE_NAME} from './def' +import {makeQBOClient} from './QBOClient' + +export const qboServer = { + sourceSync: ({config, settings}) => { + const qbo = makeQBOClient(config, settings) + async function* iterateEntities() { + for await (const res of qbo.getAll('Account')) { + yield res.entities.map((a) => qboHelpers._opData('account', a.Id, a)) + } + const updatedSince = undefined + for (const type of Object.values(TRANSACTION_TYPE_NAME)) { + for await (const res of qbo.getAll(type, {updatedSince})) { + const entities = res.entities as QBO.Transaction[] + yield entities.map((t) => + qboHelpers._opData('transaction', t.Id, { + type: type as 'Purchase', + entity: t as QBO.Purchase, + realmId: settings.realmId, + }), + ) + } + } + } + + return rxjs + .from(iterateEntities()) + .pipe(Rx.mergeMap((ops) => rxjs.from([...ops, qboHelpers._op('commit')]))) + }, +} satisfies IntegrationServer + +export default qboServer From 742a4f31c913b3d3809c79114e56e93045c6f401 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 18:04:41 +0800 Subject: [PATCH 04/80] Migrated integration moota --- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 +++ .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 10 +++- apps/app-config/providers.ts | 2 - .../{mootaProvider.ts => def.ts} | 56 +++++-------------- integrations/integration-moota/index.ts | 3 +- integrations/integration-moota/server.ts | 37 ++++++++++++ integrations/integration-qbo/index.ts | 1 + 9 files changed, 76 insertions(+), 45 deletions(-) rename integrations/integration-moota/{mootaProvider.ts => def.ts} (58%) create mode 100644 integrations/integration-moota/server.ts diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index 1a442a02..503b208b 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -4,6 +4,7 @@ import {default as integrationBrex} from '@usevenice/integration-brex/def' import {default as integrationHeron} from '@usevenice/integration-heron/def' import {default as integrationMercury} from '@usevenice/integration-mercury/def' import {default as integrationMerge} from '@usevenice/integration-merge/def' +import {default as integrationMoota} from '@usevenice/integration-moota/def' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/def' import {default as integrationPlaid} from '@usevenice/integration-plaid/def' import {default as integrationQbo} from '@usevenice/integration-qbo/def' @@ -19,6 +20,7 @@ export const defIntegrations = { heron: integrationHeron, mercury: integrationMercury, merge: integrationMerge, + moota: integrationMoota, onebrick: integrationOnebrick, plaid: integrationPlaid, qbo: integrationQbo, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 638ff50a..b2dea443 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -9,6 +9,8 @@ import {default as integrationMercury_def} from '@usevenice/integration-mercury/ import {default as integrationMerge_client} from '@usevenice/integration-merge/client' import {default as integrationMerge_def} from '@usevenice/integration-merge/def' import {default as integrationMerge_server} from '@usevenice/integration-merge/server' +import {default as integrationMoota_def} from '@usevenice/integration-moota/def' +import {default as integrationMoota_server} from '@usevenice/integration-moota/server' import {default as integrationOnebrick_client} from '@usevenice/integration-onebrick/client' import {default as integrationOnebrick_def} from '@usevenice/integration-onebrick/def' import {default as integrationOnebrick_server} from '@usevenice/integration-onebrick/server' @@ -54,6 +56,11 @@ const integrationMerge = { ...integrationMerge_server, } +const integrationMoota = { + ...integrationMoota_def, + ...integrationMoota_server, +} + const integrationOnebrick = { ...integrationOnebrick_def, ...integrationOnebrick_client, @@ -103,6 +110,7 @@ export const mergedIntegrations = { heron: integrationHeron, mercury: integrationMercury, merge: integrationMerge, + moota: integrationMoota, onebrick: integrationOnebrick, plaid: integrationPlaid, qbo: integrationQbo, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index fe2973bf..8e61b1eb 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -3,6 +3,7 @@ import {default as integrationAirtable} from '@usevenice/integration-airtable/se import {default as integrationBrex} from '@usevenice/integration-brex/server' import {default as integrationHeron} from '@usevenice/integration-heron/server' import {default as integrationMerge} from '@usevenice/integration-merge/server' +import {default as integrationMoota} from '@usevenice/integration-moota/server' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/server' import {default as integrationPlaid} from '@usevenice/integration-plaid/server' import {default as integrationQbo} from '@usevenice/integration-qbo/server' @@ -16,6 +17,7 @@ export const serverIntegrations = { brex: integrationBrex, heron: integrationHeron, merge: integrationMerge, + moota: integrationMoota, onebrick: integrationOnebrick, plaid: integrationPlaid, qbo: integrationQbo, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index 1d9b236e..b86e82ea 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -95,7 +95,15 @@ module.exports = [ server: '@usevenice/integration-merge/server', }, }, - {dirName: 'integration-moota', varName: 'integrationMoota', imports: {}}, + { + name: 'moota', + dirName: 'integration-moota', + varName: 'integrationMoota', + imports: { + def: '@usevenice/integration-moota/def', + server: '@usevenice/integration-moota/server', + }, + }, { name: 'onebrick', dirName: 'integration-onebrick', diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index d08b130e..6aef249f 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -9,7 +9,6 @@ import {webhookProvider} from '@usevenice/core-integration-webhook' import {beancountProvider} from '@usevenice/integration-beancount' import {foreceiptProvider} from '@usevenice/integration-foreceipt' import {lunchmoneyProvider} from '@usevenice/integration-lunchmoney' -import {mootaProvider} from '@usevenice/integration-moota' import {plaidProvider} from '@usevenice/integration-plaid' import {postgresProvider} from '@usevenice/integration-postgres' import {rampProvider} from '@usevenice/integration-ramp' @@ -48,5 +47,4 @@ export const PROVIDERS = [ foreceiptProvider, splitwiseProvider, - mootaProvider, ] as const diff --git a/integrations/integration-moota/mootaProvider.ts b/integrations/integration-moota/def.ts similarity index 58% rename from integrations/integration-moota/mootaProvider.ts rename to integrations/integration-moota/def.ts index 6986dc77..d8865c35 100644 --- a/integrations/integration-moota/mootaProvider.ts +++ b/integrations/integration-moota/def.ts @@ -1,11 +1,11 @@ -import {makeSyncProvider} from '@usevenice/cdk-core' -import {makePostingsMap, veniceProviderBase} from '@usevenice/cdk-ledger' -import {A, objectFromArray, Rx, rxjs, z, zCast} from '@usevenice/util' +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {makePostingsMap} from '@usevenice/cdk-ledger' +import {A, objectFromArray, z, zCast} from '@usevenice/util' -import {makeMootaClient, zConfig} from './mootaClient' +import {zConfig} from './mootaClient' -const _def = makeSyncProvider.def({ - ...veniceProviderBase.def, +export const mootaSchemas = { name: z.literal('moota'), integrationConfig: zConfig, sourceOutputEntity: z.discriminatedUnion('entityName', [ @@ -20,18 +20,20 @@ const _def = makeSyncProvider.def({ entity: zCast(), }), ]), -}) +} satisfies IntegrationSchemas -export const mootaProviderDef = makeSyncProvider.def.helpers(_def) +export const mootaHelpers = intHelpers(mootaSchemas) function toISO(mootaDate: string) { // Local timezone return mootaDate.replace(' ', 'T') + '+07:00' // Always Jakarta time } -export const mootaProvider = makeSyncProvider({ +export const mootaDef = { + name: 'moota', + def: mootaSchemas, metadata: {categories: ['banking'], logoUrl: '/_assets/logo-moota.png'}, - ...veniceProviderBase(mootaProviderDef, { + extension: { sourceMapEntity: { account: ({entity: a}) => ({ id: a.bank_id, @@ -72,35 +74,7 @@ export const mootaProvider = makeSyncProvider({ }, }), }, - }), - sourceSync: ({config}) => { - const moota = makeMootaClient(config) - const accounts: Moota.BankAccount[] = [] - async function* iterateEntities() { - for await (const acc of moota.iterateAllBankAccounts()) { - accounts.push(...acc.data) - yield acc.data.map((a) => - mootaProviderDef._opData('account', a.bank_id, a), - ) - } - - for (const {bank_id} of accounts) { - for await (const transaction of moota.iterateAllTransactions({ - bank_id, - })) { - yield transaction.data.map((t) => - mootaProviderDef._opData('transaction', t.mutation_id, t), - ) - } - } - } - - return rxjs - .from(iterateEntities()) - .pipe( - Rx.mergeMap((ops) => - rxjs.from([...ops, mootaProviderDef._op('commit')]), - ), - ) }, -}) +} satisfies IntegrationDef + +export default mootaDef diff --git a/integrations/integration-moota/index.ts b/integrations/integration-moota/index.ts index 8d2106a4..f6cd80a4 100644 --- a/integrations/integration-moota/index.ts +++ b/integrations/integration-moota/index.ts @@ -1,4 +1,5 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' export * from './mootaClient' -export * from './mootaProvider' +export * from './server' // codegen:end diff --git a/integrations/integration-moota/server.ts b/integrations/integration-moota/server.ts new file mode 100644 index 00000000..624ed078 --- /dev/null +++ b/integrations/integration-moota/server.ts @@ -0,0 +1,37 @@ +import type {IntegrationServer} from '@usevenice/cdk-core' +import {Rx, rxjs} from '@usevenice/util' + +import type {mootaSchemas} from './def' +import {mootaHelpers} from './def' +import {makeMootaClient} from './mootaClient' + +export const mootaServer = { + sourceSync: ({config}) => { + const moota = makeMootaClient(config) + const accounts: Moota.BankAccount[] = [] + async function* iterateEntities() { + for await (const acc of moota.iterateAllBankAccounts()) { + accounts.push(...acc.data) + yield acc.data.map((a) => mootaHelpers._opData('account', a.bank_id, a)) + } + + for (const {bank_id} of accounts) { + for await (const transaction of moota.iterateAllTransactions({ + bank_id, + })) { + yield transaction.data.map((t) => + mootaHelpers._opData('transaction', t.mutation_id, t), + ) + } + } + } + + return rxjs + .from(iterateEntities()) + .pipe( + Rx.mergeMap((ops) => rxjs.from([...ops, mootaHelpers._op('commit')])), + ) + }, +} satisfies IntegrationServer + +export default mootaServer diff --git a/integrations/integration-qbo/index.ts b/integrations/integration-qbo/index.ts index b42349ea..d064d39f 100644 --- a/integrations/integration-qbo/index.ts +++ b/integrations/integration-qbo/index.ts @@ -1,4 +1,5 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' export * from './QBOClient' export * from './server' // codegen:end From 2532b44d8b25d3d4952e45cdec52862caf5aa69b Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 18:09:02 +0800 Subject: [PATCH 05/80] Migrated splitwise --- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 +++ .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 6 +- apps/app-config/providers.ts | 3 - .../{SplitwiseProvider.ts => def.ts} | 70 ++++--------------- integrations/integration-splitwise/index.ts | 3 +- integrations/integration-splitwise/server.ts | 56 +++++++++++++++ 8 files changed, 90 insertions(+), 60 deletions(-) rename integrations/integration-splitwise/{SplitwiseProvider.ts => def.ts} (72%) create mode 100644 integrations/integration-splitwise/server.ts diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index 503b208b..88c1596e 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -9,6 +9,7 @@ import {default as integrationOnebrick} from '@usevenice/integration-onebrick/de import {default as integrationPlaid} from '@usevenice/integration-plaid/def' import {default as integrationQbo} from '@usevenice/integration-qbo/def' import {default as integrationSaltedge} from '@usevenice/integration-saltedge/def' +import {default as integrationSplitwise} from '@usevenice/integration-splitwise/def' import {default as integrationStripe} from '@usevenice/integration-stripe/def' import {default as integrationTeller} from '@usevenice/integration-teller/def' import {default as integrationVenmo} from '@usevenice/integration-venmo/def' @@ -25,6 +26,7 @@ export const defIntegrations = { plaid: integrationPlaid, qbo: integrationQbo, saltedge: integrationSaltedge, + splitwise: integrationSplitwise, stripe: integrationStripe, teller: integrationTeller, venmo: integrationVenmo, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index b2dea443..b1b20386 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -21,6 +21,8 @@ import {default as integrationQbo_def} from '@usevenice/integration-qbo/def' import {default as integrationQbo_server} from '@usevenice/integration-qbo/server' import {default as integrationSaltedge_def} from '@usevenice/integration-saltedge/def' import {default as integrationSaltedge_server} from '@usevenice/integration-saltedge/server' +import {default as integrationSplitwise_def} from '@usevenice/integration-splitwise/def' +import {default as integrationSplitwise_server} from '@usevenice/integration-splitwise/server' import {default as integrationStripe_def} from '@usevenice/integration-stripe/def' import {default as integrationStripe_server} from '@usevenice/integration-stripe/server' import {default as integrationTeller_client} from '@usevenice/integration-teller/client' @@ -83,6 +85,11 @@ const integrationSaltedge = { ...integrationSaltedge_server, } +const integrationSplitwise = { + ...integrationSplitwise_def, + ...integrationSplitwise_server, +} + const integrationStripe = { ...integrationStripe_def, ...integrationStripe_server, @@ -115,6 +122,7 @@ export const mergedIntegrations = { plaid: integrationPlaid, qbo: integrationQbo, saltedge: integrationSaltedge, + splitwise: integrationSplitwise, stripe: integrationStripe, teller: integrationTeller, venmo: integrationVenmo, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index 8e61b1eb..910ccce5 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -8,6 +8,7 @@ import {default as integrationOnebrick} from '@usevenice/integration-onebrick/se import {default as integrationPlaid} from '@usevenice/integration-plaid/server' import {default as integrationQbo} from '@usevenice/integration-qbo/server' import {default as integrationSaltedge} from '@usevenice/integration-saltedge/server' +import {default as integrationSplitwise} from '@usevenice/integration-splitwise/server' import {default as integrationStripe} from '@usevenice/integration-stripe/server' import {default as integrationTeller} from '@usevenice/integration-teller/server' import {default as integrationYodlee} from '@usevenice/integration-yodlee/server' @@ -22,6 +23,7 @@ export const serverIntegrations = { plaid: integrationPlaid, qbo: integrationQbo, saltedge: integrationSaltedge, + splitwise: integrationSplitwise, stripe: integrationStripe, teller: integrationTeller, yodlee: integrationYodlee, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index b86e82ea..e543b7bb 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -149,9 +149,13 @@ module.exports = [ }, }, { + name: 'splitwise', dirName: 'integration-splitwise', varName: 'integrationSplitwise', - imports: {}, + imports: { + def: '@usevenice/integration-splitwise/def', + server: '@usevenice/integration-splitwise/server', + }, }, { dirName: 'integration-spreadsheet', diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 6aef249f..7720d4c6 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -12,7 +12,6 @@ import {lunchmoneyProvider} from '@usevenice/integration-lunchmoney' import {plaidProvider} from '@usevenice/integration-plaid' import {postgresProvider} from '@usevenice/integration-postgres' import {rampProvider} from '@usevenice/integration-ramp' -import {splitwiseProvider} from '@usevenice/integration-splitwise' import {spreadsheetProvider} from '@usevenice/integration-spreadsheet' import {togglProvider} from '@usevenice/integration-toggl' import {wiseProvider} from '@usevenice/integration-wise' @@ -45,6 +44,4 @@ export const PROVIDERS = [ wiseProvider, togglProvider, foreceiptProvider, - - splitwiseProvider, ] as const diff --git a/integrations/integration-splitwise/SplitwiseProvider.ts b/integrations/integration-splitwise/def.ts similarity index 72% rename from integrations/integration-splitwise/SplitwiseProvider.ts rename to integrations/integration-splitwise/def.ts index 1cec00f0..03b3efd3 100644 --- a/integrations/integration-splitwise/SplitwiseProvider.ts +++ b/integrations/integration-splitwise/def.ts @@ -1,5 +1,6 @@ -import {makeSyncProvider} from '@usevenice/cdk-core' -import {makePostingsMap, veniceProviderBase} from '@usevenice/cdk-ledger' +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {makePostingsMap} from '@usevenice/cdk-ledger' import type {Standard} from '@usevenice/standard' import { A, @@ -8,17 +9,13 @@ import { math, objectFromArray, parseMoney, - Rx, - rxjs, z, } from '@usevenice/util' import type {zUser} from './splitwise-schema' import {zCurrentUser, zExpense, zGroup} from './splitwise-schema' -import {makeSplitwiseClient} from './SplitwiseClientNext' -const _def = makeSyncProvider.def({ - ...veniceProviderBase.def, +export const splitwiseSchemas = { name: z.literal('splitwise'), resourceSettings: z.object({ currentUser: zCurrentUser.nullish(), @@ -37,12 +34,15 @@ const _def = makeSyncProvider.def({ entity: zExpense.extend({group_name: z.string()}), }), ]), -}) -const def = makeSyncProvider.def.helpers(_def) +} satisfies IntegrationSchemas -export const splitwiseProvider = makeSyncProvider({ +export const splitwiseHelpers = intHelpers(splitwiseSchemas) + +export const splitwiseDef = { + name: 'splitwise', + def: splitwiseSchemas, metadata: {categories: ['personal-finance']}, - ...veniceProviderBase(def, { + extension: { sourceMapEntity: { account: ({entity: a}) => ({ id: `${a.id}`, @@ -152,54 +152,14 @@ export const splitwiseProvider = makeSyncProvider({ } }, }, - }), - - sourceSync: ({settings: {accessToken}}) => { - const splitwise = makeSplitwiseClient({accessToken}) - - async function* iterateEntities() { - const groups = await splitwise.getGroups() - - yield groups.map((a) => def._opData('account', `${a.id}`, a)) - - let offset = 0 - let limit = 100 - - while (true) { - const expenses = await splitwise.getExpenses({offset, limit}) - if (expenses.length === 0) { - break - } - yield expenses.map((t) => { - // For now it's easiest to get the group name - // TODO: Need to check for the better way to get the group name - const group_name = - groups.find( - (a) => - a.name - .toLowerCase() - .includes(formatUser(t.users[0]?.user).toLowerCase()), - // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing - )?.name || formatUser(t.users[0]?.user) - return def._opData('transaction', `${t.id}`, { - ...t, - group_name, - }) - }) - offset += expenses.length - limit = 500 - } - } - - return rxjs - .from(iterateEntities()) - .pipe(Rx.mergeMap((ops) => rxjs.from([...ops, def._op('commit')]))) }, -}) +} satisfies IntegrationDef // Helpers -function formatUser(user?: z.infer) { +export function formatUser(user?: z.infer) { return user ? `${user.first_name ?? ''} ${user.last_name ?? ''}`.trim() : 'Unnamed user' } + +export default splitwiseDef diff --git a/integrations/integration-splitwise/index.ts b/integrations/integration-splitwise/index.ts index f415d46d..d0a5e66c 100644 --- a/integrations/integration-splitwise/index.ts +++ b/integrations/integration-splitwise/index.ts @@ -1,5 +1,6 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' +export * from './server' export * from './splitwise-schema' export * from './SplitwiseClientNext' -export * from './SplitwiseProvider' // codegen:end diff --git a/integrations/integration-splitwise/server.ts b/integrations/integration-splitwise/server.ts new file mode 100644 index 00000000..67d06ead --- /dev/null +++ b/integrations/integration-splitwise/server.ts @@ -0,0 +1,56 @@ +import type {IntegrationServer} from '@usevenice/cdk-core' +import {Rx, rxjs} from '@usevenice/util' + +import type {splitwiseSchemas} from './def' +import {formatUser, splitwiseHelpers} from './def' +import {makeSplitwiseClient} from './SplitwiseClientNext' + +export const splitwiseServer = { + sourceSync: ({settings: {accessToken}}) => { + const splitwise = makeSplitwiseClient({accessToken}) + + async function* iterateEntities() { + const groups = await splitwise.getGroups() + + yield groups.map((a) => splitwiseHelpers._opData('account', `${a.id}`, a)) + + let offset = 0 + let limit = 100 + + while (true) { + const expenses = await splitwise.getExpenses({offset, limit}) + if (expenses.length === 0) { + break + } + yield expenses.map((t) => { + // For now it's easiest to get the group name + // TODO: Need to check for the better way to get the group name + const group_name = + groups.find( + (a) => + a.name + .toLowerCase() + .includes(formatUser(t.users[0]?.user).toLowerCase()), + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + )?.name || formatUser(t.users[0]?.user) + return splitwiseHelpers._opData('transaction', `${t.id}`, { + ...t, + group_name, + }) + }) + offset += expenses.length + limit = 500 + } + } + + return rxjs + .from(iterateEntities()) + .pipe( + Rx.mergeMap((ops) => + rxjs.from([...ops, splitwiseHelpers._op('commit')]), + ), + ) + }, +} satisfies IntegrationServer + +export default splitwiseServer From 81c807ea69e72ed3f9a4114574f562a19c2f8db4 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 18:15:20 +0800 Subject: [PATCH 06/80] Migrate toggl integration --- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 +++ .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 10 ++- apps/app-config/providers.ts | 1 - .../{TogglProvider.ts => def.ts} | 72 +++++-------------- integrations/integration-toggl/index.ts | 3 +- integrations/integration-toggl/server.ts | 39 ++++++++++ 8 files changed, 80 insertions(+), 57 deletions(-) rename integrations/integration-toggl/{TogglProvider.ts => def.ts} (52%) create mode 100644 integrations/integration-toggl/server.ts diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index 88c1596e..d6fdef29 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -12,6 +12,7 @@ import {default as integrationSaltedge} from '@usevenice/integration-saltedge/de import {default as integrationSplitwise} from '@usevenice/integration-splitwise/def' import {default as integrationStripe} from '@usevenice/integration-stripe/def' import {default as integrationTeller} from '@usevenice/integration-teller/def' +import {default as integrationToggl} from '@usevenice/integration-toggl/def' import {default as integrationVenmo} from '@usevenice/integration-venmo/def' import {default as integrationYodlee} from '@usevenice/integration-yodlee/def' @@ -29,6 +30,7 @@ export const defIntegrations = { splitwise: integrationSplitwise, stripe: integrationStripe, teller: integrationTeller, + toggl: integrationToggl, venmo: integrationVenmo, yodlee: integrationYodlee, } diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index b1b20386..5fd62add 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -28,6 +28,8 @@ import {default as integrationStripe_server} from '@usevenice/integration-stripe import {default as integrationTeller_client} from '@usevenice/integration-teller/client' import {default as integrationTeller_def} from '@usevenice/integration-teller/def' import {default as integrationTeller_server} from '@usevenice/integration-teller/server' +import {default as integrationToggl_def} from '@usevenice/integration-toggl/def' +import {default as integrationToggl_server} from '@usevenice/integration-toggl/server' import {default as integrationVenmo_def} from '@usevenice/integration-venmo/def' import {default as integrationYodlee_client} from '@usevenice/integration-yodlee/client' import {default as integrationYodlee_def} from '@usevenice/integration-yodlee/def' @@ -101,6 +103,11 @@ const integrationTeller = { ...integrationTeller_server, } +const integrationToggl = { + ...integrationToggl_def, + ...integrationToggl_server, +} + const integrationVenmo = { ...integrationVenmo_def, } @@ -125,6 +132,7 @@ export const mergedIntegrations = { splitwise: integrationSplitwise, stripe: integrationStripe, teller: integrationTeller, + toggl: integrationToggl, venmo: integrationVenmo, yodlee: integrationYodlee, } diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index 910ccce5..c7a5c115 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -11,6 +11,7 @@ import {default as integrationSaltedge} from '@usevenice/integration-saltedge/se import {default as integrationSplitwise} from '@usevenice/integration-splitwise/server' import {default as integrationStripe} from '@usevenice/integration-stripe/server' import {default as integrationTeller} from '@usevenice/integration-teller/server' +import {default as integrationToggl} from '@usevenice/integration-toggl/server' import {default as integrationYodlee} from '@usevenice/integration-yodlee/server' export const serverIntegrations = { @@ -26,5 +27,6 @@ export const serverIntegrations = { splitwise: integrationSplitwise, stripe: integrationStripe, teller: integrationTeller, + toggl: integrationToggl, yodlee: integrationYodlee, } diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index e543b7bb..e0b86856 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -181,7 +181,15 @@ module.exports = [ server: '@usevenice/integration-teller/server', }, }, - {dirName: 'integration-toggl', varName: 'integrationToggl', imports: {}}, + { + name: 'toggl', + dirName: 'integration-toggl', + varName: 'integrationToggl', + imports: { + def: '@usevenice/integration-toggl/def', + server: '@usevenice/integration-toggl/server', + }, + }, { name: 'venmo', dirName: 'integration-venmo', diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 7720d4c6..dde09842 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -42,6 +42,5 @@ export const PROVIDERS = [ lunchmoneyProvider, rampProvider, wiseProvider, - togglProvider, foreceiptProvider, ] as const diff --git a/integrations/integration-toggl/TogglProvider.ts b/integrations/integration-toggl/def.ts similarity index 52% rename from integrations/integration-toggl/TogglProvider.ts rename to integrations/integration-toggl/def.ts index c9cd49fc..1068537d 100644 --- a/integrations/integration-toggl/TogglProvider.ts +++ b/integrations/integration-toggl/def.ts @@ -1,18 +1,15 @@ -import {makeSyncProvider} from '@usevenice/cdk-core' -import {makePostingsMap, veniceProviderBase} from '@usevenice/cdk-ledger' -import type {Standard} from '@usevenice/standard' -import {A, R, Rx, rxjs, z} from '@usevenice/util' - import { - itemProjectResponseSchema, - itemTimeEntriesSchema, - makeTogglClient, -} from './TogglCient' + IntegrationDef, + IntegrationSchemas, + intHelpers, +} from '@usevenice/cdk-core' +import {makePostingsMap} from '@usevenice/cdk-ledger' +import type {Standard} from '@usevenice/standard' +import {A, R, z} from '@usevenice/util' -type TogglSyncOperation = (typeof def)['_opType'] +import {itemProjectResponseSchema, itemTimeEntriesSchema} from './TogglCient' -const def = makeSyncProvider.def({ - ...veniceProviderBase.def, +export const togglSchemas = { name: z.literal('toggl'), // integrationConfig: zTogglConfig, connectInput: z.object({ @@ -40,10 +37,14 @@ const def = makeSyncProvider.def({ entity: itemTimeEntriesSchema, }), ]), -}) +} satisfies IntegrationSchemas -export const togglProvider = makeSyncProvider({ - ...veniceProviderBase(def, { +export const togglHelpers = intHelpers(togglSchemas) + +export const togglDef = { + name: 'toggl', + def: togglSchemas, + extension: { sourceMapEntity: (data) => { if (data.entityName === 'account') { const a = data.entity @@ -74,44 +75,7 @@ export const togglProvider = makeSyncProvider({ } return null }, - }), - - postConnect: (input) => ({ - resourceExternalId: input.apiToken, - settings: { - apiToken: input.apiToken, - email: input.email, - password: input.password, - }, - triggerDefaultSync: true, - }), - - sourceSync: ({settings}) => { - const client = makeTogglClient({...settings}) - async function* iterateEntities() { - const user = await client.getMe() - const res = await client.getProjects(`${user.default_workspace_id}`) - yield res.map((a) => - _op({ - type: 'data', - data: {id: `${a.id}`, entity: a, entityName: 'account'}, - }), - ) - - // TODO: Need to pass params if necessary - const res2 = await client.getTimeEntries() - yield res2.map((t) => - _op({ - type: 'data', - data: {id: `${t.id}`, entity: t, entityName: 'transaction'}, - }), - ) - } - - return rxjs - .from(iterateEntities()) - .pipe(Rx.mergeMap((ops) => rxjs.from([...ops, _op({type: 'commit'})]))) }, -}) +} satisfies IntegrationDef -const _op: typeof R.identity = R.identity +export default togglDef diff --git a/integrations/integration-toggl/index.ts b/integrations/integration-toggl/index.ts index d6ac8f93..10432be2 100644 --- a/integrations/integration-toggl/index.ts +++ b/integrations/integration-toggl/index.ts @@ -1,4 +1,5 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' +export * from './server' export * from './TogglCient' -export * from './TogglProvider' // codegen:end diff --git a/integrations/integration-toggl/server.ts b/integrations/integration-toggl/server.ts new file mode 100644 index 00000000..93674624 --- /dev/null +++ b/integrations/integration-toggl/server.ts @@ -0,0 +1,39 @@ +import type {IntegrationServer} from '@usevenice/cdk-core' +import {Rx, rxjs} from '@usevenice/util' + +import type {togglSchemas} from './def' +import {togglHelpers} from './def' +import {makeTogglClient} from './TogglCient' + +export const togglServer = { + postConnect: (input) => ({ + resourceExternalId: input.apiToken, + settings: { + apiToken: input.apiToken, + email: input.email, + password: input.password, + }, + triggerDefaultSync: true, + }), + + sourceSync: ({settings}) => { + const client = makeTogglClient({...settings}) + async function* iterateEntities() { + const user = await client.getMe() + const res = await client.getProjects(`${user.default_workspace_id}`) + yield res.map((a) => togglHelpers._opData('account', `${a.id}`, a)) + + // TODO: Need to pass params if necessary + const res2 = await client.getTimeEntries() + yield res2.map((t) => togglHelpers._opData('transaction', `${t.id}`, t)) + } + + return rxjs + .from(iterateEntities()) + .pipe( + Rx.mergeMap((ops) => rxjs.from([...ops, togglHelpers._op('commit')])), + ) + }, +} satisfies IntegrationServer + +export default togglServer From 44fb7f55c2cb5c58fa935ea8ca24877780ade7d5 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 18:19:10 +0800 Subject: [PATCH 07/80] Migrate int foreceipt --- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 ++ .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 6 +- apps/app-config/providers.ts | 3 - .../{ForeceiptProvider.ts => def.ts} | 108 +++--------------- integrations/integration-foreceipt/index.ts | 3 +- integrations/integration-foreceipt/server.ts | 94 +++++++++++++++ 8 files changed, 128 insertions(+), 98 deletions(-) rename integrations/integration-foreceipt/{ForeceiptProvider.ts => def.ts} (58%) create mode 100644 integrations/integration-foreceipt/server.ts diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index d6fdef29..128a62ff 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -1,6 +1,7 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand import {default as integrationAirtable} from '@usevenice/integration-airtable/def' import {default as integrationBrex} from '@usevenice/integration-brex/def' +import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/def' import {default as integrationHeron} from '@usevenice/integration-heron/def' import {default as integrationMercury} from '@usevenice/integration-mercury/def' import {default as integrationMerge} from '@usevenice/integration-merge/def' @@ -19,6 +20,7 @@ import {default as integrationYodlee} from '@usevenice/integration-yodlee/def' export const defIntegrations = { airtable: integrationAirtable, brex: integrationBrex, + foreceipt: integrationForeceipt, heron: integrationHeron, mercury: integrationMercury, merge: integrationMerge, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 5fd62add..1fb6db45 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -3,6 +3,8 @@ import {default as integrationAirtable_def} from '@usevenice/integration-airtabl import {default as integrationAirtable_server} from '@usevenice/integration-airtable/server' import {default as integrationBrex_def} from '@usevenice/integration-brex/def' import {default as integrationBrex_server} from '@usevenice/integration-brex/server' +import {default as integrationForeceipt_def} from '@usevenice/integration-foreceipt/def' +import {default as integrationForeceipt_server} from '@usevenice/integration-foreceipt/server' import {default as integrationHeron_def} from '@usevenice/integration-heron/def' import {default as integrationHeron_server} from '@usevenice/integration-heron/server' import {default as integrationMercury_def} from '@usevenice/integration-mercury/def' @@ -45,6 +47,11 @@ const integrationBrex = { ...integrationBrex_server, } +const integrationForeceipt = { + ...integrationForeceipt_def, + ...integrationForeceipt_server, +} + const integrationHeron = { ...integrationHeron_def, ...integrationHeron_server, @@ -121,6 +128,7 @@ const integrationYodlee = { export const mergedIntegrations = { airtable: integrationAirtable, brex: integrationBrex, + foreceipt: integrationForeceipt, heron: integrationHeron, mercury: integrationMercury, merge: integrationMerge, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index c7a5c115..bf171317 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -1,6 +1,7 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand import {default as integrationAirtable} from '@usevenice/integration-airtable/server' import {default as integrationBrex} from '@usevenice/integration-brex/server' +import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/server' import {default as integrationHeron} from '@usevenice/integration-heron/server' import {default as integrationMerge} from '@usevenice/integration-merge/server' import {default as integrationMoota} from '@usevenice/integration-moota/server' @@ -17,6 +18,7 @@ import {default as integrationYodlee} from '@usevenice/integration-yodlee/server export const serverIntegrations = { airtable: integrationAirtable, brex: integrationBrex, + foreceipt: integrationForeceipt, heron: integrationHeron, merge: integrationMerge, moota: integrationMoota, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index e0b86856..044e253a 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -61,9 +61,13 @@ module.exports = [ imports: {}, }, { + name: 'foreceipt', dirName: 'integration-foreceipt', varName: 'integrationForeceipt', - imports: {}, + imports: { + def: '@usevenice/integration-foreceipt/def', + server: '@usevenice/integration-foreceipt/server', + }, }, { name: 'heron', diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index dde09842..21573ce8 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -7,13 +7,11 @@ import {mongodbProvider} from '@usevenice/core-integration-mongodb' import {corePostgresProvider} from '@usevenice/core-integration-postgres' import {webhookProvider} from '@usevenice/core-integration-webhook' import {beancountProvider} from '@usevenice/integration-beancount' -import {foreceiptProvider} from '@usevenice/integration-foreceipt' import {lunchmoneyProvider} from '@usevenice/integration-lunchmoney' import {plaidProvider} from '@usevenice/integration-plaid' import {postgresProvider} from '@usevenice/integration-postgres' import {rampProvider} from '@usevenice/integration-ramp' import {spreadsheetProvider} from '@usevenice/integration-spreadsheet' -import {togglProvider} from '@usevenice/integration-toggl' import {wiseProvider} from '@usevenice/integration-wise' import {mergedIntegrations} from './integrations/integrations.merged' @@ -42,5 +40,4 @@ export const PROVIDERS = [ lunchmoneyProvider, rampProvider, wiseProvider, - foreceiptProvider, ] as const diff --git a/integrations/integration-foreceipt/ForeceiptProvider.ts b/integrations/integration-foreceipt/def.ts similarity index 58% rename from integrations/integration-foreceipt/ForeceiptProvider.ts rename to integrations/integration-foreceipt/def.ts index 91fef16c..7f9bbfa5 100644 --- a/integrations/integration-foreceipt/ForeceiptProvider.ts +++ b/integrations/integration-foreceipt/def.ts @@ -1,21 +1,17 @@ -import {makeSyncProvider} from '@usevenice/cdk-core' -import {makePostingsMap, veniceProviderBase} from '@usevenice/cdk-ledger' +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {makePostingsMap} from '@usevenice/cdk-ledger' import type {SerializedTimestamp} from '@usevenice/core-integration-firebase' -import { - firebaseProvider, - serializeTimestamp, -} from '@usevenice/core-integration-firebase' import type {Standard} from '@usevenice/standard' import type {Merge} from '@usevenice/util' -import {A, objectFromArray, R, Rx, rxjs, z, zCast} from '@usevenice/util' +import {A, objectFromArray, R, z, zCast} from '@usevenice/util' import type {_parseResourceInfo} from './foreceipt-utils' import type {ForeceiptClientOptions} from './ForeceiptClient' import {makeForeceiptClient, zForeceiptConfig} from './ForeceiptClient' // type ForeceiptSyncOperation = typeof def['_opType'] -const _def = makeSyncProvider.def({ - ...veniceProviderBase.def, +export const foreceiptSchemas = { name: z.literal('foreceipt'), // integrationConfig: zForeceiptConfig, resourceSettings: z.object({ @@ -50,12 +46,15 @@ const _def = makeSyncProvider.def({ >(), }), ]), -}) -const def = makeSyncProvider.def.helpers(_def) +} satisfies IntegrationSchemas -export const foreceiptProvider = makeSyncProvider({ +export const foreceiptHelpers = intHelpers(foreceiptSchemas) + +export const foreceiptDef = { + name: 'foreceipt', + def: foreceiptSchemas, metadata: {categories: ['expense-management']}, - ...veniceProviderBase(def, { + extension: { sourceMapEntity: { account: ({entity: a}) => ({ id: `${a.id}`, @@ -155,84 +154,7 @@ export const foreceiptProvider = makeSyncProvider({ } }, }, - }), - // TODO: Need to check and fix the issue - // postConnect: async (input, config) => { - // const settings = def._type('resourceSettings', { - // ...input, - // }) - // const source$: rxjs.Observable = - // foreceiptProvider.sourceSync({settings, config, options: {}}) - - // return { - // externalId: `${input._id}`, - // settings, - // source$, - // } - // }, - - sourceSync: ({settings}) => { - const client = makeForeceiptClient({...settings}) - const getInfo = client.getInfo - let info: Awaited> - const raw$ = rxjs.of(client.initFb()).pipe( - Rx.mergeMap((fb) => { - console.log(client.fbSettings, '===firebase init ===') - return rxjs - .from(client.getQuery$()) - .pipe( - Rx.mergeMap(([q, res]) => { - info = res - return firebaseProvider.sourceSync({ - endUser: null, - settings: client.fbSettings, - state: {_fb: fb, _queries: Object.values(q)}, - }) - }), - ) - .pipe( - // Hack it with concatMap - // TODO: Need to get better understanding of rxjs by re-read references from @tony's, concatMap is very slow, but also cannot use the mergeMap. Need check another map - // Or we should cache the http request - Rx.mergeMap((op) => { - const r = - op.type === 'data' && op.data.entityName === 'Receipts' - ? (op.data.entity as Foreceipt.Receipt) - : null - return rxjs.of( - op.type !== 'data' - ? def._op('commit') - : def._op('data', { - data: - op.data.entityName === 'Receipts' - ? { - id: r?.content.id ?? op.data.id, - entity: { - ...r, - _docId: op.data.id, - create_time: serializeTimestamp( - r?.create_time as FirebaseFirestore.Timestamp, - ), - last_update_time: serializeTimestamp( - r?.last_update_time as FirebaseFirestore.Timestamp, - ), - } as Foreceipt.Receipt, - entityName: 'transaction', - info, - } - : { - id: op.data.id, - entity: op.data.entity as Foreceipt.Account, - entityName: 'account', - info, - }, - }), - ) - }), - ) - }), - ) - - return raw$.pipe(Rx.mergeMap((op) => rxjs.of(op))) }, -}) +} satisfies IntegrationDef + +export default foreceiptDef diff --git a/integrations/integration-foreceipt/index.ts b/integrations/integration-foreceipt/index.ts index 9776c71b..534f7fa5 100644 --- a/integrations/integration-foreceipt/index.ts +++ b/integrations/integration-foreceipt/index.ts @@ -1,5 +1,6 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' export * from './foreceipt-utils' export * from './ForeceiptClient' -export * from './ForeceiptProvider' +export * from './server' // codegen:end diff --git a/integrations/integration-foreceipt/server.ts b/integrations/integration-foreceipt/server.ts new file mode 100644 index 00000000..9c9b6d1c --- /dev/null +++ b/integrations/integration-foreceipt/server.ts @@ -0,0 +1,94 @@ +import type {IntegrationServer} from '@usevenice/cdk-core' +import { + firebaseProvider, + serializeTimestamp, +} from '@usevenice/core-integration-firebase' +import {Rx, rxjs} from '@usevenice/util' + +import type {foreceiptSchemas} from './def' +import {foreceiptHelpers} from './def' +import {makeForeceiptClient} from './ForeceiptClient' + +export const foreceiptServer = { + // TODO: Need to check and fix the issue + // postConnect: async (input, config) => { + // const settings = foreceiptHelpers._type('resourceSettings', { + // ...input, + // }) + // const source$: rxjs.Observable = + // foreceiptProvider.sourceSync({settings, config, options: {}}) + + // return { + // externalId: `${input._id}`, + // settings, + // source$, + // } + // }, + + sourceSync: ({settings}) => { + const client = makeForeceiptClient({...settings}) + const getInfo = client.getInfo + let info: Awaited> + const raw$ = rxjs.of(client.initFb()).pipe( + Rx.mergeMap((fb) => { + console.log(client.fbSettings, '===firebase init ===') + return rxjs + .from(client.getQuery$()) + .pipe( + Rx.mergeMap(([q, res]) => { + info = res + return firebaseProvider.sourceSync({ + endUser: null, + settings: client.fbSettings, + state: {_fb: fb, _queries: Object.values(q)}, + }) + }), + ) + .pipe( + // Hack it with concatMap + // TODO: Need to get better understanding of rxjs by re-read references from @tony's, concatMap is very slow, but also cannot use the mergeMap. Need check another map + // Or we should cache the http request + Rx.mergeMap((op) => { + const r = + op.type === 'data' && op.data.entityName === 'Receipts' + ? (op.data.entity as Foreceipt.Receipt) + : null + return rxjs.of( + op.type !== 'data' + ? foreceiptHelpers._op('commit') + : foreceiptHelpers._op('data', { + data: + op.data.entityName === 'Receipts' + ? { + id: r?.content.id ?? op.data.id, + entity: { + ...r, + _docId: op.data.id, + create_time: serializeTimestamp( + r?.create_time as FirebaseFirestore.Timestamp, + ), + last_update_time: serializeTimestamp( + r?.last_update_time as FirebaseFirestore.Timestamp, + ), + } as Foreceipt.Receipt, + entityName: 'transaction', + info, + } + : { + id: op.data.id, + entity: op.data.entity as Foreceipt.Account, + entityName: 'account', + info, + }, + }), + ) + }), + ) + }), + ) + + return raw$.pipe(Rx.mergeMap((op) => rxjs.of(op))) + }, +} satisfies IntegrationServer + +export default foreceiptServer From d3ee3d9f0ee85091e422d380a1fafc85d9663dda Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 18:24:30 +0800 Subject: [PATCH 08/80] MIgrate wise --- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 + .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 10 +- apps/app-config/providers.ts | 1 - integrations/integration-wise/WiseProvider.ts | 138 ------------------ integrations/integration-wise/def.ts | 86 +++++++++++ integrations/integration-wise/index.ts | 3 +- integrations/integration-wise/server.ts | 58 ++++++++ 9 files changed, 167 insertions(+), 141 deletions(-) delete mode 100644 integrations/integration-wise/WiseProvider.ts create mode 100644 integrations/integration-wise/def.ts create mode 100644 integrations/integration-wise/server.ts diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index 128a62ff..a794f5fd 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -15,6 +15,7 @@ import {default as integrationStripe} from '@usevenice/integration-stripe/def' import {default as integrationTeller} from '@usevenice/integration-teller/def' import {default as integrationToggl} from '@usevenice/integration-toggl/def' import {default as integrationVenmo} from '@usevenice/integration-venmo/def' +import {default as integrationWise} from '@usevenice/integration-wise/def' import {default as integrationYodlee} from '@usevenice/integration-yodlee/def' export const defIntegrations = { @@ -34,5 +35,6 @@ export const defIntegrations = { teller: integrationTeller, toggl: integrationToggl, venmo: integrationVenmo, + wise: integrationWise, yodlee: integrationYodlee, } diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 1fb6db45..7c847af0 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -33,6 +33,8 @@ import {default as integrationTeller_server} from '@usevenice/integration-teller import {default as integrationToggl_def} from '@usevenice/integration-toggl/def' import {default as integrationToggl_server} from '@usevenice/integration-toggl/server' import {default as integrationVenmo_def} from '@usevenice/integration-venmo/def' +import {default as integrationWise_def} from '@usevenice/integration-wise/def' +import {default as integrationWise_server} from '@usevenice/integration-wise/server' import {default as integrationYodlee_client} from '@usevenice/integration-yodlee/client' import {default as integrationYodlee_def} from '@usevenice/integration-yodlee/def' import {default as integrationYodlee_server} from '@usevenice/integration-yodlee/server' @@ -119,6 +121,11 @@ const integrationVenmo = { ...integrationVenmo_def, } +const integrationWise = { + ...integrationWise_def, + ...integrationWise_server, +} + const integrationYodlee = { ...integrationYodlee_def, ...integrationYodlee_client, @@ -142,5 +149,6 @@ export const mergedIntegrations = { teller: integrationTeller, toggl: integrationToggl, venmo: integrationVenmo, + wise: integrationWise, yodlee: integrationYodlee, } diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index bf171317..0ddf4383 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -13,6 +13,7 @@ import {default as integrationSplitwise} from '@usevenice/integration-splitwise/ import {default as integrationStripe} from '@usevenice/integration-stripe/server' import {default as integrationTeller} from '@usevenice/integration-teller/server' import {default as integrationToggl} from '@usevenice/integration-toggl/server' +import {default as integrationWise} from '@usevenice/integration-wise/server' import {default as integrationYodlee} from '@usevenice/integration-yodlee/server' export const serverIntegrations = { @@ -30,5 +31,6 @@ export const serverIntegrations = { stripe: integrationStripe, teller: integrationTeller, toggl: integrationToggl, + wise: integrationWise, yodlee: integrationYodlee, } diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index 044e253a..b06f41e7 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -200,7 +200,15 @@ module.exports = [ varName: 'integrationVenmo', imports: {def: '@usevenice/integration-venmo/def'}, }, - {dirName: 'integration-wise', varName: 'integrationWise', imports: {}}, + { + name: 'wise', + dirName: 'integration-wise', + varName: 'integrationWise', + imports: { + def: '@usevenice/integration-wise/def', + server: '@usevenice/integration-wise/server', + }, + }, { name: 'yodlee', dirName: 'integration-yodlee', diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 21573ce8..81dc333b 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -39,5 +39,4 @@ export const PROVIDERS = [ spreadsheetProvider, lunchmoneyProvider, rampProvider, - wiseProvider, ] as const diff --git a/integrations/integration-wise/WiseProvider.ts b/integrations/integration-wise/WiseProvider.ts deleted file mode 100644 index 8d323e1f..00000000 --- a/integrations/integration-wise/WiseProvider.ts +++ /dev/null @@ -1,138 +0,0 @@ -import {makeSyncProvider} from '@usevenice/cdk-core' -import {makePostingsMap, veniceProviderBase} from '@usevenice/cdk-ledger' -import {A, R, Rx, rxjs, z} from '@usevenice/util' - -import { - makeWiseClient, - profileResponseItemSchema, - transferResponseItemSchema, - zEnvName, -} from './WiseClient' - -type WiseSyncOperation = (typeof def)['_opType'] - -const def = makeSyncProvider.def({ - ...veniceProviderBase.def, - name: z.literal('wise'), - // integrationConfig: zWiseConfig, - resourceSettings: z.object({ - envName: zEnvName, - apiToken: z.string().nullish(), - }), - connectInput: z.object({ - redirectUri: z.string(), - clientId: z.string(), - envName: zEnvName, - }), - connectOutput: z.object({ - envName: zEnvName, - apiToken: z.string().nullish(), - }), - sourceOutputEntity: z.discriminatedUnion('entityName', [ - z.object({ - id: z.string(), - entityName: z.literal('account'), - entity: profileResponseItemSchema, - }), - z.object({ - id: z.string(), - entityName: z.literal('transaction'), - entity: transferResponseItemSchema, - }), - ]), -}) - -export const wiseProvider = makeSyncProvider({ - metadata: {categories: ['banking'], logoUrl: '/_assets/logo-wise.png'}, - ...veniceProviderBase(def, { - sourceMapEntity: (data) => { - if (data.entityName === 'account') { - const a = data.entity - return { - id: `${a.id}`, - entityName: 'account', - entity: { - name: `${data.entity.details.firstName} ${data.entity.details.lastName}`, - type: 'expense', - }, - } - } else if (data.entityName === 'transaction') { - const t = data.entity - - return { - id: `${t.id}`, - entityName: 'transaction', - entity: { - date: t.created, - description: t.details.reference ?? '', - postingsMap: makePostingsMap({ - main: { - accountExternalId: (data.entity.quoteUuid ?? '') as ExternalId, - amount: A( - data.entity.sourceValue ?? 0, - (data.entity.sourceCurrency ?? 'USD') as Unit, - ), - }, - }), - }, - } - } - return null - }, - }), - - postConnect: (input) => ({ - resourceExternalId: input.apiToken ?? '', - settings: { - envName: input.envName ?? '', - apiToken: input.apiToken, - }, - }), - sourceSync: ({settings}) => { - const client = makeWiseClient({...settings}) - async function* iterateEntities() { - const res = await client.getProfiles(settings.envName) - yield res.map((a) => - _op({ - type: 'data', - data: {id: `${a.id}`, entity: a, entityName: 'account'}, - }), - ) - - const combineRes = await Promise.all( - res.map( - async (a: z.infer) => - await client.getTransfers({ - envName: settings.envName, - profileId: a.id, - }), - ), - ) - // TODO: Need to check is it better than use the basic promise all - // const res2 = await rxjs.firstValueFrom( - // rxjs - // .from(res) - // .pipe( - // rxjs.mergeMap( (el: z.infer) => - // rxjs.from(client.getTransfers({envName: input.envName, profileId: el.id}),) - // ), - // rxjs.toArray() - // ), - // ) - const res2 = combineRes.flat(1) - - yield res2.map((t) => - _op({ - type: 'data', - data: {id: `${t.id}`, entity: t, entityName: 'transaction'}, - }), - ) - } - - return rxjs - .from(iterateEntities()) - .pipe(Rx.mergeMap((ops) => rxjs.from([...ops, _op({type: 'commit'})]))) - }, -}) - -const _op: typeof R.identity = R.identity diff --git a/integrations/integration-wise/def.ts b/integrations/integration-wise/def.ts new file mode 100644 index 00000000..96c56d83 --- /dev/null +++ b/integrations/integration-wise/def.ts @@ -0,0 +1,86 @@ +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {makePostingsMap} from '@usevenice/cdk-ledger' +import {A, z} from '@usevenice/util' + +import { + profileResponseItemSchema, + transferResponseItemSchema, + zEnvName, +} from './WiseClient' + +export const wiseSchemas = { + name: z.literal('wise'), + // integrationConfig: zWiseConfig, + resourceSettings: z.object({ + envName: zEnvName, + apiToken: z.string().nullish(), + }), + connectInput: z.object({ + redirectUri: z.string(), + clientId: z.string(), + envName: zEnvName, + }), + connectOutput: z.object({ + envName: zEnvName, + apiToken: z.string().nullish(), + }), + sourceOutputEntity: z.discriminatedUnion('entityName', [ + z.object({ + id: z.string(), + entityName: z.literal('account'), + entity: profileResponseItemSchema, + }), + z.object({ + id: z.string(), + entityName: z.literal('transaction'), + entity: transferResponseItemSchema, + }), + ]), +} satisfies IntegrationSchemas + +export const wiseHelpers = intHelpers(wiseSchemas) + +export const wiseDef = { + name: 'wise', + def: wiseSchemas, + metadata: {categories: ['banking'], logoUrl: '/_assets/logo-wise.png'}, + extension: { + sourceMapEntity: (data) => { + if (data.entityName === 'account') { + const a = data.entity + return { + id: `${a.id}`, + entityName: 'account', + entity: { + name: `${data.entity.details.firstName} ${data.entity.details.lastName}`, + type: 'expense', + }, + } + } else if (data.entityName === 'transaction') { + const t = data.entity + + return { + id: `${t.id}`, + entityName: 'transaction', + entity: { + date: t.created, + description: t.details.reference ?? '', + postingsMap: makePostingsMap({ + main: { + accountExternalId: (data.entity.quoteUuid ?? '') as ExternalId, + amount: A( + data.entity.sourceValue ?? 0, + (data.entity.sourceCurrency ?? 'USD') as Unit, + ), + }, + }), + }, + } + } + return null + }, + }, +} satisfies IntegrationDef + +export default wiseDef diff --git a/integrations/integration-wise/index.ts b/integrations/integration-wise/index.ts index 80b4b833..80d67897 100644 --- a/integrations/integration-wise/index.ts +++ b/integrations/integration-wise/index.ts @@ -1,4 +1,5 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' +export * from './server' export * from './WiseClient' -export * from './WiseProvider' // codegen:end diff --git a/integrations/integration-wise/server.ts b/integrations/integration-wise/server.ts new file mode 100644 index 00000000..943ad660 --- /dev/null +++ b/integrations/integration-wise/server.ts @@ -0,0 +1,58 @@ +import type {IntegrationServer} from '@usevenice/cdk-core' +import type {z} from '@usevenice/util' +import {Rx, rxjs} from '@usevenice/util' + +import type {wiseSchemas} from './def' +import {wiseHelpers} from './def' +import type {profileResponseItemSchema} from './WiseClient' +import {makeWiseClient} from './WiseClient' + +export const wiseServer = { + postConnect: (input) => ({ + resourceExternalId: input.apiToken ?? '', + settings: { + envName: input.envName ?? '', + apiToken: input.apiToken, + }, + }), + sourceSync: ({settings}) => { + const client = makeWiseClient({...settings}) + async function* iterateEntities() { + const res = await client.getProfiles(settings.envName) + wiseHelpers + yield res.map((a) => wiseHelpers._opData('account', `${a.id}`, a)) + + const combineRes = await Promise.all( + res.map( + async (a: z.infer) => + await client.getTransfers({ + envName: settings.envName, + profileId: a.id, + }), + ), + ) + // TODO: Need to check is it better than use the basic promise all + // const res2 = await rxjs.firstValueFrom( + // rxjs + // .from(res) + // .pipe( + // rxjs.mergeMap( (el: z.infer) => + // rxjs.from(client.getTransfers({envName: input.envName, profileId: el.id}),) + // ), + // rxjs.toArray() + // ), + // ) + const res2 = combineRes.flat(1) + + yield res2.map((t) => wiseHelpers._opData('transaction', `${t.id}`, t)) + } + + return rxjs + .from(iterateEntities()) + .pipe( + Rx.mergeMap((ops) => rxjs.from([...ops, wiseHelpers._op('commit')])), + ) + }, +} satisfies IntegrationServer + +export default wiseServer From 439d2af20df8ba7102306c8a923f90a953188dce Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 18:30:04 +0800 Subject: [PATCH 09/80] MIgrate ramp --- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 + .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 10 +- apps/app-config/providers.ts | 3 - integrations/integration-ramp/RampProvider.ts | 177 ------------------ integrations/integration-ramp/def.ts | 96 ++++++++++ integrations/integration-ramp/index.ts | 3 +- integrations/integration-ramp/server.ts | 70 +++++++ 9 files changed, 189 insertions(+), 182 deletions(-) delete mode 100644 integrations/integration-ramp/RampProvider.ts create mode 100644 integrations/integration-ramp/def.ts create mode 100644 integrations/integration-ramp/server.ts diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index a794f5fd..5e4f20c5 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -9,6 +9,7 @@ import {default as integrationMoota} from '@usevenice/integration-moota/def' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/def' import {default as integrationPlaid} from '@usevenice/integration-plaid/def' import {default as integrationQbo} from '@usevenice/integration-qbo/def' +import {default as integrationRamp} from '@usevenice/integration-ramp/def' import {default as integrationSaltedge} from '@usevenice/integration-saltedge/def' import {default as integrationSplitwise} from '@usevenice/integration-splitwise/def' import {default as integrationStripe} from '@usevenice/integration-stripe/def' @@ -29,6 +30,7 @@ export const defIntegrations = { onebrick: integrationOnebrick, plaid: integrationPlaid, qbo: integrationQbo, + ramp: integrationRamp, saltedge: integrationSaltedge, splitwise: integrationSplitwise, stripe: integrationStripe, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 7c847af0..280559d4 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -21,6 +21,8 @@ import {default as integrationPlaid_def} from '@usevenice/integration-plaid/def' import {default as integrationPlaid_server} from '@usevenice/integration-plaid/server' import {default as integrationQbo_def} from '@usevenice/integration-qbo/def' import {default as integrationQbo_server} from '@usevenice/integration-qbo/server' +import {default as integrationRamp_def} from '@usevenice/integration-ramp/def' +import {default as integrationRamp_server} from '@usevenice/integration-ramp/server' import {default as integrationSaltedge_def} from '@usevenice/integration-saltedge/def' import {default as integrationSaltedge_server} from '@usevenice/integration-saltedge/server' import {default as integrationSplitwise_def} from '@usevenice/integration-splitwise/def' @@ -91,6 +93,11 @@ const integrationQbo = { ...integrationQbo_server, } +const integrationRamp = { + ...integrationRamp_def, + ...integrationRamp_server, +} + const integrationSaltedge = { ...integrationSaltedge_def, ...integrationSaltedge_server, @@ -143,6 +150,7 @@ export const mergedIntegrations = { onebrick: integrationOnebrick, plaid: integrationPlaid, qbo: integrationQbo, + ramp: integrationRamp, saltedge: integrationSaltedge, splitwise: integrationSplitwise, stripe: integrationStripe, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index 0ddf4383..9a5877fc 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -8,6 +8,7 @@ import {default as integrationMoota} from '@usevenice/integration-moota/server' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/server' import {default as integrationPlaid} from '@usevenice/integration-plaid/server' import {default as integrationQbo} from '@usevenice/integration-qbo/server' +import {default as integrationRamp} from '@usevenice/integration-ramp/server' import {default as integrationSaltedge} from '@usevenice/integration-saltedge/server' import {default as integrationSplitwise} from '@usevenice/integration-splitwise/server' import {default as integrationStripe} from '@usevenice/integration-stripe/server' @@ -26,6 +27,7 @@ export const serverIntegrations = { onebrick: integrationOnebrick, plaid: integrationPlaid, qbo: integrationQbo, + ramp: integrationRamp, saltedge: integrationSaltedge, splitwise: integrationSplitwise, stripe: integrationStripe, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index b06f41e7..720dbc3e 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -142,7 +142,15 @@ module.exports = [ server: '@usevenice/integration-qbo/server', }, }, - {dirName: 'integration-ramp', varName: 'integrationRamp', imports: {}}, + { + name: 'ramp', + dirName: 'integration-ramp', + varName: 'integrationRamp', + imports: { + def: '@usevenice/integration-ramp/def', + server: '@usevenice/integration-ramp/server', + }, + }, { name: 'saltedge', dirName: 'integration-saltedge', diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 81dc333b..c8fb1c19 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -10,9 +10,7 @@ import {beancountProvider} from '@usevenice/integration-beancount' import {lunchmoneyProvider} from '@usevenice/integration-lunchmoney' import {plaidProvider} from '@usevenice/integration-plaid' import {postgresProvider} from '@usevenice/integration-postgres' -import {rampProvider} from '@usevenice/integration-ramp' import {spreadsheetProvider} from '@usevenice/integration-spreadsheet' -import {wiseProvider} from '@usevenice/integration-wise' import {mergedIntegrations} from './integrations/integrations.merged' @@ -38,5 +36,4 @@ export const PROVIDERS = [ beancountProvider, spreadsheetProvider, lunchmoneyProvider, - rampProvider, ] as const diff --git a/integrations/integration-ramp/RampProvider.ts b/integrations/integration-ramp/RampProvider.ts deleted file mode 100644 index 19b8492b..00000000 --- a/integrations/integration-ramp/RampProvider.ts +++ /dev/null @@ -1,177 +0,0 @@ -import {makeSyncProvider, zIntAuth} from '@usevenice/cdk-core' -import { - makePostingsMap, - makeStandardId, - veniceProviderBase, -} from '@usevenice/cdk-ledger' -import {A, md5Hash, R, Rx, rxjs, z} from '@usevenice/util' - -import { - businessResponseSchema, - makeRampClient, - transactionResponseItemSchema, -} from './RampClient' - -const kRamp = 'ramp' as const -type RampEntity = z.infer<(typeof def)['sourceOutputEntity']> -type RampSrcSyncOptions = z.infer<(typeof def)['sourceState']> -type RampSyncOperation = (typeof def)['_opType'] - -const def = makeSyncProvider.def({ - ...veniceProviderBase.def, - name: z.literal('ramp'), - integrationConfig: zIntAuth.oauth, - resourceSettings: z.object({ - accessToken: z.string().nullish(), - startAfterTransactionId: z.string().nullish(), - }), - connectInput: z.object({ - accessToken: z.string().nullish(), - }), - connectOutput: z.object({ - accessToken: z.string().nullish(), - clientId: z.string().nullish(), - clientSecret: z.string().nullish(), - }), - sourceOutputEntity: z.discriminatedUnion('entityName', [ - z.object({ - id: z.string(), - entityName: z.literal('account'), - entity: businessResponseSchema, - }), - z.object({ - id: z.string(), - entityName: z.literal('transaction'), - entity: transactionResponseItemSchema, - }), - ]), - sourceState: z.object({ - startAfterTransactionId: z.string().nullish(), - accessToken: z.string().nullish(), - clientId: z.string().nullish(), - clientSecret: z.string().nullish(), - }), -}) - -export const rampProvider = makeSyncProvider({ - metadata: { - categories: ['banking', 'expense-management'], - logoUrl: '/_assets/logo-ramp.png', - stage: 'beta', - }, - ...veniceProviderBase(def, { - sourceMapEntity: { - account: ({entity: a}) => ({ - id: a.id, - entityName: 'account', - entity: { - name: `${a.business_name_on_card}`, - type: 'asset/bank', - institutionName: a.business_name_legal, - }, - }), - transaction: ({entity: t}) => ({ - id: t.id, - entityName: 'transaction', - entity: { - date: t.user_transaction_time, - description: t.merchant_descriptor ?? '', - payee: t.merchant_name, - externalCategory: t.sk_category_name ?? '', - postingsMap: makePostingsMap({ - main: { - amount: A(-1 * t.amount, 'USD' as Unit), - memo: - t.memo ?? - R.compact([ - `${t.card_holder.first_name} ${t.card_holder.last_name}`, - t.merchant_category_code, - ]).join('/'), - subAccountKey: t.state.toLowerCase() ?? undefined, - }, - }), - custom: { - user: `${t.card_holder.first_name} ${t.card_holder.last_name}`, - }, - }, - }), - }, - }), - - postConnect: (input) => ({ - resourceExternalId: input.clientId ?? '', - settings: input, - triggerDefaultSync: true, - }), - - // Disable it for now until it's ready - // handleWebhook: (input) => { - // const conn = identity>({ - // clientId: '', - // clientSecret: '', - // authorizationCode: input.query['code'] as string, - // }) - // const sync$: rxjs.Observable = - // rampProvider.sourceSync(conn) - // return rxjs.concat(sync$) - // }, - sourceSync: ({settings, config}) => { - const client = makeRampClient(config.oauth) - async function* iterateEntities() { - const accessToken = await client.getAccessToken() - // const accessToken = await client.getToken({ - // code: input.authorizationCode ?? '', - // redirectUri: `${redirectUri}/api/webhook/ramp`, - // }) - // TODO: Need to do pagination for account if it necessary - const res = [await client.getBusiness(accessToken)] - yield res.map((a) => - _op({ - type: 'data', - data: {id: a.id, entity: a, entityName: 'account'}, - }), - ) - - let starting_after = settings.startAfterTransactionId ?? undefined - while (true) { - const res2 = await client.getTransactions({ - accessToken, - start: - starting_after && starting_after.length > 0 - ? starting_after - : undefined, - }) - starting_after = res2.data[res2.data.length - 1]?.id ?? '' - yield [ - ...res2.data.map((t) => opData('transaction', t.id, t)), - // Use the hashed accessToken for now until we now what the id that can we use for meta data - opMeta(md5Hash(settings.accessToken ?? ''), { - startAfterTransactionId: starting_after, - }), - ] - if (!res2.page.next) { - break - } - } - } - - return rxjs - .from(iterateEntities()) - .pipe(Rx.mergeMap((ops) => rxjs.from([...ops, _op({type: 'commit'})]))) - }, -}) - -const _op: typeof R.identity = R.identity - -const opData = ( - entityName: K, - id: string, - entity: Extract['entity'] | null, -) => ({type: 'data', data: {entity, entityName, id}} as RampSyncOperation) - -const opMeta = (id: string, sourceState: Partial) => - ({ - type: 'resoUpdate', - id: makeStandardId('reso', kRamp, id), - sourceState, - } as RampSyncOperation) diff --git a/integrations/integration-ramp/def.ts b/integrations/integration-ramp/def.ts new file mode 100644 index 00000000..c4165677 --- /dev/null +++ b/integrations/integration-ramp/def.ts @@ -0,0 +1,96 @@ +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers, zIntAuth} from '@usevenice/cdk-core' +import {makePostingsMap} from '@usevenice/cdk-ledger' +import {A, R, z} from '@usevenice/util' + +import { + businessResponseSchema, + transactionResponseItemSchema, +} from './RampClient' + +export const rampSchemas = { + name: z.literal('ramp'), + integrationConfig: zIntAuth.oauth, + resourceSettings: z.object({ + accessToken: z.string().nullish(), + startAfterTransactionId: z.string().nullish(), + }), + connectInput: z.object({ + accessToken: z.string().nullish(), + }), + connectOutput: z.object({ + accessToken: z.string().nullish(), + clientId: z.string().nullish(), + clientSecret: z.string().nullish(), + }), + sourceOutputEntity: z.discriminatedUnion('entityName', [ + z.object({ + id: z.string(), + entityName: z.literal('account'), + entity: businessResponseSchema, + }), + z.object({ + id: z.string(), + entityName: z.literal('transaction'), + entity: transactionResponseItemSchema, + }), + ]), + sourceState: z.object({ + startAfterTransactionId: z.string().nullish(), + accessToken: z.string().nullish(), + clientId: z.string().nullish(), + clientSecret: z.string().nullish(), + }), +} satisfies IntegrationSchemas + +export const rampHelpers = intHelpers(rampSchemas) + +export const rampDef = { + name: 'ramp', + def: rampSchemas, + metadata: { + categories: ['banking', 'expense-management'], + logoUrl: '/_assets/logo-ramp.png', + stage: 'beta', + }, + extension: { + sourceMapEntity: { + account: ({entity: a}) => ({ + id: a.id, + entityName: 'account', + entity: { + name: `${a.business_name_on_card}`, + type: 'asset/bank', + institutionName: a.business_name_legal, + }, + }), + transaction: ({entity: t}) => ({ + id: t.id, + entityName: 'transaction', + entity: { + date: t.user_transaction_time, + description: t.merchant_descriptor ?? '', + payee: t.merchant_name, + externalCategory: t.sk_category_name ?? '', + postingsMap: makePostingsMap({ + main: { + amount: A(-1 * t.amount, 'USD' as Unit), + memo: + t.memo ?? + R.compact([ + `${t.card_holder.first_name} ${t.card_holder.last_name}`, + t.merchant_category_code, + ]).join('/'), + subAccountKey: t.state.toLowerCase() ?? undefined, + }, + }), + custom: { + user: `${t.card_holder.first_name} ${t.card_holder.last_name}`, + }, + }, + }), + }, + }, +} satisfies IntegrationDef + +export default rampDef diff --git a/integrations/integration-ramp/index.ts b/integrations/integration-ramp/index.ts index 832a45d7..eabe8fb5 100644 --- a/integrations/integration-ramp/index.ts +++ b/integrations/integration-ramp/index.ts @@ -1,4 +1,5 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' export * from './RampClient' -export * from './RampProvider' +export * from './server' // codegen:end diff --git a/integrations/integration-ramp/server.ts b/integrations/integration-ramp/server.ts new file mode 100644 index 00000000..f707c440 --- /dev/null +++ b/integrations/integration-ramp/server.ts @@ -0,0 +1,70 @@ +import type {IntegrationServer} from '@usevenice/cdk-core' +import {Rx, rxjs} from '@usevenice/util' + +import type {rampSchemas} from './def' +import {rampHelpers} from './def' +import {makeRampClient} from './RampClient' + +export const rampServer = { + postConnect: (input) => ({ + resourceExternalId: input.clientId ?? '', + settings: input, + triggerDefaultSync: true, + }), + + // Disable it for now until it's ready + // handleWebhook: (input) => { + // const conn = identity>({ + // clientId: '', + // clientSecret: '', + // authorizationCode: input.query['code'] as string, + // }) + // const sync$: rxjs.Observable = + // rampProvider.sourceSync(conn) + // return rxjs.concat(sync$) + // }, + sourceSync: ({settings, config}) => { + const client = makeRampClient(config.oauth) + async function* iterateEntities() { + const accessToken = await client.getAccessToken() + // const accessToken = await client.getToken({ + // code: input.authorizationCode ?? '', + // redirectUri: `${redirectUri}/api/webhook/ramp`, + // }) + // TODO: Need to do pagination for account if it necessary + const res = [await client.getBusiness(accessToken)] + yield res.map((a) => rampHelpers._opData('account', a.id, a)) + + let starting_after = settings.startAfterTransactionId ?? undefined + while (true) { + const res2 = await client.getTransactions({ + accessToken, + start: + starting_after && starting_after.length > 0 + ? starting_after + : undefined, + }) + starting_after = res2.data[res2.data.length - 1]?.id ?? '' + yield [ + ...res2.data.map((t) => rampHelpers._opData('transaction', t.id, t)), + // Use the hashed accessToken for now until we now what the id that can we use for meta data + rampHelpers._opState({ + // accessToken: md5Hash(settings.accessToken ?? ''), + startAfterTransactionId: starting_after, + }) as never, // Temp hack... not sure why + ] + if (!res2.page.next) { + break + } + } + } + + return rxjs + .from(iterateEntities()) + .pipe( + Rx.mergeMap((ops) => rxjs.from([...ops, rampHelpers._op('commit')])), + ) + }, +} satisfies IntegrationServer + +export default rampServer From 36b6260700a70d52a3c4554994a02fca8c816f0c Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 18:46:27 +0800 Subject: [PATCH 10/80] Migrate lunchMoney and spreadsheet integrations --- .../integrations/integrations.def.ts | 4 ++ .../integrations/integrations.merged.ts | 16 +++++ .../integrations/integrations.server.ts | 4 ++ apps/app-config/integrations/meta.js | 12 +++- apps/app-config/providers.ts | 4 -- .../{lunchmoneyProvider.ts => def.ts} | 60 +++++-------------- integrations/integration-lunchmoney/index.ts | 3 +- integrations/integration-lunchmoney/server.ts | 46 ++++++++++++++ .../{SpreadsheetProvider.ts => def.ts} | 60 +++++++------------ integrations/integration-spreadsheet/index.ts | 3 +- .../integration-spreadsheet/server.ts | 33 ++++++++++ 11 files changed, 153 insertions(+), 92 deletions(-) rename integrations/integration-lunchmoney/{lunchmoneyProvider.ts => def.ts} (63%) create mode 100644 integrations/integration-lunchmoney/server.ts rename integrations/integration-spreadsheet/{SpreadsheetProvider.ts => def.ts} (62%) create mode 100644 integrations/integration-spreadsheet/server.ts diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index 5e4f20c5..f57595a4 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -3,6 +3,7 @@ import {default as integrationAirtable} from '@usevenice/integration-airtable/de import {default as integrationBrex} from '@usevenice/integration-brex/def' import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/def' import {default as integrationHeron} from '@usevenice/integration-heron/def' +import {default as integrationLunchmoney} from '@usevenice/integration-lunchmoney/def' import {default as integrationMercury} from '@usevenice/integration-mercury/def' import {default as integrationMerge} from '@usevenice/integration-merge/def' import {default as integrationMoota} from '@usevenice/integration-moota/def' @@ -12,6 +13,7 @@ import {default as integrationQbo} from '@usevenice/integration-qbo/def' import {default as integrationRamp} from '@usevenice/integration-ramp/def' import {default as integrationSaltedge} from '@usevenice/integration-saltedge/def' import {default as integrationSplitwise} from '@usevenice/integration-splitwise/def' +import {default as integrationSpreadsheet} from '@usevenice/integration-spreadsheet/def' import {default as integrationStripe} from '@usevenice/integration-stripe/def' import {default as integrationTeller} from '@usevenice/integration-teller/def' import {default as integrationToggl} from '@usevenice/integration-toggl/def' @@ -24,6 +26,7 @@ export const defIntegrations = { brex: integrationBrex, foreceipt: integrationForeceipt, heron: integrationHeron, + lunchmoney: integrationLunchmoney, mercury: integrationMercury, merge: integrationMerge, moota: integrationMoota, @@ -33,6 +36,7 @@ export const defIntegrations = { ramp: integrationRamp, saltedge: integrationSaltedge, splitwise: integrationSplitwise, + spreadsheet: integrationSpreadsheet, stripe: integrationStripe, teller: integrationTeller, toggl: integrationToggl, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 280559d4..9824aec7 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -7,6 +7,8 @@ import {default as integrationForeceipt_def} from '@usevenice/integration-forece import {default as integrationForeceipt_server} from '@usevenice/integration-foreceipt/server' import {default as integrationHeron_def} from '@usevenice/integration-heron/def' import {default as integrationHeron_server} from '@usevenice/integration-heron/server' +import {default as integrationLunchmoney_def} from '@usevenice/integration-lunchmoney/def' +import {default as integrationLunchmoney_server} from '@usevenice/integration-lunchmoney/server' import {default as integrationMercury_def} from '@usevenice/integration-mercury/def' import {default as integrationMerge_client} from '@usevenice/integration-merge/client' import {default as integrationMerge_def} from '@usevenice/integration-merge/def' @@ -27,6 +29,8 @@ import {default as integrationSaltedge_def} from '@usevenice/integration-saltedg import {default as integrationSaltedge_server} from '@usevenice/integration-saltedge/server' import {default as integrationSplitwise_def} from '@usevenice/integration-splitwise/def' import {default as integrationSplitwise_server} from '@usevenice/integration-splitwise/server' +import {default as integrationSpreadsheet_def} from '@usevenice/integration-spreadsheet/def' +import {default as integrationSpreadsheet_server} from '@usevenice/integration-spreadsheet/server' import {default as integrationStripe_def} from '@usevenice/integration-stripe/def' import {default as integrationStripe_server} from '@usevenice/integration-stripe/server' import {default as integrationTeller_client} from '@usevenice/integration-teller/client' @@ -61,6 +65,11 @@ const integrationHeron = { ...integrationHeron_server, } +const integrationLunchmoney = { + ...integrationLunchmoney_def, + ...integrationLunchmoney_server, +} + const integrationMercury = { ...integrationMercury_def, } @@ -108,6 +117,11 @@ const integrationSplitwise = { ...integrationSplitwise_server, } +const integrationSpreadsheet = { + ...integrationSpreadsheet_def, + ...integrationSpreadsheet_server, +} + const integrationStripe = { ...integrationStripe_def, ...integrationStripe_server, @@ -144,6 +158,7 @@ export const mergedIntegrations = { brex: integrationBrex, foreceipt: integrationForeceipt, heron: integrationHeron, + lunchmoney: integrationLunchmoney, mercury: integrationMercury, merge: integrationMerge, moota: integrationMoota, @@ -153,6 +168,7 @@ export const mergedIntegrations = { ramp: integrationRamp, saltedge: integrationSaltedge, splitwise: integrationSplitwise, + spreadsheet: integrationSpreadsheet, stripe: integrationStripe, teller: integrationTeller, toggl: integrationToggl, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index 9a5877fc..566fd74b 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -3,6 +3,7 @@ import {default as integrationAirtable} from '@usevenice/integration-airtable/se import {default as integrationBrex} from '@usevenice/integration-brex/server' import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/server' import {default as integrationHeron} from '@usevenice/integration-heron/server' +import {default as integrationLunchmoney} from '@usevenice/integration-lunchmoney/server' import {default as integrationMerge} from '@usevenice/integration-merge/server' import {default as integrationMoota} from '@usevenice/integration-moota/server' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/server' @@ -11,6 +12,7 @@ import {default as integrationQbo} from '@usevenice/integration-qbo/server' import {default as integrationRamp} from '@usevenice/integration-ramp/server' import {default as integrationSaltedge} from '@usevenice/integration-saltedge/server' import {default as integrationSplitwise} from '@usevenice/integration-splitwise/server' +import {default as integrationSpreadsheet} from '@usevenice/integration-spreadsheet/server' import {default as integrationStripe} from '@usevenice/integration-stripe/server' import {default as integrationTeller} from '@usevenice/integration-teller/server' import {default as integrationToggl} from '@usevenice/integration-toggl/server' @@ -22,6 +24,7 @@ export const serverIntegrations = { brex: integrationBrex, foreceipt: integrationForeceipt, heron: integrationHeron, + lunchmoney: integrationLunchmoney, merge: integrationMerge, moota: integrationMoota, onebrick: integrationOnebrick, @@ -30,6 +33,7 @@ export const serverIntegrations = { ramp: integrationRamp, saltedge: integrationSaltedge, splitwise: integrationSplitwise, + spreadsheet: integrationSpreadsheet, stripe: integrationStripe, teller: integrationTeller, toggl: integrationToggl, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index 720dbc3e..b5b43935 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -79,9 +79,13 @@ module.exports = [ }, }, { + name: 'lunchmoney', dirName: 'integration-lunchmoney', varName: 'integrationLunchmoney', - imports: {}, + imports: { + def: '@usevenice/integration-lunchmoney/def', + server: '@usevenice/integration-lunchmoney/server', + }, }, { name: 'mercury', @@ -170,9 +174,13 @@ module.exports = [ }, }, { + name: 'spreadsheet', dirName: 'integration-spreadsheet', varName: 'integrationSpreadsheet', - imports: {}, + imports: { + def: '@usevenice/integration-spreadsheet/def', + server: '@usevenice/integration-spreadsheet/server', + }, }, { name: 'stripe', diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index c8fb1c19..79dfb0e4 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -7,10 +7,8 @@ import {mongodbProvider} from '@usevenice/core-integration-mongodb' import {corePostgresProvider} from '@usevenice/core-integration-postgres' import {webhookProvider} from '@usevenice/core-integration-webhook' import {beancountProvider} from '@usevenice/integration-beancount' -import {lunchmoneyProvider} from '@usevenice/integration-lunchmoney' import {plaidProvider} from '@usevenice/integration-plaid' import {postgresProvider} from '@usevenice/integration-postgres' -import {spreadsheetProvider} from '@usevenice/integration-spreadsheet' import {mergedIntegrations} from './integrations/integrations.merged' @@ -34,6 +32,4 @@ export const PROVIDERS = [ webhookProvider, beancountProvider, - spreadsheetProvider, - lunchmoneyProvider, ] as const diff --git a/integrations/integration-lunchmoney/lunchmoneyProvider.ts b/integrations/integration-lunchmoney/def.ts similarity index 63% rename from integrations/integration-lunchmoney/lunchmoneyProvider.ts rename to integrations/integration-lunchmoney/def.ts index 743bcfcc..6832d84e 100644 --- a/integrations/integration-lunchmoney/lunchmoneyProvider.ts +++ b/integrations/integration-lunchmoney/def.ts @@ -1,17 +1,16 @@ -import {makeSyncProvider} from '@usevenice/cdk-core' -import {makePostingsMap, veniceProviderBase} from '@usevenice/cdk-ledger' -import {A, parseMoney, R, Rx, rxjs, z} from '@usevenice/util' +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {makePostingsMap} from '@usevenice/cdk-ledger' +import {A, parseMoney, R, z} from '@usevenice/util' import { assetSchema, categorySchema, - makeLunchmoneyClient, transactionSchema, zConfig, } from './lunchmoneyClient' -const _def = makeSyncProvider.def({ - ...veniceProviderBase.def, +export const lunchmoneySchemas = { name: z.literal('lunchmoney'), integrationConfig: zConfig, sourceOutputEntity: z.discriminatedUnion('entityName', [ @@ -29,13 +28,15 @@ const _def = makeSyncProvider.def({ entity: transactionSchema, }), ]), -}) +} satisfies IntegrationSchemas -export const lunchmoneyProviderDef = makeSyncProvider.def.helpers(_def) +export const lunchmoneyHelpers = intHelpers(lunchmoneySchemas) -export const lunchmoneyProvider = makeSyncProvider({ +export const lunchmoneyDef = { + name: 'lunchmoney', + def: lunchmoneySchemas, metadata: {categories: ['personal-finance']}, - ...veniceProviderBase(lunchmoneyProviderDef, { + extension: { sourceMapEntity: { account: ({entity: a}) => ({ id: `${a.id}`, @@ -90,40 +91,7 @@ export const lunchmoneyProvider = makeSyncProvider({ }, }), }, - }), - sourceSync: ({config}) => { - const lunchmoney = makeLunchmoneyClient(config) - async function* iterateEntities() { - const assets = await lunchmoney.getAssets() - const categories = await lunchmoney.getCategories() - yield assets.map((a) => - lunchmoneyProviderDef._opData('account', `${a.id}`, { - ...a, - _type: 'asset', - }), - ) - - yield categories.map((c) => - lunchmoneyProviderDef._opData('account', `${c.id}`, { - ...c, - _type: 'category', - }), - ) - - for await (const transactions of lunchmoney.iterateAllTransactions({ - debit_as_negative: true, - })) { - yield transactions.map((t) => - lunchmoneyProviderDef._opData('transaction', `${t.id}`, t), - ) - } - } - return rxjs - .from(iterateEntities()) - .pipe( - Rx.mergeMap((ops) => - rxjs.from([...ops, lunchmoneyProviderDef._op('commit')]), - ), - ) }, -}) +} satisfies IntegrationDef + +export default lunchmoneyDef diff --git a/integrations/integration-lunchmoney/index.ts b/integrations/integration-lunchmoney/index.ts index cf57b80d..3700652e 100644 --- a/integrations/integration-lunchmoney/index.ts +++ b/integrations/integration-lunchmoney/index.ts @@ -1,4 +1,5 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' export * from './lunchmoneyClient' -export * from './lunchmoneyProvider' +export * from './server' // codegen:end diff --git a/integrations/integration-lunchmoney/server.ts b/integrations/integration-lunchmoney/server.ts new file mode 100644 index 00000000..a29631d0 --- /dev/null +++ b/integrations/integration-lunchmoney/server.ts @@ -0,0 +1,46 @@ +import type {IntegrationServer} from '@usevenice/cdk-core' +import {Rx, rxjs} from '@usevenice/util' + +import type {lunchmoneySchemas} from './def' +import {lunchmoneyHelpers} from './def' +import {makeLunchmoneyClient} from './lunchmoneyClient' + +export const lunchmoneyServer = { + sourceSync: ({config}) => { + const lunchmoney = makeLunchmoneyClient(config) + async function* iterateEntities() { + const assets = await lunchmoney.getAssets() + const categories = await lunchmoney.getCategories() + yield assets.map((a) => + lunchmoneyHelpers._opData('account', `${a.id}`, { + ...a, + _type: 'asset', + }), + ) + + yield categories.map((c) => + lunchmoneyHelpers._opData('account', `${c.id}`, { + ...c, + _type: 'category', + }), + ) + + for await (const transactions of lunchmoney.iterateAllTransactions({ + debit_as_negative: true, + })) { + yield transactions.map((t) => + lunchmoneyHelpers._opData('transaction', `${t.id}`, t), + ) + } + } + return rxjs + .from(iterateEntities()) + .pipe( + Rx.mergeMap((ops) => + rxjs.from([...ops, lunchmoneyHelpers._op('commit')]), + ), + ) + }, +} satisfies IntegrationServer + +export default lunchmoneyServer diff --git a/integrations/integration-spreadsheet/SpreadsheetProvider.ts b/integrations/integration-spreadsheet/def.ts similarity index 62% rename from integrations/integration-spreadsheet/SpreadsheetProvider.ts rename to integrations/integration-spreadsheet/def.ts index ec5b12ee..53a6e82b 100644 --- a/integrations/integration-spreadsheet/SpreadsheetProvider.ts +++ b/integrations/integration-spreadsheet/def.ts @@ -1,8 +1,7 @@ -import type {SyncOperation} from '@usevenice/cdk-core' -import {makeSyncProvider} from '@usevenice/cdk-core' -import {veniceProviderBase} from '@usevenice/cdk-ledger' +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' import type {UnionToIntersection} from '@usevenice/util' -import {Rx, rxjs, z} from '@usevenice/util' +import {z} from '@usevenice/util' import { formatAlliantCreditUnion, @@ -23,7 +22,11 @@ import {makeImportFormatMap} from './makeImportFormat' // MARK: - Importing all supported formats -const {formats, zSpreadsheetEntity, zPreset} = makeImportFormatMap({ +const { + formats: spreadsheetFormats, + zSpreadsheetEntity, + zPreset, +} = makeImportFormatMap({ ramp: rampFormat, 'apple-card': formatAppleCard, 'alliant-credit-union': formatAlliantCreditUnion, @@ -39,7 +42,7 @@ const {formats, zSpreadsheetEntity, zPreset} = makeImportFormatMap({ wise: formatWise, }) -type SpreadsheetEntity = z.infer +export {spreadsheetFormats} const zSrcEntitySchema = z.object({ /** Row number */ @@ -48,9 +51,6 @@ const zSrcEntitySchema = z.object({ entityName: z.string(), entity: zSpreadsheetEntity, }) - -type SpreadsheetSyncOperation = SyncOperation> - // MARK: - /** Not implemented yet */ @@ -62,8 +62,7 @@ const zConfig = z }) .nullish() -const def = makeSyncProvider.def({ - ...veniceProviderBase.def, +export const spreadsheetSchemas = { name: z.literal('spreadsheet'), resourceSettings: z.object({ preset: zPreset, @@ -76,42 +75,27 @@ const def = makeSyncProvider.def({ // csvString belongs in syncState because among other things we can actually naturally // persist the csvString used for every single sync as part of the pipeline_jobs table! sourceState: z.object({csvString: z.string()}), -}) +} satisfies IntegrationSchemas -export const spreadsheetProvider = makeSyncProvider({ +export const spreadsheetHelpers = intHelpers(spreadsheetSchemas) + +export const spreadsheetDef = { + name: 'spreadsheet', + def: spreadsheetSchemas, metadata: { displayName: 'Spreadsheet (CSV, Google Sheets, Excel)', categories: ['flat-files-and-spreadsheets'], logoUrl: '/_assets/logo-spreadsheet.png', }, - ...veniceProviderBase(def, { + extension: { // what do we do with the fact that conn has preset and entity itself has preset? sourceMapEntity: ({entity}, conn) => - formats[entity.preset].mapEntity( + spreadsheetFormats[entity.preset].mapEntity( // A bit of a type hack... but needed entity.row as UnionToIntersection<(typeof entity)['row']>, conn.accountExternalId as ExternalId, ), - }), - sourceSync: ({settings, state}) => - rxjs.from(formats[settings.preset].parseRows(state.csvString)).pipe( - Rx.map( - (row, index): SpreadsheetSyncOperation => ({ - // This part is rather generic. we don't know what a row represents just yet - // At some point we can extract core-integration-csv out of integration-csv - type: 'data', - data: { - id: `row_${index}`, - entityName: 'csv_row', - entity: {preset: settings.preset, row} as SpreadsheetEntity, - }, - }), - ), - Rx.concatWith( - rxjs.from([ - {type: 'commit'}, - {type: 'ready'}, - ]), - ), - ), -}) + }, +} satisfies IntegrationDef + +export default spreadsheetDef diff --git a/integrations/integration-spreadsheet/index.ts b/integrations/integration-spreadsheet/index.ts index e4a1a4a9..ddb34532 100644 --- a/integrations/integration-spreadsheet/index.ts +++ b/integrations/integration-spreadsheet/index.ts @@ -1,10 +1,11 @@ import Papa from 'papaparse' // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' export * from './formats/index' export * from './import-format-utils' export * from './makeImportFormat' export * from './RowIdMaker' -export * from './SpreadsheetProvider' +export * from './server' // codegen:end export {Papa} diff --git a/integrations/integration-spreadsheet/server.ts b/integrations/integration-spreadsheet/server.ts new file mode 100644 index 00000000..f211085f --- /dev/null +++ b/integrations/integration-spreadsheet/server.ts @@ -0,0 +1,33 @@ +import type {IntegrationServer} from '@usevenice/cdk-core' +import {Rx, rxjs} from '@usevenice/util' + +import type {spreadsheetSchemas} from './def' +import {spreadsheetFormats, spreadsheetHelpers} from './def' + +// MARK: - Importing all supported formats + +export const spreadsheetServer = { + sourceSync: ({settings, state}) => + rxjs + .from(spreadsheetFormats[settings.preset].parseRows(state.csvString)) + .pipe( + Rx.map((row, index) => + // This part is rather generic. we don't know what a row represents just yet + // At some point we can extract core-integration-csv out of integration-csv + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + spreadsheetHelpers._opData('csv_row', `row_${index}`, { + preset: settings.preset, + row, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any), + ), + Rx.concatWith( + rxjs.from([ + spreadsheetHelpers._op('commit'), + spreadsheetHelpers._op('ready'), + ]), + ), + ), +} satisfies IntegrationServer + +export default spreadsheetServer From 77ccdcab06fb5f5cd4da51c59c11140e5a5e188f Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 18:51:23 +0800 Subject: [PATCH 11/80] Migrate beancont integration --- .../integrations/integrations.def.ts | 2 ++ .../integrations/integrations.merged.ts | 8 +++++ .../integrations/integrations.server.ts | 2 ++ apps/app-config/integrations/meta.js | 6 +++- apps/app-config/providers.ts | 6 ++-- integrations/integration-beancount/def.ts | 29 +++++++++++++++++ integrations/integration-beancount/index.ts | 3 +- .../{BeancountProvider.ts => server.ts} | 31 +++++-------------- 8 files changed, 57 insertions(+), 30 deletions(-) create mode 100644 integrations/integration-beancount/def.ts rename integrations/integration-beancount/{BeancountProvider.ts => server.ts} (68%) diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index f57595a4..bf9e6151 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -1,5 +1,6 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand import {default as integrationAirtable} from '@usevenice/integration-airtable/def' +import {default as integrationBeancount} from '@usevenice/integration-beancount/def' import {default as integrationBrex} from '@usevenice/integration-brex/def' import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/def' import {default as integrationHeron} from '@usevenice/integration-heron/def' @@ -23,6 +24,7 @@ import {default as integrationYodlee} from '@usevenice/integration-yodlee/def' export const defIntegrations = { airtable: integrationAirtable, + beancount: integrationBeancount, brex: integrationBrex, foreceipt: integrationForeceipt, heron: integrationHeron, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 9824aec7..61f6f4ac 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -1,6 +1,8 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand import {default as integrationAirtable_def} from '@usevenice/integration-airtable/def' import {default as integrationAirtable_server} from '@usevenice/integration-airtable/server' +import {default as integrationBeancount_def} from '@usevenice/integration-beancount/def' +import {default as integrationBeancount_server} from '@usevenice/integration-beancount/server' import {default as integrationBrex_def} from '@usevenice/integration-brex/def' import {default as integrationBrex_server} from '@usevenice/integration-brex/server' import {default as integrationForeceipt_def} from '@usevenice/integration-foreceipt/def' @@ -50,6 +52,11 @@ const integrationAirtable = { ...integrationAirtable_server, } +const integrationBeancount = { + ...integrationBeancount_def, + ...integrationBeancount_server, +} + const integrationBrex = { ...integrationBrex_def, ...integrationBrex_server, @@ -155,6 +162,7 @@ const integrationYodlee = { export const mergedIntegrations = { airtable: integrationAirtable, + beancount: integrationBeancount, brex: integrationBrex, foreceipt: integrationForeceipt, heron: integrationHeron, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index 566fd74b..01213d89 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -1,5 +1,6 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand import {default as integrationAirtable} from '@usevenice/integration-airtable/server' +import {default as integrationBeancount} from '@usevenice/integration-beancount/server' import {default as integrationBrex} from '@usevenice/integration-brex/server' import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/server' import {default as integrationHeron} from '@usevenice/integration-heron/server' @@ -21,6 +22,7 @@ import {default as integrationYodlee} from '@usevenice/integration-yodlee/server export const serverIntegrations = { airtable: integrationAirtable, + beancount: integrationBeancount, brex: integrationBrex, foreceipt: integrationForeceipt, heron: integrationHeron, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index b5b43935..119db7f2 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -42,9 +42,13 @@ module.exports = [ imports: {}, }, { + name: 'beancount', dirName: 'integration-beancount', varName: 'integrationBeancount', - imports: {}, + imports: { + def: '@usevenice/integration-beancount/def', + server: '@usevenice/integration-beancount/server', + }, }, { name: 'brex', diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 79dfb0e4..132ec21f 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -6,7 +6,6 @@ import {fsProvider} from '@usevenice/core-integration-fs' import {mongodbProvider} from '@usevenice/core-integration-mongodb' import {corePostgresProvider} from '@usevenice/core-integration-postgres' import {webhookProvider} from '@usevenice/core-integration-webhook' -import {beancountProvider} from '@usevenice/integration-beancount' import {plaidProvider} from '@usevenice/integration-plaid' import {postgresProvider} from '@usevenice/integration-postgres' @@ -23,13 +22,12 @@ export const DOCUMENTED_PROVIDERS = [ export const PROVIDERS = [ ...DOCUMENTED_PROVIDERS, // TODO: Migrate these over to the new paradigm - debugProvider, fsProvider, - postgresProvider, firebaseProvider, mongodbProvider, corePostgresProvider, + debugProvider, + postgresProvider, webhookProvider, - beancountProvider, ] as const diff --git a/integrations/integration-beancount/def.ts b/integrations/integration-beancount/def.ts new file mode 100644 index 00000000..5c3d1fb1 --- /dev/null +++ b/integrations/integration-beancount/def.ts @@ -0,0 +1,29 @@ +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {zEntityPayload} from '@usevenice/cdk-ledger' +import {isAmountUnit, z} from '@usevenice/util' + +export type BeancountDestOptions = z.infer +export const zBeancountDestOptions = z.object({ + outPath: z.string(), + separateByPeriod: z.boolean().optional(), + saveStdJson: z.boolean().optional(), + debugSaveBeanJson: z.boolean().optional(), + operatingCurrency: z.string().refine(isAmountUnit).optional(), +}) + +export const beancountSchemas = { + name: z.literal('beancount'), + destinationState: zBeancountDestOptions, + destinationInputEntity: zEntityPayload, +} satisfies IntegrationSchemas + +export const beancountHelpers = intHelpers(beancountSchemas) + +export const beancountDef = { + name: 'beancount', + def: beancountSchemas, + metadata: {categories: ['personal-finance'], platforms: ['local']}, +} satisfies IntegrationDef + +export default beancountDef diff --git a/integrations/integration-beancount/index.ts b/integrations/integration-beancount/index.ts index 45b8efce..01bec61d 100644 --- a/integrations/integration-beancount/index.ts +++ b/integrations/integration-beancount/index.ts @@ -1,5 +1,6 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} export * from './bean-fs-utils' export * from './beancountConverters' -export * from './BeancountProvider' +export * from './def' +export * from './server' // codegen:end diff --git a/integrations/integration-beancount/BeancountProvider.ts b/integrations/integration-beancount/server.ts similarity index 68% rename from integrations/integration-beancount/BeancountProvider.ts rename to integrations/integration-beancount/server.ts index 4dbe1b6a..10ceeaeb 100644 --- a/integrations/integration-beancount/BeancountProvider.ts +++ b/integrations/integration-beancount/server.ts @@ -1,41 +1,24 @@ -import {makeSyncProvider} from '@usevenice/cdk-core' +import type {IntegrationServer} from '@usevenice/cdk-core' import type {StdCache} from '@usevenice/cdk-ledger' -import {cachingLink, veniceProviderBase} from '@usevenice/cdk-ledger' +import {cachingLink} from '@usevenice/cdk-ledger' import type {Standard} from '@usevenice/standard' import { $writeFile, fromCompletion, - isAmountUnit, objectEntries, stableStringify, - z, } from '@usevenice/util' import {beanJsonToDir} from './bean-fs-utils' import {convBeanFile, convBeanJsonToStdJson} from './beancountConverters' +import type {BeancountDestOptions, beancountSchemas} from './def' -export type BeancountDestOptions = z.infer -export const zBeancountDestOptions = z.object({ - outPath: z.string(), - separateByPeriod: z.boolean().optional(), - saveStdJson: z.boolean().optional(), - debugSaveBeanJson: z.boolean().optional(), - operatingCurrency: z.string().refine(isAmountUnit).optional(), -}) - -const def = makeSyncProvider.def({ - ...veniceProviderBase.def, - name: z.literal('beancount'), - destinationState: zBeancountDestOptions, -}) - -export const beancountProvider = makeSyncProvider({ - metadata: {categories: ['personal-finance'], platforms: ['local']}, - ...veniceProviderBase(def, {sourceMapEntity: undefined}), +export const beancountServer = { destinationSync: ({state: options}) => cachingLink((cache) => fromCompletion(outputBeanFiles(cache, options))), - sourceSync: undefined, -}) +} satisfies IntegrationServer + +export default beancountServer export async function outputBeanFiles( cache: StdCache, From a0bedb90389cdcf4c0332e47d040d0d4c571a7ba Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 18:55:10 +0800 Subject: [PATCH 12/80] Migrate postgres integration --- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 +++ .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 6 +- apps/app-config/providers.ts | 7 +- integrations/integration-postgres/def.ts | 62 ++++++++++++++++++ integrations/integration-postgres/index.ts | 2 +- .../{postgresProvider.ts => server.ts} | 65 +++---------------- 8 files changed, 91 insertions(+), 63 deletions(-) create mode 100644 integrations/integration-postgres/def.ts rename integrations/integration-postgres/{postgresProvider.ts => server.ts} (76%) diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index bf9e6151..6396e2f2 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -10,6 +10,7 @@ import {default as integrationMerge} from '@usevenice/integration-merge/def' import {default as integrationMoota} from '@usevenice/integration-moota/def' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/def' import {default as integrationPlaid} from '@usevenice/integration-plaid/def' +import {default as integrationPostgres} from '@usevenice/integration-postgres/def' import {default as integrationQbo} from '@usevenice/integration-qbo/def' import {default as integrationRamp} from '@usevenice/integration-ramp/def' import {default as integrationSaltedge} from '@usevenice/integration-saltedge/def' @@ -34,6 +35,7 @@ export const defIntegrations = { moota: integrationMoota, onebrick: integrationOnebrick, plaid: integrationPlaid, + postgres: integrationPostgres, qbo: integrationQbo, ramp: integrationRamp, saltedge: integrationSaltedge, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 61f6f4ac..bebf43cd 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -23,6 +23,8 @@ import {default as integrationOnebrick_server} from '@usevenice/integration-oneb import {default as integrationPlaid_client} from '@usevenice/integration-plaid/client' import {default as integrationPlaid_def} from '@usevenice/integration-plaid/def' import {default as integrationPlaid_server} from '@usevenice/integration-plaid/server' +import {default as integrationPostgres_def} from '@usevenice/integration-postgres/def' +import {default as integrationPostgres_server} from '@usevenice/integration-postgres/server' import {default as integrationQbo_def} from '@usevenice/integration-qbo/def' import {default as integrationQbo_server} from '@usevenice/integration-qbo/server' import {default as integrationRamp_def} from '@usevenice/integration-ramp/def' @@ -104,6 +106,11 @@ const integrationPlaid = { ...integrationPlaid_server, } +const integrationPostgres = { + ...integrationPostgres_def, + ...integrationPostgres_server, +} + const integrationQbo = { ...integrationQbo_def, ...integrationQbo_server, @@ -172,6 +179,7 @@ export const mergedIntegrations = { moota: integrationMoota, onebrick: integrationOnebrick, plaid: integrationPlaid, + postgres: integrationPostgres, qbo: integrationQbo, ramp: integrationRamp, saltedge: integrationSaltedge, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index 01213d89..a1e24e6d 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -9,6 +9,7 @@ import {default as integrationMerge} from '@usevenice/integration-merge/server' import {default as integrationMoota} from '@usevenice/integration-moota/server' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/server' import {default as integrationPlaid} from '@usevenice/integration-plaid/server' +import {default as integrationPostgres} from '@usevenice/integration-postgres/server' import {default as integrationQbo} from '@usevenice/integration-qbo/server' import {default as integrationRamp} from '@usevenice/integration-ramp/server' import {default as integrationSaltedge} from '@usevenice/integration-saltedge/server' @@ -31,6 +32,7 @@ export const serverIntegrations = { moota: integrationMoota, onebrick: integrationOnebrick, plaid: integrationPlaid, + postgres: integrationPostgres, qbo: integrationQbo, ramp: integrationRamp, saltedge: integrationSaltedge, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index 119db7f2..4b6435cc 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -137,9 +137,13 @@ module.exports = [ }, }, { + name: 'postgres', dirName: 'integration-postgres', varName: 'integrationPostgres', - imports: {}, + imports: { + def: '@usevenice/integration-postgres/def', + server: '@usevenice/integration-postgres/server', + }, }, { name: 'qbo', diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 132ec21f..89ff6820 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -7,7 +7,6 @@ import {mongodbProvider} from '@usevenice/core-integration-mongodb' import {corePostgresProvider} from '@usevenice/core-integration-postgres' import {webhookProvider} from '@usevenice/core-integration-webhook' import {plaidProvider} from '@usevenice/integration-plaid' -import {postgresProvider} from '@usevenice/integration-postgres' import {mergedIntegrations} from './integrations/integrations.merged' @@ -25,9 +24,7 @@ export const PROVIDERS = [ fsProvider, firebaseProvider, mongodbProvider, - corePostgresProvider, - - debugProvider, - postgresProvider, webhookProvider, + debugProvider, + corePostgresProvider, ] as const diff --git a/integrations/integration-postgres/def.ts b/integrations/integration-postgres/def.ts new file mode 100644 index 00000000..af34b54c --- /dev/null +++ b/integrations/integration-postgres/def.ts @@ -0,0 +1,62 @@ +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import type {EntityPayloadWithExternal, ZCommon} from '@usevenice/cdk-ledger' +import {zPgConfig} from '@usevenice/core-integration-postgres' +import {z, zCast} from '@usevenice/util' + +export {makePostgresClient} from '@usevenice/core-integration-postgres' + +export const postgresSchemas = { + name: z.literal('postgres'), + // TODO: Should postgres use integration config or resourceSettings? + // if it's resourceSettings then it doesn't make as much sense to configure + // in the list of integrations... + // How do we create default resources for integrations that are basically single resource? + resourceSettings: zPgConfig.pick({databaseUrl: true}).extend({ + // gotta make sourceQueries a Textarea + + sourceQueries: z + .object({ + invoice: z + .string() + .nullish() + .describe('Should order by lastModifiedAt and id descending'), + }) + // .nullish() does not translate well to jsonSchema + // @see https://share.cleanshot.com/w0KVx1Y2 + .optional(), + }), + destinationInputEntity: zCast(), + sourceOutputEntity: zCast(), + sourceState: z + .object({ + invoice: z + .object({ + lastModifiedAt: z.string().optional(), + lastRowId: z.string().optional(), + }) + .optional(), + }) + .optional(), +} satisfies IntegrationSchemas + +export const postgresHelpers = intHelpers(postgresSchemas) + +export const postgresDef = { + name: 'postgres', + metadata: { + categories: ['database'], + logoUrl: '/_assets/logo-postgres.png', + stage: 'ga', + }, + + def: postgresSchemas, + standardMappers: { + resource: (_settings) => ({ + displayName: 'Postgres', + status: 'healthy', + }), + }, +} satisfies IntegrationDef + +export default postgresDef diff --git a/integrations/integration-postgres/index.ts b/integrations/integration-postgres/index.ts index 49a8780d..9b870a40 100644 --- a/integrations/integration-postgres/index.ts +++ b/integrations/integration-postgres/index.ts @@ -2,5 +2,5 @@ // Previously this used to be called `integration-alka`, but given alka is no longer relevant, how do we want to define this? // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} -export * from './postgresProvider' +export * from './server' // codegen:end diff --git a/integrations/integration-postgres/postgresProvider.ts b/integrations/integration-postgres/server.ts similarity index 76% rename from integrations/integration-postgres/postgresProvider.ts rename to integrations/integration-postgres/server.ts index f82c0c3c..cdc40cfa 100644 --- a/integrations/integration-postgres/postgresProvider.ts +++ b/integrations/integration-postgres/server.ts @@ -2,68 +2,19 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import {extractId, handlersLink, makeSyncProvider} from '@usevenice/cdk-core' -import type {EntityPayloadWithExternal, ZCommon} from '@usevenice/cdk-ledger' +import type {IntegrationServer} from '@usevenice/cdk-core' +import {extractId, handlersLink} from '@usevenice/cdk-core' import { makePostgresClient, upsertByIdQuery, - zPgConfig, } from '@usevenice/core-integration-postgres' -import {R, Rx, rxjs, z, zCast} from '@usevenice/util' +import {R, Rx, rxjs} from '@usevenice/util' -export {makePostgresClient} from '@usevenice/core-integration-postgres' - -const _def = makeSyncProvider.def({ - ...makeSyncProvider.def.defaults, - name: z.literal('postgres'), - // TODO: Should postgres use integration config or resourceSettings? - // if it's resourceSettings then it doesn't make as much sense to configure - // in the list of integrations... - // How do we create default resources for integrations that are basically single resource? - resourceSettings: zPgConfig.pick({databaseUrl: true}).extend({ - // gotta make sourceQueries a Textarea - - sourceQueries: z - .object({ - invoice: z - .string() - .nullish() - .describe('Should order by lastModifiedAt and id descending'), - }) - // .nullish() does not translate well to jsonSchema - // @see https://share.cleanshot.com/w0KVx1Y2 - .optional(), - }), - destinationInputEntity: zCast(), - sourceOutputEntity: zCast(), - sourceState: z - .object({ - invoice: z - .object({ - lastModifiedAt: z.string().optional(), - lastRowId: z.string().optional(), - }) - .optional(), - }) - .optional(), -}) +import type {postgresSchemas} from './def' -const def = makeSyncProvider.def.helpers(_def) +export {makePostgresClient} from '@usevenice/core-integration-postgres' -export const postgresProvider = makeSyncProvider({ - metadata: { - categories: ['database'], - logoUrl: '/_assets/logo-postgres.png', - stage: 'ga', - }, - ...makeSyncProvider.defaults, - def, - standardMappers: { - resource: (_settings) => ({ - displayName: 'Postgres', - status: 'healthy', - }), - }, +export const postgresServer = { // TODO: // 1) Implement pagination // 2) Impelemnt incremental Sync @@ -232,4 +183,6 @@ export const postgresProvider = makeSyncProvider({ }, }) }, -}) +} satisfies IntegrationServer + +export default postgresServer From 596c9c35d00d490701e46785eda8a9248ea82c53 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Jun 2023 19:01:17 +0800 Subject: [PATCH 13/80] Partially fixing typechecking issues --- apps/app-config/providers.ts | 3 --- apps/cli/airbyte-plaid-connector.ts | 2 +- apps/cli/package.json | 1 + apps/cli/sync-test-old.ts | 3 ++- apps/cli/sync-test.ts | 3 +++ integrations/integration-postgres/index.ts | 9 +++++++++ integrations/integration-postgres/server.ts | 15 ++++++++++----- pnpm-lock.yaml | 3 +++ 8 files changed, 29 insertions(+), 10 deletions(-) diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 89ff6820..6ca2ea12 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -6,12 +6,9 @@ import {fsProvider} from '@usevenice/core-integration-fs' import {mongodbProvider} from '@usevenice/core-integration-mongodb' import {corePostgresProvider} from '@usevenice/core-integration-postgres' import {webhookProvider} from '@usevenice/core-integration-webhook' -import {plaidProvider} from '@usevenice/integration-plaid' import {mergedIntegrations} from './integrations/integrations.merged' -export {plaidProvider, fsProvider} - export const DOCUMENTED_PROVIDERS = [ ...(Object.values(mergedIntegrations) as unknown as Array< typeof debugProvider diff --git a/apps/cli/airbyte-plaid-connector.ts b/apps/cli/airbyte-plaid-connector.ts index 36d46641..b7447fe8 100644 --- a/apps/cli/airbyte-plaid-connector.ts +++ b/apps/cli/airbyte-plaid-connector.ts @@ -1,7 +1,7 @@ import '@usevenice/app-config/register.node' import {makeAirbyteConnector} from '@usevenice/airbyte/makeAirbyteConnector' -import {plaidProvider} from '@usevenice/app-config/providers' +import {plaidProvider} from '@usevenice/integration-plaid' import {cliFromRouter} from './cli-utils' diff --git a/apps/cli/package.json b/apps/cli/package.json index 285fd1e0..644dc072 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -8,6 +8,7 @@ "@usevenice/app-config": "workspace:*", "@usevenice/cdk-core": "workspace:*", "@usevenice/cdk-ledger": "workspace:*", + "@usevenice/core-integration-fs": "workspace:*", "@usevenice/core-integration-firebase": "workspace:*", "@usevenice/core-integration-postgres": "workspace:*", "@usevenice/engine-backend": "workspace:*", diff --git a/apps/cli/sync-test-old.ts b/apps/cli/sync-test-old.ts index ebab6cf4..021ad1ad 100644 --- a/apps/cli/sync-test-old.ts +++ b/apps/cli/sync-test-old.ts @@ -4,8 +4,8 @@ import '@usevenice/app-config/register.node' import readline from 'node:readline' -import {fsProvider} from '@usevenice/app-config/providers' import {sync} from '@usevenice/cdk-core' +import {fsProvider} from '@usevenice/core-integration-fs' import {brexImpl} from '@usevenice/integration-brex' import {heronImpl} from '@usevenice/integration-heron' import {mergeImpl} from '@usevenice/integration-merge' @@ -80,6 +80,7 @@ switch (process.argv[2]) { sync({ source: postgresProvider.sourceSync({ state: undefined, + config: {}, endUser: null, settings: { databaseUrl: process.env['POSTGRES_OR_WEBHOOK_URL'] ?? '', diff --git a/apps/cli/sync-test.ts b/apps/cli/sync-test.ts index 501a792a..c9448caf 100644 --- a/apps/cli/sync-test.ts +++ b/apps/cli/sync-test.ts @@ -28,6 +28,7 @@ function getSource(name: string) { }) case 'postgres': return postgresProvider.sourceSync({ + config: {}, endUser: null, state: {}, settings: { @@ -72,6 +73,8 @@ function getDestination(name: string | undefined) { }) case 'postgres': return postgresProvider.destinationSync({ + config: {}, + state: {}, endUser: null, settings: { databaseUrl: diff --git a/integrations/integration-postgres/index.ts b/integrations/integration-postgres/index.ts index 9b870a40..d424d44f 100644 --- a/integrations/integration-postgres/index.ts +++ b/integrations/integration-postgres/index.ts @@ -1,6 +1,15 @@ // Should this actually be called `integration-export` or `integration-standard`? Similar to `integration-spreadsheet`? // Previously this used to be called `integration-alka`, but given alka is no longer relevant, how do we want to define this? +import postgresDef from './def' +import postgresServer from './server' + // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' export * from './server' // codegen:end + +export const postgresProvider = { + ...postgresDef, + ...postgresServer, +} diff --git a/integrations/integration-postgres/server.ts b/integrations/integration-postgres/server.ts index cdc40cfa..3af914af 100644 --- a/integrations/integration-postgres/server.ts +++ b/integrations/integration-postgres/server.ts @@ -11,6 +11,7 @@ import { import {R, Rx, rxjs} from '@usevenice/util' import type {postgresSchemas} from './def' +import {postgresHelpers} from './def' export {makePostgresClient} from '@usevenice/core-integration-postgres' @@ -64,7 +65,7 @@ export const postgresServer = { ])} WHERE end_user_id = ${endUser?.id ?? null}`, ) yield res.rows.map((row) => - def._op('data', { + postgresHelpers._op('data', { data: { entityName, entity: row.standard, @@ -101,25 +102,29 @@ export const postgresServer = { yield R.compact([ ...res.rows.map((row) => - def._op('data', { + postgresHelpers._op('data', { data: {entityName, id: `${row.id}`, entity: row}, }), ), lastRow?.modifiedAt && lastRow.id && - def._opState({ + (postgresHelpers._opState({ invoice: { lastModifiedAt: lastRow.modifiedAt, lastRowId: lastRow.id, }, - }), + }) as never), // Temp hack... ]) } } return rxjs .from(iterateEntities()) - .pipe(Rx.mergeMap((ops) => rxjs.from([...ops, def._op('commit')]))) + .pipe( + Rx.mergeMap((ops) => + rxjs.from([...ops, postgresHelpers._op('commit')]), + ), + ) }, destinationSync: ({endUser, settings: {databaseUrl}}) => { console.log('[destinationSync] Will makePostgresClient', { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e1a849d1..c20a819b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -318,6 +318,9 @@ importers: '@usevenice/core-integration-firebase': specifier: workspace:* version: link:../../integrations/core-integration-firebase + '@usevenice/core-integration-fs': + specifier: workspace:* + version: link:../../integrations/core-integration-fs '@usevenice/core-integration-postgres': specifier: workspace:* version: link:../../integrations/core-integration-postgres From 0bc859741bdad1ec0db3c5ca80a37ecaf23bac66 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 9 Jun 2023 00:34:02 +0800 Subject: [PATCH 14/80] Migrate integrationFs --- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 ++++ .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 10 ++++- apps/app-config/package.json | 2 +- apps/app-config/providers.ts | 2 +- apps/cli/package.json | 2 +- apps/cli/sync-test-old.ts | 2 +- integrations/integration-fs/def.ts | 44 +++++++++++++++++++ .../index.ts | 3 +- .../makeFsKVStore.ts | 0 .../package.json | 2 +- .../server.ts} | 43 +++--------------- packages/airbyte/Dockerfile | 2 +- pnpm-lock.yaml | 30 ++++++------- 15 files changed, 95 insertions(+), 59 deletions(-) create mode 100644 integrations/integration-fs/def.ts rename integrations/{core-integration-fs => integration-fs}/index.ts (79%) rename integrations/{core-integration-fs => integration-fs}/makeFsKVStore.ts (100%) rename integrations/{core-integration-fs => integration-fs}/package.json (84%) rename integrations/{core-integration-fs/FSProvider.ts => integration-fs/server.ts} (75%) diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index 6396e2f2..629987cb 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -3,6 +3,7 @@ import {default as integrationAirtable} from '@usevenice/integration-airtable/de import {default as integrationBeancount} from '@usevenice/integration-beancount/def' import {default as integrationBrex} from '@usevenice/integration-brex/def' import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/def' +import {default as integrationFs} from '@usevenice/integration-fs/def' import {default as integrationHeron} from '@usevenice/integration-heron/def' import {default as integrationLunchmoney} from '@usevenice/integration-lunchmoney/def' import {default as integrationMercury} from '@usevenice/integration-mercury/def' @@ -28,6 +29,7 @@ export const defIntegrations = { beancount: integrationBeancount, brex: integrationBrex, foreceipt: integrationForeceipt, + fs: integrationFs, heron: integrationHeron, lunchmoney: integrationLunchmoney, mercury: integrationMercury, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index bebf43cd..bc78b7a3 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -7,6 +7,8 @@ import {default as integrationBrex_def} from '@usevenice/integration-brex/def' import {default as integrationBrex_server} from '@usevenice/integration-brex/server' import {default as integrationForeceipt_def} from '@usevenice/integration-foreceipt/def' import {default as integrationForeceipt_server} from '@usevenice/integration-foreceipt/server' +import {default as integrationFs_def} from '@usevenice/integration-fs/def' +import {default as integrationFs_server} from '@usevenice/integration-fs/server' import {default as integrationHeron_def} from '@usevenice/integration-heron/def' import {default as integrationHeron_server} from '@usevenice/integration-heron/server' import {default as integrationLunchmoney_def} from '@usevenice/integration-lunchmoney/def' @@ -69,6 +71,11 @@ const integrationForeceipt = { ...integrationForeceipt_server, } +const integrationFs = { + ...integrationFs_def, + ...integrationFs_server, +} + const integrationHeron = { ...integrationHeron_def, ...integrationHeron_server, @@ -172,6 +179,7 @@ export const mergedIntegrations = { beancount: integrationBeancount, brex: integrationBrex, foreceipt: integrationForeceipt, + fs: integrationFs, heron: integrationHeron, lunchmoney: integrationLunchmoney, mercury: integrationMercury, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index a1e24e6d..9f476fdc 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -3,6 +3,7 @@ import {default as integrationAirtable} from '@usevenice/integration-airtable/se import {default as integrationBeancount} from '@usevenice/integration-beancount/server' import {default as integrationBrex} from '@usevenice/integration-brex/server' import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/server' +import {default as integrationFs} from '@usevenice/integration-fs/server' import {default as integrationHeron} from '@usevenice/integration-heron/server' import {default as integrationLunchmoney} from '@usevenice/integration-lunchmoney/server' import {default as integrationMerge} from '@usevenice/integration-merge/server' @@ -26,6 +27,7 @@ export const serverIntegrations = { beancount: integrationBeancount, brex: integrationBrex, foreceipt: integrationForeceipt, + fs: integrationFs, heron: integrationHeron, lunchmoney: integrationLunchmoney, merge: integrationMerge, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index 4b6435cc..6f296066 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -6,7 +6,6 @@ module.exports = [ varName: 'coreIntegrationFirebase', imports: {}, }, - {dirName: 'core-integration-fs', varName: 'coreIntegrationFs', imports: {}}, { dirName: 'core-integration-mongodb', varName: 'coreIntegrationMongodb', @@ -73,6 +72,15 @@ module.exports = [ server: '@usevenice/integration-foreceipt/server', }, }, + { + name: 'fs', + dirName: 'integration-fs', + varName: 'integrationFs', + imports: { + def: '@usevenice/integration-fs/def', + server: '@usevenice/integration-fs/server', + }, + }, { name: 'heron', dirName: 'integration-heron', diff --git a/apps/app-config/package.json b/apps/app-config/package.json index ee834252..d68f8e4c 100644 --- a/apps/app-config/package.json +++ b/apps/app-config/package.json @@ -14,7 +14,7 @@ "@usevenice/cdk-ledger": "workspace:*", "@usevenice/integration-airtable": "workspace:*", "@usevenice/core-integration-firebase": "workspace:*", - "@usevenice/core-integration-fs": "workspace:*", + "@usevenice/integration-fs": "workspace:*", "@usevenice/core-integration-mongodb": "workspace:*", "@usevenice/core-integration-postgres": "workspace:*", "@usevenice/core-integration-redis": "workspace:*", diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 6ca2ea12..567767c0 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -2,7 +2,7 @@ import {debugProvider} from '@usevenice/cdk-core' import {firebaseProvider} from '@usevenice/core-integration-firebase' -import {fsProvider} from '@usevenice/core-integration-fs' +import {fsProvider} from '@usevenice/integration-fs' import {mongodbProvider} from '@usevenice/core-integration-mongodb' import {corePostgresProvider} from '@usevenice/core-integration-postgres' import {webhookProvider} from '@usevenice/core-integration-webhook' diff --git a/apps/cli/package.json b/apps/cli/package.json index 644dc072..b2aa93c9 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -8,7 +8,7 @@ "@usevenice/app-config": "workspace:*", "@usevenice/cdk-core": "workspace:*", "@usevenice/cdk-ledger": "workspace:*", - "@usevenice/core-integration-fs": "workspace:*", + "@usevenice/integration-fs": "workspace:*", "@usevenice/core-integration-firebase": "workspace:*", "@usevenice/core-integration-postgres": "workspace:*", "@usevenice/engine-backend": "workspace:*", diff --git a/apps/cli/sync-test-old.ts b/apps/cli/sync-test-old.ts index 021ad1ad..50cdae12 100644 --- a/apps/cli/sync-test-old.ts +++ b/apps/cli/sync-test-old.ts @@ -5,7 +5,7 @@ import '@usevenice/app-config/register.node' import readline from 'node:readline' import {sync} from '@usevenice/cdk-core' -import {fsProvider} from '@usevenice/core-integration-fs' +import {fsProvider} from '@usevenice/integration-fs' import {brexImpl} from '@usevenice/integration-brex' import {heronImpl} from '@usevenice/integration-heron' import {mergeImpl} from '@usevenice/integration-merge' diff --git a/integrations/integration-fs/def.ts b/integrations/integration-fs/def.ts new file mode 100644 index 00000000..0cf90989 --- /dev/null +++ b/integrations/integration-fs/def.ts @@ -0,0 +1,44 @@ +import type { + AnyEntityPayload, + IntegrationDef, + IntegrationSchemas, +} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {z, zCast} from '@usevenice/util' + +// MARK: - Source Sync + +export const zWatchPathsInput = z.object({ + basePath: z.string(), + paths: z.array(z.string()).optional(), +}) + +export const fsSchemas = { + name: z.literal('fs'), + resourceSettings: zWatchPathsInput.pick({basePath: true}), + /** + * `paths` only used for sourceSync, not destSync. Though these are not technically states... + * And they are not safe to just erase if fullSync = true. + * TODO: Introduce a separate sourceOptions / destinationOptions type later when it becomes an + * actual problem... for now this issue only impacts FirebaseProvider and FSProvider + * which are not actually being used as top level providers + */ + sourceState: zWatchPathsInput.pick({paths: true}), + sourceOutputEntity: zCast(), + destinationInputEntity: zCast(), +} satisfies IntegrationSchemas + +export const fsHelpers = intHelpers(fsSchemas) + +export const fsDef = { + name: 'fs', + metadata: { + platforms: ['local'], + displayName: 'File system', + categories: ['flat-files-and-spreadsheets'], + }, + + def: fsSchemas, +} satisfies IntegrationDef + +export default fsDef diff --git a/integrations/core-integration-fs/index.ts b/integrations/integration-fs/index.ts similarity index 79% rename from integrations/core-integration-fs/index.ts rename to integrations/integration-fs/index.ts index a3b73a54..27e7ba6b 100644 --- a/integrations/core-integration-fs/index.ts +++ b/integrations/integration-fs/index.ts @@ -1,4 +1,5 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} -export * from './FSProvider' +export * from './def' export * from './makeFsKVStore' +export * from './server' // codegen:end diff --git a/integrations/core-integration-fs/makeFsKVStore.ts b/integrations/integration-fs/makeFsKVStore.ts similarity index 100% rename from integrations/core-integration-fs/makeFsKVStore.ts rename to integrations/integration-fs/makeFsKVStore.ts diff --git a/integrations/core-integration-fs/package.json b/integrations/integration-fs/package.json similarity index 84% rename from integrations/core-integration-fs/package.json rename to integrations/integration-fs/package.json index 82014d82..e22b4705 100644 --- a/integrations/core-integration-fs/package.json +++ b/integrations/integration-fs/package.json @@ -1,5 +1,5 @@ { - "name": "@usevenice/core-integration-fs", + "name": "@usevenice/integration-fs", "version": "0.0.0", "private": true, "sideEffects": [ diff --git a/integrations/core-integration-fs/FSProvider.ts b/integrations/integration-fs/server.ts similarity index 75% rename from integrations/core-integration-fs/FSProvider.ts rename to integrations/integration-fs/server.ts index 74cb8d90..69e6b782 100644 --- a/integrations/core-integration-fs/FSProvider.ts +++ b/integrations/integration-fs/server.ts @@ -1,5 +1,6 @@ -import type {AnyEntityPayload, SyncOperation} from '@usevenice/cdk-core' -import {handlersLink, makeSyncProvider} from '@usevenice/cdk-core' +import type {IntegrationServer, SyncOperation} from '@usevenice/cdk-core' +import {handlersLink} from '@usevenice/cdk-core' +import type {z} from '@usevenice/util' import { $chokidar, $path, @@ -11,43 +12,12 @@ import { rxjs, safeJSONParse, writeJson, - z, - zCast, } from '@usevenice/util' +import type {fsSchemas, zWatchPathsInput} from './def' import {_pathFromId} from './makeFsKVStore' -// MARK: - Source Sync - -const zWatchPathsInput = z.object({ - basePath: z.string(), - paths: z.array(z.string()).optional(), -}) - -const def = makeSyncProvider.def({ - ...makeSyncProvider.def.defaults, - name: z.literal('fs'), - resourceSettings: zWatchPathsInput.pick({basePath: true}), - /** - * `paths` only used for sourceSync, not destSync. Though these are not technically states... - * And they are not safe to just erase if fullSync = true. - * TODO: Introduce a separate sourceOptions / destinationOptions type later when it becomes an - * actual problem... for now this issue only impacts FirebaseProvider and FSProvider - * which are not actually being used as top level providers - */ - sourceState: zWatchPathsInput.pick({paths: true}), - sourceOutputEntity: zCast(), - destinationInputEntity: zCast(), -}) - -export const fsProvider = makeSyncProvider({ - metadata: { - platforms: ['local'], - displayName: 'File system', - categories: ['flat-files-and-spreadsheets'], - }, - ...makeSyncProvider.defaults, - def, +export const fsServer = { sourceSync: ({settings, state}) => _fsWatchPaths$({...settings, ...state}).pipe(_readPathData()), destinationSync: ({settings: {basePath}}) => @@ -61,8 +31,9 @@ export const fsProvider = makeSyncProvider({ ), ), }), -}) +} satisfies IntegrationServer +export default fsServer // MARK: - Source sync helpers /** id: filename, entityName: dirname */ diff --git a/packages/airbyte/Dockerfile b/packages/airbyte/Dockerfile index 5179417f..19731238 100644 --- a/packages/airbyte/Dockerfile +++ b/packages/airbyte/Dockerfile @@ -27,7 +27,7 @@ COPY ./integrations/integration-venmo/package.json ./integrations/integration-ve COPY ./integrations/integration-splitwise/package.json ./integrations/integration-splitwise/package.json COPY ./integrations/integration-yodlee/package.json ./integrations/integration-yodlee/package.json COPY ./integrations/core-integration-mongodb/package.json ./integrations/core-integration-mongodb/package.json -COPY ./integrations/core-integration-fs/package.json ./integrations/core-integration-fs/package.json +COPY ./integrations/integration-fs/package.json ./integrations/integration-fs/package.json COPY ./integrations/integration-ramp/package.json ./integrations/integration-ramp/package.json COPY ./integrations/integration-wise/package.json ./integrations/integration-wise/package.json COPY ./integrations/integration-teller/package.json ./integrations/integration-teller/package.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c20a819b..0682853a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -161,9 +161,6 @@ importers: '@usevenice/core-integration-firebase': specifier: workspace:* version: link:../../integrations/core-integration-firebase - '@usevenice/core-integration-fs': - specifier: workspace:* - version: link:../../integrations/core-integration-fs '@usevenice/core-integration-mongodb': specifier: workspace:* version: link:../../integrations/core-integration-mongodb @@ -200,6 +197,9 @@ importers: '@usevenice/integration-foreceipt': specifier: workspace:* version: link:../../integrations/integration-foreceipt + '@usevenice/integration-fs': + specifier: workspace:* + version: link:../../integrations/integration-fs '@usevenice/integration-heron': specifier: workspace:* version: link:../../integrations/integration-heron @@ -318,9 +318,6 @@ importers: '@usevenice/core-integration-firebase': specifier: workspace:* version: link:../../integrations/core-integration-firebase - '@usevenice/core-integration-fs': - specifier: workspace:* - version: link:../../integrations/core-integration-fs '@usevenice/core-integration-postgres': specifier: workspace:* version: link:../../integrations/core-integration-postgres @@ -339,6 +336,9 @@ importers: '@usevenice/integration-expensify': specifier: workspace:* version: link:../../integrations/integration-expensify + '@usevenice/integration-fs': + specifier: workspace:* + version: link:../../integrations/integration-fs '@usevenice/integration-heron': specifier: workspace:* version: link:../../integrations/integration-heron @@ -664,15 +664,6 @@ importers: specifier: 10.2.0 version: 10.2.0(@firebase/app-compat@0.1.35)(@firebase/app-types@0.8.0) - integrations/core-integration-fs: - dependencies: - '@usevenice/cdk-core': - specifier: workspace:* - version: link:../../packages/cdk-core - '@usevenice/util': - specifier: workspace:* - version: link:../../packages/util - integrations/core-integration-mongodb: dependencies: '@usevenice/cdk-core': @@ -841,6 +832,15 @@ importers: specifier: 9.8.1 version: 9.8.1(patch_hash=bmwbs4hhuxm5sycev5w2vhohk4) + integrations/integration-fs: + dependencies: + '@usevenice/cdk-core': + specifier: workspace:* + version: link:../../packages/cdk-core + '@usevenice/util': + specifier: workspace:* + version: link:../../packages/util + integrations/integration-heron: dependencies: '@usevenice/cdk-core': From bd4398278ca42e1776cc67a8d67618b913a207e8 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 9 Jun 2023 00:43:39 +0800 Subject: [PATCH 15/80] Migrate firebase integration --- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 + .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 14 +- apps/app-config/package.json | 2 +- apps/app-config/providers.ts | 4 - apps/cli/package.json | 2 +- .../MultiBatch.ts | 2 +- integrations/integration-firebase/def.ts | 93 ++++++++++ .../firebase-types.ts | 0 .../firebase-utils.ts | 0 .../firebaseAuth.ts | 45 +---- .../index.ts | 3 +- .../package.json | 2 +- .../register.node.ts | 2 +- .../server.ts} | 54 +----- .../integration-foreceipt/ForeceiptClient.ts | 2 +- integrations/integration-foreceipt/def.ts | 2 +- .../integration-foreceipt/package.json | 2 +- .../integration-foreceipt/register.node.ts | 2 +- integrations/integration-foreceipt/server.ts | 2 +- integrations/integration-fs/server.ts | 3 +- packages/airbyte/Dockerfile | 2 +- pnpm-lock.yaml | 160 ++++++------------ 24 files changed, 191 insertions(+), 219 deletions(-) rename integrations/{core-integration-firebase => integration-firebase}/MultiBatch.ts (99%) create mode 100644 integrations/integration-firebase/def.ts rename integrations/{core-integration-firebase => integration-firebase}/firebase-types.ts (100%) rename integrations/{core-integration-firebase => integration-firebase}/firebase-utils.ts (100%) rename integrations/{core-integration-firebase => integration-firebase}/firebaseAuth.ts (67%) rename integrations/{core-integration-firebase => integration-firebase}/index.ts (85%) rename integrations/{core-integration-firebase => integration-firebase}/package.json (89%) rename integrations/{core-integration-firebase => integration-firebase}/register.node.ts (74%) rename integrations/{core-integration-firebase/FirebaseProvider.ts => integration-firebase/server.ts} (75%) diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index 629987cb..73ceb965 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -2,6 +2,7 @@ import {default as integrationAirtable} from '@usevenice/integration-airtable/def' import {default as integrationBeancount} from '@usevenice/integration-beancount/def' import {default as integrationBrex} from '@usevenice/integration-brex/def' +import {default as integrationFirebase} from '@usevenice/integration-firebase/def' import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/def' import {default as integrationFs} from '@usevenice/integration-fs/def' import {default as integrationHeron} from '@usevenice/integration-heron/def' @@ -28,6 +29,7 @@ export const defIntegrations = { airtable: integrationAirtable, beancount: integrationBeancount, brex: integrationBrex, + firebase: integrationFirebase, foreceipt: integrationForeceipt, fs: integrationFs, heron: integrationHeron, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index bc78b7a3..0b4849dc 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -5,6 +5,8 @@ import {default as integrationBeancount_def} from '@usevenice/integration-beanco import {default as integrationBeancount_server} from '@usevenice/integration-beancount/server' import {default as integrationBrex_def} from '@usevenice/integration-brex/def' import {default as integrationBrex_server} from '@usevenice/integration-brex/server' +import {default as integrationFirebase_def} from '@usevenice/integration-firebase/def' +import {default as integrationFirebase_server} from '@usevenice/integration-firebase/server' import {default as integrationForeceipt_def} from '@usevenice/integration-foreceipt/def' import {default as integrationForeceipt_server} from '@usevenice/integration-foreceipt/server' import {default as integrationFs_def} from '@usevenice/integration-fs/def' @@ -66,6 +68,11 @@ const integrationBrex = { ...integrationBrex_server, } +const integrationFirebase = { + ...integrationFirebase_def, + ...integrationFirebase_server, +} + const integrationForeceipt = { ...integrationForeceipt_def, ...integrationForeceipt_server, @@ -178,6 +185,7 @@ export const mergedIntegrations = { airtable: integrationAirtable, beancount: integrationBeancount, brex: integrationBrex, + firebase: integrationFirebase, foreceipt: integrationForeceipt, fs: integrationFs, heron: integrationHeron, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index 9f476fdc..9e693f0e 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -2,6 +2,7 @@ import {default as integrationAirtable} from '@usevenice/integration-airtable/server' import {default as integrationBeancount} from '@usevenice/integration-beancount/server' import {default as integrationBrex} from '@usevenice/integration-brex/server' +import {default as integrationFirebase} from '@usevenice/integration-firebase/server' import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/server' import {default as integrationFs} from '@usevenice/integration-fs/server' import {default as integrationHeron} from '@usevenice/integration-heron/server' @@ -26,6 +27,7 @@ export const serverIntegrations = { airtable: integrationAirtable, beancount: integrationBeancount, brex: integrationBrex, + firebase: integrationFirebase, foreceipt: integrationForeceipt, fs: integrationFs, heron: integrationHeron, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index 6f296066..069627d1 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -1,11 +1,6 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand module.exports = [ - { - dirName: 'core-integration-firebase', - varName: 'coreIntegrationFirebase', - imports: {}, - }, { dirName: 'core-integration-mongodb', varName: 'coreIntegrationMongodb', @@ -63,6 +58,15 @@ module.exports = [ varName: 'integrationExpensify', imports: {}, }, + { + name: 'firebase', + dirName: 'integration-firebase', + varName: 'integrationFirebase', + imports: { + def: '@usevenice/integration-firebase/def', + server: '@usevenice/integration-firebase/server', + }, + }, { name: 'foreceipt', dirName: 'integration-foreceipt', diff --git a/apps/app-config/package.json b/apps/app-config/package.json index d68f8e4c..cd53d500 100644 --- a/apps/app-config/package.json +++ b/apps/app-config/package.json @@ -13,7 +13,7 @@ "@usevenice/cdk-core": "workspace:*", "@usevenice/cdk-ledger": "workspace:*", "@usevenice/integration-airtable": "workspace:*", - "@usevenice/core-integration-firebase": "workspace:*", + "@usevenice/integration-firebase": "workspace:*", "@usevenice/integration-fs": "workspace:*", "@usevenice/core-integration-mongodb": "workspace:*", "@usevenice/core-integration-postgres": "workspace:*", diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 567767c0..303bc0a4 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -1,8 +1,6 @@ // @deprecated. Soon to be fully replaced by integrations.merge.ts import {debugProvider} from '@usevenice/cdk-core' -import {firebaseProvider} from '@usevenice/core-integration-firebase' -import {fsProvider} from '@usevenice/integration-fs' import {mongodbProvider} from '@usevenice/core-integration-mongodb' import {corePostgresProvider} from '@usevenice/core-integration-postgres' import {webhookProvider} from '@usevenice/core-integration-webhook' @@ -18,8 +16,6 @@ export const DOCUMENTED_PROVIDERS = [ export const PROVIDERS = [ ...DOCUMENTED_PROVIDERS, // TODO: Migrate these over to the new paradigm - fsProvider, - firebaseProvider, mongodbProvider, webhookProvider, debugProvider, diff --git a/apps/cli/package.json b/apps/cli/package.json index b2aa93c9..c1a4b57a 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -9,7 +9,7 @@ "@usevenice/cdk-core": "workspace:*", "@usevenice/cdk-ledger": "workspace:*", "@usevenice/integration-fs": "workspace:*", - "@usevenice/core-integration-firebase": "workspace:*", + "@usevenice/integration-firebase": "workspace:*", "@usevenice/core-integration-postgres": "workspace:*", "@usevenice/engine-backend": "workspace:*", "@usevenice/integration-alphavantage": "workspace:*", diff --git a/integrations/core-integration-firebase/MultiBatch.ts b/integrations/integration-firebase/MultiBatch.ts similarity index 99% rename from integrations/core-integration-firebase/MultiBatch.ts rename to integrations/integration-firebase/MultiBatch.ts index 0a397a88..72a55817 100644 --- a/integrations/core-integration-firebase/MultiBatch.ts +++ b/integrations/integration-firebase/MultiBatch.ts @@ -19,7 +19,7 @@ import type { SetOptions, } from './firebase-types' import {fieldPath} from './firebase-utils' -import type {WrappedFirebase} from './FirebaseProvider' +import type {WrappedFirebase} from './server' type Instruction = | [ diff --git a/integrations/integration-firebase/def.ts b/integrations/integration-firebase/def.ts new file mode 100644 index 00000000..d261a38f --- /dev/null +++ b/integrations/integration-firebase/def.ts @@ -0,0 +1,93 @@ +import type { + AnyEntityPayload, + IntegrationDef, + IntegrationSchemas, +} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {z, zCast} from '@usevenice/util' + +import type {AnyQuery} from './firebase-types' +import type {WrappedFirebase} from './server' + +export const zFirebaseConfig = z.object({ + projectId: z.string(), + apiKey: z.string(), + appId: z.string(), + authDomain: z.string(), + databaseURL: z.string(), + storageBucket: z.string().optional(), + measurementId: z.string().optional(), + messagingSenderId: z.string().optional(), +}) + +/** + * Can be obtained by executing the following in the browser + * `console.log(JSON.stringify(fba.auth().currentUser.toJSON(), null, 2))` + */ +export type AuthUserJson = z.infer +export const zAuthUserJson = z + .object({ + uid: z.string(), + appName: z.string(), + stsTokenManager: z.record(z.unknown()), + }) + .catchall(z.unknown()) + +export const zAuthData = z.discriminatedUnion('method', [ + z.object({method: z.literal('userJson'), userJson: zAuthUserJson}), + z.object({method: z.literal('customToken'), customToken: z.string()}), + z.object({ + method: z.literal('emailPassword'), + email: z.string(), + password: z.string(), + }), +]) + +export const zFirebaseUserConfig = z.object({ + firebaseConfig: zFirebaseConfig, + authData: zAuthData, +}) + +export const zServiceAccount = z + .object({project_id: z.string()}) + .catchall(z.unknown()) + +export const zSettings = z.discriminatedUnion('role', [ + z.object({role: z.literal('admin'), serviceAccount: zServiceAccount}), + z.object({role: z.literal('user')}).merge(zFirebaseUserConfig), +]) + +export const firebaseSchemas = { + name: z.literal('firebase'), + resourceSettings: zSettings, + sourceState: z.object({ + /** + * Only used for sourceSync, not destSync. Though these are not technically states... + * And they are not safe to just erase if fullSync = true. + * TODO: Introduce a separate sourceOptions / destinationOptions type later when it becomes an + * actual problem... for now this issue only impacts FirebaseProvider and FSProvider + * which are not actually being used as top level providers + */ + collectionPaths: z.array(z.string()).nullish(), + /** + * NEXT: We should use a JSON representation of query so that it can be transfered over the network + * However firestore doesn't expose serialize / deserialize, therefore using Query type directly as a short + * term hack... + * @see https://firebase.google.com/docs/firestore/reference/rest/v1/StructuredQuery + */ + _queries: zCast(), + _fb: zCast(), + }), + sourceOutputEntity: zCast(), + destinationInputEntity: zCast(), +} satisfies IntegrationSchemas + +export const firebaseHelpers = intHelpers(firebaseSchemas) + +export const firebaseDef = { + name: 'firebase', + metadata: {categories: ['database'], logoUrl: '/_assets/logo-firebase.png'}, + def: firebaseSchemas, +} satisfies IntegrationDef + +export default firebaseDef diff --git a/integrations/core-integration-firebase/firebase-types.ts b/integrations/integration-firebase/firebase-types.ts similarity index 100% rename from integrations/core-integration-firebase/firebase-types.ts rename to integrations/integration-firebase/firebase-types.ts diff --git a/integrations/core-integration-firebase/firebase-utils.ts b/integrations/integration-firebase/firebase-utils.ts similarity index 100% rename from integrations/core-integration-firebase/firebase-utils.ts rename to integrations/integration-firebase/firebase-utils.ts diff --git a/integrations/core-integration-firebase/firebaseAuth.ts b/integrations/integration-firebase/firebaseAuth.ts similarity index 67% rename from integrations/core-integration-firebase/firebaseAuth.ts rename to integrations/integration-firebase/firebaseAuth.ts index 8e9fa955..bde663f3 100644 --- a/integrations/core-integration-firebase/firebaseAuth.ts +++ b/integrations/integration-firebase/firebaseAuth.ts @@ -1,51 +1,14 @@ -import {UserImpl} from '@firebase/auth/internal' -import firebase from 'firebase/compat/app' - -import {z, zFunction} from '@usevenice/util' - // TODO: Migrate to tree-shakable version 9 of firebase once we confirm // compat is working import 'firebase/compat/auth' import 'firebase/compat/firestore' -export const zFirebaseConfig = z.object({ - projectId: z.string(), - apiKey: z.string(), - appId: z.string(), - authDomain: z.string(), - databaseURL: z.string(), - storageBucket: z.string().optional(), - measurementId: z.string().optional(), - messagingSenderId: z.string().optional(), -}) - -/** - * Can be obtained by executing the following in the browser - * `console.log(JSON.stringify(fba.auth().currentUser.toJSON(), null, 2))` - */ -export type AuthUserJson = z.infer -export const zAuthUserJson = z - .object({ - uid: z.string(), - appName: z.string(), - stsTokenManager: z.record(z.unknown()), - }) - .catchall(z.unknown()) +import {UserImpl} from '@firebase/auth/internal' +import firebase from 'firebase/compat/app' -export const zAuthData = z.discriminatedUnion('method', [ - z.object({method: z.literal('userJson'), userJson: zAuthUserJson}), - z.object({method: z.literal('customToken'), customToken: z.string()}), - z.object({ - method: z.literal('emailPassword'), - email: z.string(), - password: z.string(), - }), -]) +import {z, zFunction} from '@usevenice/util' -export const zFirebaseUserConfig = z.object({ - firebaseConfig: zFirebaseConfig, - authData: zAuthData, -}) +import {zAuthData, zFirebaseConfig} from './def' export const makeFirebaseAuth = zFunction(zFirebaseConfig, (config) => { const fba = firebase.initializeApp(config, config.projectId) diff --git a/integrations/core-integration-firebase/index.ts b/integrations/integration-firebase/index.ts similarity index 85% rename from integrations/core-integration-firebase/index.ts rename to integrations/integration-firebase/index.ts index 55c54f9a..0f9b324f 100644 --- a/integrations/core-integration-firebase/index.ts +++ b/integrations/integration-firebase/index.ts @@ -1,7 +1,8 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' export * from './firebase-types' export * from './firebase-utils' export * from './firebaseAuth' -export * from './FirebaseProvider' export * from './MultiBatch' +export * from './server' // codegen:end diff --git a/integrations/core-integration-firebase/package.json b/integrations/integration-firebase/package.json similarity index 89% rename from integrations/core-integration-firebase/package.json rename to integrations/integration-firebase/package.json index 7c5a755c..eca91953 100644 --- a/integrations/core-integration-firebase/package.json +++ b/integrations/integration-firebase/package.json @@ -1,5 +1,5 @@ { - "name": "@usevenice/core-integration-firebase", + "name": "@usevenice/integration-firebase", "version": "0.0.0", "private": true, "sideEffects": [ diff --git a/integrations/core-integration-firebase/register.node.ts b/integrations/integration-firebase/register.node.ts similarity index 74% rename from integrations/core-integration-firebase/register.node.ts rename to integrations/integration-firebase/register.node.ts index b9555d9f..19dbc5f1 100644 --- a/integrations/core-integration-firebase/register.node.ts +++ b/integrations/integration-firebase/register.node.ts @@ -2,6 +2,6 @@ import admin from 'firebase-admin' import {implementProxyFn} from '@usevenice/util' -import {$admin} from './FirebaseProvider' +import {$admin} from './server' implementProxyFn($admin, () => admin) diff --git a/integrations/core-integration-firebase/FirebaseProvider.ts b/integrations/integration-firebase/server.ts similarity index 75% rename from integrations/core-integration-firebase/FirebaseProvider.ts rename to integrations/integration-firebase/server.ts index 04855032..469672f6 100644 --- a/integrations/core-integration-firebase/FirebaseProvider.ts +++ b/integrations/integration-firebase/server.ts @@ -1,7 +1,7 @@ import firebase from 'firebase/compat/app' -import type {AnyEntityPayload, Link, SyncOperation} from '@usevenice/cdk-core' -import {handlersLink, makeSyncProvider, mergeReady} from '@usevenice/cdk-core' +import type {IntegrationServer, Link, SyncOperation} from '@usevenice/cdk-core' +import {handlersLink, mergeReady} from '@usevenice/cdk-core' import { defineProxyFn, fromCompletion, @@ -12,10 +12,11 @@ import { rxjs, tapTeartown, z, - zCast, zFunction, } from '@usevenice/util' +import type {firebaseSchemas} from './def' +import {zSettings} from './def' import type {AnyQuery} from './firebase-types' import { getPathForQuery, @@ -23,21 +24,12 @@ import { getQuerySnapshot$, isTimestamp, } from './firebase-utils' -import {makeFirebaseAuth, zFirebaseUserConfig} from './firebaseAuth' +import {makeFirebaseAuth} from './firebaseAuth' import {MultiBatch} from './MultiBatch' export const $admin = defineProxyFn<() => typeof import('firebase-admin')>('firebase-admin') -export const zServiceAccount = z - .object({project_id: z.string()}) - .catchall(z.unknown()) - -const zSettings = z.discriminatedUnion('role', [ - z.object({role: z.literal('admin'), serviceAccount: zServiceAccount}), - z.object({role: z.literal('user')}).merge(zFirebaseUserConfig), -]) - type AuthSettings = z.infer type _query = firebase.firestore.Query @@ -81,37 +73,7 @@ function fromQuery>(query: AnyQuery) { // logLink({prefix: queryPath}), ) } - -const def = makeSyncProvider.def({ - ...makeSyncProvider.def.defaults, - name: z.literal('firebase'), - resourceSettings: zSettings, - sourceState: z.object({ - /** - * Only used for sourceSync, not destSync. Though these are not technically states... - * And they are not safe to just erase if fullSync = true. - * TODO: Introduce a separate sourceOptions / destinationOptions type later when it becomes an - * actual problem... for now this issue only impacts FirebaseProvider and FSProvider - * which are not actually being used as top level providers - */ - collectionPaths: z.array(z.string()).nullish(), - /** - * NEXT: We should use a JSON representation of query so that it can be transfered over the network - * However firestore doesn't expose serialize / deserialize, therefore using Query type directly as a short - * term hack... - * @see https://firebase.google.com/docs/firestore/reference/rest/v1/StructuredQuery - */ - _queries: zCast(), - _fb: zCast(), - }), - sourceOutputEntity: zCast(), - destinationInputEntity: zCast(), -}) - -export const firebaseProvider = makeSyncProvider({ - metadata: {categories: ['database'], logoUrl: '/_assets/logo-firebase.png'}, - ...makeSyncProvider.defaults, - def, +export const firebaseServer = { sourceSync: ({settings, state}) => { const {fst, cleanup, connect} = state._fb ?? initFirebase(settings) const queries = [ @@ -162,7 +124,7 @@ export const firebaseProvider = makeSyncProvider({ tapTeartown(cleanup), ) }, -}) +} satisfies IntegrationServer export type WrappedFirebase = ReturnType @@ -233,3 +195,5 @@ export const fromFirestore: Link = Rx.map((op) => isTimestamp(v) ? {seconds: v.seconds, nanoseconds: v.nanoseconds} : v, ), ) + +export default firebaseServer diff --git a/integrations/integration-foreceipt/ForeceiptClient.ts b/integrations/integration-foreceipt/ForeceiptClient.ts index 9714466b..a78a146a 100644 --- a/integrations/integration-foreceipt/ForeceiptClient.ts +++ b/integrations/integration-foreceipt/ForeceiptClient.ts @@ -5,7 +5,7 @@ import { initFirebase, zFirebaseUserConfig, zServiceAccount, -} from '@usevenice/core-integration-firebase' +} from '@usevenice/integration-firebase' import type {HTTPError} from '@usevenice/util' import {createHTTPClient, Rx, rxjs, z, zCast, zFunction} from '@usevenice/util' diff --git a/integrations/integration-foreceipt/def.ts b/integrations/integration-foreceipt/def.ts index 7f9bbfa5..173da125 100644 --- a/integrations/integration-foreceipt/def.ts +++ b/integrations/integration-foreceipt/def.ts @@ -1,7 +1,7 @@ import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' import {intHelpers} from '@usevenice/cdk-core' import {makePostingsMap} from '@usevenice/cdk-ledger' -import type {SerializedTimestamp} from '@usevenice/core-integration-firebase' +import type {SerializedTimestamp} from '@usevenice/integration-firebase' import type {Standard} from '@usevenice/standard' import type {Merge} from '@usevenice/util' import {A, objectFromArray, R, z, zCast} from '@usevenice/util' diff --git a/integrations/integration-foreceipt/package.json b/integrations/integration-foreceipt/package.json index a46eeef4..eccaa4be 100644 --- a/integrations/integration-foreceipt/package.json +++ b/integrations/integration-foreceipt/package.json @@ -9,7 +9,7 @@ "dependencies": { "@usevenice/cdk-core": "workspace:*", "@usevenice/cdk-ledger": "workspace:*", - "@usevenice/core-integration-firebase": "workspace:*", + "@usevenice/integration-firebase": "workspace:*", "@usevenice/standard": "workspace:*", "@usevenice/util": "workspace:*", "firebase": "9.8.1" diff --git a/integrations/integration-foreceipt/register.node.ts b/integrations/integration-foreceipt/register.node.ts index bcfa0a30..8c23a6a8 100644 --- a/integrations/integration-foreceipt/register.node.ts +++ b/integrations/integration-foreceipt/register.node.ts @@ -1 +1 @@ -import '@usevenice/core-integration-firebase/register.node' +import '@usevenice/integration-firebase/register.node' diff --git a/integrations/integration-foreceipt/server.ts b/integrations/integration-foreceipt/server.ts index 9c9b6d1c..21580183 100644 --- a/integrations/integration-foreceipt/server.ts +++ b/integrations/integration-foreceipt/server.ts @@ -2,7 +2,7 @@ import type {IntegrationServer} from '@usevenice/cdk-core' import { firebaseProvider, serializeTimestamp, -} from '@usevenice/core-integration-firebase' +} from '@usevenice/integration-firebase' import {Rx, rxjs} from '@usevenice/util' import type {foreceiptSchemas} from './def' diff --git a/integrations/integration-fs/server.ts b/integrations/integration-fs/server.ts index 69e6b782..a9cdd244 100644 --- a/integrations/integration-fs/server.ts +++ b/integrations/integration-fs/server.ts @@ -33,7 +33,6 @@ export const fsServer = { }), } satisfies IntegrationServer -export default fsServer // MARK: - Source sync helpers /** id: filename, entityName: dirname */ @@ -125,3 +124,5 @@ function _readPathData() { Rx.filter((e): e is NonNullable => !!e), ) } + +export default fsServer diff --git a/packages/airbyte/Dockerfile b/packages/airbyte/Dockerfile index 19731238..385cf6d5 100644 --- a/packages/airbyte/Dockerfile +++ b/packages/airbyte/Dockerfile @@ -39,7 +39,7 @@ COPY ./integrations/integration-beancount/package.json ./integrations/integratio COPY ./integrations/core-integration-redis/package.json ./integrations/core-integration-redis/package.json COPY ./integrations/integration-onebrick/package.json ./integrations/integration-onebrick/package.json COPY ./integrations/core-integration-postgres/package.json ./integrations/core-integration-postgres/package.json -COPY ./integrations/core-integration-firebase/package.json ./integrations/core-integration-firebase/package.json +COPY ./integrations/integration-firebase/package.json ./integrations/integration-firebase/package.json COPY ./integrations/integration-airtable/package.json ./integrations/integration-airtable/package.json COPY ./integrations/integration-plaid/package.json ./integrations/integration-plaid/package.json COPY ./integrations/integration-moota/package.json ./integrations/integration-moota/package.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0682853a..baf98107 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -158,9 +158,6 @@ importers: '@usevenice/cdk-ledger': specifier: workspace:* version: link:../../packages/cdk-ledger - '@usevenice/core-integration-firebase': - specifier: workspace:* - version: link:../../integrations/core-integration-firebase '@usevenice/core-integration-mongodb': specifier: workspace:* version: link:../../integrations/core-integration-mongodb @@ -194,6 +191,9 @@ importers: '@usevenice/integration-expensify': specifier: workspace:* version: link:../../integrations/integration-expensify + '@usevenice/integration-firebase': + specifier: workspace:* + version: link:../../integrations/integration-firebase '@usevenice/integration-foreceipt': specifier: workspace:* version: link:../../integrations/integration-foreceipt @@ -315,9 +315,6 @@ importers: '@usevenice/cdk-ledger': specifier: workspace:* version: link:../../packages/cdk-ledger - '@usevenice/core-integration-firebase': - specifier: workspace:* - version: link:../../integrations/core-integration-firebase '@usevenice/core-integration-postgres': specifier: workspace:* version: link:../../integrations/core-integration-postgres @@ -336,6 +333,9 @@ importers: '@usevenice/integration-expensify': specifier: workspace:* version: link:../../integrations/integration-expensify + '@usevenice/integration-firebase': + specifier: workspace:* + version: link:../../integrations/integration-firebase '@usevenice/integration-fs': specifier: workspace:* version: link:../../integrations/integration-fs @@ -637,33 +637,6 @@ importers: specifier: '*' version: 5.74.0(esbuild@0.17.5) - integrations/core-integration-firebase: - dependencies: - '@firebase/app': - specifier: 0.x - version: 0.8.0 - '@firebase/app-compat': - specifier: 0.x - version: 0.1.35 - '@firebase/app-types': - specifier: 0.x - version: 0.8.0 - '@firebase/auth': - specifier: '*' - version: 0.20.8(@firebase/app@0.8.0) - '@usevenice/cdk-core': - specifier: workspace:* - version: link:../../packages/cdk-core - '@usevenice/util': - specifier: workspace:* - version: link:../../packages/util - firebase: - specifier: 9.8.1 - version: 9.8.1(patch_hash=bmwbs4hhuxm5sycev5w2vhohk4) - firebase-admin: - specifier: 10.2.0 - version: 10.2.0(@firebase/app-compat@0.1.35)(@firebase/app-types@0.8.0) - integrations/core-integration-mongodb: dependencies: '@usevenice/cdk-core': @@ -811,6 +784,33 @@ importers: specifier: workspace:* version: link:../../packages/util + integrations/integration-firebase: + dependencies: + '@firebase/app': + specifier: 0.x + version: 0.7.24 + '@firebase/app-compat': + specifier: 0.x + version: 0.1.25 + '@firebase/app-types': + specifier: 0.x + version: 0.7.0 + '@firebase/auth': + specifier: '*' + version: 0.20.1(@firebase/app@0.7.24) + '@usevenice/cdk-core': + specifier: workspace:* + version: link:../../packages/cdk-core + '@usevenice/util': + specifier: workspace:* + version: link:../../packages/util + firebase: + specifier: 9.8.1 + version: 9.8.1(patch_hash=bmwbs4hhuxm5sycev5w2vhohk4) + firebase-admin: + specifier: 10.2.0 + version: 10.2.0(@firebase/app-compat@0.1.25)(@firebase/app-types@0.7.0) + integrations/integration-foreceipt: dependencies: '@usevenice/cdk-core': @@ -819,9 +819,9 @@ importers: '@usevenice/cdk-ledger': specifier: workspace:* version: link:../../packages/cdk-ledger - '@usevenice/core-integration-firebase': + '@usevenice/integration-firebase': specifier: workspace:* - version: link:../core-integration-firebase + version: link:../integration-firebase '@usevenice/standard': specifier: workspace:* version: link:../../packages/standard @@ -2691,16 +2691,6 @@ packages: tslib: 2.4.1 dev: false - /@firebase/app-compat@0.1.35: - resolution: {integrity: sha512-6ax9yXCPEBSREHxo+nCpSgSg01mGTvR4I7u/EHqVNNqG8uEWog7sUan3Y3vr3q3zH8t5BkXDGejOH9atF+XnAQ==} - dependencies: - '@firebase/app': 0.8.0 - '@firebase/component': 0.5.18 - '@firebase/logger': 0.3.3 - '@firebase/util': 1.7.0 - tslib: 2.4.0 - dev: false - /@firebase/app-types@0.7.0: resolution: {integrity: sha512-6fbHQwDv2jp/v6bXhBw2eSRbNBpxHcd1NBF864UksSMVIqIyri9qpJB1Mn6sGZE+bnDsSQBC5j2TbMxYsJQkQg==} dev: false @@ -2719,16 +2709,6 @@ packages: tslib: 2.4.1 dev: false - /@firebase/app@0.8.0: - resolution: {integrity: sha512-9kZjhIDv4u4PlrCgcQVBA2u8BZHrP8rUWDltmCUi9BLHv0tltfxLMZODV5LeuAfCJKVp2dbIrpGHPxAaLLl/ww==} - dependencies: - '@firebase/component': 0.5.18 - '@firebase/logger': 0.3.3 - '@firebase/util': 1.7.0 - idb: 7.0.1 - tslib: 2.4.0 - dev: false - /@firebase/auth-compat@0.2.14(@firebase/app-compat@0.1.25)(@firebase/app-types@0.7.0)(@firebase/app@0.7.24): resolution: {integrity: sha512-1KSNItrTQzky2d0GVCum6d7Hdj9pfNh9aGTN0uJPNk+th9XHBCy0El8Wx5yk0miiyB3h1evWAXdgnIyNs4kTEQ==} peerDependencies: @@ -2750,24 +2730,24 @@ packages: - utf-8-validate dev: false - /@firebase/auth-interop-types@0.1.6(@firebase/app-types@0.7.0)(@firebase/util@1.6.0): + /@firebase/auth-interop-types@0.1.6(@firebase/app-types@0.7.0)(@firebase/util@1.5.2): resolution: {integrity: sha512-etIi92fW3CctsmR9e3sYM3Uqnoq861M0Id9mdOPF6PWIg38BXL5k4upCNBggGUpLIS0H1grMOvy/wn1xymwe2g==} peerDependencies: '@firebase/app-types': 0.x '@firebase/util': 1.x dependencies: '@firebase/app-types': 0.7.0 - '@firebase/util': 1.6.0 + '@firebase/util': 1.5.2 dev: false - /@firebase/auth-interop-types@0.1.6(@firebase/app-types@0.8.0)(@firebase/util@1.5.2): + /@firebase/auth-interop-types@0.1.6(@firebase/app-types@0.7.0)(@firebase/util@1.6.0): resolution: {integrity: sha512-etIi92fW3CctsmR9e3sYM3Uqnoq861M0Id9mdOPF6PWIg38BXL5k4upCNBggGUpLIS0H1grMOvy/wn1xymwe2g==} peerDependencies: '@firebase/app-types': 0.x '@firebase/util': 1.x dependencies: - '@firebase/app-types': 0.8.0 - '@firebase/util': 1.5.2 + '@firebase/app-types': 0.7.0 + '@firebase/util': 1.6.0 dev: false /@firebase/auth-types@0.11.0(@firebase/app-types@0.7.0)(@firebase/util@1.6.0): @@ -2798,24 +2778,6 @@ packages: - utf-8-validate dev: false - /@firebase/auth@0.20.8(@firebase/app@0.8.0): - resolution: {integrity: sha512-ss0Uyp5sLrGRR/8bbkZTod5gmdgltqvcxQySAKYGbsyBq4j+RTjZzVqUHZHFY7f8NLm2Bz4hO1dZRaIUYW8zLw==} - peerDependencies: - '@firebase/app': 0.x - dependencies: - '@firebase/app': 0.8.0 - '@firebase/component': 0.5.18 - '@firebase/logger': 0.3.3 - '@firebase/util': 1.7.0 - node-fetch: 2.6.7 - selenium-webdriver: 4.1.2 - tslib: 2.4.0 - transitivePeerDependencies: - - bufferutil - - encoding - - utf-8-validate - dev: false - /@firebase/component@0.5.13: resolution: {integrity: sha512-hxhJtpD8Ppf/VU2Rlos6KFCEV77TGIGD5bJlkPK1+B/WUe0mC6dTjW7KhZtXTc+qRBp9nFHWcsIORnT8liHP9w==} dependencies: @@ -2830,21 +2792,14 @@ packages: tslib: 2.4.1 dev: false - /@firebase/component@0.5.18: - resolution: {integrity: sha512-worbz6idNWud/Sfpp3Lf9BE9tM8GRHhuQ4Hsqnva6ECdSRKYt8RRPg3UUSwDGa4iFpPo+gF/jKfydYN676+JmQ==} - dependencies: - '@firebase/util': 1.7.0 - tslib: 2.4.1 - dev: false - - /@firebase/database-compat@0.1.8(@firebase/app-compat@0.1.35)(@firebase/app-types@0.8.0): + /@firebase/database-compat@0.1.8(@firebase/app-compat@0.1.25)(@firebase/app-types@0.7.0): resolution: {integrity: sha512-dhXr5CSieBuKNdU96HgeewMQCT9EgOIkfF1GNy+iRrdl7BWLxmlKuvLfK319rmIytSs/vnCzcD9uqyxTeU/A3A==} peerDependencies: '@firebase/app-compat': 0.x dependencies: - '@firebase/app-compat': 0.1.35 + '@firebase/app-compat': 0.1.25 '@firebase/component': 0.5.13 - '@firebase/database': 0.12.8(@firebase/app-types@0.8.0) + '@firebase/database': 0.12.8(@firebase/app-types@0.7.0) '@firebase/database-types': 0.9.7 '@firebase/logger': 0.3.2 '@firebase/util': 1.5.2 @@ -2890,10 +2845,10 @@ packages: '@firebase/util': 1.6.0 dev: false - /@firebase/database@0.12.8(@firebase/app-types@0.8.0): + /@firebase/database@0.12.8(@firebase/app-types@0.7.0): resolution: {integrity: sha512-JBQVfFLzfhxlQbl4OU6ov9fdsddkytBQdtSSR49cz48homj38ccltAhK6seum+BI7f28cV2LFHF9672lcN+qxA==} dependencies: - '@firebase/auth-interop-types': 0.1.6(@firebase/app-types@0.8.0)(@firebase/util@1.5.2) + '@firebase/auth-interop-types': 0.1.6(@firebase/app-types@0.7.0)(@firebase/util@1.5.2) '@firebase/component': 0.5.13 '@firebase/logger': 0.3.2 '@firebase/util': 1.5.2 @@ -3019,12 +2974,6 @@ packages: tslib: 2.4.1 dev: false - /@firebase/logger@0.3.3: - resolution: {integrity: sha512-POTJl07jOKTOevLXrTvJD/VZ0M6PnJXflbAh5J9VGkmtXPXNG6MdZ9fmRgqYhXKTaDId6AQenQ262uwgpdtO0Q==} - dependencies: - tslib: 2.4.1 - dev: false - /@firebase/messaging-compat@0.1.13(@firebase/app-compat@0.1.25)(@firebase/app@0.7.24): resolution: {integrity: sha512-kGuzjpl+pcTRmEgGDjyOKQnxxQgC7wIJIIHhLMIpfxHHL5+ysN1Tjq0Ztr1t/gcdHKErtnD/n9To5eoGZHqpzA==} peerDependencies: @@ -6485,6 +6434,7 @@ packages: /@types/node@18.8.3: resolution: {integrity: sha512-0os9vz6BpGwxGe9LOhgP/ncvYN5Tx1fNcd2TM3rD/aCGBkysb+ZWpXEocG24h6ZzOi13+VB8HndAQFezsSOw1w==} + dev: true /@types/node@8.10.66: resolution: {integrity: sha512-tktOkFUA4kXx2hhhrB8bIFb5TbwzS4uOhKEmwiD+NoiL0qtP2OQ9mFldbgD4dV1djrlBYP6eBuQZiWjuHUpqFw==} @@ -9807,13 +9757,13 @@ packages: path-exists: 4.0.0 dev: true - /firebase-admin@10.2.0(@firebase/app-compat@0.1.35)(@firebase/app-types@0.8.0): + /firebase-admin@10.2.0(@firebase/app-compat@0.1.25)(@firebase/app-types@0.7.0): resolution: {integrity: sha512-6ehn5J9UEFgi4+naqYvozmGpnZae3cJLdwSkSsDc8/Y0eTBjVMFdf9N2ft7N81UNHA0N5DknOyXhlsdAdyBLCA==} engines: {node: '>=12.7.0'} dependencies: - '@firebase/database-compat': 0.1.8(@firebase/app-compat@0.1.35)(@firebase/app-types@0.8.0) + '@firebase/database-compat': 0.1.8(@firebase/app-compat@0.1.25)(@firebase/app-types@0.7.0) '@firebase/database-types': 0.9.14 - '@types/node': 18.8.3 + '@types/node': 18.11.18 dicer: 0.3.1 jsonwebtoken: 8.5.1 jwks-rsa: 2.1.4 @@ -15674,18 +15624,6 @@ packages: - utf-8-validate dev: false - /selenium-webdriver@4.1.2: - resolution: {integrity: sha512-e4Ap8vQvhipgBB8Ry9zBiKGkU6kHKyNnWiavGGLKkrdW81Zv7NVMtFOL/j3yX0G8QScM7XIXijKssNd4EUxSOw==} - engines: {node: '>= 10.15.0'} - dependencies: - jszip: 3.10.1 - tmp: 0.2.1 - ws: 8.9.0 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - dev: false - /semver-compare@1.0.0: resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==} From 14db7ef4f89bf9267b4f6179d51b3875036064fa Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 9 Jun 2023 00:50:20 +0800 Subject: [PATCH 16/80] Migrate mongo integration --- .../integrations/integrations.def.ts | 2 ++ .../integrations/integrations.merged.ts | 8 +++++ .../integrations/integrations.server.ts | 2 ++ apps/app-config/integrations/meta.js | 14 +++++---- apps/app-config/package.json | 2 +- apps/app-config/providers.ts | 2 -- integrations/integration-mongodb/def.ts | 25 ++++++++++++++++ .../index.ts | 3 +- .../package.json | 2 +- .../server.ts} | 29 +++++++----------- packages/airbyte/Dockerfile | 2 +- pnpm-lock.yaml | 30 +++++++++---------- 12 files changed, 76 insertions(+), 45 deletions(-) create mode 100644 integrations/integration-mongodb/def.ts rename integrations/{core-integration-mongodb => integration-mongodb}/index.ts (76%) rename integrations/{core-integration-mongodb => integration-mongodb}/package.json (80%) rename integrations/{core-integration-mongodb/mongoDBProvider.ts => integration-mongodb/server.ts} (73%) diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index 73ceb965..18139a63 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -9,6 +9,7 @@ import {default as integrationHeron} from '@usevenice/integration-heron/def' import {default as integrationLunchmoney} from '@usevenice/integration-lunchmoney/def' import {default as integrationMercury} from '@usevenice/integration-mercury/def' import {default as integrationMerge} from '@usevenice/integration-merge/def' +import {default as integrationMongodb} from '@usevenice/integration-mongodb/def' import {default as integrationMoota} from '@usevenice/integration-moota/def' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/def' import {default as integrationPlaid} from '@usevenice/integration-plaid/def' @@ -36,6 +37,7 @@ export const defIntegrations = { lunchmoney: integrationLunchmoney, mercury: integrationMercury, merge: integrationMerge, + mongodb: integrationMongodb, moota: integrationMoota, onebrick: integrationOnebrick, plaid: integrationPlaid, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 0b4849dc..5d6932c7 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -19,6 +19,8 @@ import {default as integrationMercury_def} from '@usevenice/integration-mercury/ import {default as integrationMerge_client} from '@usevenice/integration-merge/client' import {default as integrationMerge_def} from '@usevenice/integration-merge/def' import {default as integrationMerge_server} from '@usevenice/integration-merge/server' +import {default as integrationMongodb_def} from '@usevenice/integration-mongodb/def' +import {default as integrationMongodb_server} from '@usevenice/integration-mongodb/server' import {default as integrationMoota_def} from '@usevenice/integration-moota/def' import {default as integrationMoota_server} from '@usevenice/integration-moota/server' import {default as integrationOnebrick_client} from '@usevenice/integration-onebrick/client' @@ -103,6 +105,11 @@ const integrationMerge = { ...integrationMerge_server, } +const integrationMongodb = { + ...integrationMongodb_def, + ...integrationMongodb_server, +} + const integrationMoota = { ...integrationMoota_def, ...integrationMoota_server, @@ -192,6 +199,7 @@ export const mergedIntegrations = { lunchmoney: integrationLunchmoney, mercury: integrationMercury, merge: integrationMerge, + mongodb: integrationMongodb, moota: integrationMoota, onebrick: integrationOnebrick, plaid: integrationPlaid, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index 9e693f0e..a1382ba8 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -8,6 +8,7 @@ import {default as integrationFs} from '@usevenice/integration-fs/server' import {default as integrationHeron} from '@usevenice/integration-heron/server' import {default as integrationLunchmoney} from '@usevenice/integration-lunchmoney/server' import {default as integrationMerge} from '@usevenice/integration-merge/server' +import {default as integrationMongodb} from '@usevenice/integration-mongodb/server' import {default as integrationMoota} from '@usevenice/integration-moota/server' import {default as integrationOnebrick} from '@usevenice/integration-onebrick/server' import {default as integrationPlaid} from '@usevenice/integration-plaid/server' @@ -33,6 +34,7 @@ export const serverIntegrations = { heron: integrationHeron, lunchmoney: integrationLunchmoney, merge: integrationMerge, + mongodb: integrationMongodb, moota: integrationMoota, onebrick: integrationOnebrick, plaid: integrationPlaid, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index 069627d1..f09b5957 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -1,11 +1,6 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand module.exports = [ - { - dirName: 'core-integration-mongodb', - varName: 'coreIntegrationMongodb', - imports: {}, - }, { dirName: 'core-integration-postgres', varName: 'coreIntegrationPostgres', @@ -119,6 +114,15 @@ module.exports = [ server: '@usevenice/integration-merge/server', }, }, + { + name: 'mongodb', + dirName: 'integration-mongodb', + varName: 'integrationMongodb', + imports: { + def: '@usevenice/integration-mongodb/def', + server: '@usevenice/integration-mongodb/server', + }, + }, { name: 'moota', dirName: 'integration-moota', diff --git a/apps/app-config/package.json b/apps/app-config/package.json index cd53d500..0083620b 100644 --- a/apps/app-config/package.json +++ b/apps/app-config/package.json @@ -15,7 +15,7 @@ "@usevenice/integration-airtable": "workspace:*", "@usevenice/integration-firebase": "workspace:*", "@usevenice/integration-fs": "workspace:*", - "@usevenice/core-integration-mongodb": "workspace:*", + "@usevenice/integration-mongodb": "workspace:*", "@usevenice/core-integration-postgres": "workspace:*", "@usevenice/core-integration-redis": "workspace:*", "@usevenice/core-integration-webhook": "workspace:*", diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 303bc0a4..d0643861 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -1,7 +1,6 @@ // @deprecated. Soon to be fully replaced by integrations.merge.ts import {debugProvider} from '@usevenice/cdk-core' -import {mongodbProvider} from '@usevenice/core-integration-mongodb' import {corePostgresProvider} from '@usevenice/core-integration-postgres' import {webhookProvider} from '@usevenice/core-integration-webhook' @@ -16,7 +15,6 @@ export const DOCUMENTED_PROVIDERS = [ export const PROVIDERS = [ ...DOCUMENTED_PROVIDERS, // TODO: Migrate these over to the new paradigm - mongodbProvider, webhookProvider, debugProvider, corePostgresProvider, diff --git a/integrations/integration-mongodb/def.ts b/integrations/integration-mongodb/def.ts new file mode 100644 index 00000000..8fe56125 --- /dev/null +++ b/integrations/integration-mongodb/def.ts @@ -0,0 +1,25 @@ +import type { + AnyEntityPayload, + IntegrationDef, + IntegrationSchemas, +} from '@usevenice/cdk-core' +import {z, zCast} from '@usevenice/util' + +export const zMongoConnection = z.object({ + databaseUrl: z.string(), + databaseName: z.string(), +}) + +export const mongoSchemas = { + name: z.literal('mongodb'), + resourceSettings: zMongoConnection, + destinationInputEntity: zCast(), +} satisfies IntegrationSchemas + +export const mongoDef = { + name: 'mongodb', + def: mongoSchemas, + metadata: {categories: ['database'], logoUrl: '/_assets/logo-mongodb.png'}, +} satisfies IntegrationDef + +export default mongoDef diff --git a/integrations/core-integration-mongodb/index.ts b/integrations/integration-mongodb/index.ts similarity index 76% rename from integrations/core-integration-mongodb/index.ts rename to integrations/integration-mongodb/index.ts index 08ad799a..81f31bd7 100644 --- a/integrations/core-integration-mongodb/index.ts +++ b/integrations/integration-mongodb/index.ts @@ -1,3 +1,4 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} -export * from './mongoDBProvider' +export * from './def' +export * from './server' // codegen:end diff --git a/integrations/core-integration-mongodb/package.json b/integrations/integration-mongodb/package.json similarity index 80% rename from integrations/core-integration-mongodb/package.json rename to integrations/integration-mongodb/package.json index 4d3cbcb6..59772a20 100644 --- a/integrations/core-integration-mongodb/package.json +++ b/integrations/integration-mongodb/package.json @@ -1,5 +1,5 @@ { - "name": "@usevenice/core-integration-mongodb", + "name": "@usevenice/integration-mongodb", "version": "0.0.0", "private": true, "module": "./index.ts", diff --git a/integrations/core-integration-mongodb/mongoDBProvider.ts b/integrations/integration-mongodb/server.ts similarity index 73% rename from integrations/core-integration-mongodb/mongoDBProvider.ts rename to integrations/integration-mongodb/server.ts index 7bbae67b..212af872 100644 --- a/integrations/core-integration-mongodb/mongoDBProvider.ts +++ b/integrations/integration-mongodb/server.ts @@ -1,26 +1,14 @@ import type {Db} from 'mongodb' import {MongoClient} from 'mongodb' -import type {AnyEntityPayload} from '@usevenice/cdk-core' -import {handlersLink, makeSyncProvider} from '@usevenice/cdk-core' -import {z, zCast, zFunction} from '@usevenice/util' +import type {AnyEntityPayload, IntegrationServer} from '@usevenice/cdk-core' +import {handlersLink} from '@usevenice/cdk-core' +import {zCast, zFunction} from '@usevenice/util' -export const zMongoConnection = z.object({ - databaseUrl: z.string(), - databaseName: z.string(), -}) +import type {mongoSchemas} from './def' +import {mongoDef, zMongoConnection} from './def' -const def = makeSyncProvider.def({ - ...makeSyncProvider.def.defaults, - name: z.literal('mongodb'), - resourceSettings: zMongoConnection, - destinationInputEntity: zCast(), -}) - -export const mongodbProvider = makeSyncProvider({ - metadata: {categories: ['database'], logoUrl: '/_assets/logo-mongodb.png'}, - ...makeSyncProvider.defaults, - def, +export const mongodbProvider = { destinationSync: ({settings}) => { const mongodb = mongoDBConnection(settings) void mongodb.initMongoDB() @@ -30,7 +18,8 @@ export const mongodbProvider = makeSyncProvider({ }, }) }, -}) +} satisfies IntegrationServer + const zData = zCast() export const mongoDBConnection = zFunction( @@ -70,3 +59,5 @@ export const mongoDBConnection = zFunction( } }, ) + +export default mongoDef diff --git a/packages/airbyte/Dockerfile b/packages/airbyte/Dockerfile index 385cf6d5..2dedac35 100644 --- a/packages/airbyte/Dockerfile +++ b/packages/airbyte/Dockerfile @@ -26,7 +26,7 @@ COPY ./integrations/integration-postgres/package.json ./integrations/integration COPY ./integrations/integration-venmo/package.json ./integrations/integration-venmo/package.json COPY ./integrations/integration-splitwise/package.json ./integrations/integration-splitwise/package.json COPY ./integrations/integration-yodlee/package.json ./integrations/integration-yodlee/package.json -COPY ./integrations/core-integration-mongodb/package.json ./integrations/core-integration-mongodb/package.json +COPY ./integrations/integration-mongodb/package.json ./integrations/integration-mongodb/package.json COPY ./integrations/integration-fs/package.json ./integrations/integration-fs/package.json COPY ./integrations/integration-ramp/package.json ./integrations/integration-ramp/package.json COPY ./integrations/integration-wise/package.json ./integrations/integration-wise/package.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index baf98107..43cdcd87 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -158,9 +158,6 @@ importers: '@usevenice/cdk-ledger': specifier: workspace:* version: link:../../packages/cdk-ledger - '@usevenice/core-integration-mongodb': - specifier: workspace:* - version: link:../../integrations/core-integration-mongodb '@usevenice/core-integration-postgres': specifier: workspace:* version: link:../../integrations/core-integration-postgres @@ -212,6 +209,9 @@ importers: '@usevenice/integration-merge': specifier: workspace:* version: link:../../integrations/integration-merge + '@usevenice/integration-mongodb': + specifier: workspace:* + version: link:../../integrations/integration-mongodb '@usevenice/integration-moota': specifier: workspace:* version: link:../../integrations/integration-moota @@ -637,18 +637,6 @@ importers: specifier: '*' version: 5.74.0(esbuild@0.17.5) - integrations/core-integration-mongodb: - dependencies: - '@usevenice/cdk-core': - specifier: workspace:* - version: link:../../packages/cdk-core - '@usevenice/util': - specifier: workspace:* - version: link:../../packages/util - mongodb: - specifier: 4.9.1 - version: 4.9.1 - integrations/core-integration-postgres: dependencies: '@slonik/migrator': @@ -925,6 +913,18 @@ importers: specifier: 6.2.1 version: 6.2.1 + integrations/integration-mongodb: + dependencies: + '@usevenice/cdk-core': + specifier: workspace:* + version: link:../../packages/cdk-core + '@usevenice/util': + specifier: workspace:* + version: link:../../packages/util + mongodb: + specifier: 4.9.1 + version: 4.9.1 + integrations/integration-moota: dependencies: '@usevenice/cdk-core': From 0231fc82caa890d277525f48bea0303142b7bfa9 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 9 Jun 2023 00:55:35 +0800 Subject: [PATCH 17/80] 2 more to go --- apps/app-config/integrations/meta.js | 8 +-- apps/app-config/package.json | 4 +- apps/app-config/providers.ts | 2 - .../index.ts | 0 .../package.json | 2 +- .../redisKvStore.ts | 0 integrations/integration-webhook/def.ts | 26 ++++++++ .../index.ts | 3 +- .../package.json | 2 +- .../server.ts} | 26 +++----- packages/airbyte/Dockerfile | 4 +- pnpm-lock.yaml | 60 +++++++++---------- 12 files changed, 75 insertions(+), 62 deletions(-) rename integrations/{core-integration-redis => integration-redis}/index.ts (100%) rename integrations/{core-integration-redis => integration-redis}/package.json (85%) rename integrations/{core-integration-redis => integration-redis}/redisKvStore.ts (100%) create mode 100644 integrations/integration-webhook/def.ts rename integrations/{core-integration-webhook => integration-webhook}/index.ts (76%) rename integrations/{core-integration-webhook => integration-webhook}/package.json (80%) rename integrations/{core-integration-webhook/WebhookProvider.ts => integration-webhook/server.ts} (55%) diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index f09b5957..287d7318 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -7,12 +7,7 @@ module.exports = [ imports: {}, }, { - dirName: 'core-integration-redis', - varName: 'coreIntegrationRedis', - imports: {}, - }, - { - dirName: 'core-integration-webhook', + dirName: 'integration-webhook', varName: 'coreIntegrationWebhook', imports: {}, }, @@ -179,6 +174,7 @@ module.exports = [ server: '@usevenice/integration-ramp/server', }, }, + {dirName: 'integration-redis', varName: 'integrationRedis', imports: {}}, { name: 'saltedge', dirName: 'integration-saltedge', diff --git a/apps/app-config/package.json b/apps/app-config/package.json index 0083620b..392b43c1 100644 --- a/apps/app-config/package.json +++ b/apps/app-config/package.json @@ -17,8 +17,8 @@ "@usevenice/integration-fs": "workspace:*", "@usevenice/integration-mongodb": "workspace:*", "@usevenice/core-integration-postgres": "workspace:*", - "@usevenice/core-integration-redis": "workspace:*", - "@usevenice/core-integration-webhook": "workspace:*", + "@usevenice/integration-redis": "workspace:*", + "@usevenice/integration-webhook": "workspace:*", "@usevenice/engine-backend": "workspace:*", "@usevenice/engine-frontend": "workspace:*", "@usevenice/integration-alphavantage": "workspace:*", diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index d0643861..aad245dd 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -2,7 +2,6 @@ import {debugProvider} from '@usevenice/cdk-core' import {corePostgresProvider} from '@usevenice/core-integration-postgres' -import {webhookProvider} from '@usevenice/core-integration-webhook' import {mergedIntegrations} from './integrations/integrations.merged' @@ -15,7 +14,6 @@ export const DOCUMENTED_PROVIDERS = [ export const PROVIDERS = [ ...DOCUMENTED_PROVIDERS, // TODO: Migrate these over to the new paradigm - webhookProvider, debugProvider, corePostgresProvider, ] as const diff --git a/integrations/core-integration-redis/index.ts b/integrations/integration-redis/index.ts similarity index 100% rename from integrations/core-integration-redis/index.ts rename to integrations/integration-redis/index.ts diff --git a/integrations/core-integration-redis/package.json b/integrations/integration-redis/package.json similarity index 85% rename from integrations/core-integration-redis/package.json rename to integrations/integration-redis/package.json index b4b9d83f..0039040e 100644 --- a/integrations/core-integration-redis/package.json +++ b/integrations/integration-redis/package.json @@ -1,5 +1,5 @@ { - "name": "@usevenice/core-integration-redis", + "name": "@usevenice/integration-redis", "version": "0.0.0", "private": true, "sideEffects": [], diff --git a/integrations/core-integration-redis/redisKvStore.ts b/integrations/integration-redis/redisKvStore.ts similarity index 100% rename from integrations/core-integration-redis/redisKvStore.ts rename to integrations/integration-redis/redisKvStore.ts diff --git a/integrations/integration-webhook/def.ts b/integrations/integration-webhook/def.ts new file mode 100644 index 00000000..f6d5cdf4 --- /dev/null +++ b/integrations/integration-webhook/def.ts @@ -0,0 +1,26 @@ +import type { + AnyEntityPayload, + IntegrationDef, + IntegrationSchemas, +} from '@usevenice/cdk-core' +import {intHelpers} from '@usevenice/cdk-core' +import {z, zCast} from '@usevenice/util' + +export const webhookSchemas = { + name: z.literal('webhook'), + resourceSettings: z.object({ + destinationUrl: z.string(), + }), + destinationInputEntity: zCast(), +} satisfies IntegrationSchemas + +export const webhookHelpers = intHelpers(webhookSchemas) + +export const webhookDef = { + name: 'webhook', + metadata: {categories: ['streaming'], logoUrl: '/_assets/logo-webhook.png'}, + + def: webhookSchemas, +} satisfies IntegrationDef + +export default webhookDef diff --git a/integrations/core-integration-webhook/index.ts b/integrations/integration-webhook/index.ts similarity index 76% rename from integrations/core-integration-webhook/index.ts rename to integrations/integration-webhook/index.ts index e0d3f680..81f31bd7 100644 --- a/integrations/core-integration-webhook/index.ts +++ b/integrations/integration-webhook/index.ts @@ -1,3 +1,4 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} -export * from './WebhookProvider' +export * from './def' +export * from './server' // codegen:end diff --git a/integrations/core-integration-webhook/package.json b/integrations/integration-webhook/package.json similarity index 80% rename from integrations/core-integration-webhook/package.json rename to integrations/integration-webhook/package.json index c6c440e4..8010a6f4 100644 --- a/integrations/core-integration-webhook/package.json +++ b/integrations/integration-webhook/package.json @@ -1,5 +1,5 @@ { - "name": "@usevenice/core-integration-webhook", + "name": "@usevenice/integration-webhook", "version": "0.0.0", "private": true, "module": "./index.ts", diff --git a/integrations/core-integration-webhook/WebhookProvider.ts b/integrations/integration-webhook/server.ts similarity index 55% rename from integrations/core-integration-webhook/WebhookProvider.ts rename to integrations/integration-webhook/server.ts index 8c51f946..253e8e39 100644 --- a/integrations/core-integration-webhook/WebhookProvider.ts +++ b/integrations/integration-webhook/server.ts @@ -1,27 +1,17 @@ -import type {AnyEntityPayload} from '@usevenice/cdk-core' -import {handlersLink, makeSyncProvider} from '@usevenice/cdk-core' -import {createHTTPClient, rxjs, z, zCast} from '@usevenice/util' +import type {IntegrationServer} from '@usevenice/cdk-core' +import {handlersLink} from '@usevenice/cdk-core' +import {createHTTPClient, rxjs} from '@usevenice/util' -const webhookProviderDef = makeSyncProvider.def({ - ...makeSyncProvider.def.defaults, - name: z.literal('webhook'), - resourceSettings: z.object({ - destinationUrl: z.string(), - }), - destinationInputEntity: zCast(), -}) +import type {webhookHelpers, webhookSchemas} from './def' -export const webhookProvider = makeSyncProvider({ - metadata: {categories: ['streaming'], logoUrl: '/_assets/logo-webhook.png'}, - ...makeSyncProvider.defaults, - def: webhookProviderDef, +export const webhookServer = { destinationSync: ({settings: {destinationUrl}}) => { const http = createHTTPClient({baseURL: destinationUrl}) let batch = { resUpdates: [] as unknown[], stateUpdates: [] as unknown[], entities: [] as Array< - (typeof webhookProviderDef)['_types']['destinationInputEntity'] + (typeof webhookHelpers)['_types']['destinationInputEntity'] >, } @@ -48,4 +38,6 @@ export const webhookProvider = makeSyncProvider({ }, }) }, -}) +} satisfies IntegrationServer + +export default webhookServer diff --git a/packages/airbyte/Dockerfile b/packages/airbyte/Dockerfile index 2dedac35..b5f26cc8 100644 --- a/packages/airbyte/Dockerfile +++ b/packages/airbyte/Dockerfile @@ -18,7 +18,7 @@ COPY ./patches/ ./patches ## TODO: This list would need to be generated by _generateIntegrationsList.ts if we end up using the Dockerfile more COPY ./integrations/integration-alphavantage/package.json ./integrations/integration-alphavantage/package.json COPY ./integrations/integration-qbo/package.json ./integrations/integration-qbo/package.json -COPY ./integrations/core-integration-webhook/package.json ./integrations/core-integration-webhook/package.json +COPY ./integrations/integration-webhook/package.json ./integrations/integration-webhook/package.json COPY ./integrations/integration-spreadsheet/package.json ./integrations/integration-spreadsheet/package.json COPY ./integrations/integration-foreceipt/package.json ./integrations/integration-foreceipt/package.json COPY ./integrations/integration-stripe/package.json ./integrations/integration-stripe/package.json @@ -36,7 +36,7 @@ COPY ./integrations/integration-saltedge/package.json ./integrations/integration COPY ./integrations/integration-lunchmoney/package.json ./integrations/integration-lunchmoney/package.json COPY ./integrations/integration-toggl/package.json ./integrations/integration-toggl/package.json COPY ./integrations/integration-beancount/package.json ./integrations/integration-beancount/package.json -COPY ./integrations/core-integration-redis/package.json ./integrations/core-integration-redis/package.json +COPY ./integrations/integration-redis/package.json ./integrations/integration-redis/package.json COPY ./integrations/integration-onebrick/package.json ./integrations/integration-onebrick/package.json COPY ./integrations/core-integration-postgres/package.json ./integrations/core-integration-postgres/package.json COPY ./integrations/integration-firebase/package.json ./integrations/integration-firebase/package.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 43cdcd87..1e27b0ad 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -161,12 +161,6 @@ importers: '@usevenice/core-integration-postgres': specifier: workspace:* version: link:../../integrations/core-integration-postgres - '@usevenice/core-integration-redis': - specifier: workspace:* - version: link:../../integrations/core-integration-redis - '@usevenice/core-integration-webhook': - specifier: workspace:* - version: link:../../integrations/core-integration-webhook '@usevenice/engine-backend': specifier: workspace:* version: link:../../packages/engine-backend @@ -230,6 +224,9 @@ importers: '@usevenice/integration-ramp': specifier: workspace:* version: link:../../integrations/integration-ramp + '@usevenice/integration-redis': + specifier: workspace:* + version: link:../../integrations/integration-redis '@usevenice/integration-saltedge': specifier: workspace:* version: link:../../integrations/integration-saltedge @@ -251,6 +248,9 @@ importers: '@usevenice/integration-venmo': specifier: workspace:* version: link:../../integrations/integration-venmo + '@usevenice/integration-webhook': + specifier: workspace:* + version: link:../../integrations/integration-webhook '@usevenice/integration-wise': specifier: workspace:* version: link:../../integrations/integration-wise @@ -665,30 +665,6 @@ importers: specifier: 8.6.5 version: 8.6.5 - integrations/core-integration-redis: - dependencies: - '@usevenice/cdk-core': - specifier: workspace:* - version: link:../../packages/cdk-core - '@usevenice/util': - specifier: workspace:* - version: link:../../packages/util - handy-redis: - specifier: 2.3.1 - version: 2.3.1 - redis: - specifier: 3.1.2 - version: 3.1.2 - - integrations/core-integration-webhook: - dependencies: - '@usevenice/cdk-core': - specifier: workspace:* - version: link:../../packages/cdk-core - '@usevenice/util': - specifier: workspace:* - version: link:../../packages/util - integrations/integration-airtable: dependencies: '@usevenice/cdk-core': @@ -1049,6 +1025,21 @@ importers: specifier: workspace:* version: link:../../packages/util + integrations/integration-redis: + dependencies: + '@usevenice/cdk-core': + specifier: workspace:* + version: link:../../packages/cdk-core + '@usevenice/util': + specifier: workspace:* + version: link:../../packages/util + handy-redis: + specifier: 2.3.1 + version: 2.3.1 + redis: + specifier: 3.1.2 + version: 3.1.2 + integrations/integration-saltedge: dependencies: '@usevenice/cdk-core': @@ -1169,6 +1160,15 @@ importers: specifier: workspace:* version: link:../../packages/util + integrations/integration-webhook: + dependencies: + '@usevenice/cdk-core': + specifier: workspace:* + version: link:../../packages/cdk-core + '@usevenice/util': + specifier: workspace:* + version: link:../../packages/util + integrations/integration-wise: dependencies: '@usevenice/cdk-core': From e633ea5d2452ad9fbe75e8dfaf94d744d923f3af Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 16 Jun 2023 11:34:38 +0800 Subject: [PATCH 18/80] Merge core-postgres with just postgres --- apps/app-config/backendConfig.ts | 4 +- .../integrations/integrations.def.ts | 2 + .../integrations/integrations.merged.ts | 8 +++ .../integrations/integrations.server.ts | 2 + apps/app-config/integrations/meta.js | 19 +++--- apps/app-config/package.json | 15 ++--- apps/app-config/providers.ts | 2 - apps/cli/_cli.ts | 8 +-- apps/cli/package.json | 5 +- apps/cli/pgMigrator-cli.ts | 5 +- .../api/resources/[resourceId]/sql/route.ts | 5 +- .../web}/migrations/0_setup-ulid.sql | 0 .../web}/migrations/1_create-table.sql | 0 .../web}/migrations/2023-01-06_rls.sql | 0 .../migrations/2023-01-25_rename_to_raw.sql | 0 ...23-01-27_rename_connection_to_resource.sql | 0 .../2023-02-01_user_apikey_index.sql | 0 .../2023-02-02_add_connection_name.sql | 0 .../2023-02-03_graphql_comments.sql | 0 .../migrations/2023-02-22_institution_rls.sql | 0 .../2023-02-27_remove_database_users.sql | 0 .../migrations/2023-02-27_revoke_anon.sql | 0 ...2-28_1307_enable_realtime_publications.sql | 0 .../2023-02-28_1954_remove_public_uid.sql | 0 .../migrations/2023-04-02_0140_admin_user.sql | 0 .../2023-04-04_0211_pgrest_pre_request.sql | 0 ..._1503_add_end_user_id_remove_ledger_id.sql | 0 .../2023-04-23_0735_default_id_prefix.sql | 0 ...0753_materialized_cte_over_pre_request.sql | 0 .../2023-04-29_1549_multi_tenant.sql | 0 .../2023-05-22_2146_migrate_plaid_config.sql | 0 .../2023-01-06_connection-user.sql | 0 .../2023-01-07_create-views.sql | 0 ...-02-27_2252_postgres-follow-privileges.sql | 0 .../2023-02-28_1433_database_webhooks.sql | 0 .../2023-04-22_1501_pre_request_to_mcte.sql | 0 .../2023-04-22_2039_end_user_table.sql | 0 apps/web/package.json | 2 +- .../corePostgresProvider.ts | 29 --------- .../core-integration-postgres/index.ts | 5 -- .../core-integration-postgres/package.json | 18 ------ .../@types/slonik-interceptor-preset.d.ts | 0 integrations/integration-postgres/def.ts | 5 +- integrations/integration-postgres/index.ts | 2 + .../makePostgresClient.spec.ts} | 0 .../makePostgresClient.ts | 0 .../makePostgresMetaService.ts | 0 .../integration-postgres/package.json | 6 +- integrations/integration-postgres/server.ts | 7 +- packages/airbyte/Dockerfile | 1 - packages/airbyte/airbyteMetaService.ts | 6 +- packages/airbyte/package.json | 2 +- pnpm-lock.yaml | 64 ++++++------------- 53 files changed, 79 insertions(+), 143 deletions(-) rename {integrations/core-integration-postgres => apps/web}/migrations/0_setup-ulid.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/1_create-table.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-01-06_rls.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-01-25_rename_to_raw.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-01-27_rename_connection_to_resource.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-02-01_user_apikey_index.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-02-02_add_connection_name.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-02-03_graphql_comments.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-02-22_institution_rls.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-02-27_remove_database_users.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-02-27_revoke_anon.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-02-28_1307_enable_realtime_publications.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-02-28_1954_remove_public_uid.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-04-02_0140_admin_user.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-04-04_0211_pgrest_pre_request.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-04-22_1503_add_end_user_id_remove_ledger_id.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-04-23_0735_default_id_prefix.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-04-23_0753_materialized_cte_over_pre_request.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-04-29_1549_multi_tenant.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations/2023-05-22_2146_migrate_plaid_config.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations_wip/2023-01-06_connection-user.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations_wip/2023-01-07_create-views.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations_wip/2023-02-27_2252_postgres-follow-privileges.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations_wip/2023-02-28_1433_database_webhooks.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations_wip/2023-04-22_1501_pre_request_to_mcte.sql (100%) rename {integrations/core-integration-postgres => apps/web}/migrations_wip/2023-04-22_2039_end_user_table.sql (100%) delete mode 100644 integrations/core-integration-postgres/corePostgresProvider.ts delete mode 100644 integrations/core-integration-postgres/index.ts delete mode 100644 integrations/core-integration-postgres/package.json rename integrations/{core-integration-postgres => integration-postgres}/@types/slonik-interceptor-preset.d.ts (100%) rename integrations/{core-integration-postgres/index.spec.ts => integration-postgres/makePostgresClient.spec.ts} (100%) rename integrations/{core-integration-postgres => integration-postgres}/makePostgresClient.ts (100%) rename integrations/{core-integration-postgres => integration-postgres}/makePostgresMetaService.ts (100%) diff --git a/apps/app-config/backendConfig.ts b/apps/app-config/backendConfig.ts index d162cb0a..17206701 100644 --- a/apps/app-config/backendConfig.ts +++ b/apps/app-config/backendConfig.ts @@ -7,16 +7,16 @@ import { mapStandardEntityLink, renameAccountLink, } from '@usevenice/cdk-ledger' -import {makePostgresMetaService} from '@usevenice/core-integration-postgres' import type {PipelineInput} from '@usevenice/engine-backend' import {getContextFactory} from '@usevenice/engine-backend' +import {makePostgresMetaService} from '@usevenice/integration-postgres' import {joinPath, R, Rx} from '@usevenice/util' import {getServerUrl} from './constants' import {env} from './env' import {PROVIDERS} from './providers' -export {DatabaseError} from '@usevenice/core-integration-postgres/makePostgresClient' +export {DatabaseError} from '@usevenice/integration-postgres/makePostgresClient' export {Papa} from '@usevenice/integration-spreadsheet' export {makePostgresClient} from '@usevenice/integration-postgres' diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index 18139a63..1f42b4c0 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -23,6 +23,7 @@ import {default as integrationStripe} from '@usevenice/integration-stripe/def' import {default as integrationTeller} from '@usevenice/integration-teller/def' import {default as integrationToggl} from '@usevenice/integration-toggl/def' import {default as integrationVenmo} from '@usevenice/integration-venmo/def' +import {default as integrationWebhook} from '@usevenice/integration-webhook/def' import {default as integrationWise} from '@usevenice/integration-wise/def' import {default as integrationYodlee} from '@usevenice/integration-yodlee/def' @@ -51,6 +52,7 @@ export const defIntegrations = { teller: integrationTeller, toggl: integrationToggl, venmo: integrationVenmo, + webhook: integrationWebhook, wise: integrationWise, yodlee: integrationYodlee, } diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 5d6932c7..12358547 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -49,6 +49,8 @@ import {default as integrationTeller_server} from '@usevenice/integration-teller import {default as integrationToggl_def} from '@usevenice/integration-toggl/def' import {default as integrationToggl_server} from '@usevenice/integration-toggl/server' import {default as integrationVenmo_def} from '@usevenice/integration-venmo/def' +import {default as integrationWebhook_def} from '@usevenice/integration-webhook/def' +import {default as integrationWebhook_server} from '@usevenice/integration-webhook/server' import {default as integrationWise_def} from '@usevenice/integration-wise/def' import {default as integrationWise_server} from '@usevenice/integration-wise/server' import {default as integrationYodlee_client} from '@usevenice/integration-yodlee/client' @@ -177,6 +179,11 @@ const integrationVenmo = { ...integrationVenmo_def, } +const integrationWebhook = { + ...integrationWebhook_def, + ...integrationWebhook_server, +} + const integrationWise = { ...integrationWise_def, ...integrationWise_server, @@ -213,6 +220,7 @@ export const mergedIntegrations = { teller: integrationTeller, toggl: integrationToggl, venmo: integrationVenmo, + webhook: integrationWebhook, wise: integrationWise, yodlee: integrationYodlee, } diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index a1382ba8..a2cf9b08 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -21,6 +21,7 @@ import {default as integrationSpreadsheet} from '@usevenice/integration-spreadsh import {default as integrationStripe} from '@usevenice/integration-stripe/server' import {default as integrationTeller} from '@usevenice/integration-teller/server' import {default as integrationToggl} from '@usevenice/integration-toggl/server' +import {default as integrationWebhook} from '@usevenice/integration-webhook/server' import {default as integrationWise} from '@usevenice/integration-wise/server' import {default as integrationYodlee} from '@usevenice/integration-yodlee/server' @@ -47,6 +48,7 @@ export const serverIntegrations = { stripe: integrationStripe, teller: integrationTeller, toggl: integrationToggl, + webhook: integrationWebhook, wise: integrationWise, yodlee: integrationYodlee, } diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index 287d7318..a77e0b36 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -1,16 +1,6 @@ // generated by _generateIntegrationLists.ts. Do not modify by hand module.exports = [ - { - dirName: 'core-integration-postgres', - varName: 'coreIntegrationPostgres', - imports: {}, - }, - { - dirName: 'integration-webhook', - varName: 'coreIntegrationWebhook', - imports: {}, - }, { name: 'airtable', dirName: 'integration-airtable', @@ -236,6 +226,15 @@ module.exports = [ varName: 'integrationVenmo', imports: {def: '@usevenice/integration-venmo/def'}, }, + { + name: 'webhook', + dirName: 'integration-webhook', + varName: 'integrationWebhook', + imports: { + def: '@usevenice/integration-webhook/def', + server: '@usevenice/integration-webhook/server', + }, + }, { name: 'wise', dirName: 'integration-wise', diff --git a/apps/app-config/package.json b/apps/app-config/package.json index 392b43c1..94b976c7 100644 --- a/apps/app-config/package.json +++ b/apps/app-config/package.json @@ -12,37 +12,36 @@ "@t3-oss/env-nextjs": "0.3.1", "@usevenice/cdk-core": "workspace:*", "@usevenice/cdk-ledger": "workspace:*", - "@usevenice/integration-airtable": "workspace:*", - "@usevenice/integration-firebase": "workspace:*", - "@usevenice/integration-fs": "workspace:*", - "@usevenice/integration-mongodb": "workspace:*", - "@usevenice/core-integration-postgres": "workspace:*", - "@usevenice/integration-redis": "workspace:*", - "@usevenice/integration-webhook": "workspace:*", "@usevenice/engine-backend": "workspace:*", "@usevenice/engine-frontend": "workspace:*", + "@usevenice/integration-airtable": "workspace:*", "@usevenice/integration-alphavantage": "workspace:*", "@usevenice/integration-beancount": "workspace:*", "@usevenice/integration-brex": "workspace:*", "@usevenice/integration-expensify": "workspace:*", + "@usevenice/integration-firebase": "workspace:*", "@usevenice/integration-foreceipt": "workspace:*", + "@usevenice/integration-fs": "workspace:*", "@usevenice/integration-heron": "workspace:*", - "@usevenice/integration-spreadsheet": "workspace:*", "@usevenice/integration-lunchmoney": "workspace:*", "@usevenice/integration-mercury": "workspace:*", "@usevenice/integration-merge": "workspace:*", + "@usevenice/integration-mongodb": "workspace:*", "@usevenice/integration-moota": "workspace:*", "@usevenice/integration-onebrick": "workspace:*", "@usevenice/integration-plaid": "workspace:*", "@usevenice/integration-postgres": "workspace:*", "@usevenice/integration-qbo": "workspace:*", "@usevenice/integration-ramp": "workspace:*", + "@usevenice/integration-redis": "workspace:*", "@usevenice/integration-saltedge": "workspace:*", "@usevenice/integration-splitwise": "workspace:*", + "@usevenice/integration-spreadsheet": "workspace:*", "@usevenice/integration-stripe": "workspace:*", "@usevenice/integration-teller": "workspace:*", "@usevenice/integration-toggl": "workspace:*", "@usevenice/integration-venmo": "workspace:*", + "@usevenice/integration-webhook": "workspace:*", "@usevenice/integration-wise": "workspace:*", "@usevenice/integration-yodlee": "workspace:*", "@usevenice/util": "workspace:*", diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index aad245dd..7fe483b7 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -1,7 +1,6 @@ // @deprecated. Soon to be fully replaced by integrations.merge.ts import {debugProvider} from '@usevenice/cdk-core' -import {corePostgresProvider} from '@usevenice/core-integration-postgres' import {mergedIntegrations} from './integrations/integrations.merged' @@ -15,5 +14,4 @@ export const PROVIDERS = [ ...DOCUMENTED_PROVIDERS, // TODO: Migrate these over to the new paradigm debugProvider, - corePostgresProvider, ] as const diff --git a/apps/cli/_cli.ts b/apps/cli/_cli.ts index 6fcf1f41..c1630392 100644 --- a/apps/cli/_cli.ts +++ b/apps/cli/_cli.ts @@ -4,10 +4,6 @@ import '@usevenice/app-config/register.node' import {parseIntConfigsFromRawEnv} from '@usevenice/app-config/integration-envs' import type {PROVIDERS} from '@usevenice/app-config/providers' import {makeJwtClient} from '@usevenice/cdk-core' -import { - makePostgresClient, - makePostgresMetaService, -} from '@usevenice/core-integration-postgres' import {makeAlphavantageClient} from '@usevenice/integration-alphavantage' import {makeHeronClient} from '@usevenice/integration-heron' import {makeLunchmoneyClient} from '@usevenice/integration-lunchmoney' @@ -20,6 +16,10 @@ import {makeOneBrickClient} from '@usevenice/integration-onebrick' // Or perhaps we can make it into a register and/or loader for nodejs // much like tsx and others import {makePlaidClient} from '@usevenice/integration-plaid' +import { + makePostgresClient, + makePostgresMetaService, +} from '@usevenice/integration-postgres' import {makeRampClient} from '@usevenice/integration-ramp' import {makeSaltedgeClient} from '@usevenice/integration-saltedge' import {makeStripeClient} from '@usevenice/integration-stripe' diff --git a/apps/cli/package.json b/apps/cli/package.json index c1a4b57a..4a717da7 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -8,14 +8,13 @@ "@usevenice/app-config": "workspace:*", "@usevenice/cdk-core": "workspace:*", "@usevenice/cdk-ledger": "workspace:*", - "@usevenice/integration-fs": "workspace:*", - "@usevenice/integration-firebase": "workspace:*", - "@usevenice/core-integration-postgres": "workspace:*", "@usevenice/engine-backend": "workspace:*", "@usevenice/integration-alphavantage": "workspace:*", "@usevenice/integration-beancount": "workspace:*", "@usevenice/integration-brex": "workspace:*", "@usevenice/integration-expensify": "workspace:*", + "@usevenice/integration-firebase": "workspace:*", + "@usevenice/integration-fs": "workspace:*", "@usevenice/integration-heron": "workspace:*", "@usevenice/integration-lunchmoney": "workspace:*", "@usevenice/integration-merge": "workspace:*", diff --git a/apps/cli/pgMigrator-cli.ts b/apps/cli/pgMigrator-cli.ts index 7abbd17f..8811fc98 100644 --- a/apps/cli/pgMigrator-cli.ts +++ b/apps/cli/pgMigrator-cli.ts @@ -1,9 +1,12 @@ #!/usr/bin/env tsx import '@usevenice/app-config/register.node' -import {makePostgresClient} from '@usevenice/core-integration-postgres' +import path from 'node:path' + +import {makePostgresClient} from '@usevenice/integration-postgres' import {z} from '@usevenice/util' void makePostgresClient({ databaseUrl: z.string().parse(process.env['POSTGRES_OR_WEBHOOK_URL']), + migrationsPath: path.join(__dirname, '../web/migrations'), }).runMigratorCli() diff --git a/apps/web/app/api/resources/[resourceId]/sql/route.ts b/apps/web/app/api/resources/[resourceId]/sql/route.ts index 7424d262..6e6415f3 100644 --- a/apps/web/app/api/resources/[resourceId]/sql/route.ts +++ b/apps/web/app/api/resources/[resourceId]/sql/route.ts @@ -10,10 +10,7 @@ import { import {kAcceptUrlParam} from '@usevenice/app-config/constants' import type {Id} from '@usevenice/cdk-core' import {hasRole} from '@usevenice/cdk-core' -import { - makePostgresClient, - zPgConfig, -} from '@usevenice/core-integration-postgres' +import {makePostgresClient, zPgConfig} from '@usevenice/integration-postgres' import {R, z} from '@usevenice/util' import {serverComponentGetViewer} from '@/lib-server/server-component-helpers' diff --git a/integrations/core-integration-postgres/migrations/0_setup-ulid.sql b/apps/web/migrations/0_setup-ulid.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/0_setup-ulid.sql rename to apps/web/migrations/0_setup-ulid.sql diff --git a/integrations/core-integration-postgres/migrations/1_create-table.sql b/apps/web/migrations/1_create-table.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/1_create-table.sql rename to apps/web/migrations/1_create-table.sql diff --git a/integrations/core-integration-postgres/migrations/2023-01-06_rls.sql b/apps/web/migrations/2023-01-06_rls.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-01-06_rls.sql rename to apps/web/migrations/2023-01-06_rls.sql diff --git a/integrations/core-integration-postgres/migrations/2023-01-25_rename_to_raw.sql b/apps/web/migrations/2023-01-25_rename_to_raw.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-01-25_rename_to_raw.sql rename to apps/web/migrations/2023-01-25_rename_to_raw.sql diff --git a/integrations/core-integration-postgres/migrations/2023-01-27_rename_connection_to_resource.sql b/apps/web/migrations/2023-01-27_rename_connection_to_resource.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-01-27_rename_connection_to_resource.sql rename to apps/web/migrations/2023-01-27_rename_connection_to_resource.sql diff --git a/integrations/core-integration-postgres/migrations/2023-02-01_user_apikey_index.sql b/apps/web/migrations/2023-02-01_user_apikey_index.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-02-01_user_apikey_index.sql rename to apps/web/migrations/2023-02-01_user_apikey_index.sql diff --git a/integrations/core-integration-postgres/migrations/2023-02-02_add_connection_name.sql b/apps/web/migrations/2023-02-02_add_connection_name.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-02-02_add_connection_name.sql rename to apps/web/migrations/2023-02-02_add_connection_name.sql diff --git a/integrations/core-integration-postgres/migrations/2023-02-03_graphql_comments.sql b/apps/web/migrations/2023-02-03_graphql_comments.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-02-03_graphql_comments.sql rename to apps/web/migrations/2023-02-03_graphql_comments.sql diff --git a/integrations/core-integration-postgres/migrations/2023-02-22_institution_rls.sql b/apps/web/migrations/2023-02-22_institution_rls.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-02-22_institution_rls.sql rename to apps/web/migrations/2023-02-22_institution_rls.sql diff --git a/integrations/core-integration-postgres/migrations/2023-02-27_remove_database_users.sql b/apps/web/migrations/2023-02-27_remove_database_users.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-02-27_remove_database_users.sql rename to apps/web/migrations/2023-02-27_remove_database_users.sql diff --git a/integrations/core-integration-postgres/migrations/2023-02-27_revoke_anon.sql b/apps/web/migrations/2023-02-27_revoke_anon.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-02-27_revoke_anon.sql rename to apps/web/migrations/2023-02-27_revoke_anon.sql diff --git a/integrations/core-integration-postgres/migrations/2023-02-28_1307_enable_realtime_publications.sql b/apps/web/migrations/2023-02-28_1307_enable_realtime_publications.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-02-28_1307_enable_realtime_publications.sql rename to apps/web/migrations/2023-02-28_1307_enable_realtime_publications.sql diff --git a/integrations/core-integration-postgres/migrations/2023-02-28_1954_remove_public_uid.sql b/apps/web/migrations/2023-02-28_1954_remove_public_uid.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-02-28_1954_remove_public_uid.sql rename to apps/web/migrations/2023-02-28_1954_remove_public_uid.sql diff --git a/integrations/core-integration-postgres/migrations/2023-04-02_0140_admin_user.sql b/apps/web/migrations/2023-04-02_0140_admin_user.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-04-02_0140_admin_user.sql rename to apps/web/migrations/2023-04-02_0140_admin_user.sql diff --git a/integrations/core-integration-postgres/migrations/2023-04-04_0211_pgrest_pre_request.sql b/apps/web/migrations/2023-04-04_0211_pgrest_pre_request.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-04-04_0211_pgrest_pre_request.sql rename to apps/web/migrations/2023-04-04_0211_pgrest_pre_request.sql diff --git a/integrations/core-integration-postgres/migrations/2023-04-22_1503_add_end_user_id_remove_ledger_id.sql b/apps/web/migrations/2023-04-22_1503_add_end_user_id_remove_ledger_id.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-04-22_1503_add_end_user_id_remove_ledger_id.sql rename to apps/web/migrations/2023-04-22_1503_add_end_user_id_remove_ledger_id.sql diff --git a/integrations/core-integration-postgres/migrations/2023-04-23_0735_default_id_prefix.sql b/apps/web/migrations/2023-04-23_0735_default_id_prefix.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-04-23_0735_default_id_prefix.sql rename to apps/web/migrations/2023-04-23_0735_default_id_prefix.sql diff --git a/integrations/core-integration-postgres/migrations/2023-04-23_0753_materialized_cte_over_pre_request.sql b/apps/web/migrations/2023-04-23_0753_materialized_cte_over_pre_request.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-04-23_0753_materialized_cte_over_pre_request.sql rename to apps/web/migrations/2023-04-23_0753_materialized_cte_over_pre_request.sql diff --git a/integrations/core-integration-postgres/migrations/2023-04-29_1549_multi_tenant.sql b/apps/web/migrations/2023-04-29_1549_multi_tenant.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-04-29_1549_multi_tenant.sql rename to apps/web/migrations/2023-04-29_1549_multi_tenant.sql diff --git a/integrations/core-integration-postgres/migrations/2023-05-22_2146_migrate_plaid_config.sql b/apps/web/migrations/2023-05-22_2146_migrate_plaid_config.sql similarity index 100% rename from integrations/core-integration-postgres/migrations/2023-05-22_2146_migrate_plaid_config.sql rename to apps/web/migrations/2023-05-22_2146_migrate_plaid_config.sql diff --git a/integrations/core-integration-postgres/migrations_wip/2023-01-06_connection-user.sql b/apps/web/migrations_wip/2023-01-06_connection-user.sql similarity index 100% rename from integrations/core-integration-postgres/migrations_wip/2023-01-06_connection-user.sql rename to apps/web/migrations_wip/2023-01-06_connection-user.sql diff --git a/integrations/core-integration-postgres/migrations_wip/2023-01-07_create-views.sql b/apps/web/migrations_wip/2023-01-07_create-views.sql similarity index 100% rename from integrations/core-integration-postgres/migrations_wip/2023-01-07_create-views.sql rename to apps/web/migrations_wip/2023-01-07_create-views.sql diff --git a/integrations/core-integration-postgres/migrations_wip/2023-02-27_2252_postgres-follow-privileges.sql b/apps/web/migrations_wip/2023-02-27_2252_postgres-follow-privileges.sql similarity index 100% rename from integrations/core-integration-postgres/migrations_wip/2023-02-27_2252_postgres-follow-privileges.sql rename to apps/web/migrations_wip/2023-02-27_2252_postgres-follow-privileges.sql diff --git a/integrations/core-integration-postgres/migrations_wip/2023-02-28_1433_database_webhooks.sql b/apps/web/migrations_wip/2023-02-28_1433_database_webhooks.sql similarity index 100% rename from integrations/core-integration-postgres/migrations_wip/2023-02-28_1433_database_webhooks.sql rename to apps/web/migrations_wip/2023-02-28_1433_database_webhooks.sql diff --git a/integrations/core-integration-postgres/migrations_wip/2023-04-22_1501_pre_request_to_mcte.sql b/apps/web/migrations_wip/2023-04-22_1501_pre_request_to_mcte.sql similarity index 100% rename from integrations/core-integration-postgres/migrations_wip/2023-04-22_1501_pre_request_to_mcte.sql rename to apps/web/migrations_wip/2023-04-22_1501_pre_request_to_mcte.sql diff --git a/integrations/core-integration-postgres/migrations_wip/2023-04-22_2039_end_user_table.sql b/apps/web/migrations_wip/2023-04-22_2039_end_user_table.sql similarity index 100% rename from integrations/core-integration-postgres/migrations_wip/2023-04-22_2039_end_user_table.sql rename to apps/web/migrations_wip/2023-04-22_2039_end_user_table.sql diff --git a/apps/web/package.json b/apps/web/package.json index 5d1c8c70..a73949d9 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -26,9 +26,9 @@ "@usevenice/app-config": "workspace:*", "@usevenice/cdk-core": "workspace:*", "@usevenice/connect": "workspace:*", - "@usevenice/core-integration-postgres": "workspace:*", "@usevenice/engine-backend": "workspace:*", "@usevenice/engine-frontend": "workspace:*", + "@usevenice/integration-postgres": "workspace:*", "@usevenice/ui": "workspace:*", "@usevenice/util": "workspace:*", "commandbar": "1.7.3", diff --git a/integrations/core-integration-postgres/corePostgresProvider.ts b/integrations/core-integration-postgres/corePostgresProvider.ts deleted file mode 100644 index 19f3435e..00000000 --- a/integrations/core-integration-postgres/corePostgresProvider.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type {AnyEntityPayload} from '@usevenice/cdk-core' -import {handlersLink, makeSyncProvider} from '@usevenice/cdk-core' -import {z, zCast} from '@usevenice/util' - -import {makePostgresClient, zPgConfig} from './makePostgresClient' - -const def = makeSyncProvider.def({ - ...makeSyncProvider.def.defaults, - name: z.literal('core-postgres'), - resourceSettings: zPgConfig, - destinationInputEntity: zCast(), -}) - -export const corePostgresProvider = makeSyncProvider({ - metadata: {stage: 'hidden'}, - ...makeSyncProvider.defaults, - def, - destinationSync: ({settings: {databaseUrl}}) => { - const {upsertById} = makePostgresClient({databaseUrl}) - return handlersLink({ - data: async (op) => { - // prettier-ignore - const {data: {id, entityName, entity}} = op - await upsertById('meta', [{id: `${entityName}_${id}`, data: entity}]) - return op - }, - }) - }, -}) diff --git a/integrations/core-integration-postgres/index.ts b/integrations/core-integration-postgres/index.ts deleted file mode 100644 index 4f3f665c..00000000 --- a/integrations/core-integration-postgres/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -// codegen:start {preset: barrel, include: "./*.{ts,tsx}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} -export * from './corePostgresProvider' -export * from './makePostgresClient' -export * from './makePostgresMetaService' -// codegen:end diff --git a/integrations/core-integration-postgres/package.json b/integrations/core-integration-postgres/package.json deleted file mode 100644 index 28de788c..00000000 --- a/integrations/core-integration-postgres/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@usevenice/core-integration-postgres", - "version": "0.0.0", - "private": true, - "module": "./index.ts", - "dependencies": { - "@slonik/migrator": "0.11.3", - "@usevenice/cdk-core": "workspace:*", - "@usevenice/util": "workspace:*", - "pg": "8.8.0", - "pg-native": "3.0.1", - "slonik": "30.3.1", - "slonik-interceptor-preset": "1.2.10" - }, - "devDependencies": { - "@types/pg": "8.6.5" - } -} diff --git a/integrations/core-integration-postgres/@types/slonik-interceptor-preset.d.ts b/integrations/integration-postgres/@types/slonik-interceptor-preset.d.ts similarity index 100% rename from integrations/core-integration-postgres/@types/slonik-interceptor-preset.d.ts rename to integrations/integration-postgres/@types/slonik-interceptor-preset.d.ts diff --git a/integrations/integration-postgres/def.ts b/integrations/integration-postgres/def.ts index af34b54c..d193bde4 100644 --- a/integrations/integration-postgres/def.ts +++ b/integrations/integration-postgres/def.ts @@ -1,10 +1,11 @@ import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' import {intHelpers} from '@usevenice/cdk-core' import type {EntityPayloadWithExternal, ZCommon} from '@usevenice/cdk-ledger' -import {zPgConfig} from '@usevenice/core-integration-postgres' import {z, zCast} from '@usevenice/util' -export {makePostgresClient} from '@usevenice/core-integration-postgres' +import {zPgConfig} from './makePostgresClient' + +export {makePostgresClient} from './makePostgresClient' export const postgresSchemas = { name: z.literal('postgres'), diff --git a/integrations/integration-postgres/index.ts b/integrations/integration-postgres/index.ts index d424d44f..6f95f98e 100644 --- a/integrations/integration-postgres/index.ts +++ b/integrations/integration-postgres/index.ts @@ -6,6 +6,8 @@ import postgresServer from './server' // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} export * from './def' +export * from './makePostgresClient' +export * from './makePostgresMetaService' export * from './server' // codegen:end diff --git a/integrations/core-integration-postgres/index.spec.ts b/integrations/integration-postgres/makePostgresClient.spec.ts similarity index 100% rename from integrations/core-integration-postgres/index.spec.ts rename to integrations/integration-postgres/makePostgresClient.spec.ts diff --git a/integrations/core-integration-postgres/makePostgresClient.ts b/integrations/integration-postgres/makePostgresClient.ts similarity index 100% rename from integrations/core-integration-postgres/makePostgresClient.ts rename to integrations/integration-postgres/makePostgresClient.ts diff --git a/integrations/core-integration-postgres/makePostgresMetaService.ts b/integrations/integration-postgres/makePostgresMetaService.ts similarity index 100% rename from integrations/core-integration-postgres/makePostgresMetaService.ts rename to integrations/integration-postgres/makePostgresMetaService.ts diff --git a/integrations/integration-postgres/package.json b/integrations/integration-postgres/package.json index 597a5475..fb496497 100644 --- a/integrations/integration-postgres/package.json +++ b/integrations/integration-postgres/package.json @@ -6,12 +6,16 @@ "dependencies": { "@slonik/migrator": "0.11.3", "@usevenice/cdk-core": "workspace:*", - "@usevenice/core-integration-postgres": "workspace:*", "@usevenice/util": "workspace:*", "handlebars": "4.7.7", + "pg": "8.8.0", + "pg-native": "3.0.1", + "slonik": "30.3.1", + "slonik-interceptor-preset": "1.2.10", "zod": "3.21.4" }, "devDependencies": { + "@types/pg": "8.6.5", "@usevenice/cdk-ledger": "workspace:*" } } diff --git a/integrations/integration-postgres/server.ts b/integrations/integration-postgres/server.ts index 3af914af..bc4f607b 100644 --- a/integrations/integration-postgres/server.ts +++ b/integrations/integration-postgres/server.ts @@ -4,16 +4,11 @@ import type {IntegrationServer} from '@usevenice/cdk-core' import {extractId, handlersLink} from '@usevenice/cdk-core' -import { - makePostgresClient, - upsertByIdQuery, -} from '@usevenice/core-integration-postgres' import {R, Rx, rxjs} from '@usevenice/util' import type {postgresSchemas} from './def' import {postgresHelpers} from './def' - -export {makePostgresClient} from '@usevenice/core-integration-postgres' +import {makePostgresClient, upsertByIdQuery} from './makePostgresClient' export const postgresServer = { // TODO: diff --git a/packages/airbyte/Dockerfile b/packages/airbyte/Dockerfile index b5f26cc8..f467d7ff 100644 --- a/packages/airbyte/Dockerfile +++ b/packages/airbyte/Dockerfile @@ -38,7 +38,6 @@ COPY ./integrations/integration-toggl/package.json ./integrations/integration-to COPY ./integrations/integration-beancount/package.json ./integrations/integration-beancount/package.json COPY ./integrations/integration-redis/package.json ./integrations/integration-redis/package.json COPY ./integrations/integration-onebrick/package.json ./integrations/integration-onebrick/package.json -COPY ./integrations/core-integration-postgres/package.json ./integrations/core-integration-postgres/package.json COPY ./integrations/integration-firebase/package.json ./integrations/integration-firebase/package.json COPY ./integrations/integration-airtable/package.json ./integrations/integration-airtable/package.json COPY ./integrations/integration-plaid/package.json ./integrations/integration-plaid/package.json diff --git a/packages/airbyte/airbyteMetaService.ts b/packages/airbyte/airbyteMetaService.ts index 00ddc9d9..60e1a7e5 100644 --- a/packages/airbyte/airbyteMetaService.ts +++ b/packages/airbyte/airbyteMetaService.ts @@ -1,6 +1,8 @@ -import type {MetaService} from '@usevenice/cdk-core' -import {makePostgresMetaService} from '@usevenice/core-integration-postgres' import {z} from 'zod' + +import type {MetaService} from '@usevenice/cdk-core' +import {makePostgresMetaService} from '@usevenice/integration-postgres' + import {createApiClient} from './api/airbyte-client.gen' const zAirbyteMetaConfig = z.object({ diff --git a/packages/airbyte/package.json b/packages/airbyte/package.json index f7312917..adce88cd 100644 --- a/packages/airbyte/package.json +++ b/packages/airbyte/package.json @@ -8,7 +8,7 @@ "dependencies": { "@trpc/server": "10.21.1", "@usevenice/cdk-core": "workspace:*", - "@usevenice/core-integration-postgres": "workspace:*", + "@usevenice/integration-postgres": "workspace:*", "@usevenice/util": "workspace:*", "@zodios/core": "11.0.0-beta.18", "@zodios/fetch": "11.0.0-beta.18", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1e27b0ad..5fc2b2ae 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -158,9 +158,6 @@ importers: '@usevenice/cdk-ledger': specifier: workspace:* version: link:../../packages/cdk-ledger - '@usevenice/core-integration-postgres': - specifier: workspace:* - version: link:../../integrations/core-integration-postgres '@usevenice/engine-backend': specifier: workspace:* version: link:../../packages/engine-backend @@ -315,9 +312,6 @@ importers: '@usevenice/cdk-ledger': specifier: workspace:* version: link:../../packages/cdk-ledger - '@usevenice/core-integration-postgres': - specifier: workspace:* - version: link:../../integrations/core-integration-postgres '@usevenice/engine-backend': specifier: workspace:* version: link:../../packages/engine-backend @@ -521,15 +515,15 @@ importers: '@usevenice/connect': specifier: workspace:* version: link:../../packages/connect - '@usevenice/core-integration-postgres': - specifier: workspace:* - version: link:../../integrations/core-integration-postgres '@usevenice/engine-backend': specifier: workspace:* version: link:../../packages/engine-backend '@usevenice/engine-frontend': specifier: workspace:* version: link:../../packages/engine-frontend + '@usevenice/integration-postgres': + specifier: workspace:* + version: link:../../integrations/integration-postgres '@usevenice/ui': specifier: workspace:* version: link:../../packages/ui @@ -637,34 +631,6 @@ importers: specifier: '*' version: 5.74.0(esbuild@0.17.5) - integrations/core-integration-postgres: - dependencies: - '@slonik/migrator': - specifier: 0.11.3 - version: 0.11.3(slonik@30.3.1) - '@usevenice/cdk-core': - specifier: workspace:* - version: link:../../packages/cdk-core - '@usevenice/util': - specifier: workspace:* - version: link:../../packages/util - pg: - specifier: 8.8.0 - version: 8.8.0(pg-native@3.0.1) - pg-native: - specifier: 3.0.1 - version: 3.0.1 - slonik: - specifier: 30.3.1 - version: 30.3.1(pg-native@3.0.1) - slonik-interceptor-preset: - specifier: 1.2.10 - version: 1.2.10(pg-native@3.0.1) - devDependencies: - '@types/pg': - specifier: 8.6.5 - version: 8.6.5 - integrations/integration-airtable: dependencies: '@usevenice/cdk-core': @@ -977,19 +943,31 @@ importers: '@usevenice/cdk-core': specifier: workspace:* version: link:../../packages/cdk-core - '@usevenice/core-integration-postgres': - specifier: workspace:* - version: link:../core-integration-postgres '@usevenice/util': specifier: workspace:* version: link:../../packages/util handlebars: specifier: 4.7.7 version: 4.7.7 + pg: + specifier: 8.8.0 + version: 8.8.0(pg-native@3.0.1) + pg-native: + specifier: 3.0.1 + version: 3.0.1 + slonik: + specifier: 30.3.1 + version: 30.3.1(pg-native@3.0.1) + slonik-interceptor-preset: + specifier: 1.2.10 + version: 1.2.10(pg-native@3.0.1) zod: specifier: 3.21.4 version: 3.21.4(patch_hash=bzwjzhue3hmpww5lnv24u5k2ru) devDependencies: + '@types/pg': + specifier: 8.6.5 + version: 8.6.5 '@usevenice/cdk-ledger': specifier: workspace:* version: link:../../packages/cdk-ledger @@ -1220,9 +1198,9 @@ importers: '@usevenice/cdk-core': specifier: workspace:* version: link:../cdk-core - '@usevenice/core-integration-postgres': + '@usevenice/integration-postgres': specifier: workspace:* - version: link:../../integrations/core-integration-postgres + version: link:../../integrations/integration-postgres '@usevenice/util': specifier: workspace:* version: link:../util @@ -6460,7 +6438,7 @@ packages: /@types/pg@8.6.5: resolution: {integrity: sha512-tOkGtAqRVkHa/PVZicq67zuujI4Oorfglsr2IbKofDwBSysnaqSx7W1mDqFqdkGE6Fbgh+PZAl0r/BWON/mozw==} dependencies: - '@types/node': 18.8.3 + '@types/node': 18.11.18 pg-protocol: 1.5.0 pg-types: 2.2.0 dev: true From 1ad98c5c28614f454deca7cd5417d285f234a3ca Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 16 Jun 2023 11:52:22 +0800 Subject: [PATCH 19/80] Migrate debug provider over to new paradigm --- apps/app-config/_generateDocs.bin.ts | 4 +-- apps/app-config/integration-envs.ts | 5 ++-- .../integrations/integrations.def.ts | 2 ++ .../integrations/integrations.merged.ts | 8 +++++ .../integrations/integrations.server.ts | 2 ++ apps/app-config/integrations/meta.js | 9 ++++++ apps/app-config/package.json | 1 + apps/app-config/providers.ts | 14 +-------- integrations/integration-debug/def.ts | 25 ++++++++++++++++ integrations/integration-debug/index.ts | 4 +++ integrations/integration-debug/package.json | 12 ++++++++ integrations/integration-debug/server.ts | 17 +++++++++++ packages/cdk-core/DebugProvider.ts | 29 ------------------- packages/cdk-core/index.ts | 1 - pnpm-lock.yaml | 12 ++++++++ 15 files changed, 97 insertions(+), 48 deletions(-) create mode 100644 integrations/integration-debug/def.ts create mode 100644 integrations/integration-debug/index.ts create mode 100644 integrations/integration-debug/package.json create mode 100644 integrations/integration-debug/server.ts delete mode 100644 packages/cdk-core/DebugProvider.ts diff --git a/apps/app-config/_generateDocs.bin.ts b/apps/app-config/_generateDocs.bin.ts index 64879ded..347dee01 100644 --- a/apps/app-config/_generateDocs.bin.ts +++ b/apps/app-config/_generateDocs.bin.ts @@ -7,7 +7,7 @@ import {buildUrl, R} from '@usevenice/util' import {env, envConfig} from './env' import {parseIntConfigsFromRawEnv} from './integration-envs' -import {DOCUMENTED_PROVIDERS} from './providers' +import {PROVIDERS} from './providers' const envList = R.pipe( {...envConfig.server, ...envConfig.client}, @@ -15,7 +15,7 @@ const envList = R.pipe( R.filter( ([key]) => !key.startsWith('int') || - DOCUMENTED_PROVIDERS.some((p) => key.startsWith(`int_${p.name}`)), + PROVIDERS.some((p) => key.startsWith(`int_${p.name}`)), ), R.map(([key, schema]) => { const cmtLines = R.pipe( diff --git a/apps/app-config/integration-envs.ts b/apps/app-config/integration-envs.ts index 897a24f0..f8b7d4fe 100644 --- a/apps/app-config/integration-envs.ts +++ b/apps/app-config/integration-envs.ts @@ -2,8 +2,7 @@ import {makeId} from '@usevenice/cdk-core' import {R, z, zEnvVars, zFlattenForEnv} from '@usevenice/util' -import type {PROVIDERS} from './providers' -import {DOCUMENTED_PROVIDERS} from './providers' +import {PROVIDERS} from './providers' /** We would prefer to use `.` but vercel env var name can only be number, letter and underscore... */ const separator = '__' @@ -11,7 +10,7 @@ const getPrefix = (name: string) => makeId('int', name, '') // Should this be all providers or only dcoumented ones? -export const zFlatConfigByProvider = R.mapToObj(DOCUMENTED_PROVIDERS, (p) => [ +export const zFlatConfigByProvider = R.mapToObj(PROVIDERS, (p) => [ p.name, zFlattenForEnv(p.def.integrationConfig ?? z.unknown(), { prefix: getPrefix(p.name), diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index 1f42b4c0..c6077f97 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -2,6 +2,7 @@ import {default as integrationAirtable} from '@usevenice/integration-airtable/def' import {default as integrationBeancount} from '@usevenice/integration-beancount/def' import {default as integrationBrex} from '@usevenice/integration-brex/def' +import {default as integrationDebug} from '@usevenice/integration-debug/def' import {default as integrationFirebase} from '@usevenice/integration-firebase/def' import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/def' import {default as integrationFs} from '@usevenice/integration-fs/def' @@ -31,6 +32,7 @@ export const defIntegrations = { airtable: integrationAirtable, beancount: integrationBeancount, brex: integrationBrex, + debug: integrationDebug, firebase: integrationFirebase, foreceipt: integrationForeceipt, fs: integrationFs, diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index 12358547..dc094939 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -5,6 +5,8 @@ import {default as integrationBeancount_def} from '@usevenice/integration-beanco import {default as integrationBeancount_server} from '@usevenice/integration-beancount/server' import {default as integrationBrex_def} from '@usevenice/integration-brex/def' import {default as integrationBrex_server} from '@usevenice/integration-brex/server' +import {default as integrationDebug_def} from '@usevenice/integration-debug/def' +import {default as integrationDebug_server} from '@usevenice/integration-debug/server' import {default as integrationFirebase_def} from '@usevenice/integration-firebase/def' import {default as integrationFirebase_server} from '@usevenice/integration-firebase/server' import {default as integrationForeceipt_def} from '@usevenice/integration-foreceipt/def' @@ -72,6 +74,11 @@ const integrationBrex = { ...integrationBrex_server, } +const integrationDebug = { + ...integrationDebug_def, + ...integrationDebug_server, +} + const integrationFirebase = { ...integrationFirebase_def, ...integrationFirebase_server, @@ -199,6 +206,7 @@ export const mergedIntegrations = { airtable: integrationAirtable, beancount: integrationBeancount, brex: integrationBrex, + debug: integrationDebug, firebase: integrationFirebase, foreceipt: integrationForeceipt, fs: integrationFs, diff --git a/apps/app-config/integrations/integrations.server.ts b/apps/app-config/integrations/integrations.server.ts index a2cf9b08..e33aa42b 100644 --- a/apps/app-config/integrations/integrations.server.ts +++ b/apps/app-config/integrations/integrations.server.ts @@ -2,6 +2,7 @@ import {default as integrationAirtable} from '@usevenice/integration-airtable/server' import {default as integrationBeancount} from '@usevenice/integration-beancount/server' import {default as integrationBrex} from '@usevenice/integration-brex/server' +import {default as integrationDebug} from '@usevenice/integration-debug/server' import {default as integrationFirebase} from '@usevenice/integration-firebase/server' import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/server' import {default as integrationFs} from '@usevenice/integration-fs/server' @@ -29,6 +30,7 @@ export const serverIntegrations = { airtable: integrationAirtable, beancount: integrationBeancount, brex: integrationBrex, + debug: integrationDebug, firebase: integrationFirebase, foreceipt: integrationForeceipt, fs: integrationFs, diff --git a/apps/app-config/integrations/meta.js b/apps/app-config/integrations/meta.js index a77e0b36..431d6984 100644 --- a/apps/app-config/integrations/meta.js +++ b/apps/app-config/integrations/meta.js @@ -33,6 +33,15 @@ module.exports = [ server: '@usevenice/integration-brex/server', }, }, + { + name: 'debug', + dirName: 'integration-debug', + varName: 'integrationDebug', + imports: { + def: '@usevenice/integration-debug/def', + server: '@usevenice/integration-debug/server', + }, + }, { dirName: 'integration-expensify', varName: 'integrationExpensify', diff --git a/apps/app-config/package.json b/apps/app-config/package.json index 94b976c7..243ed941 100644 --- a/apps/app-config/package.json +++ b/apps/app-config/package.json @@ -18,6 +18,7 @@ "@usevenice/integration-alphavantage": "workspace:*", "@usevenice/integration-beancount": "workspace:*", "@usevenice/integration-brex": "workspace:*", + "@usevenice/integration-debug": "workspace:*", "@usevenice/integration-expensify": "workspace:*", "@usevenice/integration-firebase": "workspace:*", "@usevenice/integration-foreceipt": "workspace:*", diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts index 7fe483b7..9ed41114 100644 --- a/apps/app-config/providers.ts +++ b/apps/app-config/providers.ts @@ -1,17 +1,5 @@ // @deprecated. Soon to be fully replaced by integrations.merge.ts -import {debugProvider} from '@usevenice/cdk-core' - import {mergedIntegrations} from './integrations/integrations.merged' -export const DOCUMENTED_PROVIDERS = [ - ...(Object.values(mergedIntegrations) as unknown as Array< - typeof debugProvider - >), -] as const - -export const PROVIDERS = [ - ...DOCUMENTED_PROVIDERS, - // TODO: Migrate these over to the new paradigm - debugProvider, -] as const +export const PROVIDERS = [...Object.values(mergedIntegrations)] as const diff --git a/integrations/integration-debug/def.ts b/integrations/integration-debug/def.ts new file mode 100644 index 00000000..87a5e3b4 --- /dev/null +++ b/integrations/integration-debug/def.ts @@ -0,0 +1,25 @@ +import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' +import {intHelpers, zWebhookInput} from '@usevenice/cdk-core' +import {z} from '@usevenice/util' + +export const debugSchemas = { + name: z.literal('debug'), + webhookInput: zWebhookInput, + resourceSettings: z.unknown(), + integrationConfig: z.unknown(), + sourceOutputEntity: z.unknown(), + institutionData: z.unknown(), +} satisfies IntegrationSchemas + +export const helpers = intHelpers(debugSchemas) + +export const debugDef = { + metadata: {stage: 'hidden'}, + name: 'debug', + def: debugSchemas, + // Temporary hack to workaround assertion in mapStandardEntityLink when using debugProvider + // as a source. However we should do something so this workaround is not needed in the first place + extension: {sourceMapEntity: {}}, +} satisfies IntegrationDef + +export default debugDef diff --git a/integrations/integration-debug/index.ts b/integrations/integration-debug/index.ts new file mode 100644 index 00000000..81f31bd7 --- /dev/null +++ b/integrations/integration-debug/index.ts @@ -0,0 +1,4 @@ +// codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './def' +export * from './server' +// codegen:end diff --git a/integrations/integration-debug/package.json b/integrations/integration-debug/package.json new file mode 100644 index 00000000..08da0d50 --- /dev/null +++ b/integrations/integration-debug/package.json @@ -0,0 +1,12 @@ +{ + "name": "@usevenice/integration-debug", + "version": "0.0.0", + "private": true, + "sideEffects": [], + "module": "./index.ts", + "dependencies": { + "@usevenice/cdk-core": "workspace:*", + "@usevenice/util": "workspace:*" + }, + "devDependencies": {} +} diff --git a/integrations/integration-debug/server.ts b/integrations/integration-debug/server.ts new file mode 100644 index 00000000..f9cf8031 --- /dev/null +++ b/integrations/integration-debug/server.ts @@ -0,0 +1,17 @@ +import type {IntegrationServer} from '@usevenice/cdk-core' +import {logLink} from '@usevenice/cdk-core' +import {rxjs} from '@usevenice/util' + +import type {debugSchemas} from './def' + +export const debugServer = { + sourceSync: () => rxjs.EMPTY, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + destinationSync: () => logLink({prefix: 'debug', verbose: true}), + handleWebhook: (input) => ({ + resourceUpdates: [], + response: {body: {echo: input}}, + }), +} satisfies IntegrationServer + +export default debugServer diff --git a/packages/cdk-core/DebugProvider.ts b/packages/cdk-core/DebugProvider.ts deleted file mode 100644 index 8609ba60..00000000 --- a/packages/cdk-core/DebugProvider.ts +++ /dev/null @@ -1,29 +0,0 @@ -import {rxjs, z} from '@usevenice/util' - -import {logLink} from './base-links' -import {makeSyncProvider, zWebhookInput} from './makeSyncProvider' - -const debugProviderDef = makeSyncProvider.def({ - ...makeSyncProvider.def.defaults, - name: z.literal('debug'), - webhookInput: zWebhookInput, - resourceSettings: z.unknown(), - integrationConfig: z.unknown(), - sourceOutputEntity: z.unknown(), - institutionData: z.unknown(), -}) - -export const debugProvider = makeSyncProvider({ - ...makeSyncProvider.defaults, - def: debugProviderDef, - metadata: {stage: 'hidden'}, - sourceSync: () => rxjs.EMPTY, - destinationSync: () => logLink({prefix: 'debug', verbose: true}), - handleWebhook: (input) => ({ - resourceUpdates: [], - response: {body: {echo: input}}, - }), - // Temporary hack to workaround assertion in mapStandardEntityLink when using debugProvider - // as a source. However we should do something so this workaround is not needed in the first place - extension: {sourceMapEntity: {}}, -}) diff --git a/packages/cdk-core/index.ts b/packages/cdk-core/index.ts index 337e54f3..097166b8 100644 --- a/packages/cdk-core/index.ts +++ b/packages/cdk-core/index.ts @@ -1,6 +1,5 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{spec,test,fixture}.{ts,tsx}"} export * from './base-links' -export * from './DebugProvider' export * from './frontend-utils' export * from './id.types' export * from './integration-utils' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5fc2b2ae..a071e7cb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -176,6 +176,9 @@ importers: '@usevenice/integration-brex': specifier: workspace:* version: link:../../integrations/integration-brex + '@usevenice/integration-debug': + specifier: workspace:* + version: link:../../integrations/integration-debug '@usevenice/integration-expensify': specifier: workspace:* version: link:../../integrations/integration-expensify @@ -699,6 +702,15 @@ importers: specifier: 6.2.1 version: 6.2.1 + integrations/integration-debug: + dependencies: + '@usevenice/cdk-core': + specifier: workspace:* + version: link:../../packages/cdk-core + '@usevenice/util': + specifier: workspace:* + version: link:../../packages/util + integrations/integration-expensify: dependencies: '@usevenice/cdk-core': From 1579a4ae5133a5c24e0ca43738a5040eaab39435 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 16 Jun 2023 12:45:14 +0800 Subject: [PATCH 20/80] Fully remove previous makeSyncProvider New types now only --- apps/app-config/_generateDocs.bin.ts | 6 +- apps/app-config/backendConfig.ts | 10 +- apps/app-config/bootstrap.ts | 4 - apps/app-config/integration-envs.ts | 31 +- apps/app-config/providers.ts | 5 - apps/cli/_cli.ts | 13 +- apps/cli/sync-test-old.ts | 13 +- .../alphavantageProvider.ts | 20 - .../integration-alphavantage/index.ts | 1 - integrations/integration-foreceipt/server.ts | 5 +- integrations/integration-plaid/def.ts | 17 +- integrations/integration-stripe/def.ts | 17 +- packages/cdk-core/index.ts | 2 +- packages/cdk-core/integration.types.ts | 24 +- packages/cdk-core/makeSyncProvider.ts | 552 ------------------ packages/cdk-core/providers.types.ts | 181 ++++++ packages/cdk-ledger/entity-links.ts | 10 +- packages/cdk-ledger/veniceProviderBase.ts | 77 +-- packages/engine-backend/context.ts | 8 +- packages/engine-backend/contextHelpers.ts | 4 +- packages/engine-backend/types.ts | 34 +- packages/engine-frontend/VeniceConnect.tsx | 9 +- 22 files changed, 290 insertions(+), 753 deletions(-) delete mode 100644 apps/app-config/providers.ts delete mode 100644 integrations/integration-alphavantage/alphavantageProvider.ts delete mode 100644 packages/cdk-core/makeSyncProvider.ts create mode 100644 packages/cdk-core/providers.types.ts diff --git a/apps/app-config/_generateDocs.bin.ts b/apps/app-config/_generateDocs.bin.ts index 347dee01..7e6d210e 100644 --- a/apps/app-config/_generateDocs.bin.ts +++ b/apps/app-config/_generateDocs.bin.ts @@ -7,7 +7,7 @@ import {buildUrl, R} from '@usevenice/util' import {env, envConfig} from './env' import {parseIntConfigsFromRawEnv} from './integration-envs' -import {PROVIDERS} from './providers' +import {mergedIntegrations} from './integrations/integrations.merged' const envList = R.pipe( {...envConfig.server, ...envConfig.client}, @@ -15,7 +15,9 @@ const envList = R.pipe( R.filter( ([key]) => !key.startsWith('int') || - PROVIDERS.some((p) => key.startsWith(`int_${p.name}`)), + Object.values(mergedIntegrations).some((p) => + key.startsWith(`int_${p.name}`), + ), ), R.map(([key, schema]) => { const cmtLines = R.pipe( diff --git a/apps/app-config/backendConfig.ts b/apps/app-config/backendConfig.ts index 17206701..7f882b0b 100644 --- a/apps/app-config/backendConfig.ts +++ b/apps/app-config/backendConfig.ts @@ -14,11 +14,11 @@ import {joinPath, R, Rx} from '@usevenice/util' import {getServerUrl} from './constants' import {env} from './env' -import {PROVIDERS} from './providers' +import {mergedIntegrations} from './integrations/integrations.merged' +export {makePostgresClient} from '@usevenice/integration-postgres' export {DatabaseError} from '@usevenice/integration-postgres/makePostgresClient' export {Papa} from '@usevenice/integration-spreadsheet' -export {makePostgresClient} from '@usevenice/integration-postgres' export const backendEnv = env @@ -38,12 +38,12 @@ const usePg = env.POSTGRES_OR_WEBHOOK_URL.startsWith('postgres') // VeniceRouter['_def']['mutations']['syncPipeline'] // >[0] export type VeniceInput = PipelineInput< - (typeof PROVIDERS)[number], - (typeof PROVIDERS)[number] + (typeof mergedIntegrations)[keyof typeof mergedIntegrations], + (typeof mergedIntegrations)[keyof typeof mergedIntegrations] > export const contextFactory = getContextFactory({ - providers: PROVIDERS, + providers: Object.values(mergedIntegrations), // routerUrl: 'http://localhost:3010/api', // apiUrl? apiUrl: joinPath(getServerUrl(null), '/api/trpc'), jwtSecret: env.JWT_SECRET_OR_PUBLIC_KEY, diff --git a/apps/app-config/bootstrap.ts b/apps/app-config/bootstrap.ts index 73923c30..606706ed 100644 --- a/apps/app-config/bootstrap.ts +++ b/apps/app-config/bootstrap.ts @@ -1,15 +1,11 @@ import type {Id} from '@usevenice/cdk-core' import {extractId, makeId} from '@usevenice/cdk-core' -import type {IntegrationInput} from '@usevenice/engine-backend' import {flatRouter} from '@usevenice/engine-backend' import {getEnvVar} from '@usevenice/util' import {contextFactory} from './backendConfig' -import type {PROVIDERS} from './providers' import {parseIntConfigsFromRawEnv} from './integration-envs' -export type _ResourceInput = IntegrationInput<(typeof PROVIDERS)[number]> - // TODO: Is this file needed? We can most likely just // embed the functionality into venice cli directly... export async function bootstrap() { diff --git a/apps/app-config/integration-envs.ts b/apps/app-config/integration-envs.ts index f8b7d4fe..56d17bd8 100644 --- a/apps/app-config/integration-envs.ts +++ b/apps/app-config/integration-envs.ts @@ -1,8 +1,9 @@ /** @deprecated. We no longer initialize integration from ENVs, but maybe in clis still? */ +import type {IntegrationSchemas, IntHelpers} from '@usevenice/cdk-core' import {makeId} from '@usevenice/cdk-core' import {R, z, zEnvVars, zFlattenForEnv} from '@usevenice/util' -import {PROVIDERS} from './providers' +import {defIntegrations} from './integrations/integrations.def' /** We would prefer to use `.` but vercel env var name can only be number, letter and underscore... */ const separator = '__' @@ -10,13 +11,15 @@ const getPrefix = (name: string) => makeId('int', name, '') // Should this be all providers or only dcoumented ones? -export const zFlatConfigByProvider = R.mapToObj(PROVIDERS, (p) => [ - p.name, - zFlattenForEnv(p.def.integrationConfig ?? z.unknown(), { - prefix: getPrefix(p.name), - separator, - }), -]) +export const zFlatConfigByProvider = R.mapValues(defIntegrations, (def, name) => + zFlattenForEnv( + (def.def as IntegrationSchemas)?.integrationConfig ?? z.unknown(), + { + prefix: getPrefix(name), + separator, + }, + ), +) export const zIntegrationEnv = zEnvVars( R.pipe( @@ -59,9 +62,13 @@ export function parseIntConfigsFromRawEnv( }), (configMap) => R.pickBy(configMap, (val) => val !== undefined), ) as { - [k in (typeof PROVIDERS)[number]['name']]?: Extract< - (typeof PROVIDERS)[number], - {name: k} - >['def']['_types']['integrationConfig'] + [k in keyof typeof defIntegrations]?: GetIntConfig< + IntHelpers<(typeof defIntegrations)[k]['def']>['_types'] + > } } + +/** Feels like bit of a hack... */ +type GetIntConfig = T extends {integrationConfig: unknown} + ? T['integrationConfig'] + : {} diff --git a/apps/app-config/providers.ts b/apps/app-config/providers.ts deleted file mode 100644 index 9ed41114..00000000 --- a/apps/app-config/providers.ts +++ /dev/null @@ -1,5 +0,0 @@ -// @deprecated. Soon to be fully replaced by integrations.merge.ts - -import {mergedIntegrations} from './integrations/integrations.merged' - -export const PROVIDERS = [...Object.values(mergedIntegrations)] as const diff --git a/apps/cli/_cli.ts b/apps/cli/_cli.ts index c1630392..63218045 100644 --- a/apps/cli/_cli.ts +++ b/apps/cli/_cli.ts @@ -2,7 +2,7 @@ import '@usevenice/app-config/register.node' import {parseIntConfigsFromRawEnv} from '@usevenice/app-config/integration-envs' -import type {PROVIDERS} from '@usevenice/app-config/providers' +import type {defIntegrations} from '@usevenice/app-config/integrations/integrations.def' import {makeJwtClient} from '@usevenice/cdk-core' import {makeAlphavantageClient} from '@usevenice/integration-alphavantage' import {makeHeronClient} from '@usevenice/integration-heron' @@ -44,13 +44,8 @@ function env() { .env as typeof import('@usevenice/app-config/env')['env'] } -// Hack for plaid for now... -type hackName = 'plaid' | 'yodlee' | 'onebrick' | 'teller' | 'ramp' | 'brex' -function intConfig( - name: T, -) { - const config = - parseIntConfigsFromRawEnv()[name as Exclude] +function intConfig(name: T) { + const config = parseIntConfigsFromRawEnv()[name] if (!config) { throw new Error(`${name} provider is not configured`) } @@ -81,7 +76,7 @@ if (require.main === module) { teller: () => makeTellerClient(intConfig('teller')), stripe: () => makeStripeClient({apiKey: process.env['STRIPE_TEST_SECRET_KEY']!}), - ramp: () => makeRampClient(intConfig('ramp')), + ramp: () => makeRampClient(intConfig('ramp').oauth), wise: () => makeWiseClient(intConfig('wise')), toggl: () => makeTogglClient(intConfig('toggl')), yodlee: () => diff --git a/apps/cli/sync-test-old.ts b/apps/cli/sync-test-old.ts index 50cdae12..79c1fc67 100644 --- a/apps/cli/sync-test-old.ts +++ b/apps/cli/sync-test-old.ts @@ -5,8 +5,8 @@ import '@usevenice/app-config/register.node' import readline from 'node:readline' import {sync} from '@usevenice/cdk-core' -import {fsProvider} from '@usevenice/integration-fs' import {brexImpl} from '@usevenice/integration-brex' +import {fsServer} from '@usevenice/integration-fs' import {heronImpl} from '@usevenice/integration-heron' import {mergeImpl} from '@usevenice/integration-merge' import {postgresProvider} from '@usevenice/integration-postgres' @@ -197,8 +197,10 @@ switch (process.argv[2]) { obs.complete() }) }), - destination: fsProvider.destinationSync({ + destination: fsServer.destinationSync({ endUser: null, + config: {}, + state: {}, settings: {basePath: destPath}, }), }).catch(console.error) @@ -208,14 +210,17 @@ switch (process.argv[2]) { case 'direct': { console.log('direct mode') sync({ - source: fsProvider.sourceSync({ + source: fsServer.sourceSync({ endUser: null, settings: {basePath: srcPath}, + config: {}, state: {}, }), - destination: fsProvider.destinationSync({ + destination: fsServer.destinationSync({ endUser: null, settings: {basePath: destPath}, + config: {}, + state: {}, }), }).catch(console.error) } diff --git a/integrations/integration-alphavantage/alphavantageProvider.ts b/integrations/integration-alphavantage/alphavantageProvider.ts deleted file mode 100644 index e2692dc9..00000000 --- a/integrations/integration-alphavantage/alphavantageProvider.ts +++ /dev/null @@ -1,20 +0,0 @@ -import {makeSyncProvider} from '@usevenice/cdk-core' -import {veniceProviderBase} from '@usevenice/cdk-ledger' -import {z} from '@usevenice/util' - -import {zConfig} from './alphavantageClient' - -const _def = makeSyncProvider.def({ - ...veniceProviderBase.def, - name: z.literal('alphavantage'), - integrationConfig: zConfig, - sourceOutputEntity: z.object({ - id: z.string(), - entityName: z.literal('commodity'), - entity: z.any(), // TODO Replace with correct type - }), -}) - -export const alphavantageProviderDef = makeSyncProvider.def.helpers(_def) - -// TODO: convert the old provider with the current format diff --git a/integrations/integration-alphavantage/index.ts b/integrations/integration-alphavantage/index.ts index e06da09f..7c35feed 100644 --- a/integrations/integration-alphavantage/index.ts +++ b/integrations/integration-alphavantage/index.ts @@ -1,4 +1,3 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} export * from './alphavantageClient' -export * from './alphavantageProvider' // codegen:end diff --git a/integrations/integration-foreceipt/server.ts b/integrations/integration-foreceipt/server.ts index 21580183..17c4a03c 100644 --- a/integrations/integration-foreceipt/server.ts +++ b/integrations/integration-foreceipt/server.ts @@ -1,6 +1,6 @@ import type {IntegrationServer} from '@usevenice/cdk-core' import { - firebaseProvider, + firebaseServer, serializeTimestamp, } from '@usevenice/integration-firebase' import {Rx, rxjs} from '@usevenice/util' @@ -37,8 +37,9 @@ export const foreceiptServer = { .pipe( Rx.mergeMap(([q, res]) => { info = res - return firebaseProvider.sourceSync({ + return firebaseServer.sourceSync({ endUser: null, + config: {}, settings: client.fbSettings, state: {_fb: fb, _queries: Object.values(q)}, }) diff --git a/integrations/integration-plaid/def.ts b/integrations/integration-plaid/def.ts index 0e96feb5..f66b6fbe 100644 --- a/integrations/integration-plaid/def.ts +++ b/integrations/integration-plaid/def.ts @@ -8,11 +8,7 @@ import type { import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' import {intHelpers, zWebhookInput} from '@usevenice/cdk-core' -import { - baseIntegrationSchemas, - makePostingsMap, - makeStandardId, -} from '@usevenice/cdk-ledger' +import {makePostingsMap, makeStandardId, zStream} from '@usevenice/cdk-ledger' import {A, z, zCast} from '@usevenice/util' import { @@ -26,7 +22,6 @@ import type {ErrorShape} from './plaid.types' import {zCountryCode, zLanguage, zPlaidEnvName, zProducts} from './PlaidClient' export const plaidSchemas = { - ...baseIntegrationSchemas, name: z.literal('plaid'), // There is a mixing of cases here... Unfortunately... integrationConfig: z.object({ @@ -78,9 +73,13 @@ export const plaidSchemas = { meta: zCast().optional(), }), /** "Manually" extending for now, this will get better / safer */ - sourceState: baseIntegrationSchemas.sourceState - .removeDefault() - .extend({ + sourceState: z + .object({ + streams: z.array(zStream).nullish(), + /** Account ids to sync */ + accountIds: z.array(z.string()).nullish(), + /** Date to sync since */ + sinceDate: z.string().nullish() /** ISO8601 */, transactionSyncCursor: z.string().nullish(), /** ISO8601 */ investmentTransactionEndDate: z.string().nullish(), diff --git a/integrations/integration-stripe/def.ts b/integrations/integration-stripe/def.ts index 6c3a7d68..fd2b5114 100644 --- a/integrations/integration-stripe/def.ts +++ b/integrations/integration-stripe/def.ts @@ -1,10 +1,6 @@ import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' import {intHelpers, zIntAuth} from '@usevenice/cdk-core' -import { - makePostingsMap, - veniceProviderBase, - zCommon, -} from '@usevenice/cdk-ledger' +import {makePostingsMap, zCommon} from '@usevenice/cdk-ledger' import {A, z, zCast} from '@usevenice/util' import type {components} from './stripe.gen' @@ -25,9 +21,14 @@ export const stripeSchemas = { entity: zCast(), }), ]), - sourceState: veniceProviderBase.def.sourceState - .removeDefault() - .extend({transactionSyncCursor: z.string().nullish()}) + sourceState: z + .object({ + /** Account ids to sync */ + accountIds: z.array(z.string()).nullish(), + /** Date to sync since */ + sinceDate: z.string().nullish() /** ISO8601 */, + transactionSyncCursor: z.string().nullish(), + }) .default({}), destinationInputEntity: zCommon.Entity, } satisfies IntegrationSchemas diff --git a/packages/cdk-core/index.ts b/packages/cdk-core/index.ts index 097166b8..e8ff803f 100644 --- a/packages/cdk-core/index.ts +++ b/packages/cdk-core/index.ts @@ -5,11 +5,11 @@ export * from './id.types' export * from './integration-utils' export * from './integration.types' export * from './kvStore' -export * from './makeSyncProvider' export * from './meta.types' export * from './metaService' export * from './NoopMetaService' export * from './protocol' +export * from './providers.types' export * from './sync' export * from './viewer' // codegen:end diff --git a/packages/cdk-core/integration.types.ts b/packages/cdk-core/integration.types.ts index 68767d90..9cbcdece 100644 --- a/packages/cdk-core/integration.types.ts +++ b/packages/cdk-core/integration.types.ts @@ -3,6 +3,14 @@ import {R} from '@usevenice/util' import type {EndUserId} from './id.types' import {makeId} from './id.types' +import type {ZStandard} from './meta.types' +import type { + Destination, + ResoUpdateData, + Source, + StateUpdateData, + SyncOperation, +} from './protocol' import type { CheckResourceContext, CheckResourceOptions, @@ -12,15 +20,7 @@ import type { OpenDialogFn, ResourceUpdate, WebhookReturnType, -} from './makeSyncProvider' -import type {ZStandard} from './meta.types' -import type { - Destination, - ResoUpdateData, - Source, - StateUpdateData, - SyncOperation, -} from './protocol' +} from './providers.types' /** Maybe this should be renamed to `schemas` */ export interface IntegrationSchemas { @@ -40,6 +40,8 @@ export interface IntegrationSchemas { destinationInputEntity?: z.ZodTypeAny } +export type IntHelpers = + ReturnType> export interface IntegrationDef< TSchemas extends IntegrationSchemas = IntegrationSchemas, T extends IntHelpers = IntHelpers, @@ -88,10 +90,6 @@ export interface IntegrationDef< } } -export type IntHelpers = ReturnType< - typeof intHelpers -> - export interface IntegrationClient< TDef extends IntegrationSchemas = IntegrationSchemas, T extends IntHelpers = IntHelpers, diff --git a/packages/cdk-core/makeSyncProvider.ts b/packages/cdk-core/makeSyncProvider.ts deleted file mode 100644 index dea8bff7..00000000 --- a/packages/cdk-core/makeSyncProvider.ts +++ /dev/null @@ -1,552 +0,0 @@ -import type {MaybePromise} from '@usevenice/util' -import { - castIs, - R, - titleCase, - urlFromImage, - z, - zodToJsonSchema, -} from '@usevenice/util' - -import type {EndUserId, ExtEndUserId, ExternalId} from './id.types' -import {makeId, zExternalId} from './id.types' -import type {IntegrationSchemas} from './integration.types' -import type {ZStandard} from './meta.types' -import type { - AnyEntityPayload, - Destination, - ResoUpdateData, - Source, - StateUpdateData, - SyncOperation, -} from './protocol' - -// eslint-disable-next-line @typescript-eslint/consistent-type-definitions -type JSONSchema = {} // ReturnType | JSONSchema7Definition - -export const metaForProvider = (provider: AnySyncProvider) => ({ - // ...provider, - __typename: 'provider' as const, - name: provider.name, - displayName: provider.metadata?.displayName ?? titleCase(provider.name), - logoUrl: provider.metadata?.logoSvg - ? urlFromImage({type: 'svg', data: provider.metadata?.logoSvg}) - : provider.metadata?.logoUrl, - stage: provider.metadata?.stage ?? 'alpha', - platforms: provider.metadata?.platforms ?? ['cloud', 'local'], - categories: provider.metadata?.categories ?? ['other'], - supportedModes: R.compact([ - provider.sourceSync ? ('source' as const) : null, - provider.destinationSync ? ('destination' as const) : null, - ]), - hasPreConnect: provider.preConnect != null, - hasUseConnectHook: provider.useConnectHook != null, - hasPostConnect: provider.postConnect != null, - schemas: R.mapValues(provider.def ?? {}, (schema) => - schema instanceof z.ZodSchema ? zodToJsonSchema(schema) : undefined, - ) as Record, -}) - -export const zIntegrationCategory = z.enum([ - 'banking', - 'accounting', - 'commerce', - 'expense-management', - 'enrichment', - 'database', - 'flat-files-and-spreadsheets', - 'streaming', - 'personal-finance', - 'other', -]) - -export const zIntegrationStage = z.enum(['hidden', 'alpha', 'beta', 'ga']) - -export interface IntegrationMetadata { - logoUrl?: string - logoSvg?: string - displayName?: string - /** @deprecated way to indicate an integration outputs raw rather than standardized data */ - layer?: 'core' | 'ledger' - platforms?: Array<'cloud' | 'local'> - stage?: z.infer - // labels?: Array<'featured' | 'banking' | 'accounting' | 'enrichment'> - categories?: Array> -} - -// MARK: - Shared connect types - -/** Useful for establishing the initial pipeline when creating a connection for the first time */ - -export type ConnectOptions = z.input -export const zConnectOptions = z.object({ - // userId: UserId, - /** Noop if `connectionId` is specified */ - institutionExternalId: zExternalId.nullish(), - resourceExternalId: zExternalId.nullish(), -}) - -export const zPostConnectOptions = zConnectOptions.extend({ - syncInBand: z.boolean().nullish(), -}) - -// MARK: - Client side connect types - -export type OpenDialogFn = ( - Component: React.ComponentType<{close: () => void}>, - options?: { - dismissOnClickOutside?: boolean - onClose?: () => void - }, -) => void - -export type UseConnectHook = (scope: { - openDialog: OpenDialogFn -}) => ( - connectInput: T['_types']['connectInput'], - context: ConnectOptions, -) => Promise - -// MARK: - Server side connect types - -export interface CheckResourceContext { - webhookBaseUrl: string -} - -/** Context providers get during the connection establishing phase */ -export interface ConnectContext - extends Omit, - CheckResourceContext { - extEndUserId: ExtEndUserId - /** Used for OAuth based integrations, e.g. https://plaid.com/docs/link/oauth/#create-and-register-a-redirect-uri */ - redirectUrl?: string - resource?: { - externalId: ExternalId - settings: TSettings - } | null -} - -// TODO: We should rename `provider` to `integration` given that they are both -// Sources AND destinations. Provider only makes sense for sources. -// An integration can have `connect[UI]`, `src[Connector]` and `dest[Connector]` -export type CheckResourceOptions = z.infer -export const zCheckResourceOptions = z.object({ - /** - * Always make a request to the provider. Perhaps should be the default? - * Will have to refactor `checkResource` to be a bit different - */ - skipCache: z.boolean().nullish(), - /** Persist input into connection storage */ - import: z.boolean().nullish(), - /** - * Update the webhook associated with this connection to based on webhookBaseUrl - */ - updateWebhook: z.boolean().nullish(), - /** Fire webhook for default data updates */ - sandboxSimulateUpdate: z.boolean().nullish(), - /** For testing out disconnection handling */ - sandboxSimulateDisconnect: z.boolean().nullish(), -}) - -/** Extra props not on ResoUpdateData */ -export interface ResourceUpdate< - TEntity extends AnyEntityPayload = AnyEntityPayload, - TSettings = unknown, - > - // make `ResoUpdateData.id` not prefixed so we can have better inheritance - extends Omit, 'id'> { - // Subset of resoUpdate - resourceExternalId: ExternalId - // Can we inherit types used by metaLinks? - /** If missing it means do not change the userId... */ - endUserId?: EndUserId | null - - source$?: Source - triggerDefaultSync?: boolean -} - -export type WebhookInput = z.infer -export const zWebhookInput = z.object({ - headers: z - .record(z.unknown()) - .refine(castIs()), - query: z.record(z.unknown()), - body: z.unknown(), -}) - -export interface WebhookReturnType< - TEntity extends AnyEntityPayload, - TSettings, -> { - resourceUpdates: Array> - /** HTTP Response body */ - response?: { - body: Record - } -} - -// MARK: - Provider def - -type _opt = T | undefined -type _infer = T extends z.ZodTypeAny ? z.infer : never - -/** Surprisingly tricky, see. https://www.zhenghao.io/posts/ts-never */ -type NeverKeys = Exclude< - {[K in keyof T]: [T[K]] extends [never] ? K : never}[keyof T], - undefined -> - -type OmitNever = Omit> // & {[k in NeverKeys]?: undefined} - -export type AnyProviderDef = ReturnType -function makeSyncProviderDef< - TName extends string, - ZIntConfig extends _opt, - ZResSettings extends _opt, - ZInsData extends _opt, - // How do we enforce that the input type of ZWebhookInput must be a subset - // of the input type of typeof zWebhookInput? - // Right now things like zWebhookInput.extend({whatev: z.string()}) is allowed https://share.cleanshot.com/9PImJE - // But that is not really going to be valid. - ZWebhookInput extends _opt< - // eslint-disable-next-line @typescript-eslint/no-explicit-any - z.ZodType> - >, - // Sync - ZSrcState extends _opt, - ZSrcOutEntity extends _opt, - ZDestState extends _opt, - ZDestInEntity extends _opt, - // Connect - ZPreConnInput extends _opt, - ZConnInput extends _opt, - ZConnOutput extends _opt, ->(schemas: { - name: z.ZodLiteral - integrationConfig: ZIntConfig - resourceSettings: ZResSettings - institutionData: ZInsData - webhookInput: ZWebhookInput - preConnectInput: ZPreConnInput - connectInput: ZConnInput - connectOutput: ZConnOutput - /** Maybe can be derived from webhookInput | postConnOutput | inlineInput? */ - sourceState: ZSrcState - sourceOutputEntity: ZSrcOutEntity - destinationState: ZDestState - destinationInputEntity: ZDestInEntity -}) { - type Schemas = typeof schemas - type _types = {[k in keyof Schemas]: _infer} - type InsOpData = Extract< - SyncOperation<{ - id: string - entityName: 'institution' - entity: _types['institutionData'] - }>, - {type: 'data'} - > - type resoUpdate = ResoUpdateData< - _types['resourceSettings'], - _types['institutionData'] - > - type StateUpdate = StateUpdateData< - _types['sourceState'], - _types['destinationState'] - > - type Op = SyncOperation<_types['sourceOutputEntity'], resoUpdate, StateUpdate> - type Src = Source<_types['sourceOutputEntity'], resoUpdate, StateUpdate> - - return { - ...schemas, - _types: {} as _types, - _resUpdateType: {} as resoUpdate, - _stateUpdateType: {} as StateUpdate, - _sourceType: {} as Src, - // destination type is a function and thus causes issues... - _opType: {} as Op, - _insOpType: {} as InsOpData, - _resourceUpdateType: {} as ResourceUpdate< - _types['sourceOutputEntity'], - _types['resourceSettings'] - >, - _webhookReturnType: {} as WebhookReturnType< - _types['sourceOutputEntity'], - _types['resourceSettings'] - >, - } -} - -makeSyncProviderDef.helpers = (def: T) => { - type _types = T['_types'] - type InsOpData = T['_insOpType'] - type Op = T['_opType'] - type OpData = Extract - type OpRes = Extract - type OpState = Extract - return { - ...def, - _type: (_k: K, v: _types[K]) => v, - _op: ( - ...args: {} extends Omit, 'type'> - ? [K] - : [K, Omit, 'type'>] - ) => ({...args[1], type: args[0]} as unknown as Extract), - _opRes: (id: string, rest: Omit) => - R.identity({ - // We don't prefix in `_opData`, should we actually prefix here? - ...rest, - // TODO: ok so this is a sign that we should be prefixing using a link of some kind... - id: makeId('reso', def.name.value, id), - type: 'resoUpdate', - }) as OpRes, - _opState: ( - sourceState?: OpState['sourceState'], - destinationState?: OpState['destinationState'], - ) => - R.identity({ - sourceState, - destinationState, - type: 'stateUpdate', - }) as OpState, - _opData: ( - entityName: K, - id: string, - entity: Extract['entity'] | null, - ) => - R.identity({ - // TODO: Figure out why we need an `unknown` cast here - data: {entityName, id, entity} as unknown as OpData['data'], - type: 'data', - }) as OpData, - _insOpData: ( - id: ExternalId, - insitutionData: _types['institutionData'], - ): InsOpData => ({ - type: 'data', - data: { - // We don't prefix in `_opData`, should we actually prefix here? - id: makeId('ins', def.name.value, id), - entityName: 'institution', - entity: insitutionData, - }, - }), - _webhookReturn: ( - resourceExternalId: T['_resourceUpdateType']['resourceExternalId'], - rest: Omit, - ): T['_webhookReturnType'] => ({ - resourceUpdates: [{...rest, resourceExternalId}], - }), - } -} - -makeSyncProviderDef.defaults = makeSyncProviderDef({ - name: z.literal('noop'), - integrationConfig: undefined, - resourceSettings: undefined, - institutionData: undefined, - webhookInput: undefined, - preConnectInput: undefined, - connectInput: undefined, - connectOutput: undefined, - sourceOutputEntity: undefined, - sourceState: undefined, - destinationState: undefined, - destinationInputEntity: undefined, -}) - -// MARK: - Provider - -export type AnySyncProvider = ReturnType - -/** - * Provider definition should be universal. - * Need to figure out pattern for platform specific code - */ -export function makeSyncProvider< - T extends AnyProviderDef, - // Sync - TSrcSync extends _opt< - ( - input: OmitNever<{ - endUser: {id: EndUserId} | null | undefined - config: T['_types']['integrationConfig'] - settings: T['_types']['resourceSettings'] - state: T['_types']['sourceState'] - }>, - ) => Source - >, - TDestSync extends _opt< - ( - input: OmitNever<{ - endUser: {id: EndUserId} | null | undefined - config: T['_types']['integrationConfig'] - settings: T['_types']['resourceSettings'] - state: T['_types']['destinationState'] - }>, - ) => Destination - >, - TMetaSync extends _opt< - ( - input: OmitNever<{ - config: T['_types']['integrationConfig'] - // options: T['_types']['sourceState'] - }>, - ) => Source - >, - // Connect - TMappers extends _opt<{ - institution?: ( - data: T['_types']['institutionData'], - ) => Omit - resource: ( - settings: T['_types']['resourceSettings'], - ) => Omit - }>, - // TODO: Consider modeling after classes. Separating `static` from `instance` methods - // by introducing a separate `instance` method that accepts config - // and returns functions that already have access to config via the scope - // one challenge is that it would make the logic somewhat less clear - // as UseConnectHook would be a static method while pre/post conn are going to - // be instance methods. - TPreConn extends _opt< - ( - config: T['_types']['integrationConfig'], - context: ConnectContext, - // TODO: Turn this into an object instead - input: T['_types']['preConnectInput'], - ) => Promise - >, - TUseConnHook extends _opt>, - TPostConn extends _opt< - ( - connectOutput: T['_types']['connectOutput'], - config: T['_types']['integrationConfig'], - context: ConnectContext, - ) => MaybePromise< - Omit< - ResourceUpdate< - T['_types']['sourceOutputEntity'], - T['_types']['resourceSettings'] - >, - 'endUserId' - > - > - >, - TCheckRes extends _opt< - ( - input: OmitNever<{ - settings: T['_types']['resourceSettings'] - config: T['_types']['integrationConfig'] - options: CheckResourceOptions - context: CheckResourceContext - }>, - ) => MaybePromise< - Omit< - ResourceUpdate< - T['_types']['sourceOutputEntity'], - T['_types']['resourceSettings'] - >, - 'endUserId' - > - > - >, - // This probably need to also return an observable - TRevokeRes extends _opt< - ( - settings: T['_types']['resourceSettings'], - config: T['_types']['integrationConfig'], - ) => Promise - >, - // MARK - Webhook - // Need to add a input schema for each provider to verify the shape of the received - // webhook requests... - THandleWebhook extends _opt< - ( - webhookInput: T['_types']['webhookInput'], - config: T['_types']['integrationConfig'], - ) => MaybePromise< - WebhookReturnType< - T['_types']['sourceOutputEntity'], - T['_types']['resourceSettings'] - > - > - >, - TExtension, ->(impl: { - def: T - metadata?: IntegrationMetadata - standardMappers: TMappers - - // MARK: - Connection management - // Consider combining these into a single function with union input to make the - // provider interface less verbose. e.g. phase: 'will' | 'did' | 'revoke' - - /** e.g. Generating public token to use for connection */ - preConnect: TPreConn - - /** - * React hook called client side - * Remember to respect rule of hooks, never call conditionally and never call inside loops - * Must be called synchronously, unconditionally and with a fixed number of invocations - */ - useConnectHook: TUseConnHook - - /** aka `postConnect` e.g. Exchange public token for access token and persist connection */ - postConnect: TPostConn - - /** Notably may be used to update webhook */ - checkResource: TCheckRes - - /** Well, what it says... */ - revokeResource: TRevokeRes - - handleWebhook: THandleWebhook - - // MARK: - Synchronization - - /** - * Can be either provider initiated or venice initiated - * Venice initiated (may not explicitly complete) - * - Previusly iterateEntities. Filter for `ready` event to complete initial sync - * in case the sync is listen based rather than poll based (e.g. watchChanges) - * Provider initiated (always completes) - * - handlePushData ? - * - handleOauthCallback ? - */ - sourceSync: TSrcSync - - /** - * Always venice initiated. Handles a list of operations and may also emit - * progress events. `ready` event lets engine know that it may terminate if needed - */ - destinationSync: TDestSync - - metaSync: TMetaSync - - /** Allow core sync to be extended, for exampke, by ledger sync */ - extension: TExtension -}) { - return {...impl, name: impl.def.name.value as T['_types']['name']} -} - -makeSyncProvider.def = makeSyncProviderDef - -/** - * Can be used as makeProviderNext({...makeProviderNext.noop, yourCode...}) - * @see https://tkdodo.eu/blog/optional-vs-undefined for inspiration - */ -makeSyncProvider.defaults = makeSyncProvider({ - def: makeSyncProvider.def.defaults, - standardMappers: undefined, - preConnect: undefined, - useConnectHook: undefined, - postConnect: undefined, - checkResource: undefined, - revokeResource: undefined, - handleWebhook: undefined, - sourceSync: undefined, - destinationSync: undefined, - metaSync: undefined, - extension: undefined, -}) diff --git a/packages/cdk-core/providers.types.ts b/packages/cdk-core/providers.types.ts new file mode 100644 index 00000000..b964d79b --- /dev/null +++ b/packages/cdk-core/providers.types.ts @@ -0,0 +1,181 @@ +import { + castIs, + R, + titleCase, + urlFromImage, + z, + zodToJsonSchema, +} from '@usevenice/util' + +import type {EndUserId, ExtEndUserId, ExternalId} from './id.types' +import {zExternalId} from './id.types' +import type { + AnyIntegrationImpl, + IntegrationSchemas, + IntHelpers, +} from './integration.types' +import type {AnyEntityPayload, ResoUpdateData, Source} from './protocol' + +// eslint-disable-next-line @typescript-eslint/consistent-type-definitions +type JSONSchema = {} // ReturnType | JSONSchema7Definition + +export const metaForProvider = (provider: AnyIntegrationImpl) => ({ + // ...provider, + __typename: 'provider' as const, + name: provider.name, + displayName: provider.metadata?.displayName ?? titleCase(provider.name), + logoUrl: provider.metadata?.logoSvg + ? urlFromImage({type: 'svg', data: provider.metadata?.logoSvg}) + : provider.metadata?.logoUrl, + stage: provider.metadata?.stage ?? 'alpha', + platforms: provider.metadata?.platforms ?? ['cloud', 'local'], + categories: provider.metadata?.categories ?? ['other'], + supportedModes: R.compact([ + provider.sourceSync ? ('source' as const) : null, + provider.destinationSync ? ('destination' as const) : null, + ]), + hasPreConnect: provider.preConnect != null, + hasUseConnectHook: provider.useConnectHook != null, + hasPostConnect: provider.postConnect != null, + schemas: R.mapValues(provider.def ?? {}, (schema) => + schema instanceof z.ZodSchema ? zodToJsonSchema(schema) : undefined, + ) as Record, +}) + +export const zIntegrationCategory = z.enum([ + 'banking', + 'accounting', + 'commerce', + 'expense-management', + 'enrichment', + 'database', + 'flat-files-and-spreadsheets', + 'streaming', + 'personal-finance', + 'other', +]) + +export const zIntegrationStage = z.enum(['hidden', 'alpha', 'beta', 'ga']) + +export interface IntegrationMetadata { + logoUrl?: string + logoSvg?: string + displayName?: string + /** @deprecated way to indicate an integration outputs raw rather than standardized data */ + layer?: 'core' | 'ledger' + platforms?: Array<'cloud' | 'local'> + stage?: z.infer + // labels?: Array<'featured' | 'banking' | 'accounting' | 'enrichment'> + categories?: Array> +} + +// MARK: - Shared connect types + +/** Useful for establishing the initial pipeline when creating a connection for the first time */ + +export type ConnectOptions = z.input +export const zConnectOptions = z.object({ + // userId: UserId, + /** Noop if `connectionId` is specified */ + institutionExternalId: zExternalId.nullish(), + resourceExternalId: zExternalId.nullish(), +}) + +export const zPostConnectOptions = zConnectOptions.extend({ + syncInBand: z.boolean().nullish(), +}) + +// MARK: - Client side connect types + +export type OpenDialogFn = ( + Component: React.ComponentType<{close: () => void}>, + options?: { + dismissOnClickOutside?: boolean + onClose?: () => void + }, +) => void + +export type UseConnectHook = (scope: { + openDialog: OpenDialogFn +}) => ( + connectInput: T['_types']['connectInput'], + context: ConnectOptions, +) => Promise + +// MARK: - Server side connect types + +export interface CheckResourceContext { + webhookBaseUrl: string +} + +/** Context providers get during the connection establishing phase */ +export interface ConnectContext + extends Omit, + CheckResourceContext { + extEndUserId: ExtEndUserId + /** Used for OAuth based integrations, e.g. https://plaid.com/docs/link/oauth/#create-and-register-a-redirect-uri */ + redirectUrl?: string + resource?: { + externalId: ExternalId + settings: TSettings + } | null +} + +// TODO: We should rename `provider` to `integration` given that they are both +// Sources AND destinations. Provider only makes sense for sources. +// An integration can have `connect[UI]`, `src[Connector]` and `dest[Connector]` +export type CheckResourceOptions = z.infer +export const zCheckResourceOptions = z.object({ + /** + * Always make a request to the provider. Perhaps should be the default? + * Will have to refactor `checkResource` to be a bit different + */ + skipCache: z.boolean().nullish(), + /** Persist input into connection storage */ + import: z.boolean().nullish(), + /** + * Update the webhook associated with this connection to based on webhookBaseUrl + */ + updateWebhook: z.boolean().nullish(), + /** Fire webhook for default data updates */ + sandboxSimulateUpdate: z.boolean().nullish(), + /** For testing out disconnection handling */ + sandboxSimulateDisconnect: z.boolean().nullish(), +}) + +/** Extra props not on ResoUpdateData */ +export interface ResourceUpdate< + TEntity extends AnyEntityPayload = AnyEntityPayload, + TSettings = unknown, + > + // make `ResoUpdateData.id` not prefixed so we can have better inheritance + extends Omit, 'id'> { + // Subset of resoUpdate + resourceExternalId: ExternalId + // Can we inherit types used by metaLinks? + /** If missing it means do not change the userId... */ + endUserId?: EndUserId | null + + source$?: Source + triggerDefaultSync?: boolean +} + +export type WebhookInput = z.infer +export const zWebhookInput = z.object({ + headers: z + .record(z.unknown()) + .refine(castIs()), + query: z.record(z.unknown()), + body: z.unknown(), +}) + +export interface WebhookReturnType< + TEntity extends AnyEntityPayload, + TSettings, +> { + resourceUpdates: Array> + /** HTTP Response body */ + response?: { + body: Record + } +} diff --git a/packages/cdk-ledger/entity-links.ts b/packages/cdk-ledger/entity-links.ts index 228e9203..7ec6408d 100644 --- a/packages/cdk-ledger/entity-links.ts +++ b/packages/cdk-ledger/entity-links.ts @@ -1,7 +1,7 @@ import type { AnyEntityPayload, - AnySyncProvider, Id, + IntegrationDef, Link, } from '@usevenice/cdk-core' import {handlersLink, transformLink} from '@usevenice/cdk-core' @@ -27,7 +27,6 @@ import type { StdSyncOperation, } from './entity-link-types' import {makeStandardId, zStandardEntityPrefixFromName} from './utils' -import {isVeniceProvider} from './veniceProviderBase' // TODO: Can we use the `parsedReso` type here? export function mapStandardEntityLink({ @@ -35,11 +34,12 @@ export function mapStandardEntityLink({ settings: initialSettings, id: sourceId, }: { - integration: {provider: AnySyncProvider} + integration: {provider: IntegrationDef} settings: unknown id: Id['reso'] | undefined }): Link { - if (!isVeniceProvider(provider)) { + const sourceMapEntity = provider.extension?.sourceMapEntity + if (!sourceMapEntity) { throw new Error('Expecting VeniceProvider in mapStandardEntityLink') } return Rx.mergeMap((op) => { @@ -48,7 +48,7 @@ export function mapStandardEntityLink({ } // TODO: Update the initialReso as we receive resource updates - const payload = R.pipe(provider.extension.sourceMapEntity, (map) => + const payload = R.pipe(sourceMapEntity, (map) => typeof map === 'function' ? map(op.data, initialSettings) : map?.[op.data.entityName]?.(op.data, initialSettings), diff --git a/packages/cdk-ledger/veniceProviderBase.ts b/packages/cdk-ledger/veniceProviderBase.ts index e1e7ab63..3f6d5f12 100644 --- a/packages/cdk-ledger/veniceProviderBase.ts +++ b/packages/cdk-ledger/veniceProviderBase.ts @@ -1,14 +1,10 @@ -import type {AnyProviderDef, AnySyncProvider} from '@usevenice/cdk-core' -import {makeSyncProvider, zStandard} from '@usevenice/cdk-core' +import {zStandard} from '@usevenice/cdk-core' import {R, z} from '@usevenice/util' -import type {EntityPayload} from './entity-link-types' -import {zEntityName, zEntityPayload} from './entity-link-types' +import {zEntityName} from './entity-link-types' // NEXT: add institution, etc. -type _opt = T | undefined - /** Aka EntityName + resource + institution, See Airbyte docs on streams */ export const zStream = z.enum([ // TODO: Merge these different references to entity names together in one place... @@ -16,73 +12,8 @@ export const zStream = z.enum([ ...(R.keys(zStandard) as Array), ]) -export const baseIntegrationSchemas = { - sourceState: z - .object({ - /** If missing, means sync all streams */ - streams: z.array(zStream).nullish(), - /** Account ids to sync */ - accountIds: z.array(z.string()).nullish(), - /** Date to sync since */ - sinceDate: z.string().nullish() /** ISO8601 */, - }) - .default({}), - // How do we omit destination defs for source only providers and vice versa? - destinationInputEntity: zEntityPayload, -} - -/** - * TODO: Narrow the type of AnyProviderDef to only those whose `sourceState` - * and `destinationInputEntity` match the type needed for venice - */ -export const veniceProviderBase = < - T extends AnyProviderDef, - TSourceMapEntity extends _opt< - // Simpler - | Partial<{ - [k in T['_types']['sourceOutputEntity']['entityName']]: ( - entity: Extract, - settings: T['_types']['resourceSettings'], - ) => EntityPayload | null - }> - // More powerful - | (( - entity: T['_types']['sourceOutputEntity'], - settings: T['_types']['resourceSettings'], - ) => EntityPayload | null) - >, ->( - def: T, - extension: {sourceMapEntity: TSourceMapEntity}, -) => makeSyncProvider({...makeSyncProvider.defaults, def, extension}) - -veniceProviderBase.def = makeSyncProvider.def({ - ...makeSyncProvider.def.defaults, - sourceState: z - .object({ - /** If missing, means sync all streams */ - streams: z.array(zStream).nullish(), - /** Account ids to sync */ - accountIds: z.array(z.string()).nullish(), - /** Date to sync since */ - sinceDate: z.string().nullish() /** ISO8601 */, - }) - .default({}), - // How do we omit destination defs for source only providers and vice versa? - destinationInputEntity: zEntityPayload, -}) - -export type VeniceProvider = ReturnType - -export type VeniceSourceState = - (typeof veniceProviderBase.def)['_types']['sourceState'] - -export function isVeniceProvider( - provider: AnySyncProvider, -): provider is VeniceProvider { - return typeof provider.extension === 'object' && provider.extension - ? 'sourceMapEntity' in provider.extension - : false +export interface VeniceSourceState { + streams?: Array> | null } export function shouldSync( diff --git a/packages/engine-backend/context.ts b/packages/engine-backend/context.ts index f5ecbb80..6f6a197d 100644 --- a/packages/engine-backend/context.ts +++ b/packages/engine-backend/context.ts @@ -1,7 +1,7 @@ import {TRPCError} from '@trpc/server' import type { - AnySyncProvider, + AnyIntegrationImpl, EndUserId, Link, LinkFactory, @@ -26,7 +26,7 @@ export interface RouterContext { as(role: R, data: Omit, 'role'>): Helpers // Non-viewer dependent - providerMap: Record + providerMap: Record jwt: JWTClient /** * Base url of the engine-backend router when deployed, e.g. `localhost:3000/api/usevenice` @@ -42,7 +42,7 @@ export interface RouterContext { } export interface ContextFactoryOptions< - TProviders extends readonly AnySyncProvider[], + TProviders extends readonly AnyIntegrationImpl[], TLinks extends Record, > extends Pick { providers: TProviders @@ -62,7 +62,7 @@ export interface ContextFactoryOptions< } export function getContextFactory< - TProviders extends readonly AnySyncProvider[], + TProviders extends readonly AnyIntegrationImpl[], TLinks extends Record, >(config: ContextFactoryOptions) { const { diff --git a/packages/engine-backend/contextHelpers.ts b/packages/engine-backend/contextHelpers.ts index aac63606..c057c23c 100644 --- a/packages/engine-backend/contextHelpers.ts +++ b/packages/engine-backend/contextHelpers.ts @@ -3,7 +3,7 @@ import {TRPCError} from '@trpc/server' import type { AnyEntityPayload, - AnySyncProvider, + AnyIntegrationImpl, Destination, Id, IDS, @@ -27,7 +27,7 @@ export function getContextHelpers({ getLinksForPipeline, }: { metaService: MetaService - providerMap: Record + providerMap: Record // TODO: Fix any type getLinksForPipeline?: (pipeline: any) => Link[] }) { diff --git a/packages/engine-backend/types.ts b/packages/engine-backend/types.ts index 1f075cd2..9653849d 100644 --- a/packages/engine-backend/types.ts +++ b/packages/engine-backend/types.ts @@ -1,4 +1,4 @@ -import type {AnySyncProvider, Id, LinkFactory} from '@usevenice/cdk-core' +import type {AnyIntegrationImpl, Id, LinkFactory} from '@usevenice/cdk-core' import {z} from '@usevenice/util' // MARK: - Input types @@ -7,17 +7,18 @@ type _inferInput = T extends z.ZodTypeAny ? z.input : never // Would be nice to improve the typing of this... Make stuff non-optional /** https://www.typescriptlang.org/docs/handbook/2/conditional-types.html#distributive-conditional-types */ -export type IntegrationInput = - T extends AnySyncProvider - ? { - id: Id['int'] - config?: Partial<_inferInput> - } - : never +export type IntegrationInput< + T extends AnyIntegrationImpl = AnyIntegrationImpl, +> = T extends AnyIntegrationImpl + ? { + id: Id['int'] + config?: Partial<_inferInput> + } + : never // Is there a way to infer this? Or would that be too much? -export type ResourceInput = - T extends AnySyncProvider +export type ResourceInput = + T extends AnyIntegrationImpl ? { id: Id['reso'] integrationId?: Id['int'] @@ -27,17 +28,17 @@ export type ResourceInput = : never export interface PipelineInput< - PSrc extends AnySyncProvider = AnySyncProvider, - PDest extends AnySyncProvider = AnySyncProvider, + PSrc extends AnyIntegrationImpl = AnyIntegrationImpl, + PDest extends AnyIntegrationImpl = AnyIntegrationImpl, TLinks extends Record = {}, > { id: Id['pipe'] - source?: PSrc extends AnySyncProvider ? ResourceInput : never - sourceState?: PSrc extends AnySyncProvider + source?: PSrc extends AnyIntegrationImpl ? ResourceInput : never + sourceState?: PSrc extends AnyIntegrationImpl ? Partial<_inferInput> : never - destination?: PDest extends AnySyncProvider ? ResourceInput : never - destinationState?: PDest extends AnySyncProvider + destination?: PDest extends AnyIntegrationImpl ? ResourceInput : never + destinationState?: PDest extends AnyIntegrationImpl ? Partial<_inferInput> : never /** Used to initialize links */ @@ -52,7 +53,6 @@ export interface PipelineInput< watch?: boolean } - export const zSyncOptions = z.object({ /** Only sync resource metadata and skip pipelines */ metaOnly: z.boolean().nullish(), diff --git a/packages/engine-frontend/VeniceConnect.tsx b/packages/engine-frontend/VeniceConnect.tsx index 8f4d16bf..a80e9ebb 100644 --- a/packages/engine-frontend/VeniceConnect.tsx +++ b/packages/engine-frontend/VeniceConnect.tsx @@ -5,7 +5,6 @@ import {Link2, Loader2, RefreshCw, Trash2} from 'lucide-react' import React from 'react' import type { - AnyProviderDef, Id, IntegrationClient, OpenDialogFn, @@ -54,7 +53,7 @@ export interface VeniceConnectProps extends UIPropsNoChildren { onEvent?: (event: {type: ConnectEventType; intId: Id['int']}) => void } -type UseConnectScope = Parameters>[0] +type UseConnectScope = Parameters[0] interface DialogConfig { Component: Parameters[0] options: Parameters[1] @@ -282,7 +281,7 @@ export const ProviderConnectButton = ({ }: UIProps & { integration: {id: Id['int']; provider: ProviderMeta} resource?: Resource - connectFn?: ReturnType> + connectFn?: ReturnType onEvent?: (event: {type: ConnectEventType}) => void }) => ( @@ -318,7 +317,7 @@ export const WithProviderConnect = ({ }: { integration: {id: Id['int']; provider: ProviderMeta} resource?: Resource - connectFn?: ReturnType> + connectFn?: ReturnType onEvent?: (event: {type: ConnectEventType}) => void children: (props: { openConnect: () => void @@ -464,7 +463,7 @@ export function ResourceDropdownMenu( provider: ProviderMeta } resource: Resource - connectFn?: ReturnType> + connectFn?: ReturnType onEvent?: (event: {type: ConnectEventType}) => void }, ) { From aed0dbe21388a48ce8e151c827f20ba9602817ec Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 16 Jun 2023 13:27:12 +0800 Subject: [PATCH 21/80] No longer importing server integrations into client --- apps/app-config/_generateDocs.bin.ts | 4 ++-- apps/app-config/_generateIntegrationLists.ts | 6 +++++- apps/app-config/integrations/integrations.merged.ts | 10 ---------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/apps/app-config/_generateDocs.bin.ts b/apps/app-config/_generateDocs.bin.ts index 7e6d210e..52dbeb80 100644 --- a/apps/app-config/_generateDocs.bin.ts +++ b/apps/app-config/_generateDocs.bin.ts @@ -7,7 +7,7 @@ import {buildUrl, R} from '@usevenice/util' import {env, envConfig} from './env' import {parseIntConfigsFromRawEnv} from './integration-envs' -import {mergedIntegrations} from './integrations/integrations.merged' +import {defIntegrations} from './integrations/integrations.def' const envList = R.pipe( {...envConfig.server, ...envConfig.client}, @@ -15,7 +15,7 @@ const envList = R.pipe( R.filter( ([key]) => !key.startsWith('int') || - Object.values(mergedIntegrations).some((p) => + Object.values(defIntegrations).some((p) => key.startsWith(`int_${p.name}`), ), ), diff --git a/apps/app-config/_generateIntegrationLists.ts b/apps/app-config/_generateIntegrationLists.ts index fc3dbfeb..d74eb55d 100644 --- a/apps/app-config/_generateIntegrationLists.ts +++ b/apps/app-config/_generateIntegrationLists.ts @@ -98,7 +98,11 @@ writePretty( `${mergedlist .flatMap((int) => { const validImports = Object.fromEntries( - Object.entries(int.imports).filter(([, v]) => !!v), + Object.entries(int.imports) + .filter(([, v]) => !!v) + // Temp hack because mergedIntegrations are only ever used server side + // This avoids server needing to import client side code unnecessarily + .filter(([k]) => k !== 'client'), ) return [ Object.entries(validImports) diff --git a/apps/app-config/integrations/integrations.merged.ts b/apps/app-config/integrations/integrations.merged.ts index dc094939..1556523d 100644 --- a/apps/app-config/integrations/integrations.merged.ts +++ b/apps/app-config/integrations/integrations.merged.ts @@ -18,17 +18,14 @@ import {default as integrationHeron_server} from '@usevenice/integration-heron/s import {default as integrationLunchmoney_def} from '@usevenice/integration-lunchmoney/def' import {default as integrationLunchmoney_server} from '@usevenice/integration-lunchmoney/server' import {default as integrationMercury_def} from '@usevenice/integration-mercury/def' -import {default as integrationMerge_client} from '@usevenice/integration-merge/client' import {default as integrationMerge_def} from '@usevenice/integration-merge/def' import {default as integrationMerge_server} from '@usevenice/integration-merge/server' import {default as integrationMongodb_def} from '@usevenice/integration-mongodb/def' import {default as integrationMongodb_server} from '@usevenice/integration-mongodb/server' import {default as integrationMoota_def} from '@usevenice/integration-moota/def' import {default as integrationMoota_server} from '@usevenice/integration-moota/server' -import {default as integrationOnebrick_client} from '@usevenice/integration-onebrick/client' import {default as integrationOnebrick_def} from '@usevenice/integration-onebrick/def' import {default as integrationOnebrick_server} from '@usevenice/integration-onebrick/server' -import {default as integrationPlaid_client} from '@usevenice/integration-plaid/client' import {default as integrationPlaid_def} from '@usevenice/integration-plaid/def' import {default as integrationPlaid_server} from '@usevenice/integration-plaid/server' import {default as integrationPostgres_def} from '@usevenice/integration-postgres/def' @@ -45,7 +42,6 @@ import {default as integrationSpreadsheet_def} from '@usevenice/integration-spre import {default as integrationSpreadsheet_server} from '@usevenice/integration-spreadsheet/server' import {default as integrationStripe_def} from '@usevenice/integration-stripe/def' import {default as integrationStripe_server} from '@usevenice/integration-stripe/server' -import {default as integrationTeller_client} from '@usevenice/integration-teller/client' import {default as integrationTeller_def} from '@usevenice/integration-teller/def' import {default as integrationTeller_server} from '@usevenice/integration-teller/server' import {default as integrationToggl_def} from '@usevenice/integration-toggl/def' @@ -55,7 +51,6 @@ import {default as integrationWebhook_def} from '@usevenice/integration-webhook/ import {default as integrationWebhook_server} from '@usevenice/integration-webhook/server' import {default as integrationWise_def} from '@usevenice/integration-wise/def' import {default as integrationWise_server} from '@usevenice/integration-wise/server' -import {default as integrationYodlee_client} from '@usevenice/integration-yodlee/client' import {default as integrationYodlee_def} from '@usevenice/integration-yodlee/def' import {default as integrationYodlee_server} from '@usevenice/integration-yodlee/server' @@ -110,7 +105,6 @@ const integrationMercury = { const integrationMerge = { ...integrationMerge_def, - ...integrationMerge_client, ...integrationMerge_server, } @@ -126,13 +120,11 @@ const integrationMoota = { const integrationOnebrick = { ...integrationOnebrick_def, - ...integrationOnebrick_client, ...integrationOnebrick_server, } const integrationPlaid = { ...integrationPlaid_def, - ...integrationPlaid_client, ...integrationPlaid_server, } @@ -173,7 +165,6 @@ const integrationStripe = { const integrationTeller = { ...integrationTeller_def, - ...integrationTeller_client, ...integrationTeller_server, } @@ -198,7 +189,6 @@ const integrationWise = { const integrationYodlee = { ...integrationYodlee_def, - ...integrationYodlee_client, ...integrationYodlee_server, } From b8bf51c55dd5eb6f9d713cbb4de511eb7efcf664 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 16 Jun 2023 13:39:16 +0800 Subject: [PATCH 22/80] Fix migration cli --- apps/cli/pgMigrator-cli.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/cli/pgMigrator-cli.ts b/apps/cli/pgMigrator-cli.ts index 8811fc98..93028524 100644 --- a/apps/cli/pgMigrator-cli.ts +++ b/apps/cli/pgMigrator-cli.ts @@ -9,4 +9,5 @@ import {z} from '@usevenice/util' void makePostgresClient({ databaseUrl: z.string().parse(process.env['POSTGRES_OR_WEBHOOK_URL']), migrationsPath: path.join(__dirname, '../web/migrations'), + migrationTableName: '_migrations', }).runMigratorCli() From 2d43ed5cccb8b7eb5c80f0a28befe5aaf5c63485 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 16 Jun 2023 13:42:59 +0800 Subject: [PATCH 23/80] Fix to use _migrations table --- apps/web/migrations/2023-02-27_revoke_anon.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/migrations/2023-02-27_revoke_anon.sql b/apps/web/migrations/2023-02-27_revoke_anon.sql index 714dc960..eaff364e 100644 --- a/apps/web/migrations/2023-02-27_revoke_anon.sql +++ b/apps/web/migrations/2023-02-27_revoke_anon.sql @@ -9,7 +9,7 @@ BEGIN WHERE rolname = 'anon') THEN RAISE NOTICE 'Role "anon" does not exists. Skipping grant.'; ELSE - REVOKE ALL PRIVILEGES ON public.migrations from anon, authenticated; + REVOKE ALL PRIVILEGES ON public._migrations from anon, authenticated; END IF; END $do$; From c3e22adfe5aedf0d821a14270c73c6313385f61a Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 13 Sep 2023 15:15:22 -0700 Subject: [PATCH 24/80] [WIP] POC getting transactions from copilot via reverse engineer firestore calls --- .gitignore | 2 + .../integration-copilot/CopilotClient.ts | 123 ++++++++++++++++++ integrations/integration-copilot/package.json | 16 +++ pnpm-lock.yaml | 19 +++ 4 files changed, 160 insertions(+) create mode 100644 integrations/integration-copilot/CopilotClient.ts create mode 100644 integrations/integration-copilot/package.json diff --git a/.gitignore b/.gitignore index 5debc00c..7c7a410f 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,5 @@ yarn-error.log* /apps/web/__generated__/tailwind.css *.secret.json + +temp/ diff --git a/integrations/integration-copilot/CopilotClient.ts b/integrations/integration-copilot/CopilotClient.ts new file mode 100644 index 00000000..55f1325b --- /dev/null +++ b/integrations/integration-copilot/CopilotClient.ts @@ -0,0 +1,123 @@ +import {initializeApp} from 'firebase/app' +import {getAuth, signInWithEmailAndPassword} from 'firebase/auth' +import {getFirestore, limit} from 'firebase/firestore' +import { + collection, + collectionGroup, + doc, + getDoc, + getDocs, + query, + setDoc, + where, +} from 'firebase/firestore' + +// TODO: Replace the following with your app's Firebase project configuration + +const app = initializeApp({ + apiKey: 'AIzaSyBi2Ht5k9K94Yi6McMSGyKeOcHC7vEsN_I', + projectId: 'copilot-production-22904', + databaseURL: 'https://copilot-production-22904.firebaseio.com', + storageBucket: 'copilot-production-22904.appspot.com', + messagingSenderId: '445606440735', + // APIKey: AIzaSyBi2Ht5k9K94Yi6McMSGyKeOcHC7vEsN_I + // bundleID: com.copilot.production + // clientID: 445606440735-nmgdjiedjntfcdf7gt3iie43v1r0sp5t.apps.googleusercontent.com + // GCMSenderID: 445606440735 + // projectID: copilot-production-22904 + // databaseURL: https://copilot-production-22904.firebaseio.com + // storageBucket: copilot-production-22904.appspot.com +}) + +const auth = getAuth(app) +const userId = process.env['COPILOT_USER_ID']! + +signInWithEmailAndPassword( + auth, + process.env['COPILOT_EMAIL']!, + process.env['COPILOT_PASSWORD']!, +) + + .then(async (creds) => { + console.log('creds', creds) + const db = getFirestore(app) + // const col = collection(db, 'items') + // getDocs( + // query( + // col, + // where('user_id', '==', userId), + // limit(10), + // ), + // ) + // .then((snapshot) => { + // console.log( + // 'snap', + // snapshot.docs.map((doc) => doc.data()), + // ) + // }) + // .catch((err) => { + // console.error(err) + // }) + + const col = collection( + db, + 'items', + 'MybOEQl9OKbjZPnBkzoh', + 'accounts', + 'bFnoccIZuY11EmmwGtAM', + 'transactions', + ) + getDocs( + query( + col, + where('user_id', '==', userId), + limit(10), + ), + ) + .then((snapshot) => { + console.log( + 'snap', + snapshot.docs.map((doc) => doc.data()), + ) + }) + .catch((err) => { + console.error(err) + }) + + // Does not work... + // const colgrp = collectionGroup(db, 'transactions') + // getDocs( + // query(colgrp, where('user_id', '==', userId)), + // ) + // .then((snapshot) => { + // console.log( + // 'snap', + // snapshot.docs.map((doc) => doc.data()), + // ) + // }) + // .catch((err) => { + // console.error(err) + // }) + + // const docRef = doc(db, 'items', 'aZO65ebnyauromy7Ex4Zirnrq0ELr1FZ1w6BY') + // const docSnap = await getDoc(docRef); + + // if (docSnap.exists()) { + // console.log("Document data:", docSnap.data()); + // } else { + // // docSnap.data() will be undefined in this case + // console.log("No such document!"); + // } + // const docRef = doc(db, 'changes', userId) + // const docSnap = await getDoc(docRef); + + // if (docSnap.exists()) { + // console.log("Document data:", docSnap.data()); + // } else { + // // docSnap.data() will be undefined in this case + // console.log("No such document!"); + // } + }) + .catch((err) => { + console.error(err) + }) diff --git a/integrations/integration-copilot/package.json b/integrations/integration-copilot/package.json new file mode 100644 index 00000000..c154438d --- /dev/null +++ b/integrations/integration-copilot/package.json @@ -0,0 +1,16 @@ +{ + "name": "@usevenice/integration-copilot", + "version": "0.0.0", + "private": true, + "sideEffects": [], + "module": "./index.ts", + "dependencies": { + "@usevenice/cdk-core": "workspace:*", + "@usevenice/integration-firebase": "workspace:*", + "@usevenice/standard": "workspace:*", + "@usevenice/util": "workspace:*" + }, + "devDependencies": { + "@usevenice/cdk-ledger": "workspace:*" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a071e7cb..6088e26b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -702,6 +702,25 @@ importers: specifier: 6.2.1 version: 6.2.1 + integrations/integration-copilot: + dependencies: + '@usevenice/cdk-core': + specifier: workspace:* + version: link:../../packages/cdk-core + '@usevenice/integration-firebase': + specifier: workspace:* + version: link:../integration-firebase + '@usevenice/standard': + specifier: workspace:* + version: link:../../packages/standard + '@usevenice/util': + specifier: workspace:* + version: link:../../packages/util + devDependencies: + '@usevenice/cdk-ledger': + specifier: workspace:* + version: link:../../packages/cdk-ledger + integrations/integration-debug: dependencies: '@usevenice/cdk-core': From 2157fc62571a99be36f2d243eb4bc8f7821a1512 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 13 Sep 2023 15:22:33 -0700 Subject: [PATCH 25/80] Getting categories as well --- .../integration-copilot/CopilotClient.ts | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/integrations/integration-copilot/CopilotClient.ts b/integrations/integration-copilot/CopilotClient.ts index 55f1325b..23b2c260 100644 --- a/integrations/integration-copilot/CopilotClient.ts +++ b/integrations/integration-copilot/CopilotClient.ts @@ -7,6 +7,7 @@ import { doc, getDoc, getDocs, + lis, query, setDoc, where, @@ -30,14 +31,13 @@ const app = initializeApp({ }) const auth = getAuth(app) -const userId = process.env['COPILOT_USER_ID']! +const userId = process.env['COPILOT_USER_ID']! signInWithEmailAndPassword( auth, process.env['COPILOT_EMAIL']!, process.env['COPILOT_PASSWORD']!, ) - .then(async (creds) => { console.log('creds', creds) const db = getFirestore(app) @@ -67,13 +67,24 @@ signInWithEmailAndPassword( 'bFnoccIZuY11EmmwGtAM', 'transactions', ) - getDocs( - query( - col, - where('user_id', '==', userId), - limit(10), - ), + getDocs(query(col, where('user_id', '==', userId), limit(10))) + .then((snapshot) => { + console.log( + 'snap', + snapshot.docs.map((doc) => doc.data()), + ) + }) + .catch((err) => { + console.error(err) + }) + + const col2 = collection( + db, + 'users', + userId, + 'categories', // also 'budgets' ) + getDocs(query(col2, limit(10))) .then((snapshot) => { console.log( 'snap', @@ -108,14 +119,19 @@ signInWithEmailAndPassword( // // docSnap.data() will be undefined in this case // console.log("No such document!"); // } - // const docRef = doc(db, 'changes', userId) - // const docSnap = await getDoc(docRef); + + // const docRef = doc(db, 'users', userId) + // const docSnap = await getDoc(docRef) // if (docSnap.exists()) { - // console.log("Document data:", docSnap.data()); + // console.log('Document data:', docSnap.data()) + // const collections = await docRef.listCollections() + // for (const col of collections) { + // console.log(`Found subcollection with id: ${col.path}`) + // } // } else { // // docSnap.data() will be undefined in this case - // console.log("No such document!"); + // console.log('No such document!') // } }) .catch((err) => { From 2912c598ae403e746596429435f673f6f930c276 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 12:51:38 -0700 Subject: [PATCH 26/80] Support both cmd+k and cmd+p --- packages/ui/command/command-components.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ui/command/command-components.tsx b/packages/ui/command/command-components.tsx index 33264b16..2142f4d5 100644 --- a/packages/ui/command/command-components.tsx +++ b/packages/ui/command/command-components.tsx @@ -192,7 +192,7 @@ export function CommandBar(props: CommandComponentProps) { const [open, setOpen] = React.useState(false) React.useEffect(() => { const down = (e: KeyboardEvent) => { - if (e.key === 'p' && e.metaKey) { + if ((e.key === 'p' || e.key === 'k') && e.metaKey) { setOpen((open) => !open) e.preventDefault() } From 6b67d373f1a8d7b6e66bb14b9b49947e67f7ced9 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 13:26:30 -0700 Subject: [PATCH 27/80] Fix http agent proxy support --- apps/app-config/package.json | 2 +- apps/app-config/register.node.ts | 12 ++++++++++- pnpm-lock.yaml | 34 +++++++++++++++++++++++++------- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/apps/app-config/package.json b/apps/app-config/package.json index 243ed941..442126eb 100644 --- a/apps/app-config/package.json +++ b/apps/app-config/package.json @@ -47,7 +47,7 @@ "@usevenice/integration-yodlee": "workspace:*", "@usevenice/util": "workspace:*", "chokidar": "3.5.3", - "cross-fetch": "3.1.5", + "cross-fetch": "4.0.0", "execa": "6.1.0", "find-config": "1.0.0", "read-file-safe": "1.0.10", diff --git a/apps/app-config/register.node.ts b/apps/app-config/register.node.ts index cffdc57a..6f371f68 100644 --- a/apps/app-config/register.node.ts +++ b/apps/app-config/register.node.ts @@ -49,12 +49,22 @@ if (process.env['SILENT']) { console.log('[Dep] app-config/register.node') -implementProxyFn($getFetchFn, () => globalThis.fetch ?? crossFetch, { +// Prefer crossfetch for agent aka tunneling support +// TODO: Fix me by switchig to the undici ProxyAgent. +// @see https://undici.nodejs.org/#/docs/api/ProxyAgent +// And https://github.com/nodejs/undici/issues/1489 +implementProxyFn($getFetchFn, () => crossFetch ?? globalThis.fetch, { replaceExisting: true, }) + implementProxyFn( $makeProxyAgent, (input) => { + if (globalThis.fetch !== crossFetch) { + console.warn( + '[proxy] Using proxy agent with non-polyfilled fetch may not work', + ) + } // Seems that the default value get overwritten by explicit undefined // value from envkey. Here we try to account for that // Would be nice if such hack is not required. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6088e26b..ba3f90d4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -264,8 +264,8 @@ importers: specifier: 3.5.3 version: 3.5.3 cross-fetch: - specifier: 3.1.5 - version: 3.1.5 + specifier: 4.0.0 + version: 4.0.0 execa: specifier: 6.1.0 version: 6.1.0 @@ -5344,7 +5344,7 @@ packages: dependencies: https-proxy-agent: 5.0.1 mkdirp: 0.5.6 - node-fetch: 2.6.7 + node-fetch: 2.7.0 npmlog: 4.1.2 progress: 2.0.3 proxy-from-env: 1.1.0 @@ -8354,6 +8354,14 @@ packages: - encoding dev: false + /cross-fetch@4.0.0: + resolution: {integrity: sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==} + dependencies: + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + dev: false + /cross-spawn@6.0.5: resolution: {integrity: sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==} engines: {node: '>=4.8'} @@ -10012,7 +10020,7 @@ packages: extend: 3.0.2 https-proxy-agent: 5.0.1 is-stream: 2.0.1 - node-fetch: 2.6.7 + node-fetch: 2.7.0 transitivePeerDependencies: - encoding - supports-color @@ -10278,7 +10286,7 @@ packages: fast-text-encoding: 1.0.6 google-auth-library: 7.14.1 is-stream-ended: 0.1.4 - node-fetch: 2.6.7 + node-fetch: 2.7.0 object-hash: 3.0.0 proto3-json-serializer: 0.1.9 protobufjs: 6.11.3 @@ -11275,7 +11283,7 @@ packages: /isomorphic-fetch@3.0.0: resolution: {integrity: sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==} dependencies: - node-fetch: 2.6.7 + node-fetch: 2.7.0 whatwg-fetch: 3.6.2 transitivePeerDependencies: - encoding @@ -13355,6 +13363,18 @@ packages: dependencies: whatwg-url: 5.0.0 + /node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + dependencies: + whatwg-url: 5.0.0 + dev: false + /node-forge@1.3.1: resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} engines: {node: '>= 6.13.0'} @@ -16730,7 +16750,7 @@ packages: dependencies: http-proxy-agent: 5.0.0 https-proxy-agent: 5.0.1 - node-fetch: 2.6.7 + node-fetch: 2.7.0 stream-events: 1.0.5 uuid: 8.3.2 transitivePeerDependencies: From 67fcc77380273490525af0c54e7dc34c1442733e Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 14:06:38 -0700 Subject: [PATCH 28/80] Sync merge transactions --- apps/app-config/register.node.ts | 2 +- integrations/integration-merge/def.ts | 15 ++++++++----- integrations/integration-merge/server.ts | 28 ++++++++++++++++-------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/apps/app-config/register.node.ts b/apps/app-config/register.node.ts index 6f371f68..efe0cf22 100644 --- a/apps/app-config/register.node.ts +++ b/apps/app-config/register.node.ts @@ -60,7 +60,7 @@ implementProxyFn($getFetchFn, () => crossFetch ?? globalThis.fetch, { implementProxyFn( $makeProxyAgent, (input) => { - if (globalThis.fetch !== crossFetch) { + if ($getFetchFn() !== crossFetch) { console.warn( '[proxy] Using proxy agent with non-polyfilled fetch may not work', ) diff --git a/integrations/integration-merge/def.ts b/integrations/integration-merge/def.ts index 44dffe68..0a72ffc9 100644 --- a/integrations/integration-merge/def.ts +++ b/integrations/integration-merge/def.ts @@ -2,6 +2,7 @@ import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' import {intHelpers} from '@usevenice/cdk-core' +import type {Standard} from '@usevenice/standard' import {z, zCast} from '@usevenice/util' import {mergeLogoSvg} from './merge-logo.svg' @@ -78,6 +79,7 @@ export const mergeDef = { } }, }, + // Should be called accounting... extension: { sourceMapEntity: { account: (entity) => ({ @@ -85,11 +87,14 @@ export const mergeDef = { entityName: 'account', entity: {name: entity.entity.name ?? ''}, }), - // transaction: (entity) => ({ - // id: entity.id, - // entityName: 'transaction', - // entity: {date: entity.entity.transaction_date}, - // }), + transaction: (entity) => ({ + id: entity.id, + entityName: 'transaction', + entity: { + date: entity.entity.transaction_date ?? '', + description: entity.entity.line_items?.[0]?.memo ?? '', + } satisfies Standard.Transaction, + }), }, }, } satisfies IntegrationDef diff --git a/integrations/integration-merge/server.ts b/integrations/integration-merge/server.ts index 5e1fc644..587499bb 100644 --- a/integrations/integration-merge/server.ts +++ b/integrations/integration-merge/server.ts @@ -91,16 +91,26 @@ export const mergeServer = { accountToken: settings.accountToken, }) - return rxjs - .from( - client.accounting - .get('/accounts', {}) - .then((res) => - (res.results ?? [])?.map((acct) => - helpers._opData('account', acct.id ?? '', acct), - ), + async function* iterateEntities() { + yield await client.accounting + .get('/accounts', {}) + .then((res) => + (res.results ?? [])?.map((acct) => + helpers._opData('account', acct.id ?? '', acct), ), - ) + ) + + yield await client.accounting + .get('/transactions', {}) + .then((res) => + (res.results ?? [])?.map((txn) => + helpers._opData('transaction', txn.id ?? '', txn), + ), + ) + } + + return rxjs + .from(iterateEntities()) .pipe(Rx.mergeMap((ops) => rxjs.from(ops))) }, } satisfies IntegrationServer From 8ba216e09afc0a81d21e6349daa71ddd0bede0a4 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 14:18:25 -0700 Subject: [PATCH 29/80] Try to fix pnpm version for vercel build --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8c0b0668..2c3d7b5f 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,7 @@ }, "engines": { "node": "18", - "pnpm": "8" + "pnpm": "8.6" }, "pnpm": { "allowedDeprecatedVersions": { From 4a5bbc95469bef8621942fd9bbe8b8e29989d3bc Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 14:18:58 -0700 Subject: [PATCH 30/80] Fix typing --- integrations/integration-copilot/CopilotClient.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/integrations/integration-copilot/CopilotClient.ts b/integrations/integration-copilot/CopilotClient.ts index 23b2c260..908cf241 100644 --- a/integrations/integration-copilot/CopilotClient.ts +++ b/integrations/integration-copilot/CopilotClient.ts @@ -1,15 +1,11 @@ import {initializeApp} from 'firebase/app' import {getAuth, signInWithEmailAndPassword} from 'firebase/auth' -import {getFirestore, limit} from 'firebase/firestore' import { collection, - collectionGroup, - doc, - getDoc, getDocs, - lis, + getFirestore, + limit, query, - setDoc, where, } from 'firebase/firestore' From 314c89f8da3ab675dd8b8ddbeda939e652825823 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 14:21:09 -0700 Subject: [PATCH 31/80] Upgrade to pnpm 8.9 and run install --- package.json | 2 +- pnpm-lock.yaml | 563 ++++++++++++++++++++++++++----------------------- 2 files changed, 303 insertions(+), 262 deletions(-) diff --git a/package.json b/package.json index 2c3d7b5f..8c0b0668 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,7 @@ }, "engines": { "node": "18", - "pnpm": "8.6" + "pnpm": "8" }, "pnpm": { "allowedDeprecatedVersions": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ba3f90d4..ae454471 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.1' +lockfileVersion: '6.0' settings: autoInstallPeers: true @@ -873,7 +873,7 @@ importers: specifier: 1.1.3 version: 1.1.3 react: - specifier: '*' + specifier: 18.2.0 version: 18.2.0 devDependencies: '@mergeapi/react-merge-link': @@ -925,7 +925,7 @@ importers: specifier: workspace:* version: link:../../packages/util react: - specifier: '*' + specifier: 18.2.0 version: 18.2.0 devDependencies: '@types/react': @@ -953,7 +953,7 @@ importers: specifier: 12.3.0 version: 12.3.0 react: - specifier: '*' + specifier: 18.2.0 version: 18.2.0 react-plaid-link: specifier: 3.3.2 @@ -1132,7 +1132,7 @@ importers: specifier: workspace:* version: link:../../packages/util react: - specifier: '*' + specifier: 18.2.0 version: 18.2.0 devDependencies: '@types/react': @@ -1205,7 +1205,7 @@ importers: specifier: workspace:* version: link:../../packages/util react: - specifier: '*' + specifier: 18.2.0 version: 18.2.0 devDependencies: '@cfworker/json-schema': @@ -1261,7 +1261,7 @@ importers: specifier: 9.0.0 version: 9.0.0 react: - specifier: '*' + specifier: 18.2.0 version: 18.2.0 react-script-hook: specifier: 1.7.2 @@ -1289,7 +1289,7 @@ importers: packages/connect: dependencies: react: - specifier: '*' + specifier: 18.2.0 version: 18.2.0 devDependencies: '@types/react': @@ -1344,7 +1344,7 @@ importers: specifier: '*' version: 0.115.0(react@18.2.0) react: - specifier: '*' + specifier: 18.2.0 version: 18.2.0 devDependencies: '@types/react': @@ -1441,10 +1441,10 @@ importers: specifier: 6.9.9 version: 6.9.9(react-dom@18.2.0)(react@18.2.0) react: - specifier: '*' + specifier: 18.2.0 version: 18.2.0 react-dom: - specifier: '*' + specifier: 18.2.0 version: 18.2.0(react@18.2.0) react-hook-form: specifier: 7.43.1 @@ -2294,7 +2294,7 @@ packages: resolution: {integrity: sha512-6zZFSvriIS0nBEM+GXFwpn1bfV+B7U/NKjVWQO/rzKldAq0Vd1hgz9aFHAamaZNjoegdpZbGyT5gPIWogz2Mqw==} engines: {node: '>=14'} peerDependencies: - react: '>=16' + react: 18.2.0 dependencies: '@clerk/shared': 0.16.1(react@18.2.0) '@clerk/types': 3.38.0 @@ -2323,8 +2323,8 @@ packages: engines: {node: '>=14'} peerDependencies: next: '>=10' - react: ^17.0.2 || ^18.0.0-0 - react-dom: ^17.0.2 || ^18.0.0-0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@clerk/backend': 0.19.1 '@clerk/clerk-react': 4.16.1(react@18.2.0) @@ -2340,7 +2340,7 @@ packages: /@clerk/shared@0.16.1(react@18.2.0): resolution: {integrity: sha512-x8DtpWb+GQkuByTMLzoWTc3gpz1+NUNCu7TDnWeFt+0PuZG5XaOb5pkC3wOfrxiMefUYr1Vi7lEYFxVmTdNMeg==} peerDependencies: - react: '>=16' + react: 18.2.0 dependencies: glob-to-regexp: 0.4.1 react: 18.2.0 @@ -3175,8 +3175,8 @@ packages: /@floating-ui/react-dom@0.7.2(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-1T0sJcpHgX/u4I1OzIEhlcrvkUN8ln39nz7fMoE/2HDHrPiMFoOGR7++GYyfUmIQHkkrTinaeQsO3XWubjSvGg==} peerDependencies: - react: '>=16.8.0' - react-dom: '>=16.8.0' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@floating-ui/dom': 0.5.4 react: 18.2.0 @@ -3189,8 +3189,8 @@ packages: /@floating-ui/react-dom@1.3.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-htwHm67Ji5E/pROEAr7f8IKFShuiCKHwUC/UY4vC3I5jiSvGFAYnSYiZO5MlGmads+QqvUkR9ANHEguGrDv72g==} peerDependencies: - react: '>=16.8.0' - react-dom: '>=16.8.0' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@floating-ui/dom': 1.2.7 react: 18.2.0 @@ -3200,8 +3200,8 @@ packages: /@floating-ui/react@0.19.2(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-JyNk4A0Ezirq8FlXECvRtQOX/iBe5Ize0W/pLkrZjfHW9GUV7Xnq6zm6fyZuQzaHHqEnVizmvlA96e1/CkZv+w==} peerDependencies: - react: '>=16.8.0' - react-dom: '>=16.8.0' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@floating-ui/react-dom': 1.3.0(react-dom@18.2.0)(react@18.2.0) aria-hidden: 1.2.1(@types/react@18.0.27)(react@18.2.0) @@ -3230,7 +3230,7 @@ packages: resolution: {integrity: sha512-uHg75Rb/XORTtVt7OS9WoK8uM276Ufi7gCzshVWkUJbHhh3svsUUeqXerrM96Wm7fRiDzfKRwSoahhMIkGAYHw==} peerDependencies: '@fortawesome/fontawesome-svg-core': ~1 || ~6 - react: '>=16.3' + react: 18.2.0 dependencies: '@fortawesome/fontawesome-svg-core': 6.3.0 prop-types: 15.8.1 @@ -3242,8 +3242,8 @@ packages: peerDependencies: lodash: ^4.17.19 marked: ^4.0.10 - react: ^16.12.0 || 17.x || 18.x - react-dom: ^16.12.0 || 17.x || 18.x + react: 18.2.0 + react-dom: 18.2.0 react-responsive-carousel: ^3.2.7 dependencies: canvas-hypertxt: 0.0.7 @@ -3273,6 +3273,7 @@ packages: /@google-cloud/paginator@3.0.7: resolution: {integrity: sha512-jJNutk0arIQhmpUUQJPJErsojqo834KcyB6X7a1mxuic8i1tKXxde8E69IZxNZawRIlZdIK2QY4WALvlK5MzYQ==} engines: {node: '>=10'} + requiresBuild: true dependencies: arrify: 2.0.1 extend: 3.0.2 @@ -3282,12 +3283,14 @@ packages: /@google-cloud/projectify@2.1.1: resolution: {integrity: sha512-+rssMZHnlh0twl122gXY4/aCrk0G1acBqkHFfYddtsqpYXGxA29nj9V5V9SfC+GyOG00l650f6lG9KL+EpFEWQ==} engines: {node: '>=10'} + requiresBuild: true dev: false optional: true /@google-cloud/promisify@2.0.4: resolution: {integrity: sha512-j8yRSSqswWi1QqUGKVEKOG03Q7qOoZP6/h2zN2YO+F5h2+DHU0bSrHCK9Y7lo2DI9fBd8qGAw795sf+3Jva4yA==} engines: {node: '>=10'} + requiresBuild: true dev: false optional: true @@ -3329,8 +3332,8 @@ packages: resolution: {integrity: sha512-kJqkdf6d4Cck05Wt5yCDZXWfs7HZgcpuoWq/v8nOa698qVaNMM3qdG4CpRsZEexku0DSSJzWWuanxd5x+sRcFg==} peerDependencies: graphql: ^15.5.0 || ^16.0.0 - react: ^16.8.0 || ^17.0.0 || ^18.0.0 - react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@graphiql/toolkit': 0.8.0(@types/node@18.11.18)(graphql-ws@5.11.3)(graphql@16.6.0) '@reach/combobox': 0.17.0(react-dom@18.2.0)(react@18.2.0) @@ -3376,6 +3379,7 @@ packages: /@grpc/grpc-js@1.6.12: resolution: {integrity: sha512-JmvQ03OTSpVd9JTlj/K3IWHSz4Gk/JMLUTtW7Zb0KvO1LcOYGATh5cNuRYzCAeDR3O8wq+q8FZe97eO9MBrkUw==} engines: {node: ^8.13.0 || >=10.10.0} + requiresBuild: true dependencies: '@grpc/proto-loader': 0.7.3 '@types/node': 18.11.18 @@ -3861,8 +3865,8 @@ packages: resolution: {integrity: sha512-Gr3uz3LYf33wlFE3eRnta4RxP5FSNxiIV9ENn2D2/rN8KgGAD8ecvcITRtsbbyuOuNkwbuHYxfeaz2Vr+CtyFA==} peerDependencies: monaco-editor: '>= 0.25.0 < 1' - react: ^16.8.0 || ^17.0.0 || ^18.0.0 - react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@monaco-editor/loader': 1.3.2(monaco-editor@0.35.0) monaco-editor: 0.35.0 @@ -4086,8 +4090,8 @@ packages: /@radix-ui/react-alert-dialog@1.0.3(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-QXFy7+bhGi0u+paF2QbJeSCHZs4gLMJIPm6sajUamyW0fro6g1CaSGc5zmc4QmK2NlSGUrq8m+UsUqJYtzvXow==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4105,8 +4109,8 @@ packages: /@radix-ui/react-arrow@1.0.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-fqYwhhI9IarZ0ll2cUSfKuXHlJK0qE4AfnRrPBbRwEH/4mGQn04/QFGomLi8TXWIdv9WJk//KgGm+aDxVIr1wA==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-primitive': 1.0.2(react-dom@18.2.0)(react@18.2.0) @@ -4117,8 +4121,8 @@ packages: /@radix-ui/react-checkbox@1.0.1(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-TisH0B8hWmYP3ONRduYCyN04rR9yLPIw/Rwyn1RoC1suSoGCa8Wn+YPdSSSarSszeIbcg3p2lBkDp2XXit4sZw==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4136,8 +4140,8 @@ packages: /@radix-ui/react-collection@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-8i1pf5dKjnq90Z8udnnXKzdCEV3/FYrfw0n/b6NvB6piXEn3fO1bOh7HBcpG8XrnIXzxlYu2oCcR38QpyLS/mg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-compose-refs': 1.0.0(react@18.2.0) @@ -4151,8 +4155,8 @@ packages: /@radix-ui/react-collection@1.0.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-s8WdQQ6wNXpaxdZ308KSr8fEWGrg4un8i4r/w7fhiS4ElRNjk5rRcl0/C6TANG2LvLOGIxtzo/jAg6Qf73TEBw==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-compose-refs': 1.0.0(react@18.2.0) @@ -4166,7 +4170,7 @@ packages: /@radix-ui/react-compose-refs@1.0.0(react@18.2.0): resolution: {integrity: sha512-0KaSv6sx787/hK3eF53iOkiSLwAGlFMx5lotrqD2pTjB18KbybKoEIgkNZTKC60YECDQTKGTRcDBILwZVqVKvA==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 react: 18.2.0 @@ -4175,7 +4179,7 @@ packages: /@radix-ui/react-context@1.0.0(react@18.2.0): resolution: {integrity: sha512-1pVM9RfOQ+n/N5PJK33kRSKsr1glNxomxONs5c49MliinBY6Yw2Q995qfBUUo0/Mbg05B/sGA0gkgPI7kmSHBg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 react: 18.2.0 @@ -4184,8 +4188,8 @@ packages: /@radix-ui/react-dialog@1.0.0(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-Yn9YU+QlHYLWwV1XfKiqnGVpWYWk6MeBVM6x/bcoyPvxgjQGoeT35482viLPctTMWoMw0PoHgqfSox7Ig+957Q==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4211,8 +4215,8 @@ packages: /@radix-ui/react-dialog@1.0.3(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-owNhq36kNPqC2/a+zJRioPg6HHnTn5B/sh/NjTY8r4W9g1L5VJlrzZIVcBr7R9Mg8iLjVmh6MGgMlfoVf/WO/A==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4238,7 +4242,7 @@ packages: /@radix-ui/react-direction@1.0.0(react@18.2.0): resolution: {integrity: sha512-2HV05lGUgYcA6xgLQ4BKPDmtL+QbIZYH5fCOTAOOcJ5O0QbWS3i9lKaurLzliYUDhORI2Qr3pyjhJh44lKA3rQ==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 react: 18.2.0 @@ -4247,8 +4251,8 @@ packages: /@radix-ui/react-dismissable-layer@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-n7kDRfx+LB1zLueRDvZ1Pd0bxdJWDUZNQ/GWoxDn2prnuJKRdxsjulejX/ePkOsLi2tTm6P24mDqlMSgQpsT6g==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4263,8 +4267,8 @@ packages: /@radix-ui/react-dismissable-layer@1.0.3(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-nXZOvFjOuHS1ovumntGV7NNoLaEp9JEvTht3MBjP44NSW5hUKj/8OnfN3+8WmB+CEhN44XaGhpHoSsUIEl5P7Q==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4279,8 +4283,8 @@ packages: /@radix-ui/react-dropdown-menu@2.0.4(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-y6AT9+MydyXcByivdK1+QpjWoKaC7MLjkS/cH1Q3keEyMvDkiY85m8o2Bi6+Z1PPUlCsMULopxagQOSfN0wahg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4299,7 +4303,7 @@ packages: /@radix-ui/react-focus-guards@1.0.0(react@18.2.0): resolution: {integrity: sha512-UagjDk4ijOAnGu4WMUPj9ahi7/zJJqNZ9ZAiGPp7waUWJO0O1aWXi/udPphI0IUjvrhBsZJGSN66dR2dsueLWQ==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 react: 18.2.0 @@ -4308,8 +4312,8 @@ packages: /@radix-ui/react-focus-scope@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-C4SWtsULLGf/2L4oGeIHlvWQx7Rf+7cX/vKOAD2dXW0A1b5QXwi3wWeaEgW+wn+SEVrraMUk05vLU9fZZz5HbQ==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-compose-refs': 1.0.0(react@18.2.0) @@ -4322,8 +4326,8 @@ packages: /@radix-ui/react-focus-scope@1.0.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-spwXlNTfeIprt+kaEWE/qYuYT3ZAqJiAGjN/JgdvgVDTu8yc+HuX+WOWXrKliKnLnwck0F6JDkqIERncnih+4A==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-compose-refs': 1.0.0(react@18.2.0) @@ -4336,7 +4340,7 @@ packages: /@radix-ui/react-id@1.0.0(react@18.2.0): resolution: {integrity: sha512-Q6iAB/U7Tq3NTolBBQbHTgclPmGWE3OlktGGqrClPozSw4vkQ1DfQAOtzgRPecKsMdJINE05iaoDUG8tRzCBjw==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-use-layout-effect': 1.0.0(react@18.2.0) @@ -4346,8 +4350,8 @@ packages: /@radix-ui/react-label@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-k+EbxeRaVbSJ4oaR9eUYuC0cDIGRB4TAPhilbFCIMpP9pXFNcyQPQUvRaVOQBrviuArYM80xh0BQR/0y3kjUdQ==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-compose-refs': 1.0.0(react@18.2.0) @@ -4361,8 +4365,8 @@ packages: /@radix-ui/react-label@2.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-7qCcZ3j2VQspWjy+gKR4W+V/z0XueQjeiZnlPOtsyiP9HaS8bfSU7ECoI3bvvdYntQj7NElW7OAYsYRW4MQvCg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-primitive': 1.0.1(react-dom@18.2.0)(react@18.2.0) @@ -4373,8 +4377,8 @@ packages: /@radix-ui/react-menu@2.0.4(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-mzKR47tZ1t193trEqlQoJvzY4u9vYfVH16ryBrVrCAGZzkgyWnMQYEZdUkM7y8ak9mrkKtJiqB47TlEnubeOFQ==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4404,8 +4408,8 @@ packages: /@radix-ui/react-popover@1.0.5(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-GRHZ8yD12MrN2NLobHPE8Rb5uHTxd9x372DE9PPNnBjpczAQHcZ5ne0KXG4xpf+RDdXSzdLv9ym6mYJCDTaUZg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4432,8 +4436,8 @@ packages: /@radix-ui/react-popper@1.1.1(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-keYDcdMPNMjSC8zTsZ8wezUMiWM9Yj14wtF3s0PTIs9srnEPC9Kt2Gny1T3T81mmSeyDjZxsD9N5WCwNNb712w==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@floating-ui/react-dom': 0.7.2(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0) @@ -4455,8 +4459,8 @@ packages: /@radix-ui/react-portal@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-a8qyFO/Xb99d8wQdu4o7qnigNjTPG123uADNecz0eX4usnQEj7o+cG4ZX4zkqq98NYekT7UoEQIjxBNWIFuqTA==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-primitive': 1.0.0(react-dom@18.2.0)(react@18.2.0) @@ -4467,8 +4471,8 @@ packages: /@radix-ui/react-portal@1.0.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-swu32idoCW7KA2VEiUZGBSu9nB6qwGdV6k6HYhUoOo3M1FFpD+VgLzUqtt3mwL1ssz7r2x8MggpLSQach2Xy/Q==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-primitive': 1.0.2(react-dom@18.2.0)(react@18.2.0) @@ -4479,8 +4483,8 @@ packages: /@radix-ui/react-presence@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-A+6XEvN01NfVWiKu38ybawfHsBjWum42MRPnEuqPsBZ4eV7e/7K321B5VgYMPv3Xx5An6o1/l9ZuDBgmcmWK3w==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-compose-refs': 1.0.0(react@18.2.0) @@ -4492,8 +4496,8 @@ packages: /@radix-ui/react-primitive@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-EyXe6mnRlHZ8b6f4ilTDrXmkLShICIuOTTj0GX4w1rp+wSxf3+TD05u1UOITC8VsJ2a9nwHvdXtOXEOl0Cw/zQ==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-slot': 1.0.0(react@18.2.0) @@ -4504,8 +4508,8 @@ packages: /@radix-ui/react-primitive@1.0.1(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-fHbmislWVkZaIdeF6GZxF0A/NH/3BjrGIYj+Ae6eTmTCr7EB0RQAAVEiqsXK6p3/JcRqVSBQoceZroj30Jj3XA==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-slot': 1.0.1(react@18.2.0) @@ -4516,8 +4520,8 @@ packages: /@radix-ui/react-primitive@1.0.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-zY6G5Qq4R8diFPNwtyoLRZBxzu1Z+SXMlfYpChN7Dv8gvmx9X3qhDqiLWvKseKVJMuedFeU/Sa0Sy/Ia+t06Dw==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-slot': 1.0.1(react@18.2.0) @@ -4528,8 +4532,8 @@ packages: /@radix-ui/react-radio-group@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-WekhUjLzH6/ZXQNAum2NNww+0zvOIfFu96+AZ1XXxWolq82vNAgqMbPrxyqij2H/sa33wbjqspErybsTLtVTDA==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4550,8 +4554,8 @@ packages: /@radix-ui/react-roving-focus@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-lHvO4MhvoWpeNbiJAoyDsEtbKqP2jkkdwsMVJ3kfqbkC71J/aXE6Th6gkZA1xHEqSku+t+UgoDjvE7Z3gsBpcg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4570,8 +4574,8 @@ packages: /@radix-ui/react-roving-focus@1.0.3(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-stjCkIoMe6h+1fWtXlA6cRfikdBzCLp3SnVk7c48cv/uy3DTGoXhN76YaOYUJuy3aEDvDIKwKR5KSmvrtPvQPQ==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4590,8 +4594,8 @@ packages: /@radix-ui/react-scroll-area@1.0.3(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-sBX9j8Q+0/jReNObEAveKIGXJtk3xUoSIx4cMKygGtO128QJyVDn01XNOFsyvihKDCTcu7SINzQ2jPAZEhIQtw==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/number': 1.0.0 @@ -4610,8 +4614,8 @@ packages: /@radix-ui/react-select@1.2.1(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-GULRMITaOHNj79BZvQs3iZO0+f2IgI8g5HDhMi7Bnc13t7IlG86NFtOCfTLme4PNZdEtU+no+oGgcl6IFiphpQ==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/number': 1.0.0 @@ -4644,8 +4648,8 @@ packages: /@radix-ui/react-separator@1.0.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-lZoAG/rS2jzb/OSvyBrpN3dmikw20ewmWx1GkM1VldbDyD0DACCbH9LIXSrqyS/2mE1VYKOHmyq5W90Dx4ryqA==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-primitive': 1.0.2(react-dom@18.2.0)(react@18.2.0) @@ -4656,7 +4660,7 @@ packages: /@radix-ui/react-slot@1.0.0(react@18.2.0): resolution: {integrity: sha512-3mrKauI/tWXo1Ll+gN5dHcxDPdm/Df1ufcDLCecn+pnCIVcdWE7CujXo8QaXOWRJyZyQWWbpB8eFwHzWXlv5mQ==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-compose-refs': 1.0.0(react@18.2.0) @@ -4666,7 +4670,7 @@ packages: /@radix-ui/react-slot@1.0.1(react@18.2.0): resolution: {integrity: sha512-avutXAFL1ehGvAXtPquu0YK5oz6ctS474iM3vNGQIkswrVhdrS52e3uoMQBzZhNRAIE0jBnUyXWNmSjGHhCFcw==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-compose-refs': 1.0.0(react@18.2.0) @@ -4676,8 +4680,8 @@ packages: /@radix-ui/react-tabs@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-oKUwEDsySVC0uuSEH7SHCVt1+ijmiDFAI9p+fHCtuZdqrRDKIFs09zp5nrmu4ggP6xqSx9lj1VSblnDH+n3IBA==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4695,8 +4699,8 @@ packages: /@radix-ui/react-toast@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-mdoF6rahgushdev0OX+9a7JKoH0xZAZBo2Ktf/s779S7EnkZeL3/MFiRIV5LpRP5CtASmfdSD3FLnEvG1RHRtQ==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/primitive': 1.0.0 @@ -4718,7 +4722,7 @@ packages: /@radix-ui/react-use-callback-ref@1.0.0(react@18.2.0): resolution: {integrity: sha512-GZtyzoHz95Rhs6S63D2t/eqvdFCm7I+yHMLVQheKM7nBD8mbZIt+ct1jz4536MDnaOGKIxynJ8eHTkVGVVkoTg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 react: 18.2.0 @@ -4727,7 +4731,7 @@ packages: /@radix-ui/react-use-controllable-state@1.0.0(react@18.2.0): resolution: {integrity: sha512-FohDoZvk3mEXh9AWAVyRTYR4Sq7/gavuofglmiXB2g1aKyboUD4YtgWxKj8O5n+Uak52gXQ4wKz5IFST4vtJHg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-use-callback-ref': 1.0.0(react@18.2.0) @@ -4737,7 +4741,7 @@ packages: /@radix-ui/react-use-escape-keydown@1.0.0(react@18.2.0): resolution: {integrity: sha512-JwfBCUIfhXRxKExgIqGa4CQsiMemo1Xt0W/B4ei3fpzpvPENKpMKQ8mZSB6Acj3ebrAEgi2xiQvcI1PAAodvyg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-use-callback-ref': 1.0.0(react@18.2.0) @@ -4747,7 +4751,7 @@ packages: /@radix-ui/react-use-escape-keydown@1.0.2(react@18.2.0): resolution: {integrity: sha512-DXGim3x74WgUv+iMNCF+cAo8xUHHeqvjx8zs7trKf+FkQKPQXLk2sX7Gx1ysH7Q76xCpZuxIJE7HLPxRE+Q+GA==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-use-callback-ref': 1.0.0(react@18.2.0) @@ -4757,7 +4761,7 @@ packages: /@radix-ui/react-use-layout-effect@1.0.0(react@18.2.0): resolution: {integrity: sha512-6Tpkq+R6LOlmQb1R5NNETLG0B4YP0wc+klfXafpUCj6JGyaUc8il7/kUZ7m59rGbXGczE9Bs+iz2qloqsZBduQ==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 react: 18.2.0 @@ -4766,7 +4770,7 @@ packages: /@radix-ui/react-use-previous@1.0.0(react@18.2.0): resolution: {integrity: sha512-RG2K8z/K7InnOKpq6YLDmT49HGjNmrK+fr82UCVKT2sW0GYfVnYp4wZWBooT/EYfQ5faA9uIjvsuMMhH61rheg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 react: 18.2.0 @@ -4775,7 +4779,7 @@ packages: /@radix-ui/react-use-rect@1.0.0(react@18.2.0): resolution: {integrity: sha512-TB7pID8NRMEHxb/qQJpvSt3hQU4sqNPM1VCTjTRjEOa7cEop/QMuq8S6fb/5Tsz64kqSvB9WnwsDHtjnrM9qew==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/rect': 1.0.0 @@ -4785,7 +4789,7 @@ packages: /@radix-ui/react-use-size@1.0.0(react@18.2.0): resolution: {integrity: sha512-imZ3aYcoYCKhhgNpkNDh/aTiU05qw9hX+HHI1QDBTyIlcFjgeFlKKySNGMwTp7nYFLQg/j0VA2FmCY4WPDDHMg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-use-layout-effect': 1.0.0(react@18.2.0) @@ -4795,8 +4799,8 @@ packages: /@radix-ui/react-visually-hidden@1.0.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-MwAhMdX+n6S4InwRKSnpUsp+lLkYG6izQF56ul6guSX2mBBLOMV9Frx7xJlkEe2GjKLzbNuHhaCS6e5gopmZNA==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-primitive': 1.0.0(react-dom@18.2.0)(react@18.2.0) @@ -4807,8 +4811,8 @@ packages: /@radix-ui/react-visually-hidden@1.0.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-qirnJxtYn73HEk1rXL12/mXnu2rwsNHDID10th2JGtdK25T9wX+mxRmGt7iPSahw512GbZOc0syZX1nLQGoEOg==} peerDependencies: - react: ^16.8 || ^17.0 || ^18.0 - react-dom: ^16.8 || ^17.0 || ^18.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 '@radix-ui/react-primitive': 1.0.2(react-dom@18.2.0)(react@18.2.0) @@ -4825,8 +4829,8 @@ packages: /@reach/auto-id@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-ud8iPwF52RVzEmkHq1twuqGuPA+moreumUHdtgvU3sr3/15BNhwp3KyDLrKKSz0LP1r3V4pSdyF9MbYM8BoSjA==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@reach/utils': 0.17.0(react-dom@18.2.0)(react@18.2.0) react: 18.2.0 @@ -4837,8 +4841,8 @@ packages: /@reach/combobox@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-2mYvU5agOBCQBMdlM4cri+P1BbNwp05P1OuDyc33xJSNiBG7BMy4+ZSHJ0X4fyle6rHwSgCAOCLOeWV1XUYjoQ==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@reach/auto-id': 0.17.0(react-dom@18.2.0)(react@18.2.0) '@reach/descendants': 0.17.0(react-dom@18.2.0)(react@18.2.0) @@ -4855,8 +4859,8 @@ packages: /@reach/descendants@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-c7lUaBfjgcmKFZiAWqhG+VnXDMEhPkI4kAav/82XKZD6NVvFjsQOTH+v3tUkskrAPV44Yuch0mFW/u5Ntifr7Q==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@reach/utils': 0.17.0(react-dom@18.2.0)(react@18.2.0) react: 18.2.0 @@ -4867,8 +4871,8 @@ packages: /@reach/dialog@0.17.0(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-AnfKXugqDTGbeG3c8xDcrQDE4h9b/vnc27Sa118oQSquz52fneUeX9MeFb5ZEiBJK8T5NJpv7QUTBIKnFCAH5A==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@reach/portal': 0.17.0(react-dom@18.2.0)(react@18.2.0) '@reach/utils': 0.17.0(react-dom@18.2.0)(react@18.2.0) @@ -4885,8 +4889,8 @@ packages: /@reach/dropdown@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-qBTIGInhxtPHtdj4Pl2XZgZMz3e37liydh0xR3qc48syu7g71sL4nqyKjOzThykyfhA3Pb3/wFgsFJKGTSdaig==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@reach/auto-id': 0.17.0(react-dom@18.2.0)(react@18.2.0) '@reach/descendants': 0.17.0(react-dom@18.2.0)(react@18.2.0) @@ -4900,8 +4904,8 @@ packages: /@reach/listbox@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-AMnH1P6/3VKy2V/nPb4Es441arYR+t4YRdh9jdcFVrCOD6y7CQrlmxsYjeg9Ocdz08XpdoEBHM3PKLJqNAUr7A==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@reach/auto-id': 0.17.0(react-dom@18.2.0)(react@18.2.0) '@reach/descendants': 0.17.0(react-dom@18.2.0)(react@18.2.0) @@ -4916,8 +4920,8 @@ packages: /@reach/machine@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-9EHnuPgXzkbRENvRUzJvVvYt+C2jp7PGN0xon7ffmKoK8rTO6eA/bb7P0xgloyDDQtu88TBUXKzW0uASqhTXGA==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@reach/utils': 0.17.0(react-dom@18.2.0)(react@18.2.0) '@xstate/fsm': 1.4.0 @@ -4929,8 +4933,8 @@ packages: /@reach/menu-button@0.17.0(react-dom@18.2.0)(react-is@17.0.2)(react@18.2.0): resolution: {integrity: sha512-YyuYVyMZKamPtivoEI6D0UEILYH3qZtg4kJzEAuzPmoR/aHN66NZO75Fx0gtjG1S6fZfbiARaCOZJC0VEiDOtQ==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 react-is: ^16.8.0 || 17.x dependencies: '@reach/dropdown': 0.17.0(react-dom@18.2.0)(react@18.2.0) @@ -4951,8 +4955,8 @@ packages: /@reach/popover@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-yYbBF4fMz4Ml4LB3agobZjcZ/oPtPsNv70ZAd7lEC2h7cvhF453pA+zOBGYTPGupKaeBvgAnrMjj7RnxDU5hoQ==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@reach/portal': 0.17.0(react-dom@18.2.0)(react@18.2.0) '@reach/rect': 0.17.0(react-dom@18.2.0)(react@18.2.0) @@ -4966,8 +4970,8 @@ packages: /@reach/portal@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-+IxsgVycOj+WOeNPL2NdgooUdHPSY285wCtj/iWID6akyr4FgGUK7sMhRM9aGFyrGpx2vzr+eggbUmAVZwOz+A==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@reach/utils': 0.17.0(react-dom@18.2.0)(react@18.2.0) react: 18.2.0 @@ -4979,8 +4983,8 @@ packages: /@reach/rect@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-3YB7KA5cLjbLc20bmPkJ06DIfXSK06Cb5BbD2dHgKXjUkT9WjZaLYIbYCO8dVjwcyO3GCNfOmPxy62VsPmZwYA==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@reach/observe-rect': 1.2.0 '@reach/utils': 0.17.0(react-dom@18.2.0)(react@18.2.0) @@ -4994,8 +4998,8 @@ packages: /@reach/tooltip@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-HP8Blordzqb/Cxg+jnhGmWQfKgypamcYLBPlcx6jconyV5iLJ5m93qipr1giK7MqKT2wlsKWy44ZcOrJ+Wrf8w==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@reach/auto-id': 0.17.0(react-dom@18.2.0)(react@18.2.0) '@reach/portal': 0.17.0(react-dom@18.2.0)(react@18.2.0) @@ -5012,8 +5016,8 @@ packages: /@reach/utils@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-M5y8fCBbrWeIsxedgcSw6oDlAMQDkl5uv3VnMVJ7guwpf4E48Xlh1v66z/1BgN/WYe2y8mB/ilFD2nysEfdGeA==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) @@ -5024,8 +5028,8 @@ packages: /@reach/visually-hidden@0.17.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-T6xF3Nv8vVnjVkGU6cm0+kWtvliLqPAo8PcZ+WxkKacZsaHTjaZb4v1PaCcyQHmuTNT/vtTVNOJLG0SjQOIb7g==} peerDependencies: - react: ^16.8.0 || 17.x - react-dom: ^16.8.0 || 17.x + react: 18.2.0 + react-dom: 18.2.0 dependencies: prop-types: 15.8.1 react: 18.2.0 @@ -5036,7 +5040,7 @@ packages: /@react-hook/debounce@3.0.0(react@18.2.0): resolution: {integrity: sha512-ir/kPrSfAzY12Gre0sOHkZ2rkEmM4fS5M5zFxCi4BnCeXh2nvx9Ujd+U4IGpKCuPA+EQD0pg1eK2NGLvfWejag==} peerDependencies: - react: '>=16.8' + react: 18.2.0 dependencies: '@react-hook/latest': 1.0.3(react@18.2.0) react: 18.2.0 @@ -5045,7 +5049,7 @@ packages: /@react-hook/event@1.2.6(react@18.2.0): resolution: {integrity: sha512-JUL5IluaOdn5w5Afpe/puPa1rj8X6udMlQ9dt4hvMuKmTrBS1Ya6sb4sVgvfe2eU4yDuOfAhik8xhbcCekbg9Q==} peerDependencies: - react: '>=16.8' + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -5053,7 +5057,7 @@ packages: /@react-hook/latest@1.0.3(react@18.2.0): resolution: {integrity: sha512-dy6duzl+JnAZcDbNTfmaP3xHiKtbXYOaz3G51MGVljh548Y8MWzTr+PHLOfvpypEVW9zwvl+VyKjbWKEVbV1Rg==} peerDependencies: - react: '>=16.8' + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -5061,7 +5065,7 @@ packages: /@react-hook/passive-layout-effect@1.2.1(react@18.2.0): resolution: {integrity: sha512-IwEphTD75liO8g+6taS+4oqz+nnroocNfWVHWz7j+N+ZO2vYrc6PV1q7GQhuahL0IOR7JccFTsFKQ/mb6iZWAg==} peerDependencies: - react: '>=16.8' + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -5069,7 +5073,7 @@ packages: /@react-hook/resize-observer@1.2.6(react@18.2.0): resolution: {integrity: sha512-DlBXtLSW0DqYYTW3Ft1/GQFZlTdKY5VAFIC4+km6IK5NiPPDFchGbEJm1j6pSgMqPRHbUQgHJX7RaR76ic1LWA==} peerDependencies: - react: '>=16.8' + react: 18.2.0 dependencies: '@juggle/resize-observer': 3.4.0 '@react-hook/latest': 1.0.3(react@18.2.0) @@ -5080,7 +5084,7 @@ packages: /@react-hook/size@2.1.2(react@18.2.0): resolution: {integrity: sha512-BmE5asyRDxSuQ9p14FUKJ0iBRgV9cROjqNG9jT/EjCM+xHha1HVqbPoT+14FQg1K7xIydabClCibUY4+1tw/iw==} peerDependencies: - react: '>=16.8' + react: 18.2.0 dependencies: '@react-hook/passive-layout-effect': 1.2.1(react@18.2.0) '@react-hook/resize-observer': 1.2.6(react@18.2.0) @@ -5090,7 +5094,7 @@ packages: /@react-hook/throttle@2.2.0(react@18.2.0): resolution: {integrity: sha512-LJ5eg+yMV8lXtqK3lR+OtOZ2WH/EfWvuiEEu0M3bhR7dZRfTyEJKxH1oK9uyBxiXPtWXiQggWbZirMCXam51tg==} peerDependencies: - react: '>=16.8' + react: 18.2.0 dependencies: '@react-hook/latest': 1.0.3(react@18.2.0) react: 18.2.0 @@ -5099,7 +5103,7 @@ packages: /@react-hook/window-size@3.1.1(react@18.2.0): resolution: {integrity: sha512-yWnVS5LKnOUIrEsI44oz3bIIUYqflamPL27n+k/PC//PsX/YeWBky09oPeAoc9As6jSH16Wgo8plI+ECZaHk3g==} peerDependencies: - react: '>=16.8' + react: 18.2.0 dependencies: '@react-hook/debounce': 3.0.0(react@18.2.0) '@react-hook/event': 1.2.6(react@18.2.0) @@ -5115,8 +5119,8 @@ packages: resolution: {integrity: sha512-YZR/Zh5un5c7Nfch2iSRdBn91jTiByyzd2HY+HsFX8UQcN+xurfamKKWg9+/cudY9azcH3Y/f6Hi+Rm0m7LhHg==} peerDependencies: js-cookie: ^3.0.5 - react: ^16.8 || ^17 || ^18 - react-dom: ^16.8 || ^17 || ^18 + react: 18.2.0 + react-dom: 18.2.0 peerDependenciesMeta: js-cookie: optional: true @@ -5129,7 +5133,7 @@ packages: /@react-types/button@3.4.1(react@18.2.0): resolution: {integrity: sha512-B54M84LxdEppwjXNlkBEJyMfe9fd+bvFV7R6+NJvupGrZm/LuFNYjFcHk7yjMKWTdWm6DbpIuQz54n5qTW7Vlg==} peerDependencies: - react: ^16.8.0 || ^17.0.0-rc.1 + react: 18.2.0 dependencies: '@react-types/shared': 3.17.0(react@18.2.0) react: 18.2.0 @@ -5138,7 +5142,7 @@ packages: /@react-types/checkbox@3.4.2(react@18.2.0): resolution: {integrity: sha512-/NWFCEQLvVgo25afPt2jv4syxYvZeY/D/n2Y92IGtoNV4akdz4AuQ65+1X+JOhQc/ZbAblWw5fFWUZoQs3CLZg==} peerDependencies: - react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 + react: 18.2.0 dependencies: '@react-types/shared': 3.17.0(react@18.2.0) react: 18.2.0 @@ -5147,7 +5151,7 @@ packages: /@react-types/radio@3.1.2(react@18.2.0): resolution: {integrity: sha512-vkIic8abrVUyl/YjKU3yTVwn8QgebzuadfV89PsaKc3hdmSiHhDsln5wYsfWOEotqMwPrG1aEv9yRMYO78OQXQ==} peerDependencies: - react: ^16.8.0 || ^17.0.0-rc.1 + react: 18.2.0 dependencies: '@react-types/shared': 3.17.0(react@18.2.0) react: 18.2.0 @@ -5156,7 +5160,7 @@ packages: /@react-types/shared@3.17.0(react@18.2.0): resolution: {integrity: sha512-1SNZ/RhVrrQ1e6yE0bPV7d5Sfp+Uv0dfUEhwF9MAu2v5msu7AMewnsiojKNA0QA6Ing1gpDLjHCxtayQfuxqcg==} peerDependencies: - react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -5164,7 +5168,7 @@ packages: /@react-types/shared@3.9.0(react@18.2.0): resolution: {integrity: sha512-YYksINfR6q92P10AhPEGo47Hd7oz1hrnZ6Vx8Gsrq62IbqDdv1XOTzPBaj17Z1ymNY2pitLUSEXsLmozt4wxxQ==} peerDependencies: - react: ^16.8.0 || ^17.0.0-rc.1 + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -5172,7 +5176,7 @@ packages: /@react-types/switch@3.1.2(react@18.2.0): resolution: {integrity: sha512-EaYWoLvUCpOnt//Ov8VBxOjbs4hBpYE/rBAzzIknXaFvKOu867iZBFL7FJbcemOgC8/dwyaj6GUZ1Gw3Z1g59w==} peerDependencies: - react: ^16.8.0 || ^17.0.0-rc.1 + react: 18.2.0 dependencies: '@react-types/checkbox': 3.4.2(react@18.2.0) '@react-types/shared': 3.17.0(react@18.2.0) @@ -5182,7 +5186,7 @@ packages: /@react-types/textfield@3.3.0(react@18.2.0): resolution: {integrity: sha512-lOf0tx3c3dVaomH/uvKpOKFVTXQ232kLnMhOJTtj97JDX7fTr3SNhDUV0G8Zf4M0vr+l+xkTrJkywYE23rzliw==} peerDependencies: - react: ^16.8.0 || ^17.0.0-rc.1 + react: 18.2.0 dependencies: '@react-types/shared': 3.17.0(react@18.2.0) react: 18.2.0 @@ -5191,7 +5195,7 @@ packages: /@rehooks/component-size@1.0.3(react@18.2.0): resolution: {integrity: sha512-pnYld+8SSF2vXwdLOqBGUyOrv/SjzwLjIUcs/4c1JJgR0q4E9eBtBfuZMD6zUD51fvSehSsbnlQMzotSmPTXPg==} peerDependencies: - react: '>=16.8.0' + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -5201,7 +5205,7 @@ packages: engines: {node: '>=14'} peerDependencies: '@rjsf/utils': 5.6.x - react: ^16.14.0 || >=17 + react: 18.2.0 dependencies: '@rjsf/utils': 5.6.2(react@18.2.0) lodash: 4.17.21 @@ -5216,7 +5220,7 @@ packages: resolution: {integrity: sha512-uditIzGKHRV5vI9HOz4spSoGplm3zrifsInn+izCWETJfLgV5961hUYHaS21vcD+VTp7qUcTZChfmhiRekSrLA==} engines: {node: '>=14'} peerDependencies: - react: ^16.14.0 || >=17 + react: 18.2.0 dependencies: json-schema-merge-allof: 0.8.1 jsonpointer: 5.0.1 @@ -5423,7 +5427,7 @@ packages: engines: {node: '>=8'} peerDependencies: next: ^10.0.8 || ^11.0 || ^12.0 || ^13.0 - react: 16.x || 17.x || 18.x + react: 18.2.0 webpack: '>= 4.0.0' peerDependenciesMeta: webpack: @@ -5469,7 +5473,7 @@ packages: resolution: {integrity: sha512-VzJeBg/v41jfxUYPkH2WYrKjWc4YiMLzDX0f4Zf6WkJ4v3IlDDSkX6DfmWekjTKBho6wiMkSNy2hJ1dHfGZ9jA==} engines: {node: '>=6'} peerDependencies: - react: 15.x || 16.x || 17.x || 18.x + react: 18.2.0 dependencies: '@sentry/browser': 6.19.7 '@sentry/minimal': 6.19.7 @@ -5484,7 +5488,7 @@ packages: resolution: {integrity: sha512-CSafT8yTtvI0j73ppDRvhL/pIl+jaQVFAHmGJXEVfVxTw/2VK1R+8yEhP83x4mRlM34sFZYWQe86OUc8BXaRgw==} engines: {node: '>=8'} peerDependencies: - react: 15.x || 16.x || 17.x || 18.x + react: 18.2.0 dependencies: '@sentry/browser': 7.53.0 '@sentry/types': 7.53.0 @@ -5563,7 +5567,7 @@ packages: /@slonik/migrator@0.11.3(slonik@30.3.1): resolution: {integrity: sha512-lzu1WmoPRWWQPfFmU4alvEuV7ZS50W5QBvsr70+x6+sv5gXXNzzKGwGSfNkyDmlVRhNUqqv2CnGaZaetI5isZg==} peerDependencies: - slonik: 27 - 29 || 30 + slonik: 30.3.1 || 30 dependencies: '@rushstack/ts-command-line': 4.12.4 slonik: 30.3.1(pg-native@3.0.1) @@ -5578,8 +5582,8 @@ packages: resolution: {integrity: sha512-Z/cj6EtYxfT4bDC9DA9WyNUQdhahiPqJcNsWnC/+yPSAqAnqDfzlkVT0iyiRxOfF0OzVfw3rfzadbYlcx/PNAw==} engines: {node: '>=14.13'} peerDependencies: - react: '>=16.8' - react-dom: '>=16.8' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@stoplight/http-spec': 5.5.6 '@stoplight/json': 3.20.2 @@ -5630,8 +5634,8 @@ packages: resolution: {integrity: sha512-smyiiYmV7y5olasCY2JKFLqZJEKE3f6zAgF1nPuo0x5+kDR1laHvdNgMTuqyywXMwx8ezo51QZj9LwBfL9UApw==} engines: {node: '>=14.13'} peerDependencies: - react: '>=16.8' - react-dom: '>=16.8' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@stoplight/elements-core': 7.7.10(@babel/core@7.21.0)(@tanstack/query-core@4.27.0)(mkdirp@0.5.6)(react-dom@18.2.0)(react@18.2.0) '@stoplight/http-spec': 5.5.6 @@ -5740,8 +5744,8 @@ packages: '@stoplight/markdown-viewer': ^5 '@stoplight/mosaic': ^1.32 '@stoplight/mosaic-code-viewer': ^1.32 - react: '>=16.8' - react-dom: '>=16.8' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@stoplight/json': 3.20.2 '@stoplight/json-schema-tree': 2.2.2 @@ -5794,8 +5798,8 @@ packages: peerDependencies: '@stoplight/mosaic': ^1.24.4 '@stoplight/mosaic-code-viewer': ^1.24.4 - react: '>=16.14' - react-dom: '>=16.14' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@rehooks/component-size': 1.0.3(react@18.2.0) '@stoplight/markdown': 3.2.0 @@ -5844,7 +5848,7 @@ packages: /@stoplight/mosaic-code-editor@1.40.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-KU9Qj+JeXA6V4PtqJa8nV6W5scml92Bvbwc+//zd13MR5iUbbEKux9kzwtB1OJxvgSRsDI/5oP5yq1RzPLLQuw==} peerDependencies: - react: '>= 16.14' + react: 18.2.0 dependencies: '@fortawesome/fontawesome-svg-core': 6.3.0 '@fortawesome/react-fontawesome': 0.2.0(@fortawesome/fontawesome-svg-core@6.3.0)(react@18.2.0) @@ -5878,7 +5882,7 @@ packages: /@stoplight/mosaic-code-viewer@1.40.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-beseNsIl3MT868HXgLDB1GNExiOpBwNPJttrOOceR3SXv1Xa/oTEVh2ADhgiW010uWuwYLq9QMllEeXJg/YX/Q==} peerDependencies: - react: '>= 16.14' + react: 18.2.0 dependencies: '@fortawesome/fontawesome-svg-core': 6.3.0 '@fortawesome/react-fontawesome': 0.2.0(@fortawesome/fontawesome-svg-core@6.3.0)(react@18.2.0) @@ -5911,7 +5915,7 @@ packages: /@stoplight/mosaic@1.40.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-hHEb0Cc4TzaPRjt9h5Ll7FuuVedvW6ZPa5HIxIMhPj4jgy5G3AaC1CCXzdRlwicBkUIp6lNkBOpB7cHadpXp1w==} peerDependencies: - react: '>= 16.14' + react: 18.2.0 dependencies: '@fortawesome/fontawesome-svg-core': 6.3.0 '@fortawesome/react-fontawesome': 0.2.0(@fortawesome/fontawesome-svg-core@6.3.0)(react@18.2.0) @@ -5956,8 +5960,8 @@ packages: resolution: {integrity: sha512-r9cyaaH2h0kFe5c0aP+yJuY9CyXgfbBaMO6660M/wRQXqM49K5Ul7kexE4ei2cqYgo+Cd6ALl6RXSZFYwf2kCA==} engines: {node: '>=10'} peerDependencies: - react: '>=16.8' - react-dom: '>=16.8' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@sentry/react': 6.19.7(react@18.2.0) react: 18.2.0 @@ -6064,8 +6068,8 @@ packages: /@tanstack/react-query@4.28.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-8cGBV5300RHlvYdS4ea+G1JcZIt5CIuprXYFnsWggkmGoC0b5JaqG0fIX3qwDL9PTNkKvG76NGThIWbpXivMrQ==} peerDependencies: - react: ^16.8.0 || ^17.0.0 || ^18.0.0 - react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 react-native: '*' peerDependenciesMeta: react-dom: @@ -6082,8 +6086,8 @@ packages: resolution: {integrity: sha512-yHs2m6lk5J5RNcu2dNtsDGux66wNXZjEgzxos6MRCX8gL+nqxeW3ZglqP6eANN0bGElPnjvqiUHGQvdACOr3Cw==} engines: {node: '>=12'} peerDependencies: - react: '>=16' - react-dom: '>=16' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@tanstack/table-core': 8.9.1 react: 18.2.0 @@ -6098,14 +6102,15 @@ packages: /@tootallnate/once@2.0.0: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} engines: {node: '>= 10'} + requiresBuild: true dev: false optional: true /@tremor/react@2.4.0(@types/react@18.0.27)(prop-types@15.8.1)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-UszaUHKdkNDbb9DX8Eqrcy04kDGwRHO3osy5JQChbVNFYcoeBG/1scD4vCPrRak86MdNk2vleA6V48qkJZDzvQ==} peerDependencies: - react: ^18.0.0 - react-dom: '>=16.6.0' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@floating-ui/react': 0.19.2(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0) date-fns: 2.29.3 @@ -6130,11 +6135,11 @@ packages: /@trpc/react-query@10.21.1(@tanstack/react-query@4.28.0)(@trpc/client@10.21.1)(@trpc/server@10.21.1)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-sCLJW9v79sJhVj3L6Ck74mImvKBXrq2KOKtdguxaZZ5rgeqMyQuc8bGHRpf9yfeV08uMNl4Svz1QTee9PjdSqg==} peerDependencies: - '@tanstack/react-query': ^4.18.0 + '@tanstack/react-query': 4.28.0 '@trpc/client': 10.21.1 '@trpc/server': 10.21.1 - react: '>=16.8.0' - react-dom: '>=16.8.0' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@tanstack/react-query': 4.28.0(react-dom@18.2.0)(react@18.2.0) '@trpc/client': 10.21.1(@trpc/server@10.21.1) @@ -6902,7 +6907,7 @@ packages: /@zodios/core@10.9.0(axios@0.21.1)(zod@3.21.4): resolution: {integrity: sha512-vfj1vbVuA+S7sRX7nSzkW4cf35R1AFFHHYfCb1rUdEHWlBrc7YLjbjoQiijHEVloCX97TtuVrzSYCU0yu8bmrQ==} peerDependencies: - axios: ^0.x || ^1.0.0 + axios: 0.21.1 zod: ^3.x dependencies: axios: 0.21.1 @@ -7186,8 +7191,8 @@ packages: resolution: {integrity: sha512-PN344VAf9j1EAi+jyVHOJ8XidQdPVssGco39eNcsGdM4wcsILtxrKLkbuiMfLWYROK1FjRQasMWCBttrhjnr6A==} engines: {node: '>=10'} peerDependencies: - '@types/react': ^16.9.0 || ^17.0.0 || ^18.0.0 - react: ^16.9.0 || ^17.0.0 || ^18.0.0 + '@types/react': 18.0.27 + react: 18.2.0 peerDependenciesMeta: '@types/react': optional: true @@ -7264,6 +7269,7 @@ packages: /arrify@2.0.1: resolution: {integrity: sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==} engines: {node: '>=8'} + requiresBuild: true dev: false optional: true @@ -7308,6 +7314,7 @@ packages: /async-retry@1.3.3: resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + requiresBuild: true dependencies: retry: 0.13.1 dev: false @@ -7574,6 +7581,7 @@ packages: /bignumber.js@9.1.0: resolution: {integrity: sha512-4LwHK4nfDOraBCtst+wOWIHbu1vhvAPJK8g8nROd4iuc3PSEjWif/qwbkh8jwCJz6yDBvtU4KPynETgrfh7y3A==} + requiresBuild: true dev: false optional: true @@ -8063,8 +8071,8 @@ packages: /cmdk@0.2.0(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-JQpKvEOb86SnvMZbYaFKYhvzFntWBeSZdyii0rZPhKJj9uwJBxu4DaVYDrRN7r3mPop56oPhRw+JYWTKs66TYw==} peerDependencies: - react: ^18.0.0 - react-dom: ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@radix-ui/react-dialog': 1.0.0(@types/react@18.0.27)(react-dom@18.2.0)(react@18.2.0) command-score: 0.1.2 @@ -8167,7 +8175,7 @@ packages: /commandbar-launcher@1.1.9(react@18.2.0): resolution: {integrity: sha512-sumGhBp0B7dEL+kCtXgPtAUF12u1znmE0HI3jkcugxFZnznQWdhs8N7snrc1abMsYjtlBjoDH/WuD7uZYXvUdA==} peerDependencies: - react: 16.x + react: 18.2.0 dependencies: react: 18.2.0 react-icons: 3.11.0(react@18.2.0) @@ -8207,6 +8215,7 @@ packages: /compressible@2.0.18: resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} engines: {node: '>= 0.6'} + requiresBuild: true dependencies: mime-db: 1.52.0 dev: false @@ -8255,6 +8264,7 @@ packages: /configstore@5.0.1: resolution: {integrity: sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==} engines: {node: '>=8'} + requiresBuild: true dependencies: dot-prop: 5.3.0 graceful-fs: 4.2.10 @@ -8384,6 +8394,7 @@ packages: /crypto-random-string@2.0.0: resolution: {integrity: sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==} engines: {node: '>=8'} + requiresBuild: true dev: false optional: true @@ -8748,6 +8759,7 @@ packages: /dot-prop@5.3.0: resolution: {integrity: sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==} engines: {node: '>=8'} + requiresBuild: true dependencies: is-obj: 2.0.0 dev: false @@ -8759,6 +8771,7 @@ packages: /duplexify@4.1.2: resolution: {integrity: sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==} + requiresBuild: true dependencies: end-of-stream: 1.4.4 inherits: 2.0.4 @@ -8851,6 +8864,7 @@ packages: /ent@2.2.0: resolution: {integrity: sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==} + requiresBuild: true dev: false optional: true @@ -8957,7 +8971,7 @@ packages: /esbuild-jest@0.5.0(patch_hash=qbdhvjs7sktehqpuch2x3qhd4y)(esbuild@0.17.5): resolution: {integrity: sha512-AMZZCdEpXfNVOIDvURlqYyHwC8qC1/BFjgsrOiSL1eyiIArVtHL8YAC83Shhn16cYYoAWEW17yZn0W/RJKJKHQ==} peerDependencies: - esbuild: '>=0.8.50' + esbuild: 0.17.5 dependencies: '@babel/core': 7.19.3 '@babel/plugin-transform-modules-commonjs': 7.18.6(@babel/core@7.19.3) @@ -9663,6 +9677,7 @@ packages: /fast-text-encoding@1.0.6: resolution: {integrity: sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==} + requiresBuild: true dev: false optional: true @@ -10015,6 +10030,7 @@ packages: /gaxios@4.3.3: resolution: {integrity: sha512-gSaYYIO1Y3wUtdfHmjDUZ8LWaxJQpiavzbF5Kq53akSzvmVg0RfyOcFDbO1KJ/KCGRFz2qG+lS81F0nkr7cRJA==} engines: {node: '>=10'} + requiresBuild: true dependencies: abort-controller: 3.0.0 extend: 3.0.2 @@ -10030,6 +10046,7 @@ packages: /gcp-metadata@4.3.1: resolution: {integrity: sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A==} engines: {node: '>=10'} + requiresBuild: true dependencies: gaxios: 4.3.3 json-bigint: 1.0.0 @@ -10257,6 +10274,7 @@ packages: /google-auth-library@7.14.1: resolution: {integrity: sha512-5Rk7iLNDFhFeBYc3s8l1CqzbEBcdhwR193RlD4vSNFajIcINKI8W8P0JLmBpwymHqqWbX34pJDQu39cSy/6RsA==} engines: {node: '>=10'} + requiresBuild: true dependencies: arrify: 2.0.1 base64-js: 1.5.1 @@ -10277,6 +10295,7 @@ packages: resolution: {integrity: sha512-Jey13YrAN2hfpozHzbtrwEfEHdStJh1GwaQ2+Akh1k0Tv/EuNVSuBtHZoKSBm5wBMvNsxTsEIZ/152NrYyZgxQ==} engines: {node: '>=10'} hasBin: true + requiresBuild: true dependencies: '@grpc/grpc-js': 1.6.12 '@grpc/proto-loader': 0.6.13 @@ -10301,6 +10320,7 @@ packages: resolution: {integrity: sha512-HHuHmkLgwjdmVRngf5+gSmpkyaRI6QmOg77J8tkNBHhNEI62sGHyw4/+UkgyZEI7h84NbWprXDJ+sa3xOYFvTg==} engines: {node: '>=10'} hasBin: true + requiresBuild: true dependencies: node-forge: 1.3.1 dev: false @@ -10339,8 +10359,8 @@ packages: resolution: {integrity: sha512-w1ujpCKMlkwkoUjeg0HpRiBBTm1WHAjHNkFv1TbMu6trjzz63mQ48GLZlmyQY1yhwmc+diCcvmmAt+AyvKLWWA==} peerDependencies: graphql: ^15.5.0 || ^16.0.0 - react: ^16.8.0 || ^17.0.0 || ^18.0.0 - react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@graphiql/react': 0.15.0(@codemirror/language@6.0.0)(@types/node@18.11.18)(@types/react@18.0.27)(graphql-ws@5.11.3)(graphql@16.6.0)(react-dom@18.2.0)(react-is@17.0.2)(react@18.2.0) '@graphiql/toolkit': 0.8.0(@types/node@18.11.18)(graphql-ws@5.11.3)(graphql@16.6.0) @@ -10386,6 +10406,7 @@ packages: /gtoken@5.3.2: resolution: {integrity: sha512-gkvEKREW7dXWF8NV8pVrKfW7WqReAmjjkMBh6lNCCGOM4ucS0r0YyXXl0r/9Yj8wcW/32ISkfc8h5mPTDbtifQ==} engines: {node: '>=10'} + requiresBuild: true dependencies: gaxios: 4.3.3 google-p12-pem: 3.1.4 @@ -10515,6 +10536,7 @@ packages: /hash-stream-validation@0.2.4: resolution: {integrity: sha512-Gjzu0Xn7IagXVkSu9cSFuK1fqzwtLwFhNhVL8IFJijRNMgUttFbBSIAzKuSIrsFMO1+g1RlsoN49zPIbwPDMGQ==} + requiresBuild: true dev: false optional: true @@ -10657,6 +10679,7 @@ packages: /http-proxy-agent@5.0.0: resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==} engines: {node: '>= 6'} + requiresBuild: true dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 @@ -11138,6 +11161,7 @@ packages: /is-obj@2.0.0: resolution: {integrity: sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==} engines: {node: '>=8'} + requiresBuild: true dev: false optional: true @@ -11188,6 +11212,7 @@ packages: /is-stream-ended@0.1.4: resolution: {integrity: sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==} + requiresBuild: true dev: false optional: true @@ -11948,7 +11973,7 @@ packages: '@urql/core': '*' immer: '*' optics-ts: '*' - react: '>=16.8' + react: 18.2.0 react-query: '*' valtio: '*' wonka: '*' @@ -11988,7 +12013,7 @@ packages: '@urql/core': '*' immer: '*' optics-ts: '*' - react: '>=16.8' + react: 18.2.0 valtio: '*' wonka: '*' xstate: '*' @@ -12054,6 +12079,7 @@ packages: /json-bigint@1.0.0: resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + requiresBuild: true dependencies: bignumber.js: 9.1.0 dev: false @@ -12233,6 +12259,7 @@ packages: /jwa@2.0.0: resolution: {integrity: sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==} + requiresBuild: true dependencies: buffer-equal-constant-time: 1.0.1 ecdsa-sig-formatter: 1.0.11 @@ -12263,6 +12290,7 @@ packages: /jws@4.0.0: resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==} + requiresBuild: true dependencies: jwa: 2.0.0 safe-buffer: 5.2.1 @@ -12622,7 +12650,7 @@ packages: /lucide-react@0.115.0(react@18.2.0): resolution: {integrity: sha512-VGL7jBKN6pEXi6peXoyn9t9O7olvinaAonnoe+iFi/F2q7/yQzAFM5KR1OY15u9PlrB3scI9HvcTFoOrIjUaVQ==} peerDependencies: - react: ^16.5.1 || ^17.0.0 || ^18.0.0 + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -12630,7 +12658,7 @@ packages: /lucide-react@0.192.0(react@18.2.0): resolution: {integrity: sha512-ThZ1jGJhUzDP8oQbHtvvG/xh0uodewEt0G6aM/pAtC9RRKPZWNHKmDfoc5sbBC5hU9CHAYknq4/R/JY/QzLOjg==} peerDependencies: - react: ^16.5.1 || ^17.0.0 || ^18.0.0 + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -12710,7 +12738,7 @@ packages: resolution: {integrity: sha512-3l4/Bigjm4bEqjCR6Xr+d4DtM1X6vvtGsMGSjJYyep8RjjIvcWtrXBS8Wbfe1/P+atKNMccpsraESIaWVplzVg==} engines: {node: '>= 10'} peerDependencies: - react: '>= 0.14.0' + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -12860,6 +12888,7 @@ packages: /memory-pager@1.5.0: resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + requiresBuild: true dev: false optional: true @@ -13018,6 +13047,7 @@ packages: resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==} engines: {node: '>=10.0.0'} hasBin: true + requiresBuild: true dev: false optional: true @@ -13145,8 +13175,8 @@ packages: /nano-css@5.3.5(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-vSB9X12bbNu4ALBu7nigJgRViZ6ja3OU7CeuiV1zMIbXOdmkLahgtPmh3GBOlDxbKY0CitqlPdOReGlBLSp+yg==} peerDependencies: - react: '*' - react-dom: '*' + react: 18.2.0 + react-dom: 18.2.0 dependencies: css-tree: 1.1.3 csstype: 3.1.1 @@ -13227,8 +13257,8 @@ packages: resolution: {integrity: sha512-B+AKNfYNIzh0vqQQKqQItTS8evEouKD7H5Hj3kmuPERwddR2TxvDSFZuTj6T7Jfn1oyeUyJMydPl1Bkxkh0W7A==} peerDependencies: next: '*' - react: '*' - react-dom: '*' + react: 18.2.0 + react-dom: 18.2.0 dependencies: next: 13.4.3(@babel/core@7.21.0)(react-dom@18.2.0)(react@18.2.0) react: 18.2.0 @@ -13247,8 +13277,8 @@ packages: '@opentelemetry/api': ^1.1.0 fibers: '>= 3.1.0' node-sass: ^6.0.0 || ^7.0.0 - react: ^18.2.0 - react-dom: ^18.2.0 + react: 18.2.0 + react-dom: 18.2.0 sass: ^1.3.0 peerDependenciesMeta: '@opentelemetry/api': @@ -13288,8 +13318,8 @@ packages: resolution: {integrity: sha512-FS5+zCdNHLq0YA+SijkbDifOMFNayKfxkCLTiM6TIrvSaf6kAzCGlNjwgVpoG3zpampVMQSVf5L/jrWFJgs23g==} peerDependencies: next: '>= 6.0.0' - react: '>= 16.0.0' - react-dom: '>= 16.0.0' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@types/nprogress': 0.2.0 next: 13.4.3(@babel/core@7.21.0)(react-dom@18.2.0)(react@18.2.0) @@ -13932,7 +13962,7 @@ packages: resolution: {integrity: sha512-VW3BlFr4aqazNaHOdf2Yg/GY8JRiMDnGExFCHXN+dm6yz8b5uZk9WuJk3YwhRPyzFjuZPnoxt8aFe5fX4gAStw==} engines: {node: '>=14.x'} peerDependencies: - react: '>=17' + react: 18.2.0 xstate: '>=4.32.1' peerDependenciesMeta: react: @@ -14571,7 +14601,7 @@ packages: /prism-react-renderer@1.3.5(react@18.2.0): resolution: {integrity: sha512-IJ+MSwBWKG+SM3b2SUfdrhC+gu01QkV2KmRQgREThBfSQRoufqRfxfHUxpG1WcaFjP+kojcFyO9Qqtpgt3qLCg==} peerDependencies: - react: '>=0.14.9' + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -14614,6 +14644,7 @@ packages: /proto3-json-serializer@0.1.9: resolution: {integrity: sha512-A60IisqvnuI45qNRygJjrnNjX2TMdQGMY+57tR3nul3ZgO2zXkR9OGR8AXxJhkqx84g0FTnrfi3D5fWMSdANdQ==} + requiresBuild: true dependencies: protobufjs: 6.11.3 dev: false @@ -14677,6 +14708,7 @@ packages: /pumpify@2.0.1: resolution: {integrity: sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==} + requiresBuild: true dependencies: duplexify: 4.1.2 inherits: 2.0.4 @@ -14775,8 +14807,8 @@ packages: /re-resizable@6.9.9(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-l+MBlKZffv/SicxDySKEEh42hR6m5bAHfNu3Tvxks2c4Ah+ldnWjfnVRwxo/nxF27SsUsxDS0raAzFuJNKABXA==} peerDependencies: - react: ^16.13.1 || ^17.0.0 || ^18.0.0 - react-dom: ^16.13.1 || ^17.0.0 || ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) @@ -14785,7 +14817,7 @@ packages: /react-clientside-effect@1.2.6(react@18.2.0): resolution: {integrity: sha512-XGGGRQAKY+q25Lz9a/4EPqom7WRjz3z9R2k4jhVKA/puQFH/5Nt27vFZYql4m4NVNdUvX8PS3O7r/Zzm7cjUlg==} peerDependencies: - react: ^15.3.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 react: 18.2.0 @@ -14794,7 +14826,7 @@ packages: /react-dom@18.2.0(react@18.2.0): resolution: {integrity: sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==} peerDependencies: - react: ^18.2.0 + react: 18.2.0 dependencies: loose-envify: 1.4.0 react: 18.2.0 @@ -14814,8 +14846,8 @@ packages: /react-focus-lock@2.9.4(@types/react@18.0.27)(react@18.2.0): resolution: {integrity: sha512-7pEdXyMseqm3kVjhdVH18sovparAzLg5h6WvIx7/Ck3ekjhrrDMEegHSa3swwC8wgfdd7DIdUVRGeiHT9/7Sgg==} peerDependencies: - '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + '@types/react': 18.0.27 + react: 18.2.0 peerDependenciesMeta: '@types/react': optional: true @@ -14834,7 +14866,7 @@ packages: resolution: {integrity: sha512-+s3+s8LLytRMriwwuSqeLStVjRXFGxgjjx2jED7Z+wz1J/88vpxieRQGvJVvzrzVxshZ0BRuocFERb779m2kNg==} engines: {node: '>=12.22.0'} peerDependencies: - react: ^16.8.0 || ^17 || ^18 + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -14842,7 +14874,7 @@ packages: /react-icons@3.11.0(react@18.2.0): resolution: {integrity: sha512-JRgiI/vdF6uyBgyZhVyYJUZAop95Sy4XDe/jmT3R/bKliFWpO/uZBwvSjWEdxwzec7SYbEPNPck0Kff2tUGM2Q==} peerDependencies: - react: '*' + react: 18.2.0 dependencies: camelcase: 5.3.1 react: 18.2.0 @@ -14864,8 +14896,8 @@ packages: /react-number-format@5.1.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-NXm/MvZVjmPqrFbjAut/prCyBZ+pA+O+C12rDkWZrwS3JXz1J42RC1kpclkdnkx2KDjRCNFLb21FlwGcNBZddw==} peerDependencies: - react: ^0.14 || ^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^18.0.0 - react-dom: ^0.14 || ^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: prop-types: 15.8.1 react: 18.2.0 @@ -14876,7 +14908,7 @@ packages: resolution: {integrity: sha512-+UegukgQ10E4ll3txz4DJyrnCgZ3eDVuv5dvR8ziyG5FfgCDZcUKeKhIgbU90oyqQa21aH4oLOoGKt0TiYJRmg==} engines: {node: '>=10'} peerDependencies: - react: '>=16' + react: 18.2.0 dependencies: react: 18.2.0 react-use: 17.4.0(react-dom@18.2.0)(react@18.2.0) @@ -14887,8 +14919,8 @@ packages: /react-plaid-link@3.3.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-W2L9C4RCE/KFFia2SUFFTi8SYRQ1UMUXh4iLE/oo4EQ8UYAT6msKzWvauj8bJirlrkzeiUtY6wgkmWil/okaNg==} peerDependencies: - react: ^16.8.0 || ^17.0.0 || ^18.0.0 - react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: prop-types: 15.8.1 react: 18.2.0 @@ -14899,7 +14931,7 @@ packages: /react-query@3.39.3(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-nLfLz7GiohKTJDuT4us4X3h/8unOh+00MLb2yJoGTPjxKs2bc1iDhkNx2bd5MKklXnOD3NrVZ+J2UXujA5In4g==} peerDependencies: - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 react-dom: '*' react-native: '*' peerDependenciesMeta: @@ -14919,8 +14951,8 @@ packages: resolution: {integrity: sha512-i9GMNWwpz8XpUpQ6QlevUtFjHGqnPG4Hxs+wlIJntu/xcsZVEpJcIV71K3ZkqNy2q3GfgvkD7y6t/Sv8ofYSbw==} engines: {node: '>=10'} peerDependencies: - '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + '@types/react': 18.0.27 + react: 18.2.0 peerDependenciesMeta: '@types/react': optional: true @@ -14935,8 +14967,8 @@ packages: resolution: {integrity: sha512-xGVKJJr0SJGQVirVFAUZ2k1QLyO6m+2fy0l8Qawbp5Jgrv3DeLalrfMNBFSlmz5kriGGzsVBtGVnf4pTKIhhWA==} engines: {node: '>=10'} peerDependencies: - '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + '@types/react': 18.0.27 + react: 18.2.0 peerDependenciesMeta: '@types/react': optional: true @@ -14954,8 +14986,8 @@ packages: resolution: {integrity: sha512-ImKhrzJJsyXJfBZ4bzu8Bwpka14c/fQt0k+cyFp/PBhTfyDnU5hjOtM4AG/0AMyy8oKzOTR0lDgJIM7pYXI0kw==} engines: {node: '>=10'} peerDependencies: - '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + '@types/react': 18.0.27 + react: 18.2.0 peerDependenciesMeta: '@types/react': optional: true @@ -14972,8 +15004,8 @@ packages: /react-resize-detector@8.1.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-S7szxlaIuiy5UqLhLL1KY3aoyGHbZzsTpYal9eYMwCyKqoqoVLCmIgAgNyIM1FhnP2KyBygASJxdhejrzjMb+w==} peerDependencies: - react: ^16.0.0 || ^17.0.0 || ^18.0.0 - react-dom: ^16.0.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: lodash: 4.17.21 react: 18.2.0 @@ -14991,7 +15023,7 @@ packages: /react-router-dom@5.3.4(react@18.2.0): resolution: {integrity: sha512-m4EqFMHv/Ih4kpcBCONHbkT68KoAeHN4p3lAGoNryfHi0dMy0kCzEZakiKRsvg5wHZ/JLrLW8o8KomWiz/qbYQ==} peerDependencies: - react: '>=15' + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 history: 4.10.1 @@ -15006,7 +15038,7 @@ packages: /react-router-hash-link@2.4.3(react-router-dom@5.3.4)(react@18.2.0): resolution: {integrity: sha512-NU7GWc265m92xh/aYD79Vr1W+zAIXDWp3L2YZOYP4rCqPnJ6LI6vh3+rKgkidtYijozHclaEQTAHaAaMWPVI4A==} peerDependencies: - react: '>=15' + react: 18.2.0 react-router-dom: '>=4' dependencies: prop-types: 15.8.1 @@ -15017,7 +15049,7 @@ packages: /react-router@5.3.4(react@18.2.0): resolution: {integrity: sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA==} peerDependencies: - react: '>=15' + react: 18.2.0 dependencies: '@babel/runtime': 7.19.0 history: 4.10.1 @@ -15034,8 +15066,8 @@ packages: /react-script-hook@1.7.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-fhyCEfXb94fag34UPRF0zry1XGwmVY+79iibWwTqAoOiCzYJQOYTiWJ7CnqglA9tMSV8g45cQpHCMcBwr7dwhA==} peerDependencies: - react: ^16.8.6 || 17 - 18 - react-dom: ^16.8.6 || 17 - 18 + react: 18.2.0 + react-dom: 18.2.0 dependencies: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) @@ -15044,8 +15076,8 @@ packages: resolution: {integrity: sha512-pgqSp1q8rAGtF1bXQE0m3CHGLNfZZh5oA5o1tsPLXRHnKtkujMIJ8Ws5nO1mTySZf1c4vgwlEk+pHi3Ln6eYLw==} peerDependencies: prop-types: ^15.6.0 - react: ^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 - react-dom: ^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: fast-equals: 4.0.3 prop-types: 15.8.1 @@ -15058,8 +15090,8 @@ packages: resolution: {integrity: sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g==} engines: {node: '>=10'} peerDependencies: - '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + '@types/react': 18.0.27 + react: 18.2.0 peerDependenciesMeta: '@types/react': optional: true @@ -15074,8 +15106,8 @@ packages: /react-transition-group@2.9.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-+HzNTCHpeQyl4MJ/bdE0u6XRMe9+XG/+aL4mCxVN4DnPBQ0/5bfHWPDuOZUzYdMj94daZaZdCCc1Dzt9R/xSSg==} peerDependencies: - react: '>=15.0.0' - react-dom: '>=15.0.0' + react: 18.2.0 + react-dom: 18.2.0 dependencies: dom-helpers: 3.4.0 loose-envify: 1.4.0 @@ -15088,8 +15120,8 @@ packages: /react-transition-group@4.4.5(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==} peerDependencies: - react: '>=16.6.0' - react-dom: '>=16.6.0' + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@babel/runtime': 7.19.0 dom-helpers: 5.2.1 @@ -15102,7 +15134,7 @@ packages: /react-universal-interface@0.6.2(react@18.2.0)(tslib@2.4.1): resolution: {integrity: sha512-dg8yXdcQmvgR13RIlZbTRQOoUrDciFVoSBZILwjE2LFISxZZ8loVJKAkuzswl5js8BHda79bIb2b84ehU8IjXw==} peerDependencies: - react: '*' + react: 18.2.0 tslib: '*' dependencies: react: 18.2.0 @@ -15112,8 +15144,8 @@ packages: /react-use@17.4.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-TgbNTCA33Wl7xzIJegn1HndB4qTS9u03QUwyNycUnXaweZkE4Kq2SB+Yoxx8qbshkZGYBDvUXbXWRUmQDcZZ/Q==} peerDependencies: - react: ^16.8.0 || ^17.0.0 || ^18.0.0 - react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@types/js-cookie': 2.2.7 '@xobotyi/scrollbar-width': 1.9.5 @@ -15223,8 +15255,8 @@ packages: engines: {node: '>=12'} peerDependencies: prop-types: ^15.6.0 - react: ^16.0.0 || ^17.0.0 || ^18.0.0 - react-dom: ^16.0.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 + react-dom: 18.2.0 dependencies: classnames: 2.3.2 eventemitter3: 4.0.7 @@ -15476,6 +15508,7 @@ packages: /retry-request@4.2.2: resolution: {integrity: sha512-xA93uxUD/rogV7BV59agW/JHPGXeREMWiZc9jhcwY4YdZ7QOtC7qbomYg0n4wyk2lJhggjvKvhNX8wln/Aldhg==} engines: {node: '>=8.10.0'} + requiresBuild: true dependencies: debug: 4.3.4 extend: 3.0.2 @@ -15487,6 +15520,7 @@ packages: /retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} + requiresBuild: true dev: false optional: true @@ -15872,7 +15906,7 @@ packages: resolution: {integrity: sha512-SF7EenJQDKQBrMnnrvFPhZu2W9d8I9Egajw7hzemklqwboA7yK12jtSqKOHiFPpacH0lVP1xHdUh4aZaZKmbYw==} engines: {node: '>=8.0'} peerDependencies: - slonik: '>=27.0.0 || 30' + slonik: 30.3.1 || 30 dependencies: camelcase: 6.3.0 core-js: 3.25.5 @@ -15910,7 +15944,7 @@ packages: resolution: {integrity: sha512-6DJM3Bts8eaAZWl5jVrtkH00UGKDCKusOYVaUsgK3ghyOYvfY0Q6QtlAELRi127922ZZsmYelWWTyb2xBV22pg==} engines: {node: '>=8.0'} peerDependencies: - slonik: '>=27.0.0 || 30' + slonik: 30.3.1 || 30 dependencies: crack-json: 1.3.0 pretty-ms: 7.0.1 @@ -16130,6 +16164,7 @@ packages: /sparse-bitfield@3.0.3: resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + requiresBuild: true dependencies: memory-pager: 1.5.0 dev: false @@ -16293,6 +16328,7 @@ packages: /stream-events@1.0.5: resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==} + requiresBuild: true dependencies: stubs: 3.0.0 dev: false @@ -16300,6 +16336,7 @@ packages: /stream-shift@1.0.1: resolution: {integrity: sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==} + requiresBuild: true dev: false optional: true @@ -16503,6 +16540,7 @@ packages: /stubs@3.0.0: resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==} + requiresBuild: true dev: false optional: true @@ -16522,7 +16560,7 @@ packages: peerDependencies: '@babel/core': '*' babel-plugin-macros: '*' - react: '>= 16.8.0 || 17.x.x || ^18.0.0-0' + react: 18.2.0 peerDependenciesMeta: '@babel/core': optional: true @@ -16622,7 +16660,7 @@ packages: /swr@1.3.0(react@18.2.0): resolution: {integrity: sha512-dkghQrOl2ORX9HYrMDtPa7LTVHJjCTeZoB1dqTbnnEDlSvN8JEKpYIYurDfvbQFUUS8Cg8PceFVZNkW0KNNYPw==} peerDependencies: - react: ^16.11.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 dependencies: react: 18.2.0 dev: false @@ -16747,6 +16785,7 @@ packages: /teeny-request@7.2.0: resolution: {integrity: sha512-SyY0pek1zWsi0LRVAALem+avzMLc33MKW/JLLakdP4s9+D7+jHcy5x6P+h94g2QNZsAqQNfX5lsbd3WSeJXrrw==} engines: {node: '>=10'} + requiresBuild: true dependencies: http-proxy-agent: 5.0.0 https-proxy-agent: 5.0.1 @@ -16857,7 +16896,7 @@ packages: /tilg@0.1.1(react@18.2.0): resolution: {integrity: sha512-0uHyTAUM0tJL792LeviRPFkJtCbF6Za3/hbbnRmWGUaicOhbJ0IpvBViXiXTF7nk6R0L6vve2XLesQzn5jEVng==} peerDependencies: - react: ^18.0.0 || ^17.0.0 + react: 18.2.0 dependencies: react: 18.2.0 dev: true @@ -17232,6 +17271,7 @@ packages: /unique-string@2.0.0: resolution: {integrity: sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==} engines: {node: '>=8'} + requiresBuild: true dependencies: crypto-random-string: 2.0.0 dev: false @@ -17377,8 +17417,8 @@ packages: resolution: {integrity: sha512-3FT9PRuRdbB9HfXhEq35u4oZkvpJ5kuYbpqhCfmiZyReuRgpnhDlbr2ZEnnuS0RrJAPn6l23xjFg9kpDM+Ms7w==} engines: {node: '>=10'} peerDependencies: - '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + '@types/react': 18.0.27 + react: 18.2.0 peerDependenciesMeta: '@types/react': optional: true @@ -17392,7 +17432,7 @@ packages: resolution: {integrity: sha512-49L8yCO3iGT/ZF9QttjwLF/ZD9Iwto5LnH5LmEdk/6cFmXddqi2ulF0edxTwjj+7mqvpVVGQWvbXZdn32wRSHA==} peerDependencies: '@types/react': '*' - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 peerDependenciesMeta: '@types/react': optional: true @@ -17404,8 +17444,8 @@ packages: /use-resize-observer@9.1.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-R25VqO9Wb3asSD4eqtcxk8sJalvIOYBqS8MNZlpDSQ4l4xMQxC/J7Id9HoTqPq8FwULIn0PVW+OAqF2dyYbjow==} peerDependencies: - react: 16.8.0 - 18 - react-dom: 16.8.0 - 18 + react: 18.2.0 + react-dom: 18.2.0 dependencies: '@juggle/resize-observer': 3.4.0 react: 18.2.0 @@ -17416,8 +17456,8 @@ packages: resolution: {integrity: sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw==} engines: {node: '>=10'} peerDependencies: - '@types/react': ^16.9.0 || ^17.0.0 || ^18.0.0 - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + '@types/react': 18.0.27 + react: 18.2.0 peerDependenciesMeta: '@types/react': optional: true @@ -17431,7 +17471,7 @@ packages: /use-sync-external-store@1.2.0(react@18.2.0): resolution: {integrity: sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==} peerDependencies: - react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react: 18.2.0 dependencies: react: 18.2.0 @@ -17894,6 +17934,7 @@ packages: /xdg-basedir@4.0.0: resolution: {integrity: sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==} engines: {node: '>=8'} + requiresBuild: true dev: false optional: true @@ -18024,7 +18065,7 @@ packages: resolution: {integrity: sha512-PIJDIZKtokhof+9+60cpockVOq05sJzHCriyvaLBmEJixseQ1a5Kdov6fWZfWOu5SK9c+FhH1jU0tntLxRJYMA==} engines: {node: '>=12.7.0'} peerDependencies: - react: '>=16.8' + react: 18.2.0 peerDependenciesMeta: react: optional: true From f1d1a3b5d15901a3be3af12811efd91fec2b38e0 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 14:25:49 -0700 Subject: [PATCH 32/80] Fix migration sql --- apps/web/migrations/2023-01-06_rls.sql | 2 +- apps/web/migrations/2023-04-02_0140_admin_user.sql | 8 ++++---- apps/web/migrations/2023-04-29_1549_multi_tenant.sql | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/web/migrations/2023-01-06_rls.sql b/apps/web/migrations/2023-01-06_rls.sql index 41e91871..970ee68f 100644 --- a/apps/web/migrations/2023-01-06_rls.sql +++ b/apps/web/migrations/2023-01-06_rls.sql @@ -67,7 +67,7 @@ CREATE POLICY "connection_creator_access" ON "public"."pipeline" -- Contains secrets that shouldn't be publicly available ALTER TABLE "public"."integration" ENABLE ROW LEVEL SECURITY; -ALTER TABLE "public"."migrations" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "public"."_migrations" ENABLE ROW LEVEL SECURITY; ALTER TABLE "public"."institution" ENABLE ROW LEVEL SECURITY; -- Should this be allowed? CREATE POLICY "public_readable" ON public.institution FOR SELECT USING (true); diff --git a/apps/web/migrations/2023-04-02_0140_admin_user.sql b/apps/web/migrations/2023-04-02_0140_admin_user.sql index 61f07ad9..e624cc89 100644 --- a/apps/web/migrations/2023-04-02_0140_admin_user.sql +++ b/apps/web/migrations/2023-04-02_0140_admin_user.sql @@ -20,9 +20,9 @@ $$; -- even though empirically it works because this is security we are gonna be more cautious. -- Will see if the supabase team has any ideas... -- @see https://usevenice.slack.com/archives/C04NUANB7FW/p1680462683033239 -CREATE OR REPLACE FUNCTION auth.is_admin() - RETURNS boolean - LANGUAGE sql +CREATE OR REPLACE FUNCTION auth.is_admin() + RETURNS boolean + LANGUAGE sql STABLE AS $function$ select nullif(current_setting('request.jwt.claims', true), '')::jsonb #> '{app_metadata,isAdmin}' = 'true'::jsonb @@ -37,7 +37,7 @@ CREATE POLICY "admin_access" ON "public"."integration" USING (auth.is_admin()); CREATE POLICY "admin_access" ON "public"."resource" USING (auth.is_admin()); CREATE POLICY "admin_access" ON "public"."pipeline" USING (auth.is_admin()); -CREATE POLICY "admin_access" ON "public"."migrations" USING (auth.is_admin()); +CREATE POLICY "admin_access" ON "public"."_migrations" USING (auth.is_admin()); DO $$ BEGIN diff --git a/apps/web/migrations/2023-04-29_1549_multi_tenant.sql b/apps/web/migrations/2023-04-29_1549_multi_tenant.sql index fcbf3650..2f098519 100644 --- a/apps/web/migrations/2023-04-29_1549_multi_tenant.sql +++ b/apps/web/migrations/2023-04-29_1549_multi_tenant.sql @@ -7,7 +7,7 @@ DROP POLICY IF EXISTS admin_access ON institution; DROP POLICY IF EXISTS admin_access ON integration; DROP POLICY IF EXISTS admin_access ON resource; DROP POLICY IF EXISTS admin_access ON pipeline; -DROP POLICY IF EXISTS admin_access ON migrations; +DROP POLICY IF EXISTS admin_access ON _migrations; DROP FUNCTION IF EXISTS auth.is_admin; DROP PROCEDURE IF EXISTS auth.set_user_admin; From 1d3e7edd3c2e9dee67f8739e09f3eb83c586af51 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 14:56:58 -0700 Subject: [PATCH 33/80] Better json schema title --- packages/util/zod-jsonschema-utils.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/util/zod-jsonschema-utils.ts b/packages/util/zod-jsonschema-utils.ts index b72f5975..ac2b7afd 100644 --- a/packages/util/zod-jsonschema-utils.ts +++ b/packages/util/zod-jsonschema-utils.ts @@ -9,15 +9,17 @@ import {jsonSchemaWalkNodes} from './jsonschema-nodewalker' export function defaultTitleAsJsonPath(jsonSchema: T) { jsonSchemaWalkNodes(jsonSchema, (node, meta) => { // Skip if we already have one.. - if (node.title) { - return - } + // TODO: We can also handle json metadata here as desired - const title = [...(meta?.path ?? []), meta.name] + const jsonPath = [...(meta?.path ?? []), meta.name] .filter((n) => !!n) // Filter out nesting from things like anyOf .join('.') - if (title && !title.endsWith('.')) { - node.title = title + + if (node.title) { + // @see https://share.cleanshot.com/16sDgL6D + node.title = `${jsonPath}: ${node.title}` + } else if (jsonPath && !jsonPath.endsWith('.')) { + node.title = jsonPath } }) return jsonSchema From 1e1735dcf97d2a3a7138eadc401048951c2bf6b7 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 16:33:38 -0700 Subject: [PATCH 34/80] Keeping nango integrations in sync with oauth-based venice integrations --- apps/app-config/backendConfig.ts | 1 + apps/app-config/env.ts | 2 + apps/cli/_cli.ts | 4 +- integrations/integration-qbo/def.ts | 1 + packages/cdk-core/index.ts | 1 + packages/cdk-core/integration.types.ts | 1 + packages/cdk-core/nango.ts | 214 ++++++++++++++++++ packages/cdk-core/providers.types.ts | 3 + packages/engine-backend/context.ts | 5 + packages/engine-backend/router/adminRouter.ts | 36 ++- 10 files changed, 263 insertions(+), 5 deletions(-) create mode 100644 packages/cdk-core/nango.ts diff --git a/apps/app-config/backendConfig.ts b/apps/app-config/backendConfig.ts index 7f882b0b..3aa1747d 100644 --- a/apps/app-config/backendConfig.ts +++ b/apps/app-config/backendConfig.ts @@ -47,6 +47,7 @@ export const contextFactory = getContextFactory({ // routerUrl: 'http://localhost:3010/api', // apiUrl? apiUrl: joinPath(getServerUrl(null), '/api/trpc'), jwtSecret: env.JWT_SECRET_OR_PUBLIC_KEY, + nangoSecretKey: env.NANGO_SECRET_KEY, getRedirectUrl: (_, _ctx) => joinPath(getServerUrl(null), '/'), getMetaService: (viewer) => makePostgresMetaService({databaseUrl: env.POSTGRES_OR_WEBHOOK_URL, viewer}), diff --git a/apps/app-config/env.ts b/apps/app-config/env.ts index c482fd23..db1acaf5 100644 --- a/apps/app-config/env.ts +++ b/apps/app-config/env.ts @@ -19,6 +19,7 @@ Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent t INNGEST_EVENT_KEY: z.string(), INNGEST_SIGNING_KEY: z.string(), + NANGO_SECRET_KEY: z.string(), }, client: { NEXT_PUBLIC_SUPABASE_URL: z.string(), @@ -36,6 +37,7 @@ Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent t CLERK_SECRET_KEY: process.env['CLERK_SECRET_KEY'], INNGEST_EVENT_KEY: process.env['INNGEST_EVENT_KEY'], INNGEST_SIGNING_KEY: process.env['INNGEST_SIGNING_KEY'], + NANGO_SECRET_KEY: process.env['NANGO_SECRET_KEY'], JWT_SECRET_OR_PUBLIC_KEY: process.env['JWT_SECRET_OR_PUBLIC_KEY'], NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: process.env['NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY'], diff --git a/apps/cli/_cli.ts b/apps/cli/_cli.ts index 63218045..99aa735f 100644 --- a/apps/cli/_cli.ts +++ b/apps/cli/_cli.ts @@ -3,7 +3,7 @@ import '@usevenice/app-config/register.node' import {parseIntConfigsFromRawEnv} from '@usevenice/app-config/integration-envs' import type {defIntegrations} from '@usevenice/app-config/integrations/integrations.def' -import {makeJwtClient} from '@usevenice/cdk-core' +import {makeJwtClient, makeNangoClient} from '@usevenice/cdk-core' import {makeAlphavantageClient} from '@usevenice/integration-alphavantage' import {makeHeronClient} from '@usevenice/integration-heron' import {makeLunchmoneyClient} from '@usevenice/integration-lunchmoney' @@ -103,6 +103,8 @@ if (require.main === module) { accountToken: process.env['MERGE_TEST_LINKED_ACCOUNT_TOKEN'] ?? '', }).integrations, heron: () => makeHeronClient({apiKey: process.env['HERON_API_KEY']!}), + nango: () => + makeNangoClient({secretKey: process.env['_NANGO_SECRET_KEY']!}), } const clientFactory = z diff --git a/integrations/integration-qbo/def.ts b/integrations/integration-qbo/def.ts index 927266c6..16ac1377 100644 --- a/integrations/integration-qbo/def.ts +++ b/integrations/integration-qbo/def.ts @@ -54,6 +54,7 @@ export const qboDef = { stage: 'beta', categories: ['accounting'], logoUrl: '/_assets/logo-qbo.svg', + nangoProvider: 'quickbooks', }, extension: { sourceMapEntity: { diff --git a/packages/cdk-core/index.ts b/packages/cdk-core/index.ts index e8ff803f..ac7ab355 100644 --- a/packages/cdk-core/index.ts +++ b/packages/cdk-core/index.ts @@ -7,6 +7,7 @@ export * from './integration.types' export * from './kvStore' export * from './meta.types' export * from './metaService' +export * from './nango' export * from './NoopMetaService' export * from './protocol' export * from './providers.types' diff --git a/packages/cdk-core/integration.types.ts b/packages/cdk-core/integration.types.ts index 9cbcdece..3aa2b461 100644 --- a/packages/cdk-core/integration.types.ts +++ b/packages/cdk-core/integration.types.ts @@ -47,6 +47,7 @@ export interface IntegrationDef< T extends IntHelpers = IntHelpers, > { name: TSchemas['name']['_def']['value'] + // TODO: Rename def to schemas... def: TSchemas metadata?: IntegrationMetadata diff --git a/packages/cdk-core/nango.ts b/packages/cdk-core/nango.ts new file mode 100644 index 00000000..dd198975 --- /dev/null +++ b/packages/cdk-core/nango.ts @@ -0,0 +1,214 @@ +import type {Endpoints, InfoFromEndpoints} from '@usevenice/util' +import {makeOpenApiClient, z} from '@usevenice/util' + +const zNangoProvider = z.enum([ + 'accelo', + 'adobe', + 'aircall', + 'airtable', + 'amazon', + 'amplitude', + 'asana', + 'ashby', + 'atlassian', + 'bamboohr', + 'battlenet', + 'bitbucket', + 'boldsign', + 'box', + 'braintree', + 'braintree-sandbox', + 'brex', + 'brex-staging', + 'calendly', + 'clickup', + 'confluence', + 'contentstack', + 'deel', + 'deel-sandbox', + 'digitalocean', + 'discord', + 'docusign', + 'docusign-sandbox', + 'dropbox', + 'epic-games', + 'evaluagent', + 'eventbrite', + 'exact-online', + 'factorial', + 'facebook', + 'figjam', + 'figma', + 'fitbit', + 'freshbooks', + 'freshservice', + 'front', + 'github', + 'github-app', + 'gitlab', + 'gong', + 'google', + 'google-calendar', + 'google-mail', + 'google-sheet', + 'gorgias', + 'greenhouse', + 'gumroad', + 'gusto', + 'health-gorilla', + 'highlevel', + 'hubspot', + 'instagram', + 'intercom', + 'intuit', + 'jira', + 'keap', + 'lever', + 'linear', + 'linkedin', + 'linkhut', + 'mailchimp', + 'microsoft-teams', + 'mixpanel', + 'miro', + 'monday', + 'mural', + 'nationbuilder', + 'netsuite', + 'notion', + 'one-drive', + 'osu', + 'outreach', + 'pagerduty', + 'pandadoc', + 'payfit', + 'pennylane', + 'pipedrive', + 'qualtrics', + 'quickbooks', + 'ramp', + 'ramp-sandbox', + 'reddit', + 'ring-central', + 'ring-central-sandbox', + 'segment', + 'sage', + 'salesforce', + 'salesforce-sandbox', + 'salesloft', + 'servicem8', + 'shopify', + 'shortcut', + 'slack', + 'smugmug', + 'splitwise', + 'spotify', + 'squareup', + 'squareup-sandbox', + 'stackexchange', + 'strava', + 'stripe', + 'stripe-express', + 'survey-monkey', + 'teamwork', + 'timely', + 'trello', + 'todoist', + 'twitch', + 'twitter', + 'twitter-v2', + 'twinfield', + 'typeform', + 'uber', + 'unauthenticated', + 'wakatime', + 'wave-accounting', + 'wildix-pbx', + 'workable', + 'xero', + 'yahoo', + 'yandex', + 'youtube', + 'zapier-nla', + 'zendesk', + 'zenefits', + 'zoho', + 'zoho-books', + 'zoho-crm', + 'zoho-desk', + 'zoho-inventory', + 'zoho-invoice', + 'zoom', +]) + +export type NangoProvider = z.infer + +export const zIntegrationShort = z.object({ + provider: zNangoProvider, + /** aka providerConfigKey */ + unique_key: z.string(), +}) + +export const zIntegration = zIntegrationShort.extend({ + client_id: z.string(), + client_secret: z.string(), + scopes: z.string(), + app_link: z.string().nullish(), + auth_mode: z.enum(['OAUTH2', 'OAUTH1', 'BASIC']), +}) + +export const zUpsertIntegration = zIntegration + .omit({ + unique_key: true, + client_id: true, + client_secret: true, + scopes: true, + }) + .extend({ + provider_config_key: z.string(), + oauth_client_id: z.string(), + oauth_client_secret: z.string(), + oauth_scopes: z.string().optional(), + }) + .partial({auth_mode: true}) + +export const endpoints = { + get: { + '/config': {input: {}, output: z.array(zIntegrationShort)}, + '/config/{providerConfigKey}': { + input: { + path: z.object({providerConfigKey: z.string()}), + query: z.object({include_creds: z.boolean().optional()}), + }, + output: z.union([zIntegration, zIntegrationShort]), + }, + }, + post: { + '/config': {input: {bodyJson: zUpsertIntegration}, output: z.undefined()}, + }, + put: { + '/config': {input: {bodyJson: zUpsertIntegration}, output: z.undefined()}, + }, + delete: { + '/config/{providerConfigKey}': { + input: {path: z.object({providerConfigKey: z.string()})}, + output: z.undefined(), + }, + }, +} satisfies Endpoints + +// --- + +export const zNangoConfig = z.object({ + secretKey: z.string(), +}) + +export function makeNangoClient(config: z.infer) { + const client = makeOpenApiClient>({ + baseUrl: 'https://api.nango.dev', + auth: {bearerToken: config.secretKey}, + }) + return client +} + +export type NangoClient = ReturnType diff --git a/packages/cdk-core/providers.types.ts b/packages/cdk-core/providers.types.ts index b964d79b..350afe17 100644 --- a/packages/cdk-core/providers.types.ts +++ b/packages/cdk-core/providers.types.ts @@ -14,6 +14,7 @@ import type { IntegrationSchemas, IntHelpers, } from './integration.types' +import type {NangoProvider} from './nango' import type {AnyEntityPayload, ResoUpdateData, Source} from './protocol' // eslint-disable-next-line @typescript-eslint/consistent-type-definitions @@ -67,6 +68,8 @@ export interface IntegrationMetadata { stage?: z.infer // labels?: Array<'featured' | 'banking' | 'accounting' | 'enrichment'> categories?: Array> + /** Whether this is an oauth integration? */ + nangoProvider?: NangoProvider } // MARK: - Shared connect types diff --git a/packages/engine-backend/context.ts b/packages/engine-backend/context.ts index 6f6a197d..4a76d715 100644 --- a/packages/engine-backend/context.ts +++ b/packages/engine-backend/context.ts @@ -1,11 +1,13 @@ import {TRPCError} from '@trpc/server' +import {makeNangoClient} from '@usevenice/cdk-core' import type { AnyIntegrationImpl, EndUserId, Link, LinkFactory, MetaService, + NangoClient, } from '@usevenice/cdk-core' import type {JWTClient, Viewer, ViewerRole} from '@usevenice/cdk-core/viewer' import {makeJwtClient, zViewerFromJwtPayload} from '@usevenice/cdk-core/viewer' @@ -28,6 +30,7 @@ export interface RouterContext { // Non-viewer dependent providerMap: Record jwt: JWTClient + nango: NangoClient /** * Base url of the engine-backend router when deployed, e.g. `localhost:3000/api/usevenice` * This is needed for 1) server side rendering and 2) webhook handling @@ -51,6 +54,7 @@ export interface ContextFactoryOptions< /** Used for authentication */ jwtSecret: string + nangoSecretKey: string /** Used to store metadata */ getMetaService: (viewer: Viewer) => MetaService @@ -97,6 +101,7 @@ export function getContextFactory< // --- Non-viewer dependent providerMap, jwt, + nango: makeNangoClient({secretKey: config.nangoSecretKey}), apiUrl, getRedirectUrl, } diff --git a/packages/engine-backend/router/adminRouter.ts b/packages/engine-backend/router/adminRouter.ts index caf75a96..14b72bce 100644 --- a/packages/engine-backend/router/adminRouter.ts +++ b/packages/engine-backend/router/adminRouter.ts @@ -1,6 +1,7 @@ import {TRPCError} from '@trpc/server' import { + extractProviderName, handlersLink, makeId, sync, @@ -91,7 +92,7 @@ export const adminRouter = trpc.router({ // this makes me wonder if UPSERT should always be the default.... .required({orgId: true}), ) - .mutation(({input: {id: _id, providerName, ...input}, ctx}) => { + .mutation(async ({input: {id: _id, providerName, ...input}, ctx}) => { const id = _id ? _id : providerName && input.orgId @@ -103,14 +104,41 @@ export const adminRouter = trpc.router({ message: 'Missing id or providerName/orgId', }) } + const provider = ctx.providerMap[extractProviderName(id)] + + if (!provider) { + throw new TRPCError({ + code: 'BAD_REQUEST', + message: `Missing provider for ${extractProviderName(id)}`, + }) + } + if (provider.metadata?.nangoProvider) { + // Create nango integration here... + await ctx.nango.post('/config', { + bodyJson: { + provider_config_key: id, + provider: provider.metadata.nangoProvider, + // TODO: gotta fix the typing here... + oauth_client_id: (input.config as any).clientId, + oauth_client_secret: (input.config as any).clientSecret, + }, + }) + } + return ctx.helpers.patchReturning('integration', id, input) }), // Need a tuple for some reason... otherwise seems to not work in practice. adminDeleteIntegration: adminProcedure .input(z.tuple([zId('int')])) - .mutation(({input: [intId], ctx}) => - ctx.helpers.metaService.tables.integration.delete(intId), - ), + .mutation(async ({input: [intId], ctx}) => { + const provider = ctx.providerMap[extractProviderName(intId)] + if (provider?.metadata?.nangoProvider) { + await ctx.nango.delete('/config/{providerConfigKey}', { + path: {providerConfigKey: intId}, + }) + } + return ctx.helpers.metaService.tables.integration.delete(intId) + }), adminCreateConnectToken: adminProcedure .input(adminRouterSchema.adminCreateConnectToken.input) .mutation(({input: {endUserId, orgId, validityInSeconds}, ctx}) => { From 28765e93b78d2ed6998137db1fbfc84732ebb1dc Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 16:39:16 -0700 Subject: [PATCH 35/80] Adding nango public key also --- apps/app-config/env.ts | 2 ++ apps/cli/_cli.ts | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/app-config/env.ts b/apps/app-config/env.ts index db1acaf5..4d1bda78 100644 --- a/apps/app-config/env.ts +++ b/apps/app-config/env.ts @@ -31,6 +31,7 @@ Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent t NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME: z .string() .default('supabase'), + NEXT_PUBLIC_NANGO_PUBLIC_KEY: z.string(), NEXT_PUBLIC_COMMANDBAR_ORG_ID: z.string().optional(), }, runtimeEnv: overrideFromLocalStorage({ @@ -38,6 +39,7 @@ Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent t INNGEST_EVENT_KEY: process.env['INNGEST_EVENT_KEY'], INNGEST_SIGNING_KEY: process.env['INNGEST_SIGNING_KEY'], NANGO_SECRET_KEY: process.env['NANGO_SECRET_KEY'], + NEXT_PUBLIC_NANGO_PUBLIC_KEY: process.env['NEXT_PUBLIC_NANGO_PUBLIC_KEY'], JWT_SECRET_OR_PUBLIC_KEY: process.env['JWT_SECRET_OR_PUBLIC_KEY'], NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: process.env['NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY'], diff --git a/apps/cli/_cli.ts b/apps/cli/_cli.ts index 99aa735f..3778fcb2 100644 --- a/apps/cli/_cli.ts +++ b/apps/cli/_cli.ts @@ -38,7 +38,7 @@ if (getEnvVar('DEBUG_ZOD')) { } function env() { - process.env['SKIP_ENV_VALIDATION'] = 'true' + process.env['_SKIP_ENV_VALIDATION'] = 'true' // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access return require('@usevenice/app-config/env') .env as typeof import('@usevenice/app-config/env')['env'] @@ -75,7 +75,7 @@ if (require.main === module) { onebrick: () => makeOneBrickClient(intConfig('onebrick')) as {}, teller: () => makeTellerClient(intConfig('teller')), stripe: () => - makeStripeClient({apiKey: process.env['STRIPE_TEST_SECRET_KEY']!}), + makeStripeClient({apiKey: process.env['_STRIPE_TEST_SECRET_KEY']!}), ramp: () => makeRampClient(intConfig('ramp').oauth), wise: () => makeWiseClient(intConfig('wise')), toggl: () => makeTogglClient(intConfig('toggl')), @@ -94,15 +94,15 @@ if (require.main === module) { 'merge.accounting': () => makeMergeClient({ - apiKey: process.env['MERGE_TEST_API_KEY'] ?? '', - accountToken: process.env['MERGE_TEST_LINKED_ACCOUNT_TOKEN'] ?? '', + apiKey: process.env['_MERGE_TEST_API_KEY'] ?? '', + accountToken: process.env['_MERGE_TEST_LINKED_ACCOUNT_TOKEN'] ?? '', }).accounting, 'merge.integrations': () => makeMergeClient({ - apiKey: process.env['MERGE_TEST_API_KEY'] ?? '', - accountToken: process.env['MERGE_TEST_LINKED_ACCOUNT_TOKEN'] ?? '', + apiKey: process.env['_MERGE_TEST_API_KEY'] ?? '', + accountToken: process.env['_MERGE_TEST_LINKED_ACCOUNT_TOKEN'] ?? '', }).integrations, - heron: () => makeHeronClient({apiKey: process.env['HERON_API_KEY']!}), + heron: () => makeHeronClient({apiKey: process.env['_HERON_API_KEY']!}), nango: () => makeNangoClient({secretKey: process.env['_NANGO_SECRET_KEY']!}), } From 10d885cca03a9587d0574bb57947db8950766d6c Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 17:32:18 -0700 Subject: [PATCH 36/80] Fix bad import in postgres/def --- apps/app-config/backendConfig.ts | 6 ++++-- integrations/integration-postgres/def.ts | 9 ++++++--- .../integration-postgres/makePostgresClient.ts | 10 ++-------- .../integration-postgres/makePostgresMetaService.ts | 7 ++----- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/apps/app-config/backendConfig.ts b/apps/app-config/backendConfig.ts index 3aa1747d..3f060ae1 100644 --- a/apps/app-config/backendConfig.ts +++ b/apps/app-config/backendConfig.ts @@ -16,8 +16,10 @@ import {getServerUrl} from './constants' import {env} from './env' import {mergedIntegrations} from './integrations/integrations.merged' -export {makePostgresClient} from '@usevenice/integration-postgres' -export {DatabaseError} from '@usevenice/integration-postgres/makePostgresClient' +export { + DatabaseError, + makePostgresClient, +} from '@usevenice/integration-postgres/makePostgresClient' export {Papa} from '@usevenice/integration-spreadsheet' export const backendEnv = env diff --git a/integrations/integration-postgres/def.ts b/integrations/integration-postgres/def.ts index d193bde4..bacdbc75 100644 --- a/integrations/integration-postgres/def.ts +++ b/integrations/integration-postgres/def.ts @@ -3,9 +3,12 @@ import {intHelpers} from '@usevenice/cdk-core' import type {EntityPayloadWithExternal, ZCommon} from '@usevenice/cdk-ledger' import {z, zCast} from '@usevenice/util' -import {zPgConfig} from './makePostgresClient' - -export {makePostgresClient} from './makePostgresClient' +export const zPgConfig = z.object({ + databaseUrl: z.string(), + migrationsPath: z.string().optional(), + migrationTableName: z.string().optional(), + transformFieldNames: z.boolean().optional(), +}) export const postgresSchemas = { name: z.literal('postgres'), diff --git a/integrations/integration-postgres/makePostgresClient.ts b/integrations/integration-postgres/makePostgresClient.ts index 0c30f268..6605b573 100644 --- a/integrations/integration-postgres/makePostgresClient.ts +++ b/integrations/integration-postgres/makePostgresClient.ts @@ -13,18 +13,12 @@ import { memoize, R, snakeCase, - z, zFunction, } from '@usevenice/util' -export {DatabaseError} from 'pg' +import {zPgConfig} from './def' -export const zPgConfig = z.object({ - databaseUrl: z.string(), - migrationsPath: z.string().optional(), - migrationTableName: z.string().optional(), - transformFieldNames: z.boolean().optional(), -}) +export {DatabaseError} from 'pg' export const makePostgresClient = zFunction( zPgConfig, diff --git a/integrations/integration-postgres/makePostgresMetaService.ts b/integrations/integration-postgres/makePostgresMetaService.ts index 39cac157..169f132e 100644 --- a/integrations/integration-postgres/makePostgresMetaService.ts +++ b/integrations/integration-postgres/makePostgresMetaService.ts @@ -12,11 +12,8 @@ import type { import {zViewer} from '@usevenice/cdk-core' import {memoize, R, zFunction} from '@usevenice/util' -import { - applyLimitOffset, - makePostgresClient, - zPgConfig, -} from './makePostgresClient' +import {zPgConfig} from './def' +import {applyLimitOffset, makePostgresClient} from './makePostgresClient' const getPostgreClient = memoize((databaseUrl: string) => makePostgresClient({databaseUrl}), From 9f65c73b36c4f9b563a89efd1c30ba0adb6b72ad Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 17:41:34 -0700 Subject: [PATCH 37/80] [WIP] Got popup open for QBO authentication via nango --- .../resources/ResourcesPage.tsx | 8 ++++- apps/web/app/connect/ConnectPage.tsx | 11 ++++++- packages/cdk-core/integration.types.ts | 7 ++-- packages/cdk-core/nango.ts | 4 ++- packages/cdk-core/providers.types.ts | 7 ++-- packages/engine-frontend/VeniceConnect.tsx | 33 ++++++++++++++++--- packages/engine-frontend/package.json | 1 + packages/util/zod-jsonschema-utils.ts | 2 +- packages/util/zod-jsonschema.spec.ts | 4 +-- pnpm-lock.yaml | 7 ++++ 10 files changed, 69 insertions(+), 15 deletions(-) diff --git a/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx b/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx index 35ab0589..65c8ee86 100644 --- a/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx +++ b/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx @@ -5,7 +5,9 @@ import Image from 'next/image' import Link from 'next/link' import React from 'react' +import {env} from '@usevenice/app-config/env' import {clientIntegrations} from '@usevenice/app-config/integrations/integrations.client' +import {defIntegrations} from '@usevenice/app-config/integrations/integrations.def' import {extractProviderName, zRaw} from '@usevenice/cdk-core' import type {RouterOutput} from '@usevenice/engine-backend' import {_trpcReact, VeniceConnectButton} from '@usevenice/engine-frontend' @@ -54,7 +56,11 @@ export default function ResourcesPage() {

Resources

- +

Resources are created based on integration configurations

+ return ( + + ) } diff --git a/packages/cdk-core/integration.types.ts b/packages/cdk-core/integration.types.ts index 3aa2b461..8f46148f 100644 --- a/packages/cdk-core/integration.types.ts +++ b/packages/cdk-core/integration.types.ts @@ -1,7 +1,7 @@ import type {MaybePromise, z} from '@usevenice/util' import {R} from '@usevenice/util' -import type {EndUserId} from './id.types' +import type {EndUserId, Id} from './id.types' import {makeId} from './id.types' import type {ZStandard} from './meta.types' import type { @@ -100,7 +100,10 @@ export interface IntegrationClient< openDialog: OpenDialogFn }) => ( connectInput: T['_types']['connectInput'], - context: ConnectOptions, + context: ConnectOptions & { + // TODO: Does this belong here? + integrationId: Id['int'] + }, ) => Promise } diff --git a/packages/cdk-core/nango.ts b/packages/cdk-core/nango.ts index dd198975..74454f56 100644 --- a/packages/cdk-core/nango.ts +++ b/packages/cdk-core/nango.ts @@ -154,7 +154,9 @@ export const zIntegration = zIntegrationShort.extend({ client_secret: z.string(), scopes: z.string(), app_link: z.string().nullish(), - auth_mode: z.enum(['OAUTH2', 'OAUTH1', 'BASIC']), + // In practice we only use nango for oauth integrations + // but in theory we could use it for a generic secret store as well + auth_mode: z.enum(['OAUTH2', 'OAUTH1', 'BASIC', 'API_KEY']), }) export const zUpsertIntegration = zIntegration diff --git a/packages/cdk-core/providers.types.ts b/packages/cdk-core/providers.types.ts index 350afe17..d768d55c 100644 --- a/packages/cdk-core/providers.types.ts +++ b/packages/cdk-core/providers.types.ts @@ -7,7 +7,7 @@ import { zodToJsonSchema, } from '@usevenice/util' -import type {EndUserId, ExtEndUserId, ExternalId} from './id.types' +import type {EndUserId, ExtEndUserId, ExternalId, Id} from './id.types' import {zExternalId} from './id.types' import type { AnyIntegrationImpl, @@ -102,7 +102,10 @@ export type UseConnectHook = (scope: { openDialog: OpenDialogFn }) => ( connectInput: T['_types']['connectInput'], - context: ConnectOptions, + context: ConnectOptions & { + // TODO: Does this belong here? + integrationId: Id['int'] + }, ) => Promise // MARK: - Server side connect types diff --git a/packages/engine-frontend/VeniceConnect.tsx b/packages/engine-frontend/VeniceConnect.tsx index a80e9ebb..48b5042b 100644 --- a/packages/engine-frontend/VeniceConnect.tsx +++ b/packages/engine-frontend/VeniceConnect.tsx @@ -1,5 +1,6 @@ 'use client' +import Nango from '@nangohq/frontend' import {useMutation} from '@tanstack/react-query' import {Link2, Loader2, RefreshCw, Trash2} from 'lucide-react' import React from 'react' @@ -7,6 +8,7 @@ import React from 'react' import type { Id, IntegrationClient, + IntegrationDef, OpenDialogFn, UseConnectHook, } from '@usevenice/cdk-core' @@ -47,9 +49,12 @@ import {_trpcReact} from './TRPCProvider' type ConnectEventType = 'open' | 'close' | 'error' export interface VeniceConnectProps extends UIPropsNoChildren { + /** Does this belong here? */ + nangoPublicKey: string /** Whether to display the existing connections */ showExisting?: boolean clientIntegrations: Record + defIntegrations: Record onEvent?: (event: {type: ConnectEventType; intId: Id['int']}) => void } @@ -132,15 +137,22 @@ type ProviderMeta = Catalog[string] export function _VeniceConnect({ catalog, clientIntegrations, + defIntegrations, onEvent, showExisting, className, integrationIds, + nangoPublicKey, ...uiProps }: VeniceConnectProps & { integrationIds: Array catalog: Catalog }) { + const nango = React.useMemo( + () => new Nango({publicKey: nangoPublicKey}), + [nangoPublicKey], + ) + // VeniceConnect should be fetching its own integrationIds as well as resources // this way it can esure those are refreshed as operations take place // This is esp true when we are operating in client envs (js embed) @@ -194,10 +206,21 @@ export function _VeniceConnect({ integrationIds, R.map(extractProviderName), R.uniq, - R.mapToObj((name: string) => [ - name, - clientIntegrations[name]?.useConnectHook?.({openDialog}), - ]), + R.mapToObj((name: string) => { + let fn = clientIntegrations[name]?.useConnectHook?.({openDialog}) + const nangoProvider = defIntegrations[name]?.metadata?.nangoProvider + if (!fn && nangoProvider) { + console.log('adding nnango provider for', nangoProvider) + + fn = async (_, {integrationId}) => { + console.log('inputs', integrationId) + await nango.auth(integrationId, 'conn_test').then((r) => { + console.log('auth', r) + }) + } + } + return [name, fn] + }), ) const categories = zIntegrationCategory.options @@ -359,7 +382,7 @@ export const WithProviderConnect = ({ // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const connOutput = connectFn - ? await connectFn?.(connInput, {}) + ? await connectFn?.(connInput, {integrationId: int.id}) : connInput console.log(`[VeniceConnect] ${int.id} connOutput`, connOutput) diff --git a/packages/engine-frontend/package.json b/packages/engine-frontend/package.json index 6b5f84a0..2e522986 100644 --- a/packages/engine-frontend/package.json +++ b/packages/engine-frontend/package.json @@ -5,6 +5,7 @@ "sideEffects": false, "module": "./index.ts", "dependencies": { + "@nangohq/frontend": "0.33.8", "@tanstack/react-query": "*", "@trpc/client": "10.21.1", "@trpc/react-query": "10.21.1", diff --git a/packages/util/zod-jsonschema-utils.ts b/packages/util/zod-jsonschema-utils.ts index ac2b7afd..6d451502 100644 --- a/packages/util/zod-jsonschema-utils.ts +++ b/packages/util/zod-jsonschema-utils.ts @@ -15,7 +15,7 @@ export function defaultTitleAsJsonPath(jsonSchema: T) { .filter((n) => !!n) // Filter out nesting from things like anyOf .join('.') - if (node.title) { + if (node.title && jsonPath) { // @see https://share.cleanshot.com/16sDgL6D node.title = `${jsonPath}: ${node.title}` } else if (jsonPath && !jsonPath.endsWith('.')) { diff --git a/packages/util/zod-jsonschema.spec.ts b/packages/util/zod-jsonschema.spec.ts index 019b6acd..2c2c0d81 100644 --- a/packages/util/zod-jsonschema.spec.ts +++ b/packages/util/zod-jsonschema.spec.ts @@ -140,11 +140,11 @@ test('enum description to title 2', () => { "clientId", "clientSecret", ], - "title": "Enable", + "title": "oauth: Enable", "type": "object", }, { - "title": "Disable", + "title": "oauth: Disable", "type": "null", }, ], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ae454471..fdeb4230 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1316,6 +1316,9 @@ importers: packages/engine-frontend: dependencies: + '@nangohq/frontend': + specifier: 0.33.8 + version: 0.33.8 '@tanstack/react-query': specifier: 4.28.0 version: 4.28.0(react-dom@18.2.0)(react@18.2.0) @@ -3880,6 +3883,10 @@ packages: engines: {node: '>=12'} dev: false + /@nangohq/frontend@0.33.8: + resolution: {integrity: sha512-iKTo4BDEXaKG6FbZzT9iACr07lGDBhmaS90+tsd92XrFpO8Gk7JGMp13lUS66mfwTcCXqPBXFVSkBvE0QyJi8Q==} + dev: false + /@next/env@13.4.3: resolution: {integrity: sha512-pa1ErjyFensznttAk3EIv77vFbfSYT6cLzVRK5jx4uiRuCQo+m2wCFAREaHKIy63dlgvOyMlzh6R8Inu8H3KrQ==} dev: false From d07c82f9236591821f9dc118cfecccd372e39a74 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 11 Oct 2023 23:40:05 -0700 Subject: [PATCH 38/80] [wip] Document the connection endpoints --- .../cdk-core/{nango.ts => NangoClient.ts} | 41 ++++++++++++++++--- packages/cdk-core/index.ts | 2 +- packages/cdk-core/providers.types.ts | 2 +- 3 files changed, 38 insertions(+), 7 deletions(-) rename packages/cdk-core/{nango.ts => NangoClient.ts} (75%) diff --git a/packages/cdk-core/nango.ts b/packages/cdk-core/NangoClient.ts similarity index 75% rename from packages/cdk-core/nango.ts rename to packages/cdk-core/NangoClient.ts index 74454f56..96e906a3 100644 --- a/packages/cdk-core/nango.ts +++ b/packages/cdk-core/NangoClient.ts @@ -145,10 +145,22 @@ export type NangoProvider = z.infer export const zIntegrationShort = z.object({ provider: zNangoProvider, - /** aka providerConfigKey */ + /** aka provider_config_key */ unique_key: z.string(), }) +export const zConnectionShort = z.object({ + /** + * This is a very mis-leading property name. It is typically the userId. Notably this means each user would only be able + * to connect a single connection for each provider + */ + connection_id: z.string(), + created: z.string().datetime(), + /** Now this is actually the unique id for the connection */ + id: z.number(), + provider: zNangoProvider, +}) + export const zIntegration = zIntegrationShort.extend({ client_id: z.string(), client_secret: z.string(), @@ -177,13 +189,25 @@ export const zUpsertIntegration = zIntegration export const endpoints = { get: { '/config': {input: {}, output: z.array(zIntegrationShort)}, - '/config/{providerConfigKey}': { + '/config/{provider_config_key}': { input: { - path: z.object({providerConfigKey: z.string()}), + path: z.object({provider_config_key: z.string()}), query: z.object({include_creds: z.boolean().optional()}), }, output: z.union([zIntegration, zIntegrationShort]), }, + '/connection': {input: {}, output: z.array(zConnectionShort)}, + '/connection/{connectionId}': { + input: { + path: z.object({connectionId: z.string()}), + query: z.object({ + provider_config_key: z.string(), + force_refresh: z.boolean().optional(), + refresh_token: z.boolean().optional(), + }), + }, + output: z.any(), + }, }, post: { '/config': {input: {bodyJson: zUpsertIntegration}, output: z.undefined()}, @@ -192,8 +216,15 @@ export const endpoints = { '/config': {input: {bodyJson: zUpsertIntegration}, output: z.undefined()}, }, delete: { - '/config/{providerConfigKey}': { - input: {path: z.object({providerConfigKey: z.string()})}, + '/config/{provider_config_key}': { + input: {path: z.object({provider_config_key: z.string()})}, + output: z.undefined(), + }, + '/connection/{connection_id}': { + input: { + path: z.object({connection_id: z.string()}), + query: z.object({provider_config_key: z.string()}), + }, output: z.undefined(), }, }, diff --git a/packages/cdk-core/index.ts b/packages/cdk-core/index.ts index ac7ab355..7350c9f3 100644 --- a/packages/cdk-core/index.ts +++ b/packages/cdk-core/index.ts @@ -7,7 +7,7 @@ export * from './integration.types' export * from './kvStore' export * from './meta.types' export * from './metaService' -export * from './nango' +export * from './NangoClient' export * from './NoopMetaService' export * from './protocol' export * from './providers.types' diff --git a/packages/cdk-core/providers.types.ts b/packages/cdk-core/providers.types.ts index d768d55c..14ffab40 100644 --- a/packages/cdk-core/providers.types.ts +++ b/packages/cdk-core/providers.types.ts @@ -14,7 +14,7 @@ import type { IntegrationSchemas, IntHelpers, } from './integration.types' -import type {NangoProvider} from './nango' +import type {NangoProvider} from './NangoClient' import type {AnyEntityPayload, ResoUpdateData, Source} from './protocol' // eslint-disable-next-line @typescript-eslint/consistent-type-definitions From ed4a2483b136158f56c749d876a03208a6d8c1fb Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 12 Oct 2023 22:10:38 -0700 Subject: [PATCH 39/80] [WIP] Got QBO connection working via Nango --- integrations/integration-qbo/QBOClient.ts | 1 + packages/cdk-core/NangoClient.ts | 32 +++++++- packages/cdk-core/providers.types.ts | 4 +- packages/engine-backend/router/adminRouter.ts | 8 +- .../engine-backend/router/endUserRouter.ts | 82 +++++++++++++------ packages/engine-frontend/VeniceConnect.tsx | 10 ++- 6 files changed, 103 insertions(+), 34 deletions(-) diff --git a/integrations/integration-qbo/QBOClient.ts b/integrations/integration-qbo/QBOClient.ts index 7df68578..adc49d6a 100644 --- a/integrations/integration-qbo/QBOClient.ts +++ b/integrations/integration-qbo/QBOClient.ts @@ -10,6 +10,7 @@ import { export const zConfig = z.object({ clientId: z.string(), clientSecret: z.string(), + scope: z.string(), url: z.string().nullish().describe('For proxies, not typically needed'), verifierToken: z.string().nullish().describe('For webhooks'), }) diff --git a/packages/cdk-core/NangoClient.ts b/packages/cdk-core/NangoClient.ts index 96e906a3..390fd452 100644 --- a/packages/cdk-core/NangoClient.ts +++ b/packages/cdk-core/NangoClient.ts @@ -141,6 +141,8 @@ const zNangoProvider = z.enum([ 'zoom', ]) +export const zAuthMode = z.enum(['OAUTH2', 'OAUTH1', 'BASIC', 'API_KEY']) + export type NangoProvider = z.infer export const zIntegrationShort = z.object({ @@ -161,6 +163,32 @@ export const zConnectionShort = z.object({ provider: zNangoProvider, }) +export const zConnection = zConnectionShort.extend({ + updated_at: z.string().datetime(), + provider_config_key: z.string(), + credentials: z.object({ + type: zAuthMode, + access_token: z.string(), + expires_at: z.string().datetime(), + raw: z.object({ + access_token: z.string(), + expires_in: z.number(), + refresh_token_expires_in: z.number(), + token_type: z.string(), //'bearer', + scope: z.string(), + expires_at: z.string().datetime(), + }), + }), + connection_config: z.record(z.unknown()), + metadata: z.record(z.unknown()).nullable(), + credentials_iv: z.string(), + credentials_tag: z.string(), + environment_id: z.number(), + deleted: z.boolean(), + deleted_at: z.string().datetime().nullable(), + last_fetched_at: z.string().datetime().nullable(), +}) + export const zIntegration = zIntegrationShort.extend({ client_id: z.string(), client_secret: z.string(), @@ -168,7 +196,7 @@ export const zIntegration = zIntegrationShort.extend({ app_link: z.string().nullish(), // In practice we only use nango for oauth integrations // but in theory we could use it for a generic secret store as well - auth_mode: z.enum(['OAUTH2', 'OAUTH1', 'BASIC', 'API_KEY']), + auth_mode: zAuthMode, }) export const zUpsertIntegration = zIntegration @@ -206,7 +234,7 @@ export const endpoints = { refresh_token: z.boolean().optional(), }), }, - output: z.any(), + output: zConnection, }, }, post: { diff --git a/packages/cdk-core/providers.types.ts b/packages/cdk-core/providers.types.ts index 14ffab40..698cb528 100644 --- a/packages/cdk-core/providers.types.ts +++ b/packages/cdk-core/providers.types.ts @@ -37,7 +37,9 @@ export const metaForProvider = (provider: AnyIntegrationImpl) => ({ ]), hasPreConnect: provider.preConnect != null, hasUseConnectHook: provider.useConnectHook != null, - hasPostConnect: provider.postConnect != null, + // TODO: Maybe nangoProvider be more explicit as a base provider? + hasPostConnect: + provider.postConnect != null || provider.metadata?.nangoProvider, schemas: R.mapValues(provider.def ?? {}, (schema) => schema instanceof z.ZodSchema ? zodToJsonSchema(schema) : undefined, ) as Record, diff --git a/packages/engine-backend/router/adminRouter.ts b/packages/engine-backend/router/adminRouter.ts index 14b72bce..d8f0c696 100644 --- a/packages/engine-backend/router/adminRouter.ts +++ b/packages/engine-backend/router/adminRouter.ts @@ -113,14 +113,16 @@ export const adminRouter = trpc.router({ }) } if (provider.metadata?.nangoProvider) { + // TODO: Should we use put vs. post? need to fix it up here... // Create nango integration here... - await ctx.nango.post('/config', { + await ctx.nango.put('/config', { bodyJson: { provider_config_key: id, provider: provider.metadata.nangoProvider, // TODO: gotta fix the typing here... oauth_client_id: (input.config as any).clientId, oauth_client_secret: (input.config as any).clientSecret, + oauth_scopes: (input.config as any).scope, }, }) } @@ -133,8 +135,8 @@ export const adminRouter = trpc.router({ .mutation(async ({input: [intId], ctx}) => { const provider = ctx.providerMap[extractProviderName(intId)] if (provider?.metadata?.nangoProvider) { - await ctx.nango.delete('/config/{providerConfigKey}', { - path: {providerConfigKey: intId}, + await ctx.nango.delete('/config/{provider_config_key}', { + path: {provider_config_key: intId}, }) } return ctx.helpers.metaService.tables.integration.delete(intId) diff --git a/packages/engine-backend/router/endUserRouter.ts b/packages/engine-backend/router/endUserRouter.ts index 14e11e73..ba467772 100644 --- a/packages/engine-backend/router/endUserRouter.ts +++ b/packages/engine-backend/router/endUserRouter.ts @@ -1,5 +1,6 @@ import type {ResourceUpdate} from '@usevenice/cdk-core' import { + extractId, makeId, zConnectOptions, zId, @@ -72,34 +73,63 @@ export const endUserRouter = trpc.router({ }) => { const int = await ctx.asOrgIfNeeded.getIntegrationOrFail(intId) console.log('didConnect start', int.provider.name, input, connCtxInput) - if (!int.provider.postConnect || !int.provider.def.connectOutput) { + + const resoUpdate = await (async () => { + if ( + !int.provider.postConnect && + int.provider.metadata?.nangoProvider + ) { + const {connectionId: resoId} = z + .object({ + providerConfigKey: z.string(), + connectionId: zId('reso'), + }) + .parse(input) + const result = await ctx.nango.get('/connection/{connectionId}', { + path: {connectionId: resoId}, + query: {provider_config_key: intId}, + }) + + return { + resourceExternalId: extractId(resoId)[2], + settings: {nango: result}, + } satisfies Omit, 'endUserId'> + } + + if (!int.provider.postConnect || !int.provider.def.connectOutput) { + return null + } + + const reso = resourceExternalId + ? await ctx.helpers.getResourceOrFail( + makeId('reso', int.provider.name, resourceExternalId), + ) + : undefined + return await int.provider.postConnect( + int.provider.def.connectOutput.parse(input), + int.config, + { + ...connCtxInput, + extEndUserId: ctx.extEndUserId, + resource: reso + ? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + {externalId: resourceExternalId!, settings: reso.settings} + : undefined, + webhookBaseUrl: joinPath( + ctx.apiUrl, + parseWebhookRequest.pathOf(int.id), + ), + redirectUrl: ctx.getRedirectUrl?.(int, { + endUserId: + ctx.viewer.role === 'end_user' ? ctx.viewer.endUserId : null, + }), + }, + ) + })() + + if (!resoUpdate) { return 'Noop' } - const reso = resourceExternalId - ? await ctx.helpers.getResourceOrFail( - makeId('reso', int.provider.name, resourceExternalId), - ) - : undefined - const resoUpdate = await int.provider.postConnect( - int.provider.def.connectOutput.parse(input), - int.config, - { - ...connCtxInput, - extEndUserId: ctx.extEndUserId, - resource: reso - ? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - {externalId: resourceExternalId!, settings: reso.settings} - : undefined, - webhookBaseUrl: joinPath( - ctx.apiUrl, - parseWebhookRequest.pathOf(int.id), - ), - redirectUrl: ctx.getRedirectUrl?.(int, { - endUserId: - ctx.viewer.role === 'end_user' ? ctx.viewer.endUserId : null, - }), - }, - ) const syncInBackground = resoUpdate.triggerDefaultSync && !connCtxInput.syncInBand diff --git a/packages/engine-frontend/VeniceConnect.tsx b/packages/engine-frontend/VeniceConnect.tsx index 48b5042b..eef9a991 100644 --- a/packages/engine-frontend/VeniceConnect.tsx +++ b/packages/engine-frontend/VeniceConnect.tsx @@ -16,6 +16,7 @@ import { CANCELLATION_TOKEN, extractId, extractProviderName, + makeId, zIntegrationCategory, } from '@usevenice/cdk-core' import type {RouterInput, RouterOutput} from '@usevenice/engine-backend' @@ -42,7 +43,7 @@ import { useToast, } from '@usevenice/ui' import {cn} from '@usevenice/ui/utils' -import {R, titleCase, z} from '@usevenice/util' +import {makeUlid, R, titleCase, z} from '@usevenice/util' import {_trpcReact} from './TRPCProvider' @@ -214,8 +215,13 @@ export function _VeniceConnect({ fn = async (_, {integrationId}) => { console.log('inputs', integrationId) - await nango.auth(integrationId, 'conn_test').then((r) => { + const resoId = makeId('reso', name, makeUlid()) + return await nango.auth(integrationId, resoId).then((r) => { console.log('auth', r) + if ('message' in r) { + throw new Error(`${r.type}: ${r.message}`) + } + return r }) } } From c6312ed78964b4c0ea4576a5f14c175ee7a26337 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 12 Oct 2023 23:28:47 -0700 Subject: [PATCH 40/80] Remove nangoPublicKey and defIntegrations from VeniceConnect props Adding getPublicEnv prodecure --- apps/app-config/backendConfig.ts | 1 + apps/app-config/env.ts | 1 + .../(authenticated)/resources/ResourcesPage.tsx | 8 +------- apps/web/app/connect/ConnectPage.tsx | 11 +---------- packages/cdk-core/providers.types.ts | 1 + packages/engine-backend/context.ts | 6 ++++++ packages/engine-backend/router/publicRouter.ts | 3 +++ packages/engine-frontend/VeniceConnect.tsx | 16 ++++++++-------- 8 files changed, 22 insertions(+), 25 deletions(-) diff --git a/apps/app-config/backendConfig.ts b/apps/app-config/backendConfig.ts index 3f060ae1..047d7fba 100644 --- a/apps/app-config/backendConfig.ts +++ b/apps/app-config/backendConfig.ts @@ -48,6 +48,7 @@ export const contextFactory = getContextFactory({ providers: Object.values(mergedIntegrations), // routerUrl: 'http://localhost:3010/api', // apiUrl? apiUrl: joinPath(getServerUrl(null), '/api/trpc'), + env, jwtSecret: env.JWT_SECRET_OR_PUBLIC_KEY, nangoSecretKey: env.NANGO_SECRET_KEY, getRedirectUrl: (_, _ctx) => joinPath(getServerUrl(null), '/'), diff --git a/apps/app-config/env.ts b/apps/app-config/env.ts index 4d1bda78..ab51d6bc 100644 --- a/apps/app-config/env.ts +++ b/apps/app-config/env.ts @@ -63,6 +63,7 @@ Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent t } satisfies Parameters[0] export const env = createEnv(envConfig) +export type Env = typeof env // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any ;(globalThis as any).env = env diff --git a/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx b/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx index 65c8ee86..35ab0589 100644 --- a/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx +++ b/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx @@ -5,9 +5,7 @@ import Image from 'next/image' import Link from 'next/link' import React from 'react' -import {env} from '@usevenice/app-config/env' import {clientIntegrations} from '@usevenice/app-config/integrations/integrations.client' -import {defIntegrations} from '@usevenice/app-config/integrations/integrations.def' import {extractProviderName, zRaw} from '@usevenice/cdk-core' import type {RouterOutput} from '@usevenice/engine-backend' import {_trpcReact, VeniceConnectButton} from '@usevenice/engine-frontend' @@ -56,11 +54,7 @@ export default function ResourcesPage() {

Resources

- +

Resources are created based on integration configurations

- ) + return } diff --git a/packages/cdk-core/providers.types.ts b/packages/cdk-core/providers.types.ts index 698cb528..13056171 100644 --- a/packages/cdk-core/providers.types.ts +++ b/packages/cdk-core/providers.types.ts @@ -40,6 +40,7 @@ export const metaForProvider = (provider: AnyIntegrationImpl) => ({ // TODO: Maybe nangoProvider be more explicit as a base provider? hasPostConnect: provider.postConnect != null || provider.metadata?.nangoProvider, + nangoProvider: provider.metadata?.nangoProvider, schemas: R.mapValues(provider.def ?? {}, (schema) => schema instanceof z.ZodSchema ? zodToJsonSchema(schema) : undefined, ) as Record, diff --git a/packages/engine-backend/context.ts b/packages/engine-backend/context.ts index 4a76d715..7cd4c03e 100644 --- a/packages/engine-backend/context.ts +++ b/packages/engine-backend/context.ts @@ -13,6 +13,8 @@ import type {JWTClient, Viewer, ViewerRole} from '@usevenice/cdk-core/viewer' import {makeJwtClient, zViewerFromJwtPayload} from '@usevenice/cdk-core/viewer' import {R} from '@usevenice/util' +import type {Env} from '../../apps/app-config/env' +// Should we actually do this hmm import type {_Integration, _Pipeline} from './contextHelpers' import {getContextHelpers} from './contextHelpers' import type {PipelineInput, ResourceInput} from './types' @@ -31,6 +33,7 @@ export interface RouterContext { providerMap: Record jwt: JWTClient nango: NangoClient + env: Env /** * Base url of the engine-backend router when deployed, e.g. `localhost:3000/api/usevenice` * This is needed for 1) server side rendering and 2) webhook handling @@ -55,6 +58,7 @@ export interface ContextFactoryOptions< /** Used for authentication */ jwtSecret: string nangoSecretKey: string + env: Env /** Used to store metadata */ getMetaService: (viewer: Viewer) => MetaService @@ -76,6 +80,7 @@ export function getContextFactory< getMetaService, providers, jwtSecret, + env, } = config for (const provider of providers) { if (typeof provider.name !== 'string') { @@ -101,6 +106,7 @@ export function getContextFactory< // --- Non-viewer dependent providerMap, jwt, + env, nango: makeNangoClient({secretKey: config.nangoSecretKey}), apiUrl, getRedirectUrl, diff --git a/packages/engine-backend/router/publicRouter.ts b/packages/engine-backend/router/publicRouter.ts index ea7c1cd8..d5f3da68 100644 --- a/packages/engine-backend/router/publicRouter.ts +++ b/packages/engine-backend/router/publicRouter.ts @@ -8,4 +8,7 @@ export const publicRouter = trpc.router({ getIntegrationCatalog: publicProcedure.query(({ctx}) => R.mapValues(ctx.providerMap, (provider) => metaForProvider(provider)), ), + getPublicEnv: publicProcedure.query(({ctx}) => + R.pick(ctx.env, ['NEXT_PUBLIC_NANGO_PUBLIC_KEY']), + ), }) diff --git a/packages/engine-frontend/VeniceConnect.tsx b/packages/engine-frontend/VeniceConnect.tsx index eef9a991..b8fa5fb5 100644 --- a/packages/engine-frontend/VeniceConnect.tsx +++ b/packages/engine-frontend/VeniceConnect.tsx @@ -8,7 +8,6 @@ import React from 'react' import type { Id, IntegrationClient, - IntegrationDef, OpenDialogFn, UseConnectHook, } from '@usevenice/cdk-core' @@ -50,12 +49,9 @@ import {_trpcReact} from './TRPCProvider' type ConnectEventType = 'open' | 'close' | 'error' export interface VeniceConnectProps extends UIPropsNoChildren { - /** Does this belong here? */ - nangoPublicKey: string /** Whether to display the existing connections */ showExisting?: boolean clientIntegrations: Record - defIntegrations: Record onEvent?: (event: {type: ConnectEventType; intId: Id['int']}) => void } @@ -138,19 +134,20 @@ type ProviderMeta = Catalog[string] export function _VeniceConnect({ catalog, clientIntegrations, - defIntegrations, onEvent, showExisting, className, integrationIds, - nangoPublicKey, ...uiProps }: VeniceConnectProps & { integrationIds: Array catalog: Catalog }) { + const nangoPublicKey = + _trpcReact.getPublicEnv.useQuery().data?.NEXT_PUBLIC_NANGO_PUBLIC_KEY + const nango = React.useMemo( - () => new Nango({publicKey: nangoPublicKey}), + () => nangoPublicKey && new Nango({publicKey: nangoPublicKey}), [nangoPublicKey], ) @@ -209,13 +206,16 @@ export function _VeniceConnect({ R.uniq, R.mapToObj((name: string) => { let fn = clientIntegrations[name]?.useConnectHook?.({openDialog}) - const nangoProvider = defIntegrations[name]?.metadata?.nangoProvider + const nangoProvider = catalog[name]?.nangoProvider if (!fn && nangoProvider) { console.log('adding nnango provider for', nangoProvider) fn = async (_, {integrationId}) => { console.log('inputs', integrationId) const resoId = makeId('reso', name, makeUlid()) + if (!nango) { + throw new Error('Missing nango public key') + } return await nango.auth(integrationId, resoId).then((r) => { console.log('auth', r) if ('message' in r) { From 3731d2a34be5de9778e37e90974fa8323ee0b24f Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 13 Oct 2023 00:39:32 -0700 Subject: [PATCH 41/80] Nango QBO POC complete With ability to sync QBO data using creds from Nango --- integrations/integration-qbo/QBOClient.ts | 1 + packages/cdk-core/NangoClient.ts | 15 ++++++++-- packages/engine-backend/router/adminRouter.ts | 29 ++++++++++++------- .../engine-backend/router/endUserRouter.ts | 10 +++++-- .../ui/domain-components/ProviderCard.tsx | 2 +- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/integrations/integration-qbo/QBOClient.ts b/integrations/integration-qbo/QBOClient.ts index adc49d6a..b6b9da82 100644 --- a/integrations/integration-qbo/QBOClient.ts +++ b/integrations/integration-qbo/QBOClient.ts @@ -11,6 +11,7 @@ export const zConfig = z.object({ clientId: z.string(), clientSecret: z.string(), scope: z.string(), + envName: z.enum(['sandbox', 'production']), url: z.string().nullish().describe('For proxies, not typically needed'), verifierToken: z.string().nullish().describe('For webhooks'), }) diff --git a/packages/cdk-core/NangoClient.ts b/packages/cdk-core/NangoClient.ts index 390fd452..f85f9024 100644 --- a/packages/cdk-core/NangoClient.ts +++ b/packages/cdk-core/NangoClient.ts @@ -173,10 +173,12 @@ export const zConnection = zConnectionShort.extend({ raw: z.object({ access_token: z.string(), expires_in: z.number(), - refresh_token_expires_in: z.number(), + expires_at: z.string().datetime(), + /** Refresh token (Only returned if the REFRESH_TOKEN boolean parameter is set to true and the refresh token is available) */ + refresh_token: z.string().nullish(), + refresh_token_expires_in: z.number().nullish(), token_type: z.string(), //'bearer', scope: z.string(), - expires_at: z.string().datetime(), }), }), connection_config: z.record(z.unknown()), @@ -192,6 +194,7 @@ export const zConnection = zConnectionShort.extend({ export const zIntegration = zIntegrationShort.extend({ client_id: z.string(), client_secret: z.string(), + /** comma deliminated scopes with no spaces in between */ scopes: z.string(), app_link: z.string().nullish(), // In practice we only use nango for oauth integrations @@ -214,6 +217,8 @@ export const zUpsertIntegration = zIntegration }) .partial({auth_mode: true}) +export type UpsertIntegration = z.infer + export const endpoints = { get: { '/config': {input: {}, output: z.array(zIntegrationShort)}, @@ -251,7 +256,11 @@ export const endpoints = { '/connection/{connection_id}': { input: { path: z.object({connection_id: z.string()}), - query: z.object({provider_config_key: z.string()}), + query: z.object({ + provider_config_key: z.string(), + force_refresh: z.boolean().optional(), + refresh_token: z.boolean().optional(), + }), }, output: z.undefined(), }, diff --git a/packages/engine-backend/router/adminRouter.ts b/packages/engine-backend/router/adminRouter.ts index d8f0c696..20059862 100644 --- a/packages/engine-backend/router/adminRouter.ts +++ b/packages/engine-backend/router/adminRouter.ts @@ -1,5 +1,7 @@ import {TRPCError} from '@trpc/server' +import type { + UpsertIntegration} from '@usevenice/cdk-core'; import { extractProviderName, handlersLink, @@ -9,7 +11,7 @@ import { zId, zRaw, } from '@usevenice/cdk-core' -import {makeUlid, rxjs, z} from '@usevenice/util' +import {HTTPError, makeUlid, rxjs, z} from '@usevenice/util' import {adminProcedure, trpc} from './_base' @@ -115,15 +117,22 @@ export const adminRouter = trpc.router({ if (provider.metadata?.nangoProvider) { // TODO: Should we use put vs. post? need to fix it up here... // Create nango integration here... - await ctx.nango.put('/config', { - bodyJson: { - provider_config_key: id, - provider: provider.metadata.nangoProvider, - // TODO: gotta fix the typing here... - oauth_client_id: (input.config as any).clientId, - oauth_client_secret: (input.config as any).clientSecret, - oauth_scopes: (input.config as any).scope, - }, + const bodyJson: UpsertIntegration = { + provider_config_key: id, + provider: provider.metadata.nangoProvider, + // TODO: gotta fix the typing here... + oauth_client_id: (input.config as any).clientId, + oauth_client_secret: (input.config as any).clientSecret, + oauth_scopes: (input.config as any).scope, + } + await ctx.nango.put('/config', {bodyJson}).catch((err) => { + if ( + err instanceof HTTPError && + err.response?.data.type === 'unknown_provider_config' + ) { + return ctx.nango.post('/config', {bodyJson}) + } + throw err }) } diff --git a/packages/engine-backend/router/endUserRouter.ts b/packages/engine-backend/router/endUserRouter.ts index ba467772..e0474e3d 100644 --- a/packages/engine-backend/router/endUserRouter.ts +++ b/packages/engine-backend/router/endUserRouter.ts @@ -87,12 +87,18 @@ export const endUserRouter = trpc.router({ .parse(input) const result = await ctx.nango.get('/connection/{connectionId}', { path: {connectionId: resoId}, - query: {provider_config_key: intId}, + query: {provider_config_key: intId, refresh_token: true}, }) return { resourceExternalId: extractId(resoId)[2], - settings: {nango: result}, + settings: { + nango: result, + accessToken: result.credentials.access_token, + accessTokenExpiresAt: result.credentials.expires_at, + refreshToken: result.credentials.raw.refresh_token, + realmId: result.connection_config['realmId'], + }, } satisfies Omit, 'endUserId'> } diff --git a/packages/ui/domain-components/ProviderCard.tsx b/packages/ui/domain-components/ProviderCard.tsx index 8d44bcdc..bebab951 100644 --- a/packages/ui/domain-components/ProviderCard.tsx +++ b/packages/ui/domain-components/ProviderCard.tsx @@ -109,7 +109,7 @@ export const IntegrationCard = ({ // Temporary hack due to presence of labels for plaid. Need better design for ProviderCard and IntegrationCard labels={ // TODO: Fix this hack soon. We should have some kind of mapStandardIntegration method - int.providerName === 'plaid' && int.config?.['envName'] + int.config?.['envName'] ? // eslint-disable-next-line @typescript-eslint/no-base-to-string [`${int.config?.['envName']}`] : [] From e19d462f69680d64b5dac9bf41d5c021ab7da358 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 13 Oct 2023 01:25:56 -0700 Subject: [PATCH 42/80] Hacking nango to catch user cancellation --- package.json | 3 +- packages/cdk-core/frontend-utils.tsx | 3 +- packages/engine-frontend/VeniceConnect.tsx | 26 ++++++--- patches/@nangohq__frontend@0.33.8.patch | 62 ++++++++++++++++++++++ pnpm-lock.yaml | 8 ++- 5 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 patches/@nangohq__frontend@0.33.8.patch diff --git a/package.json b/package.json index 8c0b0668..d926681c 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,8 @@ "esbuild-jest@0.5.0": "patches/esbuild-jest@0.5.0.patch", "firebase@9.8.1": "patches/firebase@9.8.1.patch", "micro-memoize@4.0.10": "patches/micro-memoize@4.0.10.patch", - "zod@3.21.4": "patches/zod@3.21.4.patch" + "zod@3.21.4": "patches/zod@3.21.4.patch", + "@nangohq/frontend@0.33.8": "patches/@nangohq__frontend@0.33.8.patch" }, "peerDependencyRules": { "allowedVersions": { diff --git a/packages/cdk-core/frontend-utils.tsx b/packages/cdk-core/frontend-utils.tsx index a59b96fe..faadc058 100644 --- a/packages/cdk-core/frontend-utils.tsx +++ b/packages/cdk-core/frontend-utils.tsx @@ -18,7 +18,8 @@ export function useScript(src: string) { return deferred.current.promise // TODO: return loaded / error also } -export const CANCELLATION_TOKEN = 'CANCELLED' +/** Casting to `Error` type to suppress warning */ +export const CANCELLATION_TOKEN = 'CANCELLED' as unknown as Error /** Used by yodlee container among others */ export function DivContainer(props: { diff --git a/packages/engine-frontend/VeniceConnect.tsx b/packages/engine-frontend/VeniceConnect.tsx index b8fa5fb5..800d894e 100644 --- a/packages/engine-frontend/VeniceConnect.tsx +++ b/packages/engine-frontend/VeniceConnect.tsx @@ -1,5 +1,6 @@ 'use client' +import type {AuthError} from '@nangohq/frontend' import Nango from '@nangohq/frontend' import {useMutation} from '@tanstack/react-query' import {Link2, Loader2, RefreshCw, Trash2} from 'lucide-react' @@ -46,6 +47,10 @@ import {makeUlid, R, titleCase, z} from '@usevenice/util' import {_trpcReact} from './TRPCProvider' +function isNangoAuthError(err: unknown): err is AuthError { + return typeof err === 'object' && err != null && 'type' in err +} + type ConnectEventType = 'open' | 'close' | 'error' export interface VeniceConnectProps extends UIPropsNoChildren { @@ -130,6 +135,11 @@ export function VeniceConnect(props: VeniceConnectProps) { type Catalog = RouterOutput['getIntegrationCatalog'] type ProviderMeta = Catalog[string] +// TODOD: Dedupe this with app-config/constants +const __DEBUG__ = Boolean( + typeof window !== 'undefined' && window.location.hostname === 'localhost', +) + /** Need _VeniceConnect integrationIds to not have useConnectHook execute unreliably */ export function _VeniceConnect({ catalog, @@ -147,7 +157,9 @@ export function _VeniceConnect({ _trpcReact.getPublicEnv.useQuery().data?.NEXT_PUBLIC_NANGO_PUBLIC_KEY const nango = React.useMemo( - () => nangoPublicKey && new Nango({publicKey: nangoPublicKey}), + () => + nangoPublicKey && + new Nango({publicKey: nangoPublicKey, debug: __DEBUG__}), [nangoPublicKey], ) @@ -216,12 +228,14 @@ export function _VeniceConnect({ if (!nango) { throw new Error('Missing nango public key') } - return await nango.auth(integrationId, resoId).then((r) => { - console.log('auth', r) - if ('message' in r) { - throw new Error(`${r.type}: ${r.message}`) + return await nango.auth(integrationId, resoId).catch((err) => { + if (isNangoAuthError(err)) { + if (err.type === 'user_cancelled') { + throw CANCELLATION_TOKEN + } + throw new Error(`${err.type}: ${err.message}`) } - return r + throw err }) } } diff --git a/patches/@nangohq__frontend@0.33.8.patch b/patches/@nangohq__frontend@0.33.8.patch new file mode 100644 index 00000000..33006ba9 --- /dev/null +++ b/patches/@nangohq__frontend@0.33.8.patch @@ -0,0 +1,62 @@ +diff --git a/dist/index.d.ts b/dist/index.d.ts +index 91d370b7ada12e8b27cc6706151d676b27a72c20..bee8cc225dc1e621bbab0902e0d7648cb81e1845 100644 +--- a/dist/index.d.ts ++++ b/dist/index.d.ts +@@ -1,4 +1,5 @@ +-type AuthError = { ++/** Gets thrown by `create` and `auth` methods */ ++export type AuthError = { + message: string; + type: string; + }; +@@ -14,14 +15,16 @@ export default class Nango { + publicKey: string; + debug?: boolean; + }); ++ /** @throws `AuthError` */ + create(providerConfigKey: string, connectionId: string, connectionConfig: ConnectionConfig): Promise<{ + providerConfigKey: string; + connectionId: string; +- } | AuthError>; ++ }>; ++ /** @throws `AuthError` */ + auth(providerConfigKey: string, connectionId: string, conectionConfigOrCredentials?: ConnectionConfig | BasicApiCredentials | ApiKeyCredentials): Promise<{ + providerConfigKey: string; + connectionId: string; +- } | AuthError>; ++ }>; + convertCredentialsToConfig(credentials: BasicApiCredentials | ApiKeyCredentials): ConnectionConfig; + private apiAuth; + private toQueryString; +diff --git a/dist/index.js b/dist/index.js +index 3ee0e9e9a72cb4d3fd611066811dd494b6839c7f..0014509fc6df476c7e662f4e7a44bd5dc355523e 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -217,6 +217,7 @@ class AuthorizationModal { + this.swClient.onmessage = (message) => { + this.handleMessage(message, successHandler, errorHandler); + }; ++ this.errorHandler = errorHandler; + } + /** + * Handles the messages received from the Nango server via WebSocket. +@@ -269,6 +270,19 @@ class AuthorizationModal { + */ + open(wsClientId) { + this.modal.location = this.url + '&ws_client_id=' + wsClientId; ++ // @see https://github.com/NangoHQ/nango/pull/1073 for a real fix ++ // Can remove this patch once it merges into master and gets released ++ this.interval = setInterval(() => { ++ if (this.modal?.closed) { ++ setTimeout(() => { ++ if (this.swClient.readyState !== this.swClient.CLOSED) { ++ this.errorHandler('user_cancelled', 'User cancelled the authorization process.') ++ this.swClient.close(); ++ } ++ clearInterval(this.interval) ++ }, 500) ++ } ++ }, 500) + return this.modal; + } + /** diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fdeb4230..9eda1112 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,6 +18,9 @@ overrides: esbuild: 0.17.5 patchedDependencies: + '@nangohq/frontend@0.33.8': + hash: cscdyzeu5adridjxg5g7kdqy6q + path: patches/@nangohq__frontend@0.33.8.patch cac@6.7.12: hash: nsfmzcycabp3f3q7kw2uyer77u path: patches/cac@6.7.12.patch @@ -1318,7 +1321,7 @@ importers: dependencies: '@nangohq/frontend': specifier: 0.33.8 - version: 0.33.8 + version: 0.33.8(patch_hash=cscdyzeu5adridjxg5g7kdqy6q) '@tanstack/react-query': specifier: 4.28.0 version: 4.28.0(react-dom@18.2.0)(react@18.2.0) @@ -3883,9 +3886,10 @@ packages: engines: {node: '>=12'} dev: false - /@nangohq/frontend@0.33.8: + /@nangohq/frontend@0.33.8(patch_hash=cscdyzeu5adridjxg5g7kdqy6q): resolution: {integrity: sha512-iKTo4BDEXaKG6FbZzT9iACr07lGDBhmaS90+tsd92XrFpO8Gk7JGMp13lUS66mfwTcCXqPBXFVSkBvE0QyJi8Q==} dev: false + patched: true /@next/env@13.4.3: resolution: {integrity: sha512-pa1ErjyFensznttAk3EIv77vFbfSYT6cLzVRK5jx4uiRuCQo+m2wCFAREaHKIy63dlgvOyMlzh6R8Inu8H3KrQ==} From c4fbe91595b5ce6b4b8d8c9f5db7800aaa4ddbff Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 13 Oct 2023 14:07:15 -0700 Subject: [PATCH 43/80] Encapsulate all nango logic into oauthIntegration --- integrations/integration-qbo/QBOClient.ts | 348 +++++++++--------- integrations/integration-qbo/def.ts | 7 +- integrations/integration-qbo/server.ts | 3 +- packages/cdk-core/index.ts | 4 +- packages/cdk-core/{ => oauth}/NangoClient.ts | 0 packages/cdk-core/oauth/oauthIntegration.ts | 108 ++++++ packages/cdk-core/package.json | 1 + packages/cdk-core/providers.types.ts | 2 +- packages/engine-backend/router/adminRouter.ts | 32 +- .../engine-backend/router/endUserRouter.ts | 33 +- packages/engine-frontend/VeniceConnect.tsx | 40 +- pnpm-lock.yaml | 4 +- 12 files changed, 335 insertions(+), 247 deletions(-) rename packages/cdk-core/{ => oauth}/NangoClient.ts (100%) create mode 100644 packages/cdk-core/oauth/oauthIntegration.ts diff --git a/integrations/integration-qbo/QBOClient.ts b/integrations/integration-qbo/QBOClient.ts index b6b9da82..3c22d0e3 100644 --- a/integrations/integration-qbo/QBOClient.ts +++ b/integrations/integration-qbo/QBOClient.ts @@ -1,3 +1,4 @@ +import {oauthBaseSchema} from '@usevenice/cdk-core' import { createHTTPClient, DateTime, @@ -7,197 +8,210 @@ import { zFunction, } from '@usevenice/util' -export const zConfig = z.object({ - clientId: z.string(), - clientSecret: z.string(), - scope: z.string(), +export const zConfig = oauthBaseSchema.integrationConfig.extend({ envName: z.enum(['sandbox', 'production']), url: z.string().nullish().describe('For proxies, not typically needed'), verifierToken: z.string().nullish().describe('For webhooks'), }) -export const zCreds = z.object({ - sandbox: z.boolean().nullish(), - realmId: z.string(), - refreshToken: z.string(), - accessToken: z.string().nullish(), - accessTokenExpiresAt: z.string().nullish(), // ISODateTime - /** Informational for nwo. */ - refreshTokenExpiresAt: z.string().nullish(), // ISODateTime +const oReso = oauthBaseSchema.resourceSettings +/** Very verbose definition... Do we want it a bit simpler maybe? */ +export const zSettings = oReso.extend({ + oauth: oReso.shape.oauth.extend({ + connection_config: z.object({ + realmId: z.string(), + }), + credentials: oReso.shape.oauth.shape.credentials.extend({ + raw: oReso.shape.oauth.shape.credentials.shape.raw.extend({ + refresh_token: z.string(), + }), + }), + }), }) -export const makeQBOClient = zFunction([zConfig, zCreds], (config, creds) => { - const apiHost = - config.url ?? - (creds.sandbox - ? 'https://sandbox-quickbooks.api.intuit.com' - : 'https://quickbooks.api.intuit.com') +export const makeQBOClient = zFunction( + [zConfig, zSettings], + (config, settings) => { + const apiHost = + config.url ?? + (config.envName === 'sandbox' + ? 'https://sandbox-quickbooks.api.intuit.com' + : 'https://quickbooks.api.intuit.com') - function oauth2(config: {clientId: string; clientSecret: string}) { - return new OAuth2Client({ - clientId: config.clientId, - clientSecret: config.clientSecret, - authorizeURL: 'https://appcenter.intuit.com/connect/oauth2', - tokenURL: 'https://oauth.platform.intuit.com/oauth2/v1/tokens/bearer', - revokeUrl: 'https://developer.api.intuit.com/v2/oauth2/tokens/revoke', - clientAuthLocation: 'header', - errorToString: (err) => - err.error_description - ? `${err.error}: ${err.error_description}` - : err.error, - }) - } + const creds = { + realmId: settings.oauth.connection_config.realmId, + accessToken: settings.oauth.credentials.access_token, + accessTokenExpiresAt: settings.oauth.credentials.expires_at, + refreshToken: settings.oauth.credentials.raw.refresh_token, + } - function parseTokens(tokens: QBOOAuthTypes['tokens']) { - // TODO: Use intuit returned Date in header if we could. Looks like - // Date: Wed, 12 Aug 2020 14:25:39 GMT - // Need access to header also here - const now = DateTime.utc() - return { - accessToken: tokens.access_token, - refreshToken: tokens.refresh_token, - accessTokenExpiresAt: now.plus({seconds: tokens.expires_in}).toISO(), - refreshTokenExpiresAt: now - .plus({seconds: tokens.x_refresh_token_expires_in}) - .toISO(), + function oauth2() { + return new OAuth2Client({ + clientId: config.oauth.client_id, + clientSecret: config.oauth.client_secret, + authorizeURL: 'https://appcenter.intuit.com/connect/oauth2', + tokenURL: 'https://oauth.platform.intuit.com/oauth2/v1/tokens/bearer', + revokeUrl: 'https://developer.api.intuit.com/v2/oauth2/tokens/revoke', + clientAuthLocation: 'header', + errorToString: (err) => + err.error_description + ? `${err.error}: ${err.error_description}` + : err.error, + }) } - } - const http = createHTTPClient({ - baseURL: `${apiHost}/v3/company/${creds.realmId}/`, - headers: {'Content-Type': 'application/json'}, - requestTransformer: (req) => { - req.headers = { - ...(req.headers as Record), - Authorization: `Bearer ${creds.accessToken}`, + function parseTokens(tokens: QBOOAuthTypes['tokens']) { + // TODO: Use intuit returned Date in header if we could. Looks like + // Date: Wed, 12 Aug 2020 14:25:39 GMT + // Need access to header also here + const now = DateTime.utc() + return { + accessToken: tokens.access_token, + refreshToken: tokens.refresh_token, + accessTokenExpiresAt: now.plus({seconds: tokens.expires_in}).toISO(), + refreshTokenExpiresAt: now + .plus({seconds: tokens.x_refresh_token_expires_in}) + .toISO(), } - return req - }, - errorTransformer: (err) => - // if (err.response && err.response.data) { - // return new YodleeError(err.response.data, err) - // } - err, - refreshAuth: { - shouldProactiveRefresh: () => - creds.accessTokenExpiresAt - ? // Proactive refresh within 30 mins. - DateTime.fromISO(creds.accessTokenExpiresAt) >= - DateTime.utc().minus({minutes: 30}) - : false, - refresh: () => - oauth2(config) - .refreshToken(creds.refreshToken) - .then((res) => { - Object.assign(creds, parseTokens(res)) - }), - }, - }) + } - /** Prefix id with realmId to get id global within QBO provider */ - function globalId(id: string) { - return `${creds.realmId}_${id}` - } + const http = createHTTPClient({ + baseURL: `${apiHost}/v3/company/${creds.realmId}/`, + headers: {'Content-Type': 'application/json'}, + requestTransformer: (req) => { + req.headers = { + ...(req.headers as Record), + Authorization: `Bearer ${creds.accessToken}`, + } + return req + }, + errorTransformer: (err) => + // if (err.response && err.response.data) { + // return new YodleeError(err.response.data, err) + // } + err, + refreshAuth: { + shouldProactiveRefresh: () => + creds.accessTokenExpiresAt + ? // Proactive refresh within 30 mins. + DateTime.fromISO(creds.accessTokenExpiresAt) >= + DateTime.utc().minus({minutes: 30}) + : false, + refresh: () => + oauth2() + .refreshToken(creds.refreshToken) + .then((res) => { + Object.assign(creds, parseTokens(res)) + }), + }, + }) + + /** Prefix id with realmId to get id global within QBO provider */ + function globalId(id: string) { + return `${creds.realmId}_${id}` + } - const query = zFunction(z.string(), async (query) => - http.get('query', {params: {query}}).then((r) => r.data), - ) + const query = zFunction(z.string(), async (query) => + http + .get('query', {params: {query}}) + .then((r) => r.data), + ) - async function* getAll( - entityName: T, - /** Range is inclusive */ - params: {updatedSince?: ISODateTime} = {}, - ) { - let startPosition = 1 // QBO is 1 index based - // Fetch 100 transactions only on the first request to optimize for incremental - // sync scenarios - let maxResults = 100 - while (true) { - const res = await query( - `SELECT * FROM ${entityName} ${ - params.updatedSince - ? `WHERE MetaData.LastUpdatedTime >='${params.updatedSince}'` - : '' - } ORDERBY MetaData.LastUpdatedTime DESC + async function* getAll( + entityName: T, + /** Range is inclusive */ + params: {updatedSince?: ISODateTime} = {}, + ) { + let startPosition = 1 // QBO is 1 index based + // Fetch 100 transactions only on the first request to optimize for incremental + // sync scenarios + let maxResults = 100 + while (true) { + const res = await query( + `SELECT * FROM ${entityName} ${ + params.updatedSince + ? `WHERE MetaData.LastUpdatedTime >='${params.updatedSince}'` + : '' + } ORDERBY MetaData.LastUpdatedTime DESC STARTPOSITION ${startPosition} MAXRESULTS ${maxResults}`, - ) + ) - const entities = res.QueryResponse[entityName] ?? [] - yield { - // Hack needed for some reason - entities: entities as Exclude, - startPosition, - maxResults, + const entities = res.QueryResponse[entityName] ?? [] + yield { + // Hack needed for some reason + entities: entities as Exclude, + startPosition, + maxResults, + } + if (entities.length === 0) { + break + } + startPosition += entities.length + maxResults = 500 // Then fetch 500 fo efficiency } - if (entities.length === 0) { - break - } - startPosition += entities.length - maxResults = 500 // Then fetch 500 fo efficiency } - } - return { - globalId, - query, - read: zFunction( - [zCast(), z.union([z.string(), z.number()])], - async (entityName, id) => - http.get(`${entityName}/${id}`).then((r) => r.data), - ), - count: zFunction(zCast(), async (entity) => - query(`SELECT count(*) FROM ${entity}`).then( - (r) => r.QueryResponse.totalCount ?? -1, + return { + globalId, + query, + read: zFunction( + [zCast(), z.union([z.string(), z.number()])], + async (entityName, id) => + http.get(`${entityName}/${id}`).then((r) => r.data), ), - ), - getAll, - getChangeDataCapture: zFunction( - z.object({ - entities: zCast(), - changedSince: z.string(), - }), - async (params) => + count: zFunction(zCast(), async (entity) => + query(`SELECT count(*) FROM ${entity}`).then( + (r) => r.QueryResponse.totalCount ?? -1, + ), + ), + getAll, + getChangeDataCapture: zFunction( + z.object({ + entities: zCast(), + changedSince: z.string(), + }), + async (params) => + http + .get('cdc', { + params: {...params, entities: params.entities.join(',')}, + }) + .then((r) => r.data), + ), + getCompanyInfo: zFunction(async () => http - .get('cdc', { - params: {...params, entities: params.entities.join(',')}, - }) + .get(`companyinfo/${creds.realmId}`) .then((r) => r.data), - ), - getCompanyInfo: zFunction(async () => - http - .get(`companyinfo/${creds.realmId}`) - .then((r) => r.data), - ), - getPreferences: zFunction(async () => - http.get('preferences').then((r) => r.data), - ), - reportTransactionsList: zFunction(async () => - http - .get('reports/TransactionList') - .then((r) => r.data), - ), - revokeAccessToken: zFunction(async () => { - const oauth = oauth2(config) - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const res = await oauth.revokeToken(creds.accessToken!) - return res - }), + ), + getPreferences: zFunction(async () => + http.get('preferences').then((r) => r.data), + ), + reportTransactionsList: zFunction(async () => + http + .get('reports/TransactionList') + .then((r) => r.data), + ), + revokeAccessToken: zFunction(async () => { + const oauth = oauth2() + const res = await oauth.revokeToken(creds.accessToken) + return res + }), - refreshAccessToken: zFunction(async () => { - const oauth = oauth2(config) - const res = await oauth.refreshToken(creds.refreshToken) - creds.accessToken = res.access_token - return res - }), - SCOPES: { - accounting: 'com.intuit.quickbooks.accounting', - payment: 'com.intuit.quickbooks.payment', - payroll: 'com.intuit.quickbooks.payroll', - timetracking: 'com.intuit.quickbooks.payroll.timetracking', - benefits: 'com.intuit.quickbooks.payroll.benefits', - }, - } -}) + refreshAccessToken: zFunction(async () => { + const oauth = oauth2() + const res = await oauth.refreshToken(creds.refreshToken) + creds.accessToken = res.access_token + return res + }), + SCOPES: { + accounting: 'com.intuit.quickbooks.accounting', + payment: 'com.intuit.quickbooks.payment', + payroll: 'com.intuit.quickbooks.payroll', + timetracking: 'com.intuit.quickbooks.payroll.timetracking', + benefits: 'com.intuit.quickbooks.payroll.benefits', + }, + } + }, +) interface QBOOAuthTypes { error: {error: string; error_description?: string} diff --git a/integrations/integration-qbo/def.ts b/integrations/integration-qbo/def.ts index 16ac1377..d345c00e 100644 --- a/integrations/integration-qbo/def.ts +++ b/integrations/integration-qbo/def.ts @@ -1,16 +1,17 @@ import type {IntegrationDef, IntegrationSchemas} from '@usevenice/cdk-core' -import {intHelpers} from '@usevenice/cdk-core' +import {intHelpers, oauthBaseSchema} from '@usevenice/cdk-core' import {makePostingsMap} from '@usevenice/cdk-ledger' import type {Standard} from '@usevenice/standard' import type {EnumOf} from '@usevenice/util' import {A, DateTime, z, zCast} from '@usevenice/util' -import {zConfig, zCreds} from './QBOClient' +import {zConfig, zSettings} from './QBOClient' export const qboSchemas = { name: z.literal('qbo'), integrationConfig: zConfig, - resourceSettings: zCreds, + resourceSettings: zSettings, + connectOutput: oauthBaseSchema.connectOutput, sourceOutputEntity: z.discriminatedUnion('entityName', [ z.object({ id: z.string(), diff --git a/integrations/integration-qbo/server.ts b/integrations/integration-qbo/server.ts index 4d2ab632..fcbed570 100644 --- a/integrations/integration-qbo/server.ts +++ b/integrations/integration-qbo/server.ts @@ -8,6 +8,7 @@ import {makeQBOClient} from './QBOClient' export const qboServer = { sourceSync: ({config, settings}) => { const qbo = makeQBOClient(config, settings) + const realmId = settings.oauth.connection_config.realmId async function* iterateEntities() { for await (const res of qbo.getAll('Account')) { yield res.entities.map((a) => qboHelpers._opData('account', a.Id, a)) @@ -20,7 +21,7 @@ export const qboServer = { qboHelpers._opData('transaction', t.Id, { type: type as 'Purchase', entity: t as QBO.Purchase, - realmId: settings.realmId, + realmId, }), ) } diff --git a/packages/cdk-core/index.ts b/packages/cdk-core/index.ts index 7350c9f3..317bedc6 100644 --- a/packages/cdk-core/index.ts +++ b/packages/cdk-core/index.ts @@ -1,3 +1,6 @@ +export * from './oauth/NangoClient' +export * from './oauth/oauthIntegration' + // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{spec,test,fixture}.{ts,tsx}"} export * from './base-links' export * from './frontend-utils' @@ -7,7 +10,6 @@ export * from './integration.types' export * from './kvStore' export * from './meta.types' export * from './metaService' -export * from './NangoClient' export * from './NoopMetaService' export * from './protocol' export * from './providers.types' diff --git a/packages/cdk-core/NangoClient.ts b/packages/cdk-core/oauth/NangoClient.ts similarity index 100% rename from packages/cdk-core/NangoClient.ts rename to packages/cdk-core/oauth/NangoClient.ts diff --git a/packages/cdk-core/oauth/oauthIntegration.ts b/packages/cdk-core/oauth/oauthIntegration.ts new file mode 100644 index 00000000..2f07b023 --- /dev/null +++ b/packages/cdk-core/oauth/oauthIntegration.ts @@ -0,0 +1,108 @@ +import type NangoFrontend from '@nangohq/frontend' +import type {AuthError} from '@nangohq/frontend' + +import {HTTPError, makeUlid, z} from '@usevenice/util' + +import {CANCELLATION_TOKEN} from '../frontend-utils' +import type {Id} from '../id.types' +import {extractId, makeId, zId} from '../id.types' +import type { + IntegrationSchemas, + IntegrationServer, + IntHelpers, +} from '../integration.types' +import type {NangoClient, NangoProvider, UpsertIntegration} from './NangoClient' +import {zConnection, zIntegration} from './NangoClient' + +export const oauthBaseSchema = { + name: z.literal('__oauth__'), // TODO: This is a noop + integrationConfig: z.object({ + oauth: zIntegration.pick({ + client_id: true, + client_secret: true, + scopes: true, + }), + }), + resourceSettings: z.object({ + oauth: zConnection.pick({ + credentials: true, + connection_config: true, + metadata: true, + }), + }), + connectOutput: z.object({ + providerConfigKey: zId('int'), + connectionId: zId('reso'), + }), +} satisfies IntegrationSchemas + +export type OauthBaseTypes = IntHelpers['_types'] + +function isNangoAuthError(err: unknown): err is AuthError { + return typeof err === 'object' && err != null && 'type' in err +} + +export function oauthConnect({ + nangoFrontend, + providerName, + integrationId, +}: { + nangoFrontend: NangoFrontend + providerName: string + integrationId: Id['int'] +}): Promise { + return nangoFrontend + .auth(integrationId, makeId('reso', providerName, makeUlid())) + .then((r) => oauthBaseSchema.connectOutput.parse(r)) + .catch((err) => { + if (isNangoAuthError(err)) { + if (err.type === 'user_cancelled') { + throw CANCELLATION_TOKEN + } + throw new Error(`${err.type}: ${err.message}`) + } + throw err + }) +} + +export function makeOauthIntegrationServer({ + nangoClient, + nangoProvider, + intId, +}: { + nangoClient: NangoClient + nangoProvider: NangoProvider + intId: Id['int'] +}) { + const intServer = { + async postConnect(connectOutput) { + const {connectionId: resoId} = connectOutput + const res = await nangoClient.get('/connection/{connectionId}', { + path: {connectionId: resoId}, + query: {provider_config_key: intId, refresh_token: true}, + }) + return {resourceExternalId: extractId(resoId)[2], settings: {oauth: res}} + }, + } satisfies IntegrationServer + return { + ...intServer, + upsertIntegration: async (config: OauthBaseTypes['integrationConfig']) => { + const bodyJson: UpsertIntegration = { + provider_config_key: intId, + provider: nangoProvider, + oauth_client_id: config.oauth.client_id, + oauth_client_secret: config.oauth.client_secret, + oauth_scopes: config.oauth.scopes, + } + await nangoClient.put('/config', {bodyJson}).catch((err) => { + if ( + err instanceof HTTPError && + err.response?.data.type === 'unknown_provider_config' + ) { + return nangoClient.post('/config', {bodyJson}) + } + throw err + }) + }, + } +} diff --git a/packages/cdk-core/package.json b/packages/cdk-core/package.json index accde330..61f0896f 100644 --- a/packages/cdk-core/package.json +++ b/packages/cdk-core/package.json @@ -11,6 +11,7 @@ "react-script-hook": "1.7.2" }, "devDependencies": { + "@nangohq/frontend": "0.33.8", "@types/jsonwebtoken": "9.0.2", "@types/react": "*" }, diff --git a/packages/cdk-core/providers.types.ts b/packages/cdk-core/providers.types.ts index 13056171..2a2600aa 100644 --- a/packages/cdk-core/providers.types.ts +++ b/packages/cdk-core/providers.types.ts @@ -14,7 +14,7 @@ import type { IntegrationSchemas, IntHelpers, } from './integration.types' -import type {NangoProvider} from './NangoClient' +import type {NangoProvider} from './oauth/NangoClient' import type {AnyEntityPayload, ResoUpdateData, Source} from './protocol' // eslint-disable-next-line @typescript-eslint/consistent-type-definitions diff --git a/packages/engine-backend/router/adminRouter.ts b/packages/engine-backend/router/adminRouter.ts index 20059862..7734f62f 100644 --- a/packages/engine-backend/router/adminRouter.ts +++ b/packages/engine-backend/router/adminRouter.ts @@ -1,17 +1,17 @@ import {TRPCError} from '@trpc/server' -import type { - UpsertIntegration} from '@usevenice/cdk-core'; import { extractProviderName, handlersLink, makeId, + makeOauthIntegrationServer, + oauthBaseSchema, sync, zEndUserId, zId, zRaw, } from '@usevenice/cdk-core' -import {HTTPError, makeUlid, rxjs, z} from '@usevenice/util' +import {makeUlid, rxjs, z} from '@usevenice/util' import {adminProcedure, trpc} from './_base' @@ -115,25 +115,13 @@ export const adminRouter = trpc.router({ }) } if (provider.metadata?.nangoProvider) { - // TODO: Should we use put vs. post? need to fix it up here... - // Create nango integration here... - const bodyJson: UpsertIntegration = { - provider_config_key: id, - provider: provider.metadata.nangoProvider, - // TODO: gotta fix the typing here... - oauth_client_id: (input.config as any).clientId, - oauth_client_secret: (input.config as any).clientSecret, - oauth_scopes: (input.config as any).scope, - } - await ctx.nango.put('/config', {bodyJson}).catch((err) => { - if ( - err instanceof HTTPError && - err.response?.data.type === 'unknown_provider_config' - ) { - return ctx.nango.post('/config', {bodyJson}) - } - throw err - }) + await makeOauthIntegrationServer({ + intId: id, + nangoClient: ctx.nango, + nangoProvider: provider.metadata.nangoProvider, + }).upsertIntegration( + oauthBaseSchema.integrationConfig.parse(input.config), + ) } return ctx.helpers.patchReturning('integration', id, input) diff --git a/packages/engine-backend/router/endUserRouter.ts b/packages/engine-backend/router/endUserRouter.ts index e0474e3d..05084e0b 100644 --- a/packages/engine-backend/router/endUserRouter.ts +++ b/packages/engine-backend/router/endUserRouter.ts @@ -1,7 +1,7 @@ -import type {ResourceUpdate} from '@usevenice/cdk-core' +import type {OauthBaseTypes, ResourceUpdate} from '@usevenice/cdk-core' import { - extractId, makeId, + makeOauthIntegrationServer, zConnectOptions, zId, zPostConnectOptions, @@ -79,27 +79,14 @@ export const endUserRouter = trpc.router({ !int.provider.postConnect && int.provider.metadata?.nangoProvider ) { - const {connectionId: resoId} = z - .object({ - providerConfigKey: z.string(), - connectionId: zId('reso'), - }) - .parse(input) - const result = await ctx.nango.get('/connection/{connectionId}', { - path: {connectionId: resoId}, - query: {provider_config_key: intId, refresh_token: true}, - }) - - return { - resourceExternalId: extractId(resoId)[2], - settings: { - nango: result, - accessToken: result.credentials.access_token, - accessTokenExpiresAt: result.credentials.expires_at, - refreshToken: result.credentials.raw.refresh_token, - realmId: result.connection_config['realmId'], - }, - } satisfies Omit, 'endUserId'> + return (await makeOauthIntegrationServer({ + nangoClient: ctx.nango, + intId, + nangoProvider: int.provider.metadata.nangoProvider, + }).postConnect(input as OauthBaseTypes['connectOutput'])) as Omit< + ResourceUpdate, + 'endUserId' + > } if (!int.provider.postConnect || !int.provider.def.connectOutput) { diff --git a/packages/engine-frontend/VeniceConnect.tsx b/packages/engine-frontend/VeniceConnect.tsx index 800d894e..aa3469a4 100644 --- a/packages/engine-frontend/VeniceConnect.tsx +++ b/packages/engine-frontend/VeniceConnect.tsx @@ -1,7 +1,6 @@ 'use client' -import type {AuthError} from '@nangohq/frontend' -import Nango from '@nangohq/frontend' +import NangoFrontend from '@nangohq/frontend' import {useMutation} from '@tanstack/react-query' import {Link2, Loader2, RefreshCw, Trash2} from 'lucide-react' import React from 'react' @@ -16,7 +15,7 @@ import { CANCELLATION_TOKEN, extractId, extractProviderName, - makeId, + oauthConnect, zIntegrationCategory, } from '@usevenice/cdk-core' import type {RouterInput, RouterOutput} from '@usevenice/engine-backend' @@ -43,14 +42,10 @@ import { useToast, } from '@usevenice/ui' import {cn} from '@usevenice/ui/utils' -import {makeUlid, R, titleCase, z} from '@usevenice/util' +import {R, titleCase, z} from '@usevenice/util' import {_trpcReact} from './TRPCProvider' -function isNangoAuthError(err: unknown): err is AuthError { - return typeof err === 'object' && err != null && 'type' in err -} - type ConnectEventType = 'open' | 'close' | 'error' export interface VeniceConnectProps extends UIPropsNoChildren { @@ -156,10 +151,10 @@ export function _VeniceConnect({ const nangoPublicKey = _trpcReact.getPublicEnv.useQuery().data?.NEXT_PUBLIC_NANGO_PUBLIC_KEY - const nango = React.useMemo( + const nangoFrontend = React.useMemo( () => nangoPublicKey && - new Nango({publicKey: nangoPublicKey, debug: __DEBUG__}), + new NangoFrontend({publicKey: nangoPublicKey, debug: __DEBUG__}), [nangoPublicKey], ) @@ -216,30 +211,19 @@ export function _VeniceConnect({ integrationIds, R.map(extractProviderName), R.uniq, - R.mapToObj((name: string) => { - let fn = clientIntegrations[name]?.useConnectHook?.({openDialog}) - const nangoProvider = catalog[name]?.nangoProvider + R.mapToObj((providerName: string) => { + let fn = clientIntegrations[providerName]?.useConnectHook?.({openDialog}) + const nangoProvider = catalog[providerName]?.nangoProvider if (!fn && nangoProvider) { console.log('adding nnango provider for', nangoProvider) - - fn = async (_, {integrationId}) => { - console.log('inputs', integrationId) - const resoId = makeId('reso', name, makeUlid()) - if (!nango) { + fn = (_, {integrationId}) => { + if (!nangoFrontend) { throw new Error('Missing nango public key') } - return await nango.auth(integrationId, resoId).catch((err) => { - if (isNangoAuthError(err)) { - if (err.type === 'user_cancelled') { - throw CANCELLATION_TOKEN - } - throw new Error(`${err.type}: ${err.message}`) - } - throw err - }) + return oauthConnect({integrationId, nangoFrontend, providerName}) } } - return [name, fn] + return [providerName, fn] }), ) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9eda1112..19fe23bb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1270,6 +1270,9 @@ importers: specifier: 1.7.2 version: 1.7.2(react-dom@18.2.0)(react@18.2.0) devDependencies: + '@nangohq/frontend': + specifier: 0.33.8 + version: 0.33.8(patch_hash=cscdyzeu5adridjxg5g7kdqy6q) '@types/jsonwebtoken': specifier: 9.0.2 version: 9.0.2 @@ -3888,7 +3891,6 @@ packages: /@nangohq/frontend@0.33.8(patch_hash=cscdyzeu5adridjxg5g7kdqy6q): resolution: {integrity: sha512-iKTo4BDEXaKG6FbZzT9iACr07lGDBhmaS90+tsd92XrFpO8Gk7JGMp13lUS66mfwTcCXqPBXFVSkBvE0QyJi8Q==} - dev: false patched: true /@next/env@13.4.3: From 2be4e00578053ecd055b05b507881adf070684d1 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 13 Oct 2023 14:55:50 -0700 Subject: [PATCH 44/80] Fix a few crashes during QBO sync --- integrations/integration-qbo/def.ts | 26 ++++++++++++++------------ integrations/integration-qbo/qbo.d.ts | 12 ++++++------ packages/cdk-core/oauth/NangoClient.ts | 2 +- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/integrations/integration-qbo/def.ts b/integrations/integration-qbo/def.ts index d345c00e..f490db25 100644 --- a/integrations/integration-qbo/def.ts +++ b/integrations/integration-qbo/def.ts @@ -89,10 +89,13 @@ export const qboDef = { } for (const l of t.entity.Line) { postings[l.Id] = { - accountExternalId: globalId( - t.realmId, - l.AccountBasedExpenseLineDetail.AccountRef.value, - ), + // TODO: Handle non-accountBasedExpenseLineDetail + accountExternalId: + l.AccountBasedExpenseLineDetail && + globalId( + t.realmId, + l.AccountBasedExpenseLineDetail.AccountRef.value, + ), amount: A(-1 * sign * l.Amount, t.entity.CurrencyRef.value), memo: l.Description, } @@ -132,10 +135,10 @@ export const qboDef = { } for (const l of t.entity.Line) { postings[l.Id] = { - accountExternalId: globalId( - t.realmId, - l.DepositLineDetail.AccountRef.value, - ), + // Handle https://gist.github.com/tonyxiao/a9873b41c2df76f4f66c226933134a82 + accountExternalId: + l.DepositLineDetail?.AccountRef && + globalId(t.realmId, l.DepositLineDetail.AccountRef.value), amount: A(-1 * l.Amount, t.entity.CurrencyRef.value), memo: l.Description, } @@ -265,10 +268,9 @@ export const qboDef = { // QBO uses a default accounts receivable account. but it does not appear possible to know // exactly the id of the default account. therefore we wiill have to make do... // https://c9.qbo.intuit.com/app/invoice?txnId=3968 for instance - accountExternalId: globalId( - t.realmId, - t.entity.DepositToAccountRef.value, - ), + accountExternalId: + t.entity.DepositToAccountRef && + globalId(t.realmId, t.entity.DepositToAccountRef.value), }, remainder: { accountType: 'asset/accounts_receivable', diff --git a/integrations/integration-qbo/qbo.d.ts b/integrations/integration-qbo/qbo.d.ts index 24eebee6..750c04e3 100644 --- a/integrations/integration-qbo/qbo.d.ts +++ b/integrations/integration-qbo/qbo.d.ts @@ -228,7 +228,7 @@ declare namespace QBO { Description: string Amount: number DetailType: string - AccountBasedExpenseLineDetail: AccountBasedExpenseLineDetail + AccountBasedExpenseLineDetail?: AccountBasedExpenseLineDetail } export interface AccountBasedExpenseLineDetail { @@ -242,7 +242,7 @@ declare namespace QBO { export interface Deposit extends _BaseEntity { CurrencyRef: CurrencyRef DepositToAccountRef: DepositToAccountRef - Line: Line[] + Line: DepositLine[] PrivateNote: string SyncToken: string TotalAmt: number @@ -256,9 +256,9 @@ declare namespace QBO { value: string } - export interface Line { + export interface DepositLine { Amount: number - DepositLineDetail: DepositLineDetail + DepositLineDetail?: DepositLineDetail Description: string DetailType: string Id: string @@ -266,7 +266,7 @@ declare namespace QBO { } export interface DepositLineDetail { - AccountRef: AccountRef + AccountRef?: AccountRef Entity?: Entity } @@ -337,7 +337,7 @@ declare namespace QBO { export interface Payment extends _BaseEntity { CurrencyRef: CurrencyRef CustomerRef: CustomerRef - DepositToAccountRef: {value: string} + DepositToAccountRef?: {value: string} Line: PaymentLine[] PrivateNote: string ProcessPayment: boolean diff --git a/packages/cdk-core/oauth/NangoClient.ts b/packages/cdk-core/oauth/NangoClient.ts index f85f9024..305ea83d 100644 --- a/packages/cdk-core/oauth/NangoClient.ts +++ b/packages/cdk-core/oauth/NangoClient.ts @@ -178,7 +178,7 @@ export const zConnection = zConnectionShort.extend({ refresh_token: z.string().nullish(), refresh_token_expires_in: z.number().nullish(), token_type: z.string(), //'bearer', - scope: z.string(), + scope: z.string().optional(), }), }), connection_config: z.record(z.unknown()), From 3e418341fee60f4aa0366b12f7bc1ac3e88277f1 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 13 Oct 2023 15:01:37 -0700 Subject: [PATCH 45/80] Allow resourceId to be passed in --- packages/cdk-core/oauth/oauthIntegration.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/cdk-core/oauth/oauthIntegration.ts b/packages/cdk-core/oauth/oauthIntegration.ts index 2f07b023..414a85ba 100644 --- a/packages/cdk-core/oauth/oauthIntegration.ts +++ b/packages/cdk-core/oauth/oauthIntegration.ts @@ -46,13 +46,16 @@ export function oauthConnect({ nangoFrontend, providerName, integrationId, + resourceId, }: { nangoFrontend: NangoFrontend providerName: string integrationId: Id['int'] + /** Should address the re-connect scenario, but let's see... */ + resourceId?: Id['reso'] }): Promise { return nangoFrontend - .auth(integrationId, makeId('reso', providerName, makeUlid())) + .auth(integrationId, resourceId ?? makeId('reso', providerName, makeUlid())) .then((r) => oauthBaseSchema.connectOutput.parse(r)) .catch((err) => { if (isNangoAuthError(err)) { From 4cbc13250437fd6e79e18c3af2251c8636ae63bf Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sat, 14 Oct 2023 00:35:44 -0700 Subject: [PATCH 46/80] Got openAPI working at /api/openapi --- apps/web/lib-server/appRouter.ts | 21 ++- apps/web/package.json | 3 +- apps/web/pages/api/openapi/[[...trpc]].ts | 27 +++ apps/web/pages/api/trpc/[...trpc].ts | 24 ++- packages/engine-backend/package.json | 4 +- packages/engine-backend/router/_base.ts | 6 +- packages/engine-backend/router/adminRouter.ts | 8 +- .../engine-backend/router/publicRouter.ts | 8 +- pnpm-lock.yaml | 162 ++++++++++++++++-- 9 files changed, 227 insertions(+), 36 deletions(-) create mode 100644 apps/web/pages/api/openapi/[[...trpc]].ts diff --git a/apps/web/lib-server/appRouter.ts b/apps/web/lib-server/appRouter.ts index 512b9dd7..2f1efa3c 100644 --- a/apps/web/lib-server/appRouter.ts +++ b/apps/web/lib-server/appRouter.ts @@ -1,8 +1,15 @@ import {clerkClient} from '@clerk/nextjs' import {TRPCError} from '@trpc/server' +import {generateOpenApiDocument} from 'trpc-openapi' +import {getServerUrl} from '@usevenice/app-config/constants' import {flatRouter} from '@usevenice/engine-backend' -import {adminProcedure, trpc} from '@usevenice/engine-backend/router/_base' +import { + adminProcedure, + publicProcedure, + trpc, +} from '@usevenice/engine-backend/router/_base' +import {z} from '@usevenice/util' import {zAuth} from '@/lib-common/schemas' @@ -16,8 +23,20 @@ const customRouter = trpc.router({ const org = await clerkClient.organizations.updateOrganization(id, update) return org }), + + getOpenapiDocument: publicProcedure + .meta({openapi: {method: 'GET', path: '/'}}) + .input(z.void()) + .output(z.unknown()) + .query(() => openApiDocument), }) export const appRouter = trpc.mergeRouters(flatRouter, customRouter) +export const openApiDocument = generateOpenApiDocument(appRouter, { + title: 'Venice OpenAPI', + version: '0.0.0', + baseUrl: getServerUrl(null) + '/api/openapi', +}) + export type AppRouter = typeof appRouter diff --git a/apps/web/package.json b/apps/web/package.json index a73949d9..c9b49a3e 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -47,7 +47,8 @@ "react": "18.2.0", "react-dom": "18.2.0", "superjson": "1.9.1", - "swagger2openapi": "7.0.8" + "swagger2openapi": "7.0.8", + "trpc-openapi": "1.2.0" }, "devDependencies": { "@sentry/cli": "2.13.0", diff --git a/apps/web/pages/api/openapi/[[...trpc]].ts b/apps/web/pages/api/openapi/[[...trpc]].ts new file mode 100644 index 00000000..e8c937a1 --- /dev/null +++ b/apps/web/pages/api/openapi/[[...trpc]].ts @@ -0,0 +1,27 @@ +import '@usevenice/app-config/register.node' + +import type {NextApiHandler} from 'next' +import {createOpenApiNextHandler} from 'trpc-openapi' + +import {respondToCORS} from '@/lib-server' +import {appRouter} from '@/lib-server/appRouter' + +import {createContext, onError} from '../trpc/[...trpc]' + +const handler = createOpenApiNextHandler({ + // not sure why this cast is needed... + router: appRouter as any, + createContext, + onError, +}) + +export default (function trpcHandler(req, res) { + if (respondToCORS(req, res)) { + return + } + // allow the root document to work. + if (!req.query['trpc']) { + req.query['trpc'] = '' + } + return handler(req, res) +} satisfies NextApiHandler) diff --git a/apps/web/pages/api/trpc/[...trpc].ts b/apps/web/pages/api/trpc/[...trpc].ts index e9996cd3..e30933bd 100644 --- a/apps/web/pages/api/trpc/[...trpc].ts +++ b/apps/web/pages/api/trpc/[...trpc].ts @@ -9,16 +9,24 @@ import {parseWebhookRequest} from '@usevenice/engine-backend' import {appRouter} from '@/lib-server/appRouter' import {respondToCORS, serverGetViewer} from '@/lib-server/server-helpers' +export const createContext: Parameters< + typeof trpcNext.createNextApiHandler +>[0]['createContext'] = async ({req, res}) => { + const viewer = await serverGetViewer({req, res}) + console.log('[trpc.createContext]', {query: req.query, viewer}) + return contextFactory.fromViewer(viewer) +} + +export const onError: Parameters< + typeof trpcNext.createNextApiHandler +>[0]['onError'] = ({error}) => { + console.warn('error', error) +} + const handler = trpcNext.createNextApiHandler({ router: appRouter, - createContext: async ({req, res}) => { - const viewer = await serverGetViewer({req, res}) - console.log('[trpc.createContext]', {query: req.query, viewer}) - return contextFactory.fromViewer(viewer) - }, - onError: ({error}) => { - console.warn('error', error) - }, + createContext, + onError, }) export default (function trpcHandler(req, res) { diff --git a/packages/engine-backend/package.json b/packages/engine-backend/package.json index b0115f8a..c292eaa2 100644 --- a/packages/engine-backend/package.json +++ b/packages/engine-backend/package.json @@ -11,5 +11,7 @@ "@usevenice/util": "workspace:*", "inngest": "1.3.1" }, - "devDependencies": {} + "devDependencies": { + "trpc-openapi": "1.2.0" + } } diff --git a/packages/engine-backend/router/_base.ts b/packages/engine-backend/router/_base.ts index 7232006a..c7bc8549 100644 --- a/packages/engine-backend/router/_base.ts +++ b/packages/engine-backend/router/_base.ts @@ -1,15 +1,15 @@ import {initTRPC, TRPCError} from '@trpc/server' +import type {OpenApiMeta} from 'trpc-openapi' import {getExtEndUserId, hasRole} from '@usevenice/cdk-core' import type {RouterContext} from '../context' -/** TODO: Use OpenApiMeta from https://github.com/jlalmes/trpc-openapi */ -interface OpenApiMeta {} +interface RouterMeta extends OpenApiMeta {} export const trpc = initTRPC .context() - .meta() + .meta() // For client side to be able to import runtime schema from server side also .create({allowOutsideOfServer: true}) diff --git a/packages/engine-backend/router/adminRouter.ts b/packages/engine-backend/router/adminRouter.ts index 7734f62f..ccd68350 100644 --- a/packages/engine-backend/router/adminRouter.ts +++ b/packages/engine-backend/router/adminRouter.ts @@ -54,9 +54,11 @@ export const adminRouterSchema = { } satisfies Record export const adminRouter = trpc.router({ - adminListIntegrations: adminProcedure.query(async ({ctx}) => - ctx.helpers.list('integration', {}), - ), + adminListIntegrations: adminProcedure + .meta({openapi: {method: 'GET', path: '/integrations'}}) + .input(z.void()) + .output(z.array(zRaw.integration)) + .query(async ({ctx}) => ctx.helpers.list('integration', {})), adminUpsertPipeline: adminProcedure .input( zRaw.pipeline diff --git a/packages/engine-backend/router/publicRouter.ts b/packages/engine-backend/router/publicRouter.ts index d5f3da68..78293626 100644 --- a/packages/engine-backend/router/publicRouter.ts +++ b/packages/engine-backend/router/publicRouter.ts @@ -1,10 +1,14 @@ import {metaForProvider} from '@usevenice/cdk-core' -import {R} from '@usevenice/util' +import {R, z} from '@usevenice/util' import {publicProcedure, trpc} from './_base' export const publicRouter = trpc.router({ - health: publicProcedure.query(() => 'Ok ' + new Date().toISOString()), + health: publicProcedure + .meta({openapi: {method: 'GET', path: '/health'}}) + .input(z.void()) + .output(z.string()) + .query(() => 'Ok ' + new Date().toISOString()), getIntegrationCatalog: publicProcedure.query(({ctx}) => R.mapValues(ctx.providerMap, (provider) => metaForProvider(provider)), ), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 19fe23bb..dcfc7143 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -587,6 +587,9 @@ importers: swagger2openapi: specifier: 7.0.8 version: 7.0.8 + trpc-openapi: + specifier: 1.2.0 + version: 1.2.0(@trpc/server@10.21.1)(zod@3.21.4) devDependencies: '@sentry/cli': specifier: 2.13.0 @@ -1319,6 +1322,10 @@ importers: inngest: specifier: 1.3.1 version: 1.3.1 + devDependencies: + trpc-openapi: + specifier: 1.2.0 + version: 1.2.0(@trpc/server@10.21.1)(zod@3.21.4) packages/engine-frontend: dependencies: @@ -6163,7 +6170,6 @@ packages: /@trpc/server@10.21.1: resolution: {integrity: sha512-4E4upTD2/Aq0Dm9HW4nbKSoIlqbV/ajRqNl1MCFBd9FjTv47qq2ncmAWMbb0AIVh2UV87PJlTjpPNNecRm6g9Q==} - dev: false /@tsconfig/strictest@1.0.2: resolution: {integrity: sha512-IRKlC8cnP7zMz1SDBjyIVyPapkEGWLZ6wkF6Z8T+xU80P9sO5uGXlIUvtzjx+7ehPJRWxkB6CeIDwUfyqNtYkQ==} @@ -6976,6 +6982,13 @@ packages: resolution: {integrity: sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ==} dev: false + /accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + /accounting@0.4.1: resolution: {integrity: sha512-RU6KY9Y5wllyaCNBo1W11ZOTnTHMMgOZkIwdOOs6W5ibMTp72i4xIbEA48djxVGqMNTUNbvrP/1nWg5Af5m2gQ==} dev: false @@ -7764,7 +7777,6 @@ packages: /bytes@3.1.0: resolution: {integrity: sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==} engines: {node: '>= 0.8'} - dev: false /cac@6.7.12(patch_hash=nsfmzcycabp3f3q7kw2uyer77u): resolution: {integrity: sha512-rM7E2ygtMkJqD9c7WnFU6fruFcN3xe4FM5yUmgxhZzIKJk4uHl9U/fhwdajGFQbQuv43FAUo1Fe8gX/oIKDeSA==} @@ -8095,6 +8107,14 @@ packages: - '@types/react' dev: false + /co-body@6.1.0: + resolution: {integrity: sha512-m7pOT6CdLN7FuXUcpuz/8lfQ/L77x8SchHCF4G0RBTJO20Wzmhn5Sp4/5WsKy8OSpifBSUrmg83qEqaDHdyFuQ==} + dependencies: + inflation: 2.1.0 + qs: 6.11.1 + raw-body: 2.4.1 + type-is: 1.6.18 + /co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} @@ -8288,10 +8308,20 @@ packages: dev: false optional: true + /consola@3.2.3: + resolution: {integrity: sha512-I5qxpzLv+sJhTVEoLYNcTW+bThDCPsit0vLNKShZx6rLtpilNpmmeTPaeqJb9ZE9dV3DGaeby6Vuhrw38WjeyQ==} + engines: {node: ^14.18.0 || >=16.10.0} + /console-control-strings@1.1.0: resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==} dev: false + /content-disposition@0.5.4: + resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} + engines: {node: '>= 0.6'} + dependencies: + safe-buffer: 5.2.1 + /content-type@1.0.4: resolution: {integrity: sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==} engines: {node: '>= 0.6'} @@ -8306,6 +8336,9 @@ packages: resolution: {integrity: sha512-RyZrFi6PNpBFbIaQjXDlFIhFVqV42QeKSZX1yQIl6ihImq6vcHNGMtqQ/QzY3RMPuYSkvsRwtnt5M9NeYxKt0g==} dev: false + /cookie-es@1.0.0: + resolution: {integrity: sha512-mWYvfOLrfEc996hlKcdABeIiPHUPC6DM2QYZdGGOvhOTbA3tjm2eBwqlJpoFdjC89NI4Qt6h0Pu06Mp+1Pj5OQ==} + /cookie@0.4.2: resolution: {integrity: sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==} engines: {node: '>= 0.6'} @@ -8650,6 +8683,9 @@ packages: isobject: 3.0.1 dev: true + /defu@6.1.2: + resolution: {integrity: sha512-+uO4+qr7msjNNWKYPHqN/3+Dx3NFkmIzayk2L1MyZQlvgZb/J1A0fo410dpKrN2SnqFjt8n4JL8fDJE0wIgjFQ==} + /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -8672,12 +8708,14 @@ packages: /depd@1.1.2: resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} engines: {node: '>= 0.6'} - dev: false /destr@1.2.2: resolution: {integrity: sha512-lrbCJwD9saUQrqUfXvl6qoM+QN3W7tLV5pAOs+OqOmopCCz/JkE05MHedJR1xfk4IAnZuJXPVuN5+7jNA2ZCiA==} dev: false + /destr@2.0.1: + resolution: {integrity: sha512-M1Ob1zPSIvlARiJUkKqvAZ3VAqQY6Jcuth/pBKQ2b1dX/Qx0OnJ8Vux6J2H5PTMQeRzWrrbTu70VxBfv/OPDJA==} + /detect-indent@6.1.0: resolution: {integrity: sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==} engines: {node: '>=8'} @@ -9957,6 +9995,10 @@ packages: map-cache: 0.2.2 dev: true + /fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + /from@0.1.7: resolution: {integrity: sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==} dev: false @@ -10439,6 +10481,18 @@ packages: ufo: 1.0.1 dev: false + /h3@1.8.2: + resolution: {integrity: sha512-1Ca0orJJlCaiFY68BvzQtP2lKLk46kcLAxVM8JgYbtm2cUg6IY7pjpYgWMwUvDO9QI30N5JAukOKoT8KD3Q0PQ==} + dependencies: + cookie-es: 1.0.0 + defu: 6.1.2 + destr: 2.0.1 + iron-webcrypto: 0.10.1 + radix3: 1.1.0 + ufo: 1.3.1 + uncrypto: 0.1.3 + unenv: 1.7.4 + /handlebars@4.7.7: resolution: {integrity: sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==} engines: {node: '>=0.4.7'} @@ -10683,7 +10737,6 @@ packages: setprototypeof: 1.1.1 statuses: 1.5.0 toidentifier: 1.0.0 - dev: false /http-parser-js@0.5.8: resolution: {integrity: sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==} @@ -10796,7 +10849,6 @@ packages: engines: {node: '>=0.10.0'} dependencies: safer-buffer: 2.1.2 - dev: false /iconv-lite@0.6.3: resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} @@ -10852,6 +10904,10 @@ packages: engines: {node: '>=8'} dev: true + /inflation@2.1.0: + resolution: {integrity: sha512-t54PPJHG1Pp7VQvxyVCJ9mBbjG3Hqryges9bXoOO6GExCPa+//i/d5GSuFtpx3ALLd7lgIAur6zrIlBQyJuMlQ==} + engines: {node: '>= 0.8.0'} + /inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} dependencies: @@ -10945,6 +11001,9 @@ packages: resolution: {integrity: sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==} dev: false + /iron-webcrypto@0.10.1: + resolution: {integrity: sha512-QGOS8MRMnj/UiOa+aMIgfyHcvkhqNUsUxb1XzskENvbo+rEfp6TOwqd1KPuDzXC4OnGHcMSVxDGRoilqB8ViqA==} + /is-accessor-descriptor@0.1.6: resolution: {integrity: sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==} engines: {node: '>=0.10.0'} @@ -12897,7 +12956,6 @@ packages: /media-typer@0.3.0: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} - dev: false /memory-pager@1.5.0: resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} @@ -12910,6 +12968,9 @@ packages: engines: {node: '>= 0.10.0'} dev: true + /merge-descriptors@1.0.1: + resolution: {integrity: sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==} + /merge-stream@2.0.0: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} @@ -12929,6 +12990,10 @@ packages: '@types/node': 18.11.18 dev: false + /methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + /micro-memoize@4.0.10(patch_hash=hko3q5jvea7ey3trpjtkqgcale): resolution: {integrity: sha512-rk0OlvEQkShjbr2EvGn1+GdCsgLDgABQyM9ZV6VoHNU7hiNM+eSOkjGWhiNabU/XWiEalWbjNQrNO+zcqd+pEA==} dev: false @@ -13056,13 +13121,16 @@ packages: dependencies: mime-db: 1.52.0 + /mime@1.6.0: + resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} + engines: {node: '>=4'} + hasBin: true + /mime@3.0.0: resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==} engines: {node: '>=10.0.0'} hasBin: true requiresBuild: true - dev: false - optional: true /mimic-fn@2.1.0: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} @@ -13263,6 +13331,10 @@ packages: railroad-diagrams: 1.0.0 randexp: 0.4.6 + /negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + /neo-async@2.6.2: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} @@ -13395,6 +13467,9 @@ packages: resolution: {integrity: sha512-VzW+TAk2wE4X9maiKMlT+GsPU4OMmR1U9CrHSmd3DFLn2IcZ9VJ6M6BBugGfYUnPCLSYxXdZy17M0BEJyhUTwg==} dev: false + /node-fetch-native@1.4.0: + resolution: {integrity: sha512-F5kfEj95kX8tkDhUCYdV8dg3/8Olx/94zB8+ZNthFs6Bz31UpUi8Xh40TN3thLwXgrwXry1pEg9lJ++tLWTcqA==} + /node-fetch@2.6.7: resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} engines: {node: 4.x || >=6.0.0} @@ -13449,6 +13524,21 @@ packages: webpack: 5.74.0(esbuild@0.17.5) dev: true + /node-mocks-http@1.13.0: + resolution: {integrity: sha512-lArD6sJMPJ53WF50GX0nJ89B1nkV1TdMvNwq8WXXFrUXF80ujSyye1T30mgiHh4h2It0/svpF3C4kZ2OAONVlg==} + engines: {node: '>=14'} + dependencies: + accepts: 1.3.8 + content-disposition: 0.5.4 + depd: 1.1.2 + fresh: 0.5.2 + merge-descriptors: 1.0.1 + methods: 1.1.2 + mime: 1.6.0 + parseurl: 1.3.3 + range-parser: 1.2.1 + type-is: 1.6.18 + /node-readfiles@0.2.0: resolution: {integrity: sha512-SU00ZarexNlE4Rjdm83vglt5Y9yiQ+XI1XpflWlb7q7UTN1JUItm69xMeiQCTxtTfnzt+83T8Cx+vI2ED++VDA==} dependencies: @@ -13725,6 +13815,9 @@ packages: resolution: {integrity: sha512-XpeCy01X6L5EpP+6Hc3jWN7rMZJ+/k1lwki/kTmWzbVhdPie3jd5O2ZtedEx8Yp58icJ0osVldLMrTB/zslQXA==} dev: true + /openapi-types@12.1.3: + resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} + /openapi-typescript-codegen@0.23.0: resolution: {integrity: sha512-gOJXy5g3H3HlLpVNN+USrNK2i2KYBmDczk9Xk34u6JorwrGiDJZUj+al4S+i9TXdfUQ/ZaLxE59Xf3wqkxGfqA==} hasBin: true @@ -13959,6 +14052,10 @@ packages: resolution: {integrity: sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==} dev: false + /parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + /pascal-case@3.1.2: resolution: {integrity: sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==} dependencies: @@ -14038,6 +14135,9 @@ packages: engines: {node: '>=8'} dev: true + /pathe@1.1.1: + resolution: {integrity: sha512-d+RQGp0MAYTIaDBIMmOfMwz3E+LOZnxx1HZd5R18mmCZY0QBlK0LDZfPc8FW8Ed2DlvsuE6PRjroDY+wg4+j/Q==} + /pause-stream@0.0.11: resolution: {integrity: sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==} dependencies: @@ -14749,7 +14849,6 @@ packages: engines: {node: '>=0.6'} dependencies: side-channel: 1.0.4 - dev: false /qs@6.5.3: resolution: {integrity: sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==} @@ -14782,6 +14881,9 @@ packages: resolution: {integrity: sha512-6n3AEXth91ASapMVKiEh2wrbFJmI+NBilrWE0AbiGgfm0xet0QXC8+a3K19r1UVYjUjctUgB053c3V/J6V0kCQ==} dev: false + /radix3@1.1.0: + resolution: {integrity: sha512-pNsHDxbGORSvuSScqNJ+3Km6QAVqk8CfsCBIEoDgpqLrkD2f3QM4I7d1ozJJ172OmIcoUcerZaNWqtLkRXTV3A==} + /railroad-diagrams@1.0.0: resolution: {integrity: sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A==} @@ -14797,6 +14899,10 @@ packages: dependencies: safe-buffer: 5.2.1 + /range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + /raw-body@2.4.1: resolution: {integrity: sha512-9WmIKF6mkvA0SLmA2Knm9+qj89e+j1zqgyn8aXGd7+nAduPoqgI9lO57SAZNn/Byzo5P7JhXTyg9PzaJbH73bA==} engines: {node: '>= 0.8'} @@ -14805,7 +14911,6 @@ packages: http-errors: 1.7.3 iconv-lite: 0.4.24 unpipe: 1.0.0 - dev: false /rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} @@ -15640,7 +15745,6 @@ packages: /safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} - dev: false /sane@4.1.0: resolution: {integrity: sha512-hhbzAgTIX8O7SHfp2c8/kREfEn4qO/9q8C9beyY6+tvZ87EpoZ3i1RIEvp27YBswnNbY9mWd6paKVmKbAgLfZA==} @@ -15774,7 +15878,6 @@ packages: /setprototypeof@1.1.1: resolution: {integrity: sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==} - dev: false /shebang-command@1.2.0: resolution: {integrity: sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==} @@ -16331,7 +16434,6 @@ packages: /statuses@1.5.0: resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==} engines: {node: '>= 0.6'} - dev: false /stream-combiner@0.0.4: resolution: {integrity: sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==} @@ -17006,7 +17108,6 @@ packages: /toidentifier@1.0.0: resolution: {integrity: sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==} engines: {node: '>=0.6'} - dev: false /tough-cookie@2.5.0: resolution: {integrity: sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==} @@ -17036,6 +17137,21 @@ packages: resolution: {integrity: sha512-rvuRbTarPXmMb79SmzEp8aqXNKcK+y0XaB298IXueQ8I2PsrATcPBCSPyK/dDNa2iWOhKlfNnOjdAOTBU/nkFA==} dev: false + /trpc-openapi@1.2.0(@trpc/server@10.21.1)(zod@3.21.4): + resolution: {integrity: sha512-pfYoCd/3KYXWXvUPZBKJw455OOwngKN/6SIcj7Yit19OMLJ+8yVZkEvGEeg5wUSwfsiTdRsKuvqkRPXVSwV7ew==} + peerDependencies: + '@trpc/server': ^10.0.0 + zod: ^3.14.4 + dependencies: + '@trpc/server': 10.21.1 + co-body: 6.1.0 + h3: 1.8.2 + lodash.clonedeep: 4.5.0 + node-mocks-http: 1.13.0 + openapi-types: 12.1.3 + zod: 3.21.4(patch_hash=bzwjzhue3hmpww5lnv24u5k2ru) + zod-to-json-schema: 3.21.1(zod@3.21.4) + /ts-brand@0.0.2: resolution: {integrity: sha512-UhSzWY4On9ZHIj6DKkRYVN/8OaprbLAZ3b/Y2AJwdl6oozSABsQ0PvwDh4vOVdkvOtWQOkIrjctZ1kj8YfF3jA==} @@ -17173,7 +17289,6 @@ packages: dependencies: media-typer: 0.3.0 mime-types: 2.1.35 - dev: false /type@1.2.0: resolution: {integrity: sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==} @@ -17211,6 +17326,9 @@ packages: resolution: {integrity: sha512-boAm74ubXHY7KJQZLlXrtMz52qFvpsbOxDcZOnw/Wf+LS4Mmyu7JxmzD4tDLtUQtmZECypJ0FrCz4QIe6dvKRA==} dev: false + /ufo@1.3.1: + resolution: {integrity: sha512-uY/99gMLIOlJPwATcMVYfqDSxUR9//AUcgZMzwfSTJPDKzA1S8mX4VLqa+fiAtveraQUBCz4FFcwVZBGbwBXIw==} + /uglify-js@3.17.3: resolution: {integrity: sha512-JmMFDME3iufZnBpyKL+uS78LRiC+mK55zWfM5f/pWBJfpOttXAqYfdDGRukYhJuyRinvPVAtUhvy7rlDybNtFg==} engines: {node: '>=0.8.0'} @@ -17245,6 +17363,9 @@ packages: which-boxed-primitive: 1.0.2 dev: true + /uncrypto@0.1.3: + resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==} + /undici@5.16.0: resolution: {integrity: sha512-KWBOXNv6VX+oJQhchXieUznEmnJMqgXMbs0xxH2t8q/FUAWSJvOSr/rMaZKnX5RIVq7JDn0JbP4BOnKG2SGXLQ==} engines: {node: '>=12.18'} @@ -17259,6 +17380,15 @@ packages: busboy: 1.6.0 dev: true + /unenv@1.7.4: + resolution: {integrity: sha512-fjYsXYi30It0YCQYqLOcT6fHfMXsBr2hw9XC7ycf8rTG7Xxpe3ZssiqUnD0khrjiZEmkBXWLwm42yCSCH46fMw==} + dependencies: + consola: 3.2.3 + defu: 6.1.2 + mime: 3.0.0 + node-fetch-native: 1.4.0 + pathe: 1.1.1 + /unified@9.2.2: resolution: {integrity: sha512-Sg7j110mtefBD+qunSLO1lqOEKdrwBFBrR6Qd8f4uwkhWNlbkaqwHse6e7QvD3AP/MNoJdEDLaf8OxYyoWgorQ==} dependencies: @@ -17387,7 +17517,6 @@ packages: /unpipe@1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} - dev: false /unset-value@1.0.0: resolution: {integrity: sha512-PcA2tsuGSF9cnySLHTLSh2qrQiJ70mn+r+Glzxv2TWZblxsxCC52BDlZoPCsz7STd9pN7EZetkWZBAvk4cgZdQ==} @@ -18068,7 +18197,6 @@ packages: zod: ^3.21.4 dependencies: zod: 3.21.4(patch_hash=bzwjzhue3hmpww5lnv24u5k2ru) - dev: false /zod@3.21.4(patch_hash=bzwjzhue3hmpww5lnv24u5k2ru): resolution: {integrity: sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==} From 86e6403bd9006d678fd23bd4b18d7eb86bab757c Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sat, 14 Oct 2023 00:53:36 -0700 Subject: [PATCH 47/80] Add endpoints for listing resources and etc. --- apps/web/package.json | 2 +- apps/web/sdk/venice.gen.ts | 1293 +++-------------- .../engine-backend/router/protectedRouter.ts | 17 +- 3 files changed, 201 insertions(+), 1111 deletions(-) diff --git a/apps/web/package.json b/apps/web/package.json index c9b49a3e..520d0a3b 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -6,7 +6,7 @@ "build": "run-s build:*", "build:migration": "if [ \"$VERCEL_ENV\" = production ] || [ \"$VERCEL_GIT_COMMIT_REF\" = staging ]; then pnpm --dir ../../ migration up; else echo 'Skip non prod/staging migration'; fi", "build:next": "next build", - "codegen:sdk": "openapi-typescript http://localhost:3000/api/rest --output ./sdk/venice.gen.ts", + "codegen:sdk": "openapi-typescript http://localhost:3000/api/openapi --output ./sdk/venice.gen.ts", "codegen:supabase": "supabase gen types typescript --linked --schema public > ./supabase/supabase.gen.ts", "dev": "next dev", "start": "next start" diff --git a/apps/web/sdk/venice.gen.ts b/apps/web/sdk/venice.gen.ts index 8a16042a..aeb93b47 100644 --- a/apps/web/sdk/venice.gen.ts +++ b/apps/web/sdk/venice.gen.ts @@ -5,1148 +5,227 @@ export interface paths { - "/": { - /** OpenAPI description (this document) */ - get: { - /** OpenAPI description (this document) */ - responses: { - /** @description OK */ - 200: never; - }; - }; + "/health": { + get: operations["health"]; }; - "/account": { - /** - * @graphql({ - * @description "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - get: { - /** - * @graphql({ - * @description "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - responses: { - /** @description OK */ - 200: { - content: { - "application/json": (components["schemas"]["account"])[]; - "application/vnd.pgrst.object+json": (components["schemas"]["account"])[]; - "text/csv": (components["schemas"]["account"])[]; - }; - }; - /** @description Partial Content */ - 206: never; - }; - }; - /** - * @graphql({ - * @description "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - post: { - /** - * @graphql({ - * @description "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - requestBody: components["requestBodies"]["account"]; - responses: { - /** @description Created */ - 201: never; - }; - }; - /** - * @graphql({ - * @description "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - delete: { - /** - * @graphql({ - * @description "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - responses: { - /** @description No Content */ - 204: never; - }; - }; - /** - * @graphql({ - * @description "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - patch: { - /** - * @graphql({ - * @description "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - requestBody: components["requestBodies"]["account"]; - responses: { - /** @description No Content */ - 204: never; - }; - }; + "/resources": { + get: operations["listResources"]; }; - "/institution": { - get: { - responses: { - /** @description OK */ - 200: { - content: { - "application/json": (components["schemas"]["institution"])[]; - "application/vnd.pgrst.object+json": (components["schemas"]["institution"])[]; - "text/csv": (components["schemas"]["institution"])[]; - }; - }; - /** @description Partial Content */ - 206: never; - }; - }; - post: { - requestBody: components["requestBodies"]["institution"]; - responses: { - /** @description Created */ - 201: never; - }; - }; - delete: { - responses: { - /** @description No Content */ - 204: never; - }; - }; - patch: { - requestBody: components["requestBodies"]["institution"]; - responses: { - /** @description No Content */ - 204: never; - }; - }; + "/pipelines": { + get: operations["listPipelines"]; }; - "/integration": { - get: { - responses: { - /** @description OK */ - 200: { - content: { - "application/json": (components["schemas"]["integration"])[]; - "application/vnd.pgrst.object+json": (components["schemas"]["integration"])[]; - "text/csv": (components["schemas"]["integration"])[]; - }; - }; - /** @description Partial Content */ - 206: never; - }; - }; - post: { - requestBody: components["requestBodies"]["integration"]; - responses: { - /** @description Created */ - 201: never; - }; - }; - delete: { - responses: { - /** @description No Content */ - 204: never; - }; - }; - patch: { - requestBody: components["requestBodies"]["integration"]; - responses: { - /** @description No Content */ - 204: never; - }; - }; + "/pipelines/{id}": { + delete: operations["deletePipeline"]; }; - "/pipeline": { - get: { - responses: { - /** @description OK */ - 200: { - content: { - "application/json": (components["schemas"]["pipeline"])[]; - "application/vnd.pgrst.object+json": (components["schemas"]["pipeline"])[]; - "text/csv": (components["schemas"]["pipeline"])[]; - }; - }; - /** @description Partial Content */ - 206: never; - }; - }; - post: { - requestBody: components["requestBodies"]["pipeline"]; - responses: { - /** @description Created */ - 201: never; - }; - }; - delete: { - responses: { - /** @description No Content */ - 204: never; - }; - }; - patch: { - requestBody: components["requestBodies"]["pipeline"]; - responses: { - /** @description No Content */ - 204: never; - }; - }; + "/resources/{id}": { + get: operations["getResource"]; }; - "/raw_account": { - get: { - responses: { - /** @description OK */ - 200: { - content: { - "application/json": (components["schemas"]["raw_account"])[]; - "application/vnd.pgrst.object+json": (components["schemas"]["raw_account"])[]; - "text/csv": (components["schemas"]["raw_account"])[]; - }; + "/integrations": { + get: operations["adminListIntegrations"]; + }; + "/": { + get: operations["getOpenapiDocument"]; + }; +} + +export type webhooks = Record; + +export interface components { + schemas: never; + responses: { + /** @description Error response */ + error: { + content: { + "application/json": { + message: string; + code: string; + issues?: ({ + message: string; + })[]; }; - /** @description Partial Content */ - 206: never; - }; - }; - post: { - requestBody: components["requestBodies"]["raw_account"]; - responses: { - /** @description Created */ - 201: never; - }; - }; - delete: { - responses: { - /** @description No Content */ - 204: never; - }; - }; - patch: { - requestBody: components["requestBodies"]["raw_account"]; - responses: { - /** @description No Content */ - 204: never; }; }; }; - "/raw_commodity": { - get: { - responses: { - /** @description OK */ - 200: { - content: { - "application/json": (components["schemas"]["raw_commodity"])[]; - "application/vnd.pgrst.object+json": (components["schemas"]["raw_commodity"])[]; - "text/csv": (components["schemas"]["raw_commodity"])[]; - }; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} + +export type external = Record; + +export interface operations { + + health: { + responses: { + /** @description Successful response */ + 200: { + content: { + "application/json": string; }; - /** @description Partial Content */ - 206: never; - }; - }; - post: { - requestBody: components["requestBodies"]["raw_commodity"]; - responses: { - /** @description Created */ - 201: never; - }; - }; - delete: { - responses: { - /** @description No Content */ - 204: never; - }; - }; - patch: { - requestBody: components["requestBodies"]["raw_commodity"]; - responses: { - /** @description No Content */ - 204: never; }; + default: components["responses"]["error"]; }; }; - "/raw_transaction": { - get: { - responses: { - /** @description OK */ - 200: { - content: { - "application/json": (components["schemas"]["raw_transaction"])[]; - "application/vnd.pgrst.object+json": (components["schemas"]["raw_transaction"])[]; - "text/csv": (components["schemas"]["raw_transaction"])[]; - }; + listResources: { + responses: { + /** @description Successful response */ + 200: { + content: { + "application/json": ({ + /** Format: date-time */ + createdAt: string; + /** Format: date-time */ + updatedAt: string; + id: string; + providerName: string; + displayName?: string | null; + endUserId?: string | null; + integrationId: string; + institutionId?: string | null; + settings?: ({ + [key: string]: ((string | number | boolean | ("null" | null)) | (Record)[] | ({ + [key: string]: unknown | undefined; + })) | undefined; + }) | null; + standard?: ({ + displayName: string; + /** @enum {string|null} */ + status?: "healthy" | "disconnected" | "error" | "manual" | null; + statusMessage?: string | null; + labels?: (string)[]; + }) | null; + })[]; }; - /** @description Partial Content */ - 206: never; - }; - }; - post: { - requestBody: components["requestBodies"]["raw_transaction"]; - responses: { - /** @description Created */ - 201: never; - }; - }; - delete: { - responses: { - /** @description No Content */ - 204: never; - }; - }; - patch: { - requestBody: components["requestBodies"]["raw_transaction"]; - responses: { - /** @description No Content */ - 204: never; }; + default: components["responses"]["error"]; }; }; - "/resource": { - get: { - responses: { - /** @description OK */ - 200: { - content: { - "application/json": (components["schemas"]["resource"])[]; - "application/vnd.pgrst.object+json": (components["schemas"]["resource"])[]; - "text/csv": (components["schemas"]["resource"])[]; - }; + listPipelines: { + responses: { + /** @description Successful response */ + 200: { + content: { + "application/json": ({ + /** Format: date-time */ + createdAt: string; + /** Format: date-time */ + updatedAt: string; + id: string; + sourceId?: string; + sourceState?: { + [key: string]: ((string | number | boolean | ("null" | null)) | (Record)[] | ({ + [key: string]: unknown | undefined; + })) | undefined; + }; + destinationId?: string; + destinationState?: { + [key: string]: ((string | number | boolean | ("null" | null)) | (Record)[] | ({ + [key: string]: unknown | undefined; + })) | undefined; + }; + linkOptions?: ((string | ([[object Object]])[] | ([[object Object], [object Object]])[])[]) | null; + /** Format: date-time */ + lastSyncStartedAt?: string | null; + /** Format: date-time */ + lastSyncCompletedAt?: string | null; + })[]; }; - /** @description Partial Content */ - 206: never; }; + default: components["responses"]["error"]; }; - post: { - requestBody: components["requestBodies"]["resource"]; - responses: { - /** @description Created */ - 201: never; - }; - }; - delete: { - responses: { - /** @description No Content */ - 204: never; + }; + deletePipeline: { + parameters: { + path: { + id: string; }; }; - patch: { - requestBody: components["requestBodies"]["resource"]; - responses: { - /** @description No Content */ - 204: never; + responses: { + /** @description Successful response */ + 200: { + content: { + "application/json": true; + }; }; + default: components["responses"]["error"]; }; }; - "/transaction": { - /** - * TODO: Add description of transaction data type here... - * @description @graphql({ - * "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "description": "Double entry transaction", - * "foreign_keys": [ - * { - * "local_name": "transactions", - * "local_columns": ["account_id"], - * "foreign_name": "account", - * "foreign_schema": "public", - * "foreign_table": "account", - * "foreign_columns": ["id"] - * } - * ] - * }) - */ - get: { - /** - * TODO: Add description of transaction data type here... - * @description @graphql({ - * "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "description": "Double entry transaction", - * "foreign_keys": [ - * { - * "local_name": "transactions", - * "local_columns": ["account_id"], - * "foreign_name": "account", - * "foreign_schema": "public", - * "foreign_table": "account", - * "foreign_columns": ["id"] - * } - * ] - * }) - */ - responses: { - /** @description OK */ - 200: { - content: { - "application/json": (components["schemas"]["transaction"])[]; - "application/vnd.pgrst.object+json": (components["schemas"]["transaction"])[]; - "text/csv": (components["schemas"]["transaction"])[]; + getResource: { + parameters: { + path: { + id: string; + }; + }; + responses: { + /** @description Successful response */ + 200: { + content: { + "application/json": { + /** Format: date-time */ + createdAt: string; + /** Format: date-time */ + updatedAt: string; + id: string; + providerName: string; + displayName?: string | null; + endUserId?: string | null; + integrationId: string; + institutionId?: string | null; + settings?: ({ + [key: string]: ((string | number | boolean | ("null" | null)) | (Record)[] | ({ + [key: string]: unknown | undefined; + })) | undefined; + }) | null; + standard?: ({ + displayName: string; + /** @enum {string|null} */ + status?: "healthy" | "disconnected" | "error" | "manual" | null; + statusMessage?: string | null; + labels?: (string)[]; + }) | null; }; }; - /** @description Partial Content */ - 206: never; - }; - }; - /** - * TODO: Add description of transaction data type here... - * @description @graphql({ - * "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "description": "Double entry transaction", - * "foreign_keys": [ - * { - * "local_name": "transactions", - * "local_columns": ["account_id"], - * "foreign_name": "account", - * "foreign_schema": "public", - * "foreign_table": "account", - * "foreign_columns": ["id"] - * } - * ] - * }) - */ - post: { - /** - * TODO: Add description of transaction data type here... - * @description @graphql({ - * "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "description": "Double entry transaction", - * "foreign_keys": [ - * { - * "local_name": "transactions", - * "local_columns": ["account_id"], - * "foreign_name": "account", - * "foreign_schema": "public", - * "foreign_table": "account", - * "foreign_columns": ["id"] - * } - * ] - * }) - */ - requestBody: components["requestBodies"]["transaction"]; - responses: { - /** @description Created */ - 201: never; - }; - }; - /** - * TODO: Add description of transaction data type here... - * @description @graphql({ - * "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "description": "Double entry transaction", - * "foreign_keys": [ - * { - * "local_name": "transactions", - * "local_columns": ["account_id"], - * "foreign_name": "account", - * "foreign_schema": "public", - * "foreign_table": "account", - * "foreign_columns": ["id"] - * } - * ] - * }) - */ - delete: { - /** - * TODO: Add description of transaction data type here... - * @description @graphql({ - * "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "description": "Double entry transaction", - * "foreign_keys": [ - * { - * "local_name": "transactions", - * "local_columns": ["account_id"], - * "foreign_name": "account", - * "foreign_schema": "public", - * "foreign_table": "account", - * "foreign_columns": ["id"] - * } - * ] - * }) - */ - responses: { - /** @description No Content */ - 204: never; - }; - }; - /** - * TODO: Add description of transaction data type here... - * @description @graphql({ - * "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "description": "Double entry transaction", - * "foreign_keys": [ - * { - * "local_name": "transactions", - * "local_columns": ["account_id"], - * "foreign_name": "account", - * "foreign_schema": "public", - * "foreign_table": "account", - * "foreign_columns": ["id"] - * } - * ] - * }) - */ - patch: { - /** - * TODO: Add description of transaction data type here... - * @description @graphql({ - * "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "description": "Double entry transaction", - * "foreign_keys": [ - * { - * "local_name": "transactions", - * "local_columns": ["account_id"], - * "foreign_name": "account", - * "foreign_schema": "public", - * "foreign_table": "account", - * "foreign_columns": ["id"] - * } - * ] - * }) - */ - requestBody: components["requestBodies"]["transaction"]; - responses: { - /** @description No Content */ - 204: never; }; + default: components["responses"]["error"]; }; }; - "/transaction_split": { - /** - * Entities summary - * @description Entities description that - * spans - * multiple lines - * - * - * @graphql({ - * "primary_key_columns": ["id", "key"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - get: { - /** - * Entities summary - * @description Entities description that - * spans - * multiple lines - * - * - * @graphql({ - * "primary_key_columns": ["id", "key"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - responses: { - /** @description OK */ - 200: { - content: { - "application/json": (components["schemas"]["transaction_split"])[]; - "application/vnd.pgrst.object+json": (components["schemas"]["transaction_split"])[]; - "text/csv": (components["schemas"]["transaction_split"])[]; - }; + adminListIntegrations: { + responses: { + /** @description Successful response */ + 200: { + content: { + "application/json": ({ + /** Format: date-time */ + createdAt: string; + /** Format: date-time */ + updatedAt: string; + id: string; + providerName: string; + config?: ({ + [key: string]: ((string | number | boolean | ("null" | null)) | (Record)[] | ({ + [key: string]: unknown | undefined; + })) | undefined; + }) | null; + /** @description Allow end user to create resources using this integration's configuration */ + endUserAccess?: boolean | null; + orgId: string; + displayName?: string | null; + })[]; }; - /** @description Partial Content */ - 206: never; }; + default: components["responses"]["error"]; }; }; -} - -export type webhooks = Record; - -export interface components { - schemas: { - /** - * @description @graphql({ - * "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - account: { - /** - * Format: character varying - * @description Note: - * This is a Primary Key. - */ - id?: string; - /** Format: text */ - name?: string; - /** Format: text */ - type?: string; - /** Format: text */ - last_four?: string; - /** Format: text */ - institution_name?: string; - /** Format: text */ - default_unit?: string; - /** Format: double precision */ - current_balance?: number; - /** Format: double precision */ - available_balance?: number; - /** Format: jsonb */ - external?: Record; - /** Format: timestamp with time zone */ - updated_at?: string; - /** Format: timestamp with time zone */ - created_at?: string; - }; - institution: { - /** - * Format: character varying - * @description Note: - * This is a Primary Key. - * @default public.generate_ulid() - */ - id: string; - /** Format: jsonb */ - standard: Record; - /** Format: jsonb */ - external: Record; - /** - * Format: timestamp with time zone - * @default now() - */ - created_at: string; - /** - * Format: timestamp with time zone - * @default now() - */ - updated_at: string; - /** - * Format: character varying - * @default split_part((id)::text, '_'::text, 2) - */ - provider_name: string; - }; - integration: { - /** - * Format: character varying - * @description Note: - * This is a Primary Key. - * @default public.generate_ulid() - */ - id: string; - /** Format: jsonb */ - config: Record; - /** - * Format: timestamp with time zone - * @default now() - */ - created_at: string; - /** - * Format: timestamp with time zone - * @default now() - */ - updated_at: string; - /** - * Format: character varying - * @default split_part((id)::text, '_'::text, 2) - */ - provider_name: string; - }; - pipeline: { - /** - * Format: character varying - * @description Note: - * This is a Primary Key. - * @default public.generate_ulid() - */ - id: string; - /** - * Format: character varying - * @description Note: - * This is a Foreign Key to `resource.id`. - */ - source_id?: string; - /** Format: jsonb */ - source_state: Record; - /** - * Format: character varying - * @description Note: - * This is a Foreign Key to `resource.id`. - */ - destination_id?: string; - /** Format: jsonb */ - destination_state: Record; - /** Format: jsonb */ - link_options: Record; - /** - * Format: timestamp with time zone - * @default now() - */ - created_at: string; - /** - * Format: timestamp with time zone - * @default now() - */ - updated_at: string; - /** Format: timestamp with time zone */ - last_sync_started_at?: string; - /** Format: timestamp with time zone */ - last_sync_completed_at?: string; - }; - raw_account: { - /** - * Format: character varying - * @description Note: - * This is a Primary Key. - * @default public.generate_ulid() - */ - id: string; - /** Format: character varying */ - source_id?: string; - /** Format: jsonb */ - standard: Record; - /** Format: jsonb */ - external: Record; - /** - * Format: timestamp with time zone - * @default now() - */ - created_at: string; - /** - * Format: timestamp with time zone - * @default now() - */ - updated_at: string; - /** - * Format: character varying - * @default split_part((id)::text, '_'::text, 2) - */ - provider_name: string; - /** - * Format: character varying - * @description Note: - * This is a Foreign Key to `resource.id`. - */ - end_user_id?: string; - }; - raw_commodity: { - /** - * Format: character varying - * @description Note: - * This is a Primary Key. - * @default public.generate_ulid() - */ - id: string; - /** Format: character varying */ - source_id?: string; - /** Format: jsonb */ - standard: Record; - /** Format: jsonb */ - external: Record; - /** - * Format: timestamp with time zone - * @default now() - */ - created_at: string; - /** - * Format: timestamp with time zone - * @default now() - */ - updated_at: string; - /** - * Format: character varying - * @default split_part((id)::text, '_'::text, 2) - */ - provider_name: string; - /** - * Format: character varying - * @description Note: - * This is a Foreign Key to `resource.id`. - */ - end_user_id?: string; - }; - raw_transaction: { - /** - * Format: character varying - * @description Note: - * This is a Primary Key. - * @default public.generate_ulid() - */ - id: string; - /** Format: character varying */ - source_id?: string; - /** Format: jsonb */ - standard: Record; - /** Format: jsonb */ - external: Record; - /** - * Format: timestamp with time zone - * @default now() - */ - created_at: string; - /** - * Format: timestamp with time zone - * @default now() - */ - updated_at: string; - /** - * Format: character varying - * @default split_part((id)::text, '_'::text, 2) - */ - provider_name: string; - /** - * Format: character varying - * @description Note: - * This is a Foreign Key to `resource.id`. - */ - end_user_id?: string; - }; - resource: { - /** - * Format: character varying - * @description Note: - * This is a Primary Key. - * @default public.generate_ulid() - */ - id: string; - /** Format: character varying */ - end_user_id?: string; - /** - * Format: character varying - * @description Note: - * This is a Foreign Key to `integration.id`. - */ - integration_id?: string; - /** - * Format: character varying - * @description Note: - * This is a Foreign Key to `institution.id`. - */ - institution_id?: string; - /** Format: character varying */ - env_name?: string; - /** Format: jsonb */ - settings: Record; - /** - * Format: timestamp with time zone - * @default now() - */ - created_at: string; - /** - * Format: timestamp with time zone - * @default now() - */ - updated_at: string; - /** - * Format: character varying - * @default split_part((id)::text, '_'::text, 2) - */ - provider_name: string; - /** Format: character varying */ - display_name?: string; - }; - /** - * @description TODO: Add description of transaction data type here... - * - * @graphql({ - * "primary_key_columns": ["id"], - * "totalCount": {"enabled": true}, - * "description": "Double entry transaction", - * "foreign_keys": [ - * { - * "local_name": "transactions", - * "local_columns": ["account_id"], - * "foreign_name": "account", - * "foreign_schema": "public", - * "foreign_table": "account", - * "foreign_columns": ["id"] - * } - * ] - * }) - */ - transaction: { - /** - * Format: character varying - * @description Note: - * This is a Primary Key. - */ - id?: string; - /** Format: text */ - date?: string; - /** Format: text */ - description?: string; - /** Format: text */ - payee?: string; - /** Format: double precision */ - amount_quantity?: number; - /** Format: text */ - amount_unit?: string; - /** Format: text */ - account_id?: string; - /** Format: text */ - external_category?: string; - /** Format: text */ - notes?: string; - /** Format: jsonb */ - splits?: Record; - /** Format: jsonb */ - external?: Record; - /** Format: timestamp with time zone */ - updated_at?: string; - /** Format: timestamp with time zone */ - created_at?: string; - }; - /** - * @description Entities summary - * - * Entities description that - * spans - * multiple lines - * - * - * @graphql({ - * "primary_key_columns": ["id", "key"], - * "totalCount": {"enabled": true}, - * "foreign_keys": [] - * }) - */ - transaction_split: { - /** - * Format: character varying - * @description Note: - * This is a Primary Key. - */ - transaction_id?: string; - /** Format: text */ - key?: string; - /** Format: text */ - amount_quantity?: string; - /** Format: text */ - amount_unit?: string; - /** Format: text */ - account_id?: string; - /** Format: jsonb */ - data?: Record; - /** Format: timestamp with time zone */ - updated_at?: string; - /** Format: timestamp with time zone */ - created_at?: string; - }; - }; - responses: never; - parameters: { - /** @description Preference */ - preferParams: "params=single-object"; - /** @description Preference */ - preferReturn: "return=representation" | "return=minimal" | "return=none"; - /** @description Preference */ - preferCount: "count=none"; - /** @description Preference */ - preferPost: "return=representation" | "return=minimal" | "return=none" | "resolution=ignore-duplicates" | "resolution=merge-duplicates"; - /** @description Filtering Columns */ - select: string; - /** @description On Conflict */ - on_conflict: string; - /** @description Ordering */ - order: string; - /** @description Limiting and Pagination */ - range: string; - /** @description Limiting and Pagination */ - rangeUnit: string; - /** @description Limiting and Pagination */ - offset: string; - /** @description Limiting and Pagination */ - limit: string; - "rowFilter.raw_transaction.id": string; - "rowFilter.raw_transaction.source_id": string; - "rowFilter.raw_transaction.standard": string; - "rowFilter.raw_transaction.external": string; - "rowFilter.raw_transaction.created_at": string; - "rowFilter.raw_transaction.updated_at": string; - "rowFilter.raw_transaction.provider_name": string; - "rowFilter.raw_transaction.end_user_id": string; - "rowFilter.account.id": string; - "rowFilter.account.name": string; - "rowFilter.account.type": string; - "rowFilter.account.last_four": string; - "rowFilter.account.institution_name": string; - "rowFilter.account.default_unit": string; - "rowFilter.account.current_balance": string; - "rowFilter.account.available_balance": string; - "rowFilter.account.external": string; - "rowFilter.account.updated_at": string; - "rowFilter.account.created_at": string; - "rowFilter.transaction.id": string; - "rowFilter.transaction.date": string; - "rowFilter.transaction.description": string; - "rowFilter.transaction.payee": string; - "rowFilter.transaction.amount_quantity": string; - "rowFilter.transaction.amount_unit": string; - "rowFilter.transaction.account_id": string; - "rowFilter.transaction.external_category": string; - "rowFilter.transaction.notes": string; - "rowFilter.transaction.splits": string; - "rowFilter.transaction.external": string; - "rowFilter.transaction.updated_at": string; - "rowFilter.transaction.created_at": string; - "rowFilter.raw_commodity.id": string; - "rowFilter.raw_commodity.source_id": string; - "rowFilter.raw_commodity.standard": string; - "rowFilter.raw_commodity.external": string; - "rowFilter.raw_commodity.created_at": string; - "rowFilter.raw_commodity.updated_at": string; - "rowFilter.raw_commodity.provider_name": string; - "rowFilter.raw_commodity.end_user_id": string; - "rowFilter.resource.id": string; - "rowFilter.resource.end_user_id": string; - "rowFilter.resource.integration_id": string; - "rowFilter.resource.institution_id": string; - "rowFilter.resource.env_name": string; - "rowFilter.resource.settings": string; - "rowFilter.resource.created_at": string; - "rowFilter.resource.updated_at": string; - "rowFilter.resource.provider_name": string; - "rowFilter.resource.display_name": string; - "rowFilter.pipeline.id": string; - "rowFilter.pipeline.source_id": string; - "rowFilter.pipeline.source_state": string; - "rowFilter.pipeline.destination_id": string; - "rowFilter.pipeline.destination_state": string; - "rowFilter.pipeline.link_options": string; - "rowFilter.pipeline.created_at": string; - "rowFilter.pipeline.updated_at": string; - "rowFilter.pipeline.last_sync_started_at": string; - "rowFilter.pipeline.last_sync_completed_at": string; - "rowFilter.integration.id": string; - "rowFilter.integration.config": string; - "rowFilter.integration.created_at": string; - "rowFilter.integration.updated_at": string; - "rowFilter.integration.provider_name": string; - "rowFilter.transaction_split.transaction_id": string; - "rowFilter.transaction_split.key": string; - "rowFilter.transaction_split.amount_quantity": string; - "rowFilter.transaction_split.amount_unit": string; - "rowFilter.transaction_split.account_id": string; - "rowFilter.transaction_split.data": string; - "rowFilter.transaction_split.updated_at": string; - "rowFilter.transaction_split.created_at": string; - "rowFilter.raw_account.id": string; - "rowFilter.raw_account.source_id": string; - "rowFilter.raw_account.standard": string; - "rowFilter.raw_account.external": string; - "rowFilter.raw_account.created_at": string; - "rowFilter.raw_account.updated_at": string; - "rowFilter.raw_account.provider_name": string; - "rowFilter.raw_account.end_user_id": string; - "rowFilter.institution.id": string; - "rowFilter.institution.standard": string; - "rowFilter.institution.external": string; - "rowFilter.institution.created_at": string; - "rowFilter.institution.updated_at": string; - "rowFilter.institution.provider_name": string; - }; - requestBodies: { - /** @description institution */ - institution?: { - content: { - "application/json": components["schemas"]["institution"]; - "application/vnd.pgrst.object+json": components["schemas"]["institution"]; - "text/csv": components["schemas"]["institution"]; - }; - }; - /** @description raw_commodity */ - raw_commodity?: { - content: { - "application/json": components["schemas"]["raw_commodity"]; - "application/vnd.pgrst.object+json": components["schemas"]["raw_commodity"]; - "text/csv": components["schemas"]["raw_commodity"]; - }; - }; - /** @description account */ - account?: { - content: { - "application/json": components["schemas"]["account"]; - "application/vnd.pgrst.object+json": components["schemas"]["account"]; - "text/csv": components["schemas"]["account"]; - }; - }; - /** @description raw_account */ - raw_account?: { - content: { - "application/json": components["schemas"]["raw_account"]; - "application/vnd.pgrst.object+json": components["schemas"]["raw_account"]; - "text/csv": components["schemas"]["raw_account"]; - }; - }; - /** @description pipeline */ - pipeline?: { - content: { - "application/json": components["schemas"]["pipeline"]; - "application/vnd.pgrst.object+json": components["schemas"]["pipeline"]; - "text/csv": components["schemas"]["pipeline"]; - }; - }; - /** @description integration */ - integration?: { - content: { - "application/json": components["schemas"]["integration"]; - "application/vnd.pgrst.object+json": components["schemas"]["integration"]; - "text/csv": components["schemas"]["integration"]; - }; - }; - /** @description raw_transaction */ - raw_transaction?: { - content: { - "application/json": components["schemas"]["raw_transaction"]; - "application/vnd.pgrst.object+json": components["schemas"]["raw_transaction"]; - "text/csv": components["schemas"]["raw_transaction"]; - }; - }; - /** @description resource */ - resource?: { - content: { - "application/json": components["schemas"]["resource"]; - "application/vnd.pgrst.object+json": components["schemas"]["resource"]; - "text/csv": components["schemas"]["resource"]; - }; - }; - /** @description transaction */ - transaction?: { - content: { - "application/json": components["schemas"]["transaction"]; - "application/vnd.pgrst.object+json": components["schemas"]["transaction"]; - "text/csv": components["schemas"]["transaction"]; + getOpenapiDocument: { + responses: { + /** @description Successful response */ + 200: { + content: { + "application/json": Record; + }; }; + default: components["responses"]["error"]; }; }; - headers: never; - pathItems: never; } - -export type external = Record; - -export type operations = Record; diff --git a/packages/engine-backend/router/protectedRouter.ts b/packages/engine-backend/router/protectedRouter.ts index d6bbee91..5bb70606 100644 --- a/packages/engine-backend/router/protectedRouter.ts +++ b/packages/engine-backend/router/protectedRouter.ts @@ -1,10 +1,13 @@ import {TRPCError} from '@trpc/server' +import type { + ZRaw} from '@usevenice/cdk-core'; import { extractId, sync, zCheckResourceOptions, zId, + zRaw, zStandard, } from '@usevenice/cdk-core' import type {VeniceSourceState} from '@usevenice/cdk-ledger' @@ -32,22 +35,28 @@ export const protectedRouter = trpc.router({ await inngest.send(input.name, {data: input.data, user: ctx.viewer}) }), listResources: protectedProcedure + .meta({openapi: {method: 'GET', path: '/resources'}}) .input(z.object({}).optional()) + .output(z.array(zRaw.resource)) .query(async ({ctx}) => { const resources = await ctx.helpers.metaService.tables.resource.list({}) - return resources + return resources as Array }), listPipelines: protectedProcedure + .meta({openapi: {method: 'GET', path: '/pipelines'}}) .input(z.object({}).optional()) + .output(z.array(zRaw.pipeline)) .query(async ({ctx}) => { const pipelines = await ctx.helpers.metaService.tables.pipeline.list({}) - return pipelines + return pipelines as Array }), deletePipeline: protectedProcedure + .meta({openapi: {method: 'DELETE', path: '/pipelines/{id}'}}) .input(z.object({id: zId('pipe')})) + .output(z.literal(true)) .mutation(async ({ctx, input}) => { await ctx.helpers.metaService.tables.pipeline.delete(input.id) - return true + return true as const }), listConnections: protectedProcedure .input(z.object({}).optional()) @@ -185,8 +194,10 @@ export const protectedRouter = trpc.router({ getResource: protectedProcedure .meta({ description: 'Not automatically called, used for debugging for now', + openapi: {method: 'GET', path: '/resources/{id}'}, }) .input(z.object({id: zId('reso')})) + .output(zRaw.resource) // TODO: This is actually expanded... .query(async ({input, ctx}) => { const reso = await ctx.helpers.getResourceExpandedOrFail(input.id) return reso From acfccbe13d3ad188f34038493d525a1f2e853650 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sat, 14 Oct 2023 01:02:25 -0700 Subject: [PATCH 48/80] Still issue with type failing in 3rd party --- apps/cli/package.json | 2 +- apps/web/package.json | 4 +- apps/web/sdk/venice-sdk.ts | 6 +-- apps/web/sdk/venice.gen.ts | 2 +- packages/airbyte/package.json | 2 +- packages/cdk-core/meta.types.ts | 13 +++-- packages/cdk-core/package.json | 2 +- packages/engine-backend/package.json | 2 +- packages/engine-frontend/package.json | 6 +-- packages/util/package.json | 2 +- pnpm-lock.yaml | 72 +++++++++++++-------------- 11 files changed, 56 insertions(+), 57 deletions(-) diff --git a/apps/cli/package.json b/apps/cli/package.json index 4a717da7..6a1ad0dc 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "private": true, "dependencies": { - "@trpc/server": "10.21.1", + "@trpc/server": "10.40.0", "@usevenice/airbyte": "workspace:*", "@usevenice/app-config": "workspace:*", "@usevenice/cdk-core": "workspace:*", diff --git a/apps/web/package.json b/apps/web/package.json index 520d0a3b..d7c0e7d6 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -21,8 +21,8 @@ "@tanstack/query-sync-storage-persister": "4.27.1", "@tanstack/react-query": "4.28.0", "@tanstack/react-query-persist-client": "4.28.0", - "@trpc/react-query": "10.21.1", - "@trpc/server": "10.21.1", + "@trpc/react-query": "10.40.0", + "@trpc/server": "10.40.0", "@usevenice/app-config": "workspace:*", "@usevenice/cdk-core": "workspace:*", "@usevenice/connect": "workspace:*", diff --git a/apps/web/sdk/venice-sdk.ts b/apps/web/sdk/venice-sdk.ts index 73dcb655..a19637e7 100644 --- a/apps/web/sdk/venice-sdk.ts +++ b/apps/web/sdk/venice-sdk.ts @@ -12,11 +12,11 @@ fetcher.configure({ }) async function main() { - const result = await fetcher.path('/account').method('get').create()({ + // const result = await fetcher.path('/account').method('get').create()({ // TODO: This should be taking params but it is not due to // https://github.com/drwpow/openapi-typescript/issues/1040 - }) - console.log('result', result.data[0]?.name) + // }) + // console.log('result', result.data[0]?.name) // result Plaid Gold Standard 0% Interest Checking (Plaid Checking) - 0000 } diff --git a/apps/web/sdk/venice.gen.ts b/apps/web/sdk/venice.gen.ts index aeb93b47..df89e3de 100644 --- a/apps/web/sdk/venice.gen.ts +++ b/apps/web/sdk/venice.gen.ts @@ -124,7 +124,7 @@ export interface operations { [key: string]: unknown | undefined; })) | undefined; }; - linkOptions?: ((string | ([[object Object]])[] | ([[object Object], [object Object]])[])[]) | null; + linkOptions?: (Record)[] | null; /** Format: date-time */ lastSyncStartedAt?: string | null; /** Format: date-time */ diff --git a/packages/airbyte/package.json b/packages/airbyte/package.json index adce88cd..6b424f37 100644 --- a/packages/airbyte/package.json +++ b/packages/airbyte/package.json @@ -6,7 +6,7 @@ "codegen:airbyte-client": "openapi-zod-client https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-api/src/main/openapi/config.yaml -o ./api/airbyte-client.gen.ts" }, "dependencies": { - "@trpc/server": "10.21.1", + "@trpc/server": "10.40.0", "@usevenice/cdk-core": "workspace:*", "@usevenice/integration-postgres": "workspace:*", "@usevenice/util": "workspace:*", diff --git a/packages/cdk-core/meta.types.ts b/packages/cdk-core/meta.types.ts index 8bf82358..c16206a1 100644 --- a/packages/cdk-core/meta.types.ts +++ b/packages/cdk-core/meta.types.ts @@ -109,13 +109,12 @@ export const zRaw = { destinationId: zId('reso').optional(), destinationState: zJsonObject.optional(), linkOptions: z - .array( - z.union([ - z.string(), - z.tuple([z.string()]), - z.tuple([z.string(), z.unknown()]), - ]), - ) + .array(z.unknown()) + // z.union([ + // z.string(), + // z.tuple([z.string()]), + // z.tuple([z.string(), z.unknown()]), + // ]), .nullish(), // TODO: Add two separate tables sync_jobs to keep track of this instead of these two // though questionnable whether it should be in a separate database completely diff --git a/packages/cdk-core/package.json b/packages/cdk-core/package.json index 61f0896f..a29cf63c 100644 --- a/packages/cdk-core/package.json +++ b/packages/cdk-core/package.json @@ -5,7 +5,7 @@ "sideEffects": false, "module": "./index.ts", "dependencies": { - "@trpc/server": "10.21.1", + "@trpc/server": "10.40.0", "@usevenice/util": "workspace:*", "jsonwebtoken": "9.0.0", "react-script-hook": "1.7.2" diff --git a/packages/engine-backend/package.json b/packages/engine-backend/package.json index c292eaa2..6fb324ec 100644 --- a/packages/engine-backend/package.json +++ b/packages/engine-backend/package.json @@ -5,7 +5,7 @@ "sideEffects": false, "module": "./index.ts", "dependencies": { - "@trpc/server": "10.21.1", + "@trpc/server": "10.40.0", "@usevenice/cdk-core": "workspace:*", "@usevenice/cdk-ledger": "workspace:*", "@usevenice/util": "workspace:*", diff --git a/packages/engine-frontend/package.json b/packages/engine-frontend/package.json index 2e522986..20cabd70 100644 --- a/packages/engine-frontend/package.json +++ b/packages/engine-frontend/package.json @@ -7,9 +7,9 @@ "dependencies": { "@nangohq/frontend": "0.33.8", "@tanstack/react-query": "*", - "@trpc/client": "10.21.1", - "@trpc/react-query": "10.21.1", - "@trpc/server": "10.21.1", + "@trpc/client": "10.40.0", + "@trpc/react-query": "10.40.0", + "@trpc/server": "10.40.0", "@usevenice/cdk-core": "workspace:*", "@usevenice/engine-backend": "workspace:*", "@usevenice/ui": "workspace:*", diff --git a/packages/util/package.json b/packages/util/package.json index df447f49..b72f0063 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -7,7 +7,7 @@ "dependencies": { "@asteasolutions/zod-to-openapi": "5.0.0", "@blossomfinance/iso-4217-currencies": "0.2.5", - "@trpc/server": "10.21.1", + "@trpc/server": "10.40.0", "accounting": "0.4.1", "async-sema": "3.1.1", "awilix": "7.0.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dcfc7143..64fea6b8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -304,8 +304,8 @@ importers: apps/cli: dependencies: '@trpc/server': - specifier: 10.21.1 - version: 10.21.1 + specifier: 10.40.0 + version: 10.40.0 '@usevenice/airbyte': specifier: workspace:* version: link:../../packages/airbyte @@ -507,11 +507,11 @@ importers: specifier: 4.28.0 version: 4.28.0(@tanstack/react-query@4.28.0) '@trpc/react-query': - specifier: 10.21.1 - version: 10.21.1(@tanstack/react-query@4.28.0)(@trpc/client@10.21.1)(@trpc/server@10.21.1)(react-dom@18.2.0)(react@18.2.0) + specifier: 10.40.0 + version: 10.40.0(@tanstack/react-query@4.28.0)(@trpc/client@10.40.0)(@trpc/server@10.40.0)(react-dom@18.2.0)(react@18.2.0) '@trpc/server': - specifier: 10.21.1 - version: 10.21.1 + specifier: 10.40.0 + version: 10.40.0 '@usevenice/app-config': specifier: workspace:* version: link:../app-config @@ -589,7 +589,7 @@ importers: version: 7.0.8 trpc-openapi: specifier: 1.2.0 - version: 1.2.0(@trpc/server@10.21.1)(zod@3.21.4) + version: 1.2.0(@trpc/server@10.40.0)(zod@3.21.4) devDependencies: '@sentry/cli': specifier: 2.13.0 @@ -1230,8 +1230,8 @@ importers: packages/airbyte: dependencies: '@trpc/server': - specifier: 10.21.1 - version: 10.21.1 + specifier: 10.40.0 + version: 10.40.0 '@usevenice/cdk-core': specifier: workspace:* version: link:../cdk-core @@ -1258,8 +1258,8 @@ importers: packages/cdk-core: dependencies: '@trpc/server': - specifier: 10.21.1 - version: 10.21.1 + specifier: 10.40.0 + version: 10.40.0 '@usevenice/util': specifier: workspace:* version: link:../util @@ -1308,8 +1308,8 @@ importers: packages/engine-backend: dependencies: '@trpc/server': - specifier: 10.21.1 - version: 10.21.1 + specifier: 10.40.0 + version: 10.40.0 '@usevenice/cdk-core': specifier: workspace:* version: link:../cdk-core @@ -1325,7 +1325,7 @@ importers: devDependencies: trpc-openapi: specifier: 1.2.0 - version: 1.2.0(@trpc/server@10.21.1)(zod@3.21.4) + version: 1.2.0(@trpc/server@10.40.0)(zod@3.21.4) packages/engine-frontend: dependencies: @@ -1336,14 +1336,14 @@ importers: specifier: 4.28.0 version: 4.28.0(react-dom@18.2.0)(react@18.2.0) '@trpc/client': - specifier: 10.21.1 - version: 10.21.1(@trpc/server@10.21.1) + specifier: 10.40.0 + version: 10.40.0(@trpc/server@10.40.0) '@trpc/react-query': - specifier: 10.21.1 - version: 10.21.1(@tanstack/react-query@4.28.0)(@trpc/client@10.21.1)(@trpc/server@10.21.1)(react-dom@18.2.0)(react@18.2.0) + specifier: 10.40.0 + version: 10.40.0(@tanstack/react-query@4.28.0)(@trpc/client@10.40.0)(@trpc/server@10.40.0)(react-dom@18.2.0)(react@18.2.0) '@trpc/server': - specifier: 10.21.1 - version: 10.21.1 + specifier: 10.40.0 + version: 10.40.0 '@usevenice/cdk-core': specifier: workspace:* version: link:../cdk-core @@ -1494,8 +1494,8 @@ importers: specifier: 0.2.5 version: 0.2.5 '@trpc/server': - specifier: 10.21.1 - version: 10.21.1 + specifier: 10.40.0 + version: 10.40.0 accounting: specifier: 0.4.1 version: 0.4.1 @@ -6144,32 +6144,32 @@ packages: - prop-types dev: false - /@trpc/client@10.21.1(@trpc/server@10.21.1): - resolution: {integrity: sha512-oZgPbghs9y2frTCA9mZPSAlVknV9stCTKYO5nsvRr0aX+oaA0URoXJTKZTSpLZxLfuwWgxBj4iTiCWWWmaVelw==} + /@trpc/client@10.40.0(@trpc/server@10.40.0): + resolution: {integrity: sha512-bT6BcdWjj0KzGQiimE6rB2tIaRYX0Ear4Gthb5szN/c01wrP0yC1Fbz2uCcm/QTVAwu4Lve5M+YjPoEaTHG6lg==} peerDependencies: - '@trpc/server': 10.21.1 + '@trpc/server': 10.40.0 dependencies: - '@trpc/server': 10.21.1 + '@trpc/server': 10.40.0 dev: false - /@trpc/react-query@10.21.1(@tanstack/react-query@4.28.0)(@trpc/client@10.21.1)(@trpc/server@10.21.1)(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-sCLJW9v79sJhVj3L6Ck74mImvKBXrq2KOKtdguxaZZ5rgeqMyQuc8bGHRpf9yfeV08uMNl4Svz1QTee9PjdSqg==} + /@trpc/react-query@10.40.0(@tanstack/react-query@4.28.0)(@trpc/client@10.40.0)(@trpc/server@10.40.0)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-DpJrV3lmYNo9xtPtcg49lfh9CUFap3ZivjhlSmfe4QPf7H6xBjAE+ml4OdJ0RmKvSTFvbLSOiNdB1k5O8zIdzQ==} peerDependencies: '@tanstack/react-query': 4.28.0 - '@trpc/client': 10.21.1 - '@trpc/server': 10.21.1 + '@trpc/client': 10.40.0 + '@trpc/server': 10.40.0 react: 18.2.0 react-dom: 18.2.0 dependencies: '@tanstack/react-query': 4.28.0(react-dom@18.2.0)(react@18.2.0) - '@trpc/client': 10.21.1(@trpc/server@10.21.1) - '@trpc/server': 10.21.1 + '@trpc/client': 10.40.0(@trpc/server@10.40.0) + '@trpc/server': 10.40.0 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) dev: false - /@trpc/server@10.21.1: - resolution: {integrity: sha512-4E4upTD2/Aq0Dm9HW4nbKSoIlqbV/ajRqNl1MCFBd9FjTv47qq2ncmAWMbb0AIVh2UV87PJlTjpPNNecRm6g9Q==} + /@trpc/server@10.40.0: + resolution: {integrity: sha512-49SUOMWzSZtu5+OdrADmJD+u+sjSE0qj1cWgYk2FY4jLkPJunLuNRuhzM7aOeBhiUjyfhg2YTfur8FN1WBmvEw==} /@tsconfig/strictest@1.0.2: resolution: {integrity: sha512-IRKlC8cnP7zMz1SDBjyIVyPapkEGWLZ6wkF6Z8T+xU80P9sO5uGXlIUvtzjx+7ehPJRWxkB6CeIDwUfyqNtYkQ==} @@ -17137,13 +17137,13 @@ packages: resolution: {integrity: sha512-rvuRbTarPXmMb79SmzEp8aqXNKcK+y0XaB298IXueQ8I2PsrATcPBCSPyK/dDNa2iWOhKlfNnOjdAOTBU/nkFA==} dev: false - /trpc-openapi@1.2.0(@trpc/server@10.21.1)(zod@3.21.4): + /trpc-openapi@1.2.0(@trpc/server@10.40.0)(zod@3.21.4): resolution: {integrity: sha512-pfYoCd/3KYXWXvUPZBKJw455OOwngKN/6SIcj7Yit19OMLJ+8yVZkEvGEeg5wUSwfsiTdRsKuvqkRPXVSwV7ew==} peerDependencies: '@trpc/server': ^10.0.0 zod: ^3.14.4 dependencies: - '@trpc/server': 10.21.1 + '@trpc/server': 10.40.0 co-body: 6.1.0 h3: 1.8.2 lodash.clonedeep: 4.5.0 From 891da91d469037098e490762a0424eb81af2c163 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sat, 14 Oct 2023 01:02:31 -0700 Subject: [PATCH 49/80] Still issue with type failing in 3rd party --- apps/web/sdk/venice-sdk.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/web/sdk/venice-sdk.ts b/apps/web/sdk/venice-sdk.ts index a19637e7..1287321f 100644 --- a/apps/web/sdk/venice-sdk.ts +++ b/apps/web/sdk/venice-sdk.ts @@ -1,6 +1,7 @@ -import type {paths} from './venice.gen' import {Fetcher} from 'openapi-typescript-fetch' +import type {paths} from './venice.gen' + const fetcher = Fetcher.for() fetcher.configure({ baseUrl: 'http://localhost:3000/api/rest', @@ -13,8 +14,8 @@ fetcher.configure({ async function main() { // const result = await fetcher.path('/account').method('get').create()({ - // TODO: This should be taking params but it is not due to - // https://github.com/drwpow/openapi-typescript/issues/1040 + // TODO: This should be taking params but it is not due to + // https://github.com/drwpow/openapi-typescript/issues/1040 // }) // console.log('result', result.data[0]?.name) // result Plaid Gold Standard 0% Interest Checking (Plaid Checking) - 0000 From fa3fc3ae060a5dcab12b722178609dac56c1d8cd Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sat, 21 Oct 2023 20:09:39 -0700 Subject: [PATCH 50/80] fix: type failure due to trpc-openapi --- package.json | 3 ++- patches/trpc-openapi@1.2.0.patch | 15 +++++++++++++++ pnpm-lock.yaml | 10 +++++++--- 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 patches/trpc-openapi@1.2.0.patch diff --git a/package.json b/package.json index d926681c..71759543 100644 --- a/package.json +++ b/package.json @@ -99,7 +99,8 @@ "firebase@9.8.1": "patches/firebase@9.8.1.patch", "micro-memoize@4.0.10": "patches/micro-memoize@4.0.10.patch", "zod@3.21.4": "patches/zod@3.21.4.patch", - "@nangohq/frontend@0.33.8": "patches/@nangohq__frontend@0.33.8.patch" + "@nangohq/frontend@0.33.8": "patches/@nangohq__frontend@0.33.8.patch", + "trpc-openapi@1.2.0": "patches/trpc-openapi@1.2.0.patch" }, "peerDependencyRules": { "allowedVersions": { diff --git a/patches/trpc-openapi@1.2.0.patch b/patches/trpc-openapi@1.2.0.patch new file mode 100644 index 00000000..2014239b --- /dev/null +++ b/patches/trpc-openapi@1.2.0.patch @@ -0,0 +1,15 @@ +diff --git a/dist/types.d.ts b/dist/types.d.ts +index d93e38e020d89bef3e615c44fe687b10c8874417..956b349a0dda132bcdd7c255e8ae9946f41ccffd 100644 +--- a/dist/types.d.ts ++++ b/dist/types.d.ts +@@ -1,7 +1,8 @@ + import { Procedure, ProcedureParams, Router } from '@trpc/server'; + import type { RootConfig } from '@trpc/server/dist/core/internals/config'; +-import { TRPC_ERROR_CODE_KEY } from '@trpc/server/rpc'; +-import type { RouterDef } from '@trpc/server/src/core/router'; ++// PATCH: Courtesy of https://github.com/jlalmes/trpc-openapi/pull/411 ++import type { RouterDef } from '@trpc/server/dist/core/router'; ++import { TRPC_ERROR_CODE_KEY } from '@trpc/server/dist/rpc'; + import { OpenAPIV3 } from 'openapi-types'; + import { ZodIssue } from 'zod'; + export type OpenApiMethod = 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 64fea6b8..774f5eec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -33,6 +33,9 @@ patchedDependencies: micro-memoize@4.0.10: hash: hko3q5jvea7ey3trpjtkqgcale path: patches/micro-memoize@4.0.10.patch + trpc-openapi@1.2.0: + hash: zn3clwvt2tmhqbx7otjqaqekq4 + path: patches/trpc-openapi@1.2.0.patch zod@3.21.4: hash: bzwjzhue3hmpww5lnv24u5k2ru path: patches/zod@3.21.4.patch @@ -589,7 +592,7 @@ importers: version: 7.0.8 trpc-openapi: specifier: 1.2.0 - version: 1.2.0(@trpc/server@10.40.0)(zod@3.21.4) + version: 1.2.0(patch_hash=zn3clwvt2tmhqbx7otjqaqekq4)(@trpc/server@10.40.0)(zod@3.21.4) devDependencies: '@sentry/cli': specifier: 2.13.0 @@ -1325,7 +1328,7 @@ importers: devDependencies: trpc-openapi: specifier: 1.2.0 - version: 1.2.0(@trpc/server@10.40.0)(zod@3.21.4) + version: 1.2.0(patch_hash=zn3clwvt2tmhqbx7otjqaqekq4)(@trpc/server@10.40.0)(zod@3.21.4) packages/engine-frontend: dependencies: @@ -17137,7 +17140,7 @@ packages: resolution: {integrity: sha512-rvuRbTarPXmMb79SmzEp8aqXNKcK+y0XaB298IXueQ8I2PsrATcPBCSPyK/dDNa2iWOhKlfNnOjdAOTBU/nkFA==} dev: false - /trpc-openapi@1.2.0(@trpc/server@10.40.0)(zod@3.21.4): + /trpc-openapi@1.2.0(patch_hash=zn3clwvt2tmhqbx7otjqaqekq4)(@trpc/server@10.40.0)(zod@3.21.4): resolution: {integrity: sha512-pfYoCd/3KYXWXvUPZBKJw455OOwngKN/6SIcj7Yit19OMLJ+8yVZkEvGEeg5wUSwfsiTdRsKuvqkRPXVSwV7ew==} peerDependencies: '@trpc/server': ^10.0.0 @@ -17151,6 +17154,7 @@ packages: openapi-types: 12.1.3 zod: 3.21.4(patch_hash=bzwjzhue3hmpww5lnv24u5k2ru) zod-to-json-schema: 3.21.1(zod@3.21.4) + patched: true /ts-brand@0.0.2: resolution: {integrity: sha512-UhSzWY4On9ZHIj6DKkRYVN/8OaprbLAZ3b/Y2AJwdl6oozSABsQ0PvwDh4vOVdkvOtWQOkIrjctZ1kj8YfF3jA==} From 7e8608ef10245ff585be571c94f0b665b73ac9ab Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sat, 21 Oct 2023 22:49:44 -0700 Subject: [PATCH 51/80] Implement openAPI for listing resources by endUserId --- apps/cli/package.json | 2 +- .../(authenticated)/api-access/page.tsx | 5 ++- apps/web/pages/api/debug.ts | 1 + package.json | 2 +- packages/cdk-core/id.types.ts | 1 + packages/cdk-core/metaService.ts | 5 ++- packages/engine-backend/router/_schemas.ts | 6 ++++ .../engine-backend/router/protectedRouter.ts | 31 ++++++++++++------- packages/util/zod-jsonschema-utils.ts | 4 +++ patches/trpc-openapi@1.2.0.patch | 19 ++++++++++++ pnpm-lock.yaml | 29 ++++++++--------- 11 files changed, 74 insertions(+), 31 deletions(-) create mode 100644 packages/engine-backend/router/_schemas.ts diff --git a/apps/cli/package.json b/apps/cli/package.json index 6a1ad0dc..2cb16676 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -36,6 +36,6 @@ "cac": "6.7.12", "keytar": "7.9.0", "micro": "9.3.5-canary.3", - "ngrok": "4.3.3" + "ngrok": "5.0.0-beta.2" } } diff --git a/apps/web/app/(admin)/(authenticated)/api-access/page.tsx b/apps/web/app/(admin)/(authenticated)/api-access/page.tsx index acd1c0c5..c80d0c15 100644 --- a/apps/web/app/(admin)/(authenticated)/api-access/page.tsx +++ b/apps/web/app/(admin)/(authenticated)/api-access/page.tsx @@ -1,5 +1,6 @@ import Link from 'next/link' +import {kApikeyHeader, kApikeyUrlParam} from '@usevenice/app-config/constants' import {Input, Label} from '@usevenice/ui/shadcn' import {cn} from '@/lib-client/ui-utils' @@ -37,7 +38,9 @@ export default async function ApiKeyPage() { -

More docs coming soon

+

+ Use `{kApikeyHeader}` header or `{kApikeyUrlParam}` url param{' '} +

) } diff --git a/apps/web/pages/api/debug.ts b/apps/web/pages/api/debug.ts index ff0c2ca0..c35677de 100644 --- a/apps/web/pages/api/debug.ts +++ b/apps/web/pages/api/debug.ts @@ -1,5 +1,6 @@ import type {NextApiHandler} from 'next' export default ((_req, res) => { + console.log('req.query', _req.query) res.send({ok: true}) }) satisfies NextApiHandler diff --git a/package.json b/package.json index 71759543..94fdf0f6 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "jest-date-mock": "1.0.8", "jest-watch-typeahead": "2.1.1", "lint-staged": "13.0.3", - "ngrok": "4.3.3", + "ngrok": "5.0.0-beta.2", "npm-run-all": "4.1.5", "openapi-zod-client": "1.6.3", "prettier": "2.8.8", diff --git a/packages/cdk-core/id.types.ts b/packages/cdk-core/id.types.ts index d281261a..fad239af 100644 --- a/packages/cdk-core/id.types.ts +++ b/packages/cdk-core/id.types.ts @@ -46,6 +46,7 @@ export type Id = { export const zUserId = zId('user') export type UserId = z.infer +/** trpc-openapi limits us from using .brand https://share.cleanshot.com/Mf4F9xwZ */ export const zEndUserId = z.string().min(1).brand<'end_user'>() export type EndUserId = z.infer diff --git a/packages/cdk-core/metaService.ts b/packages/cdk-core/metaService.ts index 0ccfef64..ad485771 100644 --- a/packages/cdk-core/metaService.ts +++ b/packages/cdk-core/metaService.ts @@ -51,10 +51,13 @@ export interface MetaService { limit?: number offset?: number }) => Promise> + /** TODO: Implement limit & offset */ findPipelines: (options: { resourceIds?: Array secondsSinceLastSync?: number }) => Promise> /** Id is used to check RLS policy right now for end user */ - listIntegrationIds: (opts?: {id?: Id['int']}) => Promise> + listIntegrationIds: (opts?: { + id?: Id['int'] + }) => Promise> } diff --git a/packages/engine-backend/router/_schemas.ts b/packages/engine-backend/router/_schemas.ts new file mode 100644 index 00000000..4eea616d --- /dev/null +++ b/packages/engine-backend/router/_schemas.ts @@ -0,0 +1,6 @@ +import {z} from '@usevenice/util' + +export const zListParams = z.object({ + limit: z.number().optional(), + offset: z.number().optional(), +}) diff --git a/packages/engine-backend/router/protectedRouter.ts b/packages/engine-backend/router/protectedRouter.ts index 5bb70606..52b8bcbf 100644 --- a/packages/engine-backend/router/protectedRouter.ts +++ b/packages/engine-backend/router/protectedRouter.ts @@ -1,7 +1,7 @@ import {TRPCError} from '@trpc/server' -import type { - ZRaw} from '@usevenice/cdk-core'; +import type {ZRaw} from '@usevenice/cdk-core' +import {zEndUserId} from '@usevenice/cdk-core' import { extractId, sync, @@ -17,6 +17,7 @@ import {inngest, zEvent} from '../events' import {parseWebhookRequest} from '../parseWebhookRequest' import {zSyncOptions} from '../types' import {protectedProcedure, trpc} from './_base' +import {zListParams} from './_schemas' export {type inferProcedureInput} from '@trpc/server' @@ -36,18 +37,24 @@ export const protectedRouter = trpc.router({ }), listResources: protectedProcedure .meta({openapi: {method: 'GET', path: '/resources'}}) - .input(z.object({}).optional()) + .input(zListParams.extend({endUserId: zEndUserId.optional()}).optional()) .output(z.array(zRaw.resource)) - .query(async ({ctx}) => { - const resources = await ctx.helpers.metaService.tables.resource.list({}) + .query(async ({input = {}, ctx}) => { + const resources = await ctx.helpers.metaService.tables.resource.list( + input, + ) return resources as Array }), listPipelines: protectedProcedure .meta({openapi: {method: 'GET', path: '/pipelines'}}) - .input(z.object({}).optional()) + .input( + zListParams + .extend({resourceIds: z.array(zId('reso')).optional()}) + .optional(), + ) .output(z.array(zRaw.pipeline)) - .query(async ({ctx}) => { - const pipelines = await ctx.helpers.metaService.tables.pipeline.list({}) + .query(async ({input = {}, ctx}) => { + const pipelines = await ctx.helpers.metaService.findPipelines(input) return pipelines as Array }), deletePipeline: protectedProcedure @@ -59,11 +66,13 @@ export const protectedRouter = trpc.router({ return true as const }), listConnections: protectedProcedure - .input(z.object({}).optional()) - .query(async ({ctx}) => { + .input(zListParams.extend({endUserId: zEndUserId.optional()}).optional()) + .query(async ({input = {}, ctx}) => { // Add info about what it takes to `reconnect` here for resources which // has disconnected - const resources = await ctx.helpers.metaService.tables.resource.list({}) + const resources = await ctx.helpers.metaService.tables.resource.list( + input, + ) const [institutions, _pipelines] = await Promise.all([ ctx.helpers.metaService.tables.institution.list({ ids: R.compact(resources.map((c) => c.institutionId)), diff --git a/packages/util/zod-jsonschema-utils.ts b/packages/util/zod-jsonschema-utils.ts index 6d451502..01672c84 100644 --- a/packages/util/zod-jsonschema-utils.ts +++ b/packages/util/zod-jsonschema-utils.ts @@ -51,6 +51,10 @@ export function ensureNodeTitle(jsonSchema: T) { return jsonSchema } +/** + * TODO: Consider switching to this repo to allow better actual customizations + * https://github.com/anatine/zod-plugins/tree/main/packages/zod-openapi + */ export function zodToJsonSchema(schema: z.ZodTypeAny) { // Defaulting title should occur last, this way we don't end up with extraneous one return defaultTitleAsJsonPath(ensureNodeTitle(_zodToJsonSchema(schema))) diff --git a/patches/trpc-openapi@1.2.0.patch b/patches/trpc-openapi@1.2.0.patch index 2014239b..daa55e6c 100644 --- a/patches/trpc-openapi@1.2.0.patch +++ b/patches/trpc-openapi@1.2.0.patch @@ -13,3 +13,22 @@ index d93e38e020d89bef3e615c44fe687b10c8874417..956b349a0dda132bcdd7c255e8ae9946 import { OpenAPIV3 } from 'openapi-types'; import { ZodIssue } from 'zod'; export type OpenApiMethod = 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE'; +diff --git a/dist/utils/zod.js b/dist/utils/zod.js +index f9b479f56b52ceea1cab9b581c7dcd27d500d5d6..c848fa8030d6e275608863333fe9f61879692f16 100644 +--- a/dist/utils/zod.js ++++ b/dist/utils/zod.js +@@ -27,6 +27,14 @@ const instanceofZodTypeLikeVoid = (type) => { + }; + exports.instanceofZodTypeLikeVoid = instanceofZodTypeLikeVoid; + const unwrapZodType = (type, unwrapPreprocess) => { ++ // This will fail at runtime because we are not properly coercing to array type yet ++ if ((0, exports.instanceofZodTypeKind)(type, zod_1.z.ZodFirstPartyTypeKind.ZodArray)) { ++ return (0, exports.unwrapZodType)(type.element, unwrapPreprocess); ++ } ++ // Works fine ++ if ((0, exports.instanceofZodTypeKind)(type, zod_1.z.ZodFirstPartyTypeKind.ZodBrand)) { ++ return (0, exports.unwrapZodType)(type.unwrap(), unwrapPreprocess); ++ } + if ((0, exports.instanceofZodTypeKind)(type, zod_1.z.ZodFirstPartyTypeKind.ZodOptional)) { + return (0, exports.unwrapZodType)(type.unwrap(), unwrapPreprocess); + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 774f5eec..9a9d6f70 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -34,7 +34,7 @@ patchedDependencies: hash: hko3q5jvea7ey3trpjtkqgcale path: patches/micro-memoize@4.0.10.patch trpc-openapi@1.2.0: - hash: zn3clwvt2tmhqbx7otjqaqekq4 + hash: mwbhlfdju7i63ujwpiq2ctfpbe path: patches/trpc-openapi@1.2.0.patch zod@3.21.4: hash: bzwjzhue3hmpww5lnv24u5k2ru @@ -123,8 +123,8 @@ importers: specifier: 13.0.3 version: 13.0.3 ngrok: - specifier: 4.3.3 - version: 4.3.3 + specifier: 5.0.0-beta.2 + version: 5.0.0-beta.2 npm-run-all: specifier: 4.1.5 version: 4.1.5 @@ -406,8 +406,8 @@ importers: specifier: 9.3.5-canary.3 version: 9.3.5-canary.3 ngrok: - specifier: 4.3.3 - version: 4.3.3 + specifier: 5.0.0-beta.2 + version: 5.0.0-beta.2 apps/gondola: dependencies: @@ -592,7 +592,7 @@ importers: version: 7.0.8 trpc-openapi: specifier: 1.2.0 - version: 1.2.0(patch_hash=zn3clwvt2tmhqbx7otjqaqekq4)(@trpc/server@10.40.0)(zod@3.21.4) + version: 1.2.0(patch_hash=mwbhlfdju7i63ujwpiq2ctfpbe)(@trpc/server@10.40.0)(zod@3.21.4) devDependencies: '@sentry/cli': specifier: 2.13.0 @@ -1328,7 +1328,7 @@ importers: devDependencies: trpc-openapi: specifier: 1.2.0 - version: 1.2.0(patch_hash=zn3clwvt2tmhqbx7otjqaqekq4)(@trpc/server@10.40.0)(zod@3.21.4) + version: 1.2.0(patch_hash=mwbhlfdju7i63ujwpiq2ctfpbe)(@trpc/server@10.40.0)(zod@3.21.4) packages/engine-frontend: dependencies: @@ -6472,9 +6472,6 @@ packages: resolution: {integrity: sha512-0os9vz6BpGwxGe9LOhgP/ncvYN5Tx1fNcd2TM3rD/aCGBkysb+ZWpXEocG24h6ZzOi13+VB8HndAQFezsSOw1w==} dev: true - /@types/node@8.10.66: - resolution: {integrity: sha512-tktOkFUA4kXx2hhhrB8bIFb5TbwzS4uOhKEmwiD+NoiL0qtP2OQ9mFldbgD4dV1djrlBYP6eBuQZiWjuHUpqFw==} - /@types/normalize-package-data@2.4.1: resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} dev: true @@ -13417,18 +13414,17 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /ngrok@4.3.3: - resolution: {integrity: sha512-a2KApnkiG5urRxBPdDf76nNBQTnNNWXU0nXw0SsqsPI+Kmt2lGf9TdVYpYrHMnC+T9KhcNSWjCpWqBgC6QcFvw==} - engines: {node: '>=10.19.0 <14 || >=14.2'} + /ngrok@5.0.0-beta.2: + resolution: {integrity: sha512-UzsyGiJ4yTTQLCQD11k1DQaMwq2/SsztBg2b34zAqcyjS25qjDpogMKPaCKHwe/APRTHeel3iDXcVctk5CNaCQ==} + engines: {node: '>=14.2'} hasBin: true requiresBuild: true dependencies: - '@types/node': 8.10.66 extract-zip: 2.0.1 got: 11.8.5 lodash.clonedeep: 4.5.0 uuid: 8.3.2 - yaml: 1.10.2 + yaml: 2.3.0 optionalDependencies: hpagent: 0.1.2 transitivePeerDependencies: @@ -17140,7 +17136,7 @@ packages: resolution: {integrity: sha512-rvuRbTarPXmMb79SmzEp8aqXNKcK+y0XaB298IXueQ8I2PsrATcPBCSPyK/dDNa2iWOhKlfNnOjdAOTBU/nkFA==} dev: false - /trpc-openapi@1.2.0(patch_hash=zn3clwvt2tmhqbx7otjqaqekq4)(@trpc/server@10.40.0)(zod@3.21.4): + /trpc-openapi@1.2.0(patch_hash=mwbhlfdju7i63ujwpiq2ctfpbe)(@trpc/server@10.40.0)(zod@3.21.4): resolution: {integrity: sha512-pfYoCd/3KYXWXvUPZBKJw455OOwngKN/6SIcj7Yit19OMLJ+8yVZkEvGEeg5wUSwfsiTdRsKuvqkRPXVSwV7ew==} peerDependencies: '@trpc/server': ^10.0.0 @@ -18141,6 +18137,7 @@ packages: /yaml@1.10.2: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} engines: {node: '>= 6'} + dev: false /yaml@2.1.3: resolution: {integrity: sha512-AacA8nRULjKMX2DvWvOAdBZMOfQlypSFkjcOcu9FalllIDJ1kvlREzcdIZmidQUqqeMv7jorHjq2HlLv/+c2lg==} From 3973cd149ef06005481919f078eb48e38f73ef66 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 22 Oct 2023 09:58:44 -0700 Subject: [PATCH 52/80] fix: disabling test that are broken due to downed api --- .../beancountConverters.spec.ts | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/integrations/integration-beancount/beancountConverters.spec.ts b/integrations/integration-beancount/beancountConverters.spec.ts index 6daca344..f50b88b6 100644 --- a/integrations/integration-beancount/beancountConverters.spec.ts +++ b/integrations/integration-beancount/beancountConverters.spec.ts @@ -23,7 +23,26 @@ const stringSnapshotSerializer: jest.SnapshotSerializerPlugin = { expect.addSnapshotSerializer(stringSnapshotSerializer) -test('degenerate case: no transactions', async () => { +test.each([ + [ + 'General & Administrative/Business Insurance_Policy', + 'General-Administrative:Business-Insurance-Policy', + ], + ['Chase CC (2289)', 'Chase-CC-2289'], +])('cleanBeancountAccountName(%o) -> %o', (input, output) => { + expect(cleanBeancountAccountName(input)).toEqual(output) +}) + +test.each([['Assets:Wallet', {type: ACCOUNT_TYPES.asset, name: 'Wallet'}]])( + 'convAccountFullName(%o) -> %o', + (input, output) => { + expect(convAccountFullName(input)).toMatchObject(output) + }, +) + +// Disable because the json conversion api is down +// eslint-disable-next-line jest/no-disabled-tests +test.skip('degenerate case: no transactions', async () => { await expect( convBeanToStdJson .reverse({version: '1', variant: 'standard', entities: []}) @@ -31,7 +50,9 @@ test('degenerate case: no transactions', async () => { ).resolves.toMatchSnapshot() }, 60000) -test('ledger with default unit', async () => { +// Disable because the json conversion api is down +// eslint-disable-next-line jest/no-disabled-tests +test.skip('ledger with default unit', async () => { await expect( convBeanToStdJson .reverse({ @@ -44,24 +65,9 @@ test('ledger with default unit', async () => { ).resolves.toMatchSnapshot() }, 60000) -test.each([ - [ - 'General & Administrative/Business Insurance_Policy', - 'General-Administrative:Business-Insurance-Policy', - ], - ['Chase CC (2289)', 'Chase-CC-2289'], -])('cleanBeancountAccountName(%o) -> %o', (input, output) => { - expect(cleanBeancountAccountName(input)).toEqual(output) -}) - -test.each([['Assets:Wallet', {type: ACCOUNT_TYPES.asset, name: 'Wallet'}]])( - 'convAccountFullName(%o) -> %o', - (input, output) => { - expect(convAccountFullName(input)).toMatchObject(output) - }, -) - -test.each([ +// Disable because the json conversion api is down +// eslint-disable-next-line jest/no-disabled-tests +test.skip.each([ { date: '2020-01-01', description: 'Latte', From 719cf468409134eae74dccfd6b8d22b3a4bcf4d0 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 22 Oct 2023 19:24:56 -0700 Subject: [PATCH 53/80] feat: Expose openapi endpoint for all CRUD around integrations and resources --- .../integrations/IntegrationsPage.tsx | 2 +- .../resources/ResourcesPage.tsx | 2 +- packages/engine-backend/router/adminRouter.ts | 26 +++++++++------ .../engine-backend/router/endUserRouter.ts | 33 +++++++++++++++---- .../engine-backend/router/protectedRouter.ts | 33 ------------------- packages/engine-frontend/VeniceConnect.tsx | 2 +- 6 files changed, 46 insertions(+), 52 deletions(-) diff --git a/apps/web/app/(admin)/(authenticated)/integrations/IntegrationsPage.tsx b/apps/web/app/(admin)/(authenticated)/integrations/IntegrationsPage.tsx index 0be436cc..f5adbb29 100644 --- a/apps/web/app/(admin)/(authenticated)/integrations/IntegrationsPage.tsx +++ b/apps/web/app/(admin)/(authenticated)/integrations/IntegrationsPage.tsx @@ -300,7 +300,7 @@ export function IntegrationSheet({ // Specifying asChild and using this variant does not appear to be // working for some reason... variant="destructive" - onClick={() => deleteIntegration.mutate([int.id])}> + onClick={() => deleteIntegration.mutate({id: int.id})}> {deleteIntegration.isLoading && ( )} diff --git a/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx b/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx index 35ab0589..116d9d40 100644 --- a/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx +++ b/apps/web/app/(admin)/(authenticated)/resources/ResourcesPage.tsx @@ -165,7 +165,7 @@ function EditResourceSheet({ }) }, }) - const deleteResource = _trpcReact.deleteResoruce.useMutation({ + const deleteResource = _trpcReact.deleteResource.useMutation({ onSuccess: () => { setOpen(false) toast({title: 'Resource deleted', variant: 'success'}) diff --git a/packages/engine-backend/router/adminRouter.ts b/packages/engine-backend/router/adminRouter.ts index ccd68350..2c086911 100644 --- a/packages/engine-backend/router/adminRouter.ts +++ b/packages/engine-backend/router/adminRouter.ts @@ -80,6 +80,7 @@ export const adminRouter = trpc.router({ // TODO: Right now this means client has to be responsible for creating // integration IDs, we should support creating integration with providerName instead adminUpsertIntegration: adminProcedure + .meta({openapi: {method: 'POST', path: '/integrations'}}) .input( zRaw.integration .pick({ @@ -96,6 +97,7 @@ export const adminRouter = trpc.router({ // this makes me wonder if UPSERT should always be the default.... .required({orgId: true}), ) + .output(zRaw.integration) .mutation(async ({input: {id: _id, providerName, ...input}, ctx}) => { const id = _id ? _id @@ -130,8 +132,10 @@ export const adminRouter = trpc.router({ }), // Need a tuple for some reason... otherwise seems to not work in practice. adminDeleteIntegration: adminProcedure - .input(z.tuple([zId('int')])) - .mutation(async ({input: [intId], ctx}) => { + .meta({openapi: {method: 'DELETE', path: '/integrations/{id}'}}) + .input(z.object({id: zId('int')})) + .output(z.void()) + .mutation(async ({input: {id: intId}, ctx}) => { const provider = ctx.providerMap[extractProviderName(intId)] if (provider?.metadata?.nangoProvider) { await ctx.nango.delete('/config/{provider_config_key}', { @@ -141,7 +145,9 @@ export const adminRouter = trpc.router({ return ctx.helpers.metaService.tables.integration.delete(intId) }), adminCreateConnectToken: adminProcedure + .meta({openapi: {method: 'POST', path: '/connect-token'}}) .input(adminRouterSchema.adminCreateConnectToken.input) + .output(z.string()) .mutation(({input: {endUserId, orgId, validityInSeconds}, ctx}) => { if ( (ctx.viewer.role === 'user' || ctx.viewer.role === 'org') && @@ -167,14 +173,14 @@ export const adminRouter = trpc.router({ .then((rows) => rows.filter((u) => !!u.id)), ), adminGetIntegration: adminProcedure - .input(zId('int')) - .query(async ({input: intId, ctx}) => { - const int = await ctx.helpers.getIntegrationOrFail(intId) - return { - config: int.config, - provider: int.provider.name, - id: int.id, - } + .meta({openapi: {method: 'GET', path: '/integrations/{id}'}}) + .input(z.object({id: zId('int')})) + .output(zRaw.integration) + .query(async ({input: {id: intId}, ctx}) => { + const {provider: _, ...int} = await ctx.helpers.getIntegrationOrFail( + intId, + ) + return int }), adminSyncMetadata: adminProcedure .input(zId('int').nullish()) diff --git a/packages/engine-backend/router/endUserRouter.ts b/packages/engine-backend/router/endUserRouter.ts index 05084e0b..4b3c3744 100644 --- a/packages/engine-backend/router/endUserRouter.ts +++ b/packages/engine-backend/router/endUserRouter.ts @@ -145,11 +145,12 @@ export const endUserRouter = trpc.router({ }, ), createResource: protectedProcedure + .meta({openapi: {method: 'POST', path: '/resources'}}) .input(zRaw.resource.pick({integrationId: true, settings: true})) // Questionable why `zConnectContextInput` should be there. Examine whether this is actually // needed // How do we verify that the userId here is the same as the userId from preConnectOption? - + .output(z.string()) .mutation(async ({input: {integrationId, settings}, ctx}) => { // Authorization await ctx.helpers.getIntegrationInfoOrFail(integrationId) @@ -181,16 +182,36 @@ export const endUserRouter = trpc.router({ // TODO: Run server-side validation updateResource: protectedProcedure + .meta({openapi: {method: 'PATCH', path: '/resources/{id}'}}) .input(zRaw.resource.pick({id: true, settings: true, displayName: true})) + .output(zRaw.resource) .mutation(async ({input: {id, ...input}, ctx}) => // TODO: Run mapStandardResource after editing // Also we probably do not want deeply nested patch // shallow is sufficient more most situations ctx.helpers.patchReturning('resource', id, input), ), - deleteResoruce: protectedProcedure - .input(zRaw.resource.pick({id: true})) - .mutation(async ({input: {id}, ctx}) => - ctx.helpers.metaService.tables.resource.delete(id), - ), + deleteResource: protectedProcedure + .meta({openapi: {method: 'DELETE', path: '/resources/{id}'}}) + .input(z.object({id: zId('reso'), skipRevoke: z.boolean().optional()})) + .output(z.void()) + .mutation(async ({input: {id: resoId, ...opts}, ctx}) => { + if (ctx.viewer.role === 'end_user') { + await ctx.helpers.getResourceOrFail(resoId) + } + const {settings, integration, ...reso} = + await ctx.asOrgIfNeeded.getResourceExpandedOrFail(resoId) + if (!opts?.skipRevoke) { + await integration.provider.revokeResource?.( + settings, + integration.config, + ) + } + // if (opts?.todo_deleteAssociatedData) { + // TODO: Figure out how to delete... Destination is not part of meta service + // and we don't easily have the ability to handle a delete, it's not part of the sync protocol yet... + // We should probably introduce a reset / delete event... + // } + await ctx.asOrgIfNeeded.metaService.tables.resource.delete(reso.id) + }), }) diff --git a/packages/engine-backend/router/protectedRouter.ts b/packages/engine-backend/router/protectedRouter.ts index 52b8bcbf..be98d268 100644 --- a/packages/engine-backend/router/protectedRouter.ts +++ b/packages/engine-backend/router/protectedRouter.ts @@ -258,39 +258,6 @@ export const protectedRouter = trpc.router({ } return 'Ok' }), - // What about delete? Should this delete also? Or soft delete? - deleteResource: protectedProcedure - .input( - z.tuple([ - zId('reso'), - z - .object({ - skipRevoke: z.boolean().nullish(), - todo_deleteAssociatedData: z.boolean().nullish(), - }) - .optional(), - ]), - ) - .mutation(async ({input: [resoId, opts], ctx}) => { - if (ctx.viewer.role === 'end_user') { - await ctx.helpers.getResourceOrFail(resoId) - } - const {settings, integration, ...reso} = - await ctx.asOrgIfNeeded.getResourceExpandedOrFail(resoId) - if (!opts?.skipRevoke) { - await integration.provider.revokeResource?.( - settings, - integration.config, - ) - } - if (opts?.todo_deleteAssociatedData) { - // TODO: Figure out how to delete... Destination is not part of meta service - // and we don't easily have the ability to handle a delete, it's not part of the sync protocol yet... - // We should probably introduce a reset / delete event... - } - await ctx.asOrgIfNeeded.metaService.tables.resource.delete(reso.id) - }), - // MARK: - Sync syncResource: protectedProcedure diff --git a/packages/engine-frontend/VeniceConnect.tsx b/packages/engine-frontend/VeniceConnect.tsx index aa3469a4..ec021ee5 100644 --- a/packages/engine-frontend/VeniceConnect.tsx +++ b/packages/engine-frontend/VeniceConnect.tsx @@ -511,7 +511,7 @@ export function ResourceDropdownMenu( // }) // }, // }) - const deleteResource = _trpcReact.deleteResoruce.useMutation({ + const deleteResource = _trpcReact.deleteResource.useMutation({ onSuccess: () => { setOpen(false) toast({title: 'Resource deleted', variant: 'success'}) From 8abbd5306a256905ad990a9a6e30214136c7f26e Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 22 Oct 2023 20:08:13 -0700 Subject: [PATCH 54/80] Add endpoint to createMagicLink --- .../(authenticated)/end-users/page.tsx | 36 ++++---- .../(authenticated)/magic-link/page.tsx | 37 +++------ apps/web/app/connect/page.tsx | 8 +- packages/engine-backend/router/adminRouter.ts | 83 ++++++++++++------- 4 files changed, 85 insertions(+), 79 deletions(-) diff --git a/apps/web/app/(admin)/(authenticated)/end-users/page.tsx b/apps/web/app/(admin)/(authenticated)/end-users/page.tsx index b5448445..51f2dcd0 100644 --- a/apps/web/app/(admin)/(authenticated)/end-users/page.tsx +++ b/apps/web/app/(admin)/(authenticated)/end-users/page.tsx @@ -1,9 +1,7 @@ 'use client' -import {useAuth} from '@clerk/nextjs' import {Copy, MoreHorizontal, RefreshCcw} from 'lucide-react' -import {getServerUrl} from '@usevenice/app-config/constants' import type {RouterOutput} from '@usevenice/engine-backend' import {_trpcReact} from '@usevenice/engine-frontend' import { @@ -45,8 +43,7 @@ type EndUser = RouterOutput['adminSearchEndUsers'][number] function EndUserMenu({endUser}: {endUser: EndUser}) { const {toast} = useToast() - const createConnectToken = _trpcReact.adminCreateConnectToken.useMutation({}) - const {orgId} = useAuth() + const createMagicLink = _trpcReact.adminCreateMagicLink.useMutation({}) return ( @@ -70,24 +67,19 @@ function EndUserMenu({endUser}: {endUser: EndUser}) { { - if (orgId) { - createConnectToken - .mutateAsync({endUserId: endUser.id, orgId}) - .then((token) => { - // This is a problem because due to pop up blockers... - const url = new URL('/connect', getServerUrl(null)) - url.searchParams.set('token', token) - - window.open(url) - }) - .catch((err) => - toast({ - title: 'Failed to create connect token', - description: `${err}`, - variant: 'destructive', - }), - ) - } + createMagicLink + .mutateAsync({endUserId: endUser.id}) + .then((res) => { + // This is a problem because due to pop up blockers not liking it async... + window.open(res.url) + }) + .catch((err) => + toast({ + title: 'Failed to create connect token', + description: `${err}`, + variant: 'destructive', + }), + ) }}> View portal diff --git a/apps/web/app/(admin)/(authenticated)/magic-link/page.tsx b/apps/web/app/(admin)/(authenticated)/magic-link/page.tsx index deb2d033..d911d56d 100644 --- a/apps/web/app/(admin)/(authenticated)/magic-link/page.tsx +++ b/apps/web/app/(admin)/(authenticated)/magic-link/page.tsx @@ -1,22 +1,15 @@ 'use client' -import {getServerUrl} from '@usevenice/app-config/constants' import {adminRouterSchema} from '@usevenice/engine-backend/router/adminRouter' import {_trpcReact} from '@usevenice/engine-frontend' import {SchemaForm, useToast} from '@usevenice/ui' -import {useCurrengOrg} from '@/components/viewer-context' import {copyToClipboard} from '@/lib-client/copyToClipboard' -const formSchema = adminRouterSchema.adminCreateConnectToken.input.omit({ - orgId: true, -}) - export default function MagicLinkPage() { - const {orgId} = useCurrengOrg() const {toast} = useToast() - const createToken = _trpcReact.adminCreateConnectToken.useMutation({ + const createMagicLink = _trpcReact.adminCreateMagicLink.useMutation({ onError: (err) => { toast({ title: 'Error creating magic link', @@ -30,26 +23,18 @@ export default function MagicLinkPage() {

Magic link

{ - createToken.mutate( - {...values, orgId}, - { - onSuccess: async (data) => { - const url = new URL('/connect', getServerUrl(null)) - url.searchParams.set('token', data) - url.searchParams.set('displayName', values.displayName ?? '') - url.searchParams.set('redirectUrl', values.redirectUrl ?? '') - - await copyToClipboard(url.toString()) - toast({ - title: 'Magic link copied to clipboard', - variant: 'success', - }) - }, + createMagicLink.mutate(values, { + onSuccess: async (data) => { + await copyToClipboard(data.url) + toast({ + title: 'Magic link copied to clipboard', + variant: 'success', + }) }, - ) + }) }} />
diff --git a/apps/web/app/connect/page.tsx b/apps/web/app/connect/page.tsx index 1daa53d3..aeb538c4 100644 --- a/apps/web/app/connect/page.tsx +++ b/apps/web/app/connect/page.tsx @@ -2,9 +2,10 @@ import {clerkClient} from '@clerk/nextjs' import Image from 'next/image' import {getViewerId} from '@usevenice/cdk-core' +import {zConnectPageParams} from '@usevenice/engine-backend/router/adminRouter' -import {SuperHydrate} from '@/components/SuperHydrate' import {ClientRoot} from '@/components/ClientRoot' +import {SuperHydrate} from '@/components/SuperHydrate' import {createServerComponentHelpers} from '@/lib-server/server-component-helpers' import ConnectPage from './ConnectPage' @@ -31,8 +32,9 @@ export default async function ConnectPageContainer({ // @see https://github.com/vercel/next.js/issues/43704 searchParams: Record }) { + const params = zConnectPageParams.parse(searchParams) const {ssg, getDehydratedState, viewer} = await createServerComponentHelpers({ - searchParams: {_token: searchParams['token']}, + searchParams: {_token: params.token}, }) if (viewer.role !== 'end_user') { return ( @@ -57,7 +59,7 @@ export default async function ConnectPageContainer({ className="mr-4 rounded-lg" />

- {org.name} - {viewer.endUserId} + {params.displayName ?? `${org.name} - ${viewer.endUserId}`}

diff --git a/packages/engine-backend/router/adminRouter.ts b/packages/engine-backend/router/adminRouter.ts index 2c086911..db29067b 100644 --- a/packages/engine-backend/router/adminRouter.ts +++ b/packages/engine-backend/router/adminRouter.ts @@ -17,6 +17,31 @@ import {adminProcedure, trpc} from './_base' export {type inferProcedureInput} from '@trpc/server' +export const zConnectTokenPayload = z.object({ + endUserId: zEndUserId.describe( + 'Anything that uniquely identifies the end user that you will be sending the magic link to', + ), + validityInSeconds: z + .number() + .default(3600) + .describe( + 'How long the magic link will be valid for (in seconds) before it expires', + ), +}) + +export const zConnectPageParams = z.object({ + token: z.string(), + displayName: z.string().nullish().describe('What to call user by'), + redirectUrl: z + .string() + .nullish() + .describe( + 'Where to send user to after connect / if they press back button', + ), + /** Launch the integration right away */ + integrationId: z.string().nullish(), +}) + /** * Workaround to be able to re-use the schema on the frontend for now * @see https://github.com/trpc/trpc/issues/4295 @@ -30,26 +55,9 @@ export {type inferProcedureInput} from '@trpc/server' * via a trpc schema endpoint (with server side rendering of course) */ export const adminRouterSchema = { - adminCreateConnectToken: { - input: z.object({ - orgId: zId('org'), - endUserId: zEndUserId.describe( - 'Anything that uniquely identifies the end user that you will be sending the magic link to', - ), - displayName: z.string().nullish().describe('What to call user by'), - redirectUrl: z - .string() - .nullish() - .describe( - 'Where to send user to after connect / if they press back button', - ), - validityInSeconds: z - .number() - .default(3600) - .describe( - 'How long the magic link will be valid for (in seconds) before it expires', - ), - }), + adminCreateConnectToken: {input: zConnectTokenPayload}, + adminCreateMagicLink: { + input: zConnectTokenPayload.merge(zConnectPageParams.omit({token: true})), }, } satisfies Record @@ -148,21 +156,40 @@ export const adminRouter = trpc.router({ .meta({openapi: {method: 'POST', path: '/connect-token'}}) .input(adminRouterSchema.adminCreateConnectToken.input) .output(z.string()) - .mutation(({input: {endUserId, orgId, validityInSeconds}, ctx}) => { - if ( - (ctx.viewer.role === 'user' || ctx.viewer.role === 'org') && - ctx.viewer.orgId !== orgId - ) { + .mutation(({input: {endUserId, validityInSeconds}, ctx}) => { + // Figure out a better way to share code here... + if (!('orgId' in ctx.viewer) || !ctx.viewer.orgId) { throw new TRPCError({ - code: 'FORBIDDEN', - message: `${orgId} Not your org`, + code: 'BAD_REQUEST', + message: 'Current viewer missing orgId to create token', }) } return ctx.jwt.signViewer( - {role: 'end_user', endUserId, orgId}, + {role: 'end_user', endUserId, orgId: ctx.viewer.orgId}, {validityInSeconds}, ) }), + adminCreateMagicLink: adminProcedure + .meta({openapi: {method: 'POST', path: '/magic-link'}}) + .input(adminRouterSchema.adminCreateMagicLink.input) + .output(z.object({url: z.string()})) + .mutation(({input: {endUserId, validityInSeconds, ...params}, ctx}) => { + if (!('orgId' in ctx.viewer) || !ctx.viewer.orgId) { + throw new TRPCError({ + code: 'BAD_REQUEST', + message: 'Current viewer missing orgId to create token', + }) + } + const token = ctx.jwt.signViewer( + {role: 'end_user', endUserId, orgId: ctx.viewer.orgId}, + {validityInSeconds}, + ) + const url = new URL('/connect', ctx.apiUrl) // `/` will start from the root hostname itself + for (const [key, value] of Object.entries({...params, token})) { + url.searchParams.set(key, value ?? '') + } + return {url: url.toString()} + }), adminSearchEndUsers: adminProcedure .input(z.object({keywords: z.string().trim().nullish()}).optional()) .query(async ({input: {keywords} = {}, ctx}) => From ecd34ff4be6552bdf2735cf8d6c582ef70bc8744 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 22 Oct 2023 20:49:04 -0700 Subject: [PATCH 55/80] Patch trpc-openapi for zod enum also --- patches/trpc-openapi@1.2.0.patch | 13 ++++++++++--- pnpm-lock.yaml | 8 ++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/patches/trpc-openapi@1.2.0.patch b/patches/trpc-openapi@1.2.0.patch index daa55e6c..488514cd 100644 --- a/patches/trpc-openapi@1.2.0.patch +++ b/patches/trpc-openapi@1.2.0.patch @@ -14,10 +14,10 @@ index d93e38e020d89bef3e615c44fe687b10c8874417..956b349a0dda132bcdd7c255e8ae9946 import { ZodIssue } from 'zod'; export type OpenApiMethod = 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE'; diff --git a/dist/utils/zod.js b/dist/utils/zod.js -index f9b479f56b52ceea1cab9b581c7dcd27d500d5d6..c848fa8030d6e275608863333fe9f61879692f16 100644 +index f9b479f56b52ceea1cab9b581c7dcd27d500d5d6..89cc069f92ea530e94a8cb4016bbd54011aaabde 100644 --- a/dist/utils/zod.js +++ b/dist/utils/zod.js -@@ -27,6 +27,14 @@ const instanceofZodTypeLikeVoid = (type) => { +@@ -27,6 +27,21 @@ const instanceofZodTypeLikeVoid = (type) => { }; exports.instanceofZodTypeLikeVoid = instanceofZodTypeLikeVoid; const unwrapZodType = (type, unwrapPreprocess) => { @@ -26,9 +26,16 @@ index f9b479f56b52ceea1cab9b581c7dcd27d500d5d6..c848fa8030d6e275608863333fe9f618 + return (0, exports.unwrapZodType)(type.element, unwrapPreprocess); + } + // Works fine -+ if ((0, exports.instanceofZodTypeKind)(type, zod_1.z.ZodFirstPartyTypeKind.ZodBrand)) { ++ if ((0, exports.instanceofZodTypeKind)(type, zod_1.z.ZodFirstPartyTypeKind.ZodEnum)) { ++ return (0, exports.unwrapZodType)(zod_1.z.string(), unwrapPreprocess); ++ } ++ if ((0, exports.instanceofZodTypeKind)(type, zod_1.z.ZodFirstPartyTypeKind.ZodNullable)) { ++ return (0, exports.unwrapZodType)(type.unwrap(), unwrapPreprocess); ++ } ++ if ((0, exports.instanceofZodTypeKind)(type, zod_1.z.ZodFirstPartyTypeKind.ZodBranded)) { + return (0, exports.unwrapZodType)(type.unwrap(), unwrapPreprocess); + } ++ // --- end of patch --- if ((0, exports.instanceofZodTypeKind)(type, zod_1.z.ZodFirstPartyTypeKind.ZodOptional)) { return (0, exports.unwrapZodType)(type.unwrap(), unwrapPreprocess); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9a9d6f70..568189c0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -34,7 +34,7 @@ patchedDependencies: hash: hko3q5jvea7ey3trpjtkqgcale path: patches/micro-memoize@4.0.10.patch trpc-openapi@1.2.0: - hash: mwbhlfdju7i63ujwpiq2ctfpbe + hash: chm753bazxetxxpzf3uqs4fzzu path: patches/trpc-openapi@1.2.0.patch zod@3.21.4: hash: bzwjzhue3hmpww5lnv24u5k2ru @@ -592,7 +592,7 @@ importers: version: 7.0.8 trpc-openapi: specifier: 1.2.0 - version: 1.2.0(patch_hash=mwbhlfdju7i63ujwpiq2ctfpbe)(@trpc/server@10.40.0)(zod@3.21.4) + version: 1.2.0(patch_hash=chm753bazxetxxpzf3uqs4fzzu)(@trpc/server@10.40.0)(zod@3.21.4) devDependencies: '@sentry/cli': specifier: 2.13.0 @@ -1328,7 +1328,7 @@ importers: devDependencies: trpc-openapi: specifier: 1.2.0 - version: 1.2.0(patch_hash=mwbhlfdju7i63ujwpiq2ctfpbe)(@trpc/server@10.40.0)(zod@3.21.4) + version: 1.2.0(patch_hash=chm753bazxetxxpzf3uqs4fzzu)(@trpc/server@10.40.0)(zod@3.21.4) packages/engine-frontend: dependencies: @@ -17136,7 +17136,7 @@ packages: resolution: {integrity: sha512-rvuRbTarPXmMb79SmzEp8aqXNKcK+y0XaB298IXueQ8I2PsrATcPBCSPyK/dDNa2iWOhKlfNnOjdAOTBU/nkFA==} dev: false - /trpc-openapi@1.2.0(patch_hash=mwbhlfdju7i63ujwpiq2ctfpbe)(@trpc/server@10.40.0)(zod@3.21.4): + /trpc-openapi@1.2.0(patch_hash=chm753bazxetxxpzf3uqs4fzzu)(@trpc/server@10.40.0)(zod@3.21.4): resolution: {integrity: sha512-pfYoCd/3KYXWXvUPZBKJw455OOwngKN/6SIcj7Yit19OMLJ+8yVZkEvGEeg5wUSwfsiTdRsKuvqkRPXVSwV7ew==} peerDependencies: '@trpc/server': ^10.0.0 From 7156472cb7e8b451555d7d306d69b1d4dc7f99b0 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 22 Oct 2023 21:02:29 -0700 Subject: [PATCH 56/80] Adding `env_name` generated column to integration and allow end_user access to it and display_name --- .../2023-10-23_0313_integration_env_name.sql | 4 + apps/web/schema.sql | 283 ++++++++++++++---- .../makePostgresMetaService.ts | 7 +- packages/cdk-core/NoopMetaService.ts | 2 +- packages/cdk-core/meta.types.ts | 2 + packages/cdk-core/metaService.ts | 10 +- packages/engine-backend/contextHelpers.ts | 6 +- .../engine-backend/router/protectedRouter.ts | 25 +- packages/engine-frontend/VeniceConnect.tsx | 31 +- .../ui/domain-components/ProviderCard.tsx | 10 +- 10 files changed, 286 insertions(+), 94 deletions(-) create mode 100644 apps/web/migrations/2023-10-23_0313_integration_env_name.sql diff --git a/apps/web/migrations/2023-10-23_0313_integration_env_name.sql b/apps/web/migrations/2023-10-23_0313_integration_env_name.sql new file mode 100644 index 00000000..a4fda9e6 --- /dev/null +++ b/apps/web/migrations/2023-10-23_0313_integration_env_name.sql @@ -0,0 +1,4 @@ +ALTER TABLE integration ADD COLUMN env_name varchar + GENERATED ALWAYS AS (config ->> 'envName') STORED; + +GRANT SELECT(env_name, display_name) ON TABLE public.integration TO end_user; diff --git a/apps/web/schema.sql b/apps/web/schema.sql index 1f718605..2e58ed77 100644 --- a/apps/web/schema.sql +++ b/apps/web/schema.sql @@ -3,7 +3,7 @@ -- -- Dumped from database version 15.1 --- Dumped by pg_dump version 15.2 (Homebrew) +-- Dumped by pg_dump version 15.4 (Homebrew) SET statement_timeout = 0; SET lock_timeout = 0; @@ -41,12 +41,27 @@ CREATE DOMAIN public.graphql_json AS jsonb -- --- Name: archive_completed_jobs(); Type: FUNCTION; Schema: public; Owner: - +-- Name: format_relative_date(timestamp with time zone); Type: FUNCTION; Schema: public; Owner: - -- -CREATE FUNCTION public.archive_completed_jobs() RETURNS trigger +CREATE FUNCTION public.format_relative_date(date timestamp with time zone) RETURNS text LANGUAGE plpgsql - AS $$ BEGIN INSERT INTO graphile_worker.jobs_completed VALUES (OLD.*); RETURN OLD; END; $$; + AS $$ +DECLARE + time_difference INTERVAL; +BEGIN + time_difference := now() - date; + IF time_difference < INTERVAL '1 minute' THEN + RETURN EXTRACT(SECOND FROM time_difference)::INTEGER || ' seconds ago'; + ELSIF time_difference < INTERVAL '1 hour' THEN + RETURN EXTRACT(MINUTE FROM time_difference)::INTEGER || ' minutes ago'; + ELSIF time_difference < INTERVAL '1 day' THEN + RETURN EXTRACT(HOUR FROM time_difference)::INTEGER || ' hours ago'; + ELSE + RETURN EXTRACT(DAY FROM time_difference)::INTEGER || ' days ago'; + END IF; +END; +$$; -- @@ -58,13 +73,27 @@ CREATE FUNCTION public.generate_ulid() RETURNS text AS $$ DECLARE encoding BYTEA = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'; timestamp BYTEA = E'\\000\\000\\000\\000\\000\\000'; output TEXT = ''; unix_time BIGINT; ulid BYTEA; BEGIN unix_time = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT; timestamp = SET_BYTE(timestamp, 0, (unix_time >> 40)::BIT(8)::INTEGER); timestamp = SET_BYTE(timestamp, 1, (unix_time >> 32)::BIT(8)::INTEGER); timestamp = SET_BYTE(timestamp, 2, (unix_time >> 24)::BIT(8)::INTEGER); timestamp = SET_BYTE(timestamp, 3, (unix_time >> 16)::BIT(8)::INTEGER); timestamp = SET_BYTE(timestamp, 4, (unix_time >> 8)::BIT(8)::INTEGER); timestamp = SET_BYTE(timestamp, 5, unix_time::BIT(8)::INTEGER); ulid = timestamp || gen_random_bytes(10); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 0) & 224) >> 5)); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 0) & 31))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 1) & 248) >> 3)); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 1) & 7) << 2) | ((GET_BYTE(ulid, 2) & 192) >> 6))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 2) & 62) >> 1)); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 2) & 1) << 4) | ((GET_BYTE(ulid, 3) & 240) >> 4))); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 3) & 15) << 1) | ((GET_BYTE(ulid, 4) & 128) >> 7))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 4) & 124) >> 2)); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 4) & 3) << 3) | ((GET_BYTE(ulid, 5) & 224) >> 5))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 5) & 31))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 6) & 248) >> 3)); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 6) & 7) << 2) | ((GET_BYTE(ulid, 7) & 192) >> 6))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 7) & 62) >> 1)); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 7) & 1) << 4) | ((GET_BYTE(ulid, 8) & 240) >> 4))); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 8) & 15) << 1) | ((GET_BYTE(ulid, 9) & 128) >> 7))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 9) & 124) >> 2)); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 9) & 3) << 3) | ((GET_BYTE(ulid, 10) & 224) >> 5))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 10) & 31))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 11) & 248) >> 3)); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 11) & 7) << 2) | ((GET_BYTE(ulid, 12) & 192) >> 6))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 12) & 62) >> 1)); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 12) & 1) << 4) | ((GET_BYTE(ulid, 13) & 240) >> 4))); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 13) & 15) << 1) | ((GET_BYTE(ulid, 14) & 128) >> 7))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 14) & 124) >> 2)); output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 14) & 3) << 3) | ((GET_BYTE(ulid, 15) & 224) >> 5))); output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 15) & 31))); RETURN output; END $$; +-- +-- Name: jsonb_array_to_text_array(jsonb); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.jsonb_array_to_text_array(_js jsonb) RETURNS text[] + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE + AS $$SELECT ARRAY(SELECT jsonb_array_elements_text(_js))$$; + + -- -- Name: jwt_end_user_id(); Type: FUNCTION; Schema: public; Owner: - -- CREATE FUNCTION public.jwt_end_user_id() RETURNS character varying LANGUAGE sql STABLE - AS $$ select coalesce( nullif(current_setting('request.jwt.claim.end_user_id', true), ''), (nullif(current_setting('request.jwt.claims', true), '')::jsonb ->> 'end_user_id') ) $$; + AS $$ + select coalesce( + nullif(current_setting('request.jwt.claim.end_user_id', true), ''), + (nullif(current_setting('request.jwt.claims', true), '')::jsonb ->> 'end_user_id') + ) +$$; -- @@ -73,7 +102,12 @@ CREATE FUNCTION public.jwt_end_user_id() RETURNS character varying CREATE FUNCTION public.jwt_org_id() RETURNS character varying LANGUAGE sql STABLE - AS $$ select coalesce( nullif(current_setting('request.jwt.claim.org_id', true), ''), (nullif(current_setting('request.jwt.claims', true), '')::jsonb ->> 'org_id') ) $$; + AS $$ + select coalesce( + nullif(current_setting('request.jwt.claim.org_id', true), ''), + (nullif(current_setting('request.jwt.claims', true), '')::jsonb ->> 'org_id') + ) +$$; -- @@ -82,24 +116,78 @@ CREATE FUNCTION public.jwt_org_id() RETURNS character varying CREATE FUNCTION public.jwt_sub() RETURNS character varying LANGUAGE sql STABLE - AS $$ select coalesce( nullif(current_setting('request.jwt.claim.sub', true), ''), (nullif(current_setting('request.jwt.claims', true), '')::jsonb ->> 'sub') ) $$; + AS $$ + select coalesce( + nullif(current_setting('request.jwt.claim.sub', true), ''), + (nullif(current_setting('request.jwt.claims', true), '')::jsonb ->> 'sub') + ) +$$; SET default_tablespace = ''; SET default_table_access_method = heap; +-- +-- Name: _migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public._migrations ( + name text NOT NULL, + hash text NOT NULL, + date timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: raw_account; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.raw_account ( + id character varying DEFAULT concat('acct_', public.generate_ulid()) NOT NULL, + source_id character varying, + standard jsonb DEFAULT '{}'::jsonb NOT NULL, + external jsonb DEFAULT '{}'::jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, + end_user_id character varying, + CONSTRAINT raw_account_id_prefix_check CHECK (starts_with((id)::text, 'acct_'::text)) +); + + +-- +-- Name: account; Type: VIEW; Schema: public; Owner: - +-- + +CREATE VIEW public.account WITH (security_invoker='true') AS + SELECT raw_account.end_user_id, + raw_account.id, + (raw_account.standard ->> 'name'::text) AS name, + (raw_account.standard ->> 'type'::text) AS type, + (raw_account.standard ->> 'lastFour'::text) AS last_four, + (raw_account.standard ->> 'institutionName'::text) AS institution_name, + (raw_account.standard ->> 'defaultUnit'::text) AS default_unit, + ((raw_account.standard #> '{informationalBalances,current,quantity}'::text[]))::double precision AS current_balance, + ((raw_account.standard #> '{informationalBalances,available,quantity}'::text[]))::double precision AS available_balance, + (raw_account.external)::public.graphql_json AS external, + raw_account.provider_name, + raw_account.updated_at, + raw_account.created_at + FROM public.raw_account; + + -- -- Name: institution; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE public.institution ( id character varying DEFAULT concat('ins_', public.generate_ulid()) NOT NULL, - provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, standard jsonb DEFAULT '{}'::jsonb NOT NULL, external jsonb DEFAULT '{}'::jsonb NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, + provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, CONSTRAINT institution_id_prefix_check CHECK (starts_with((id)::text, 'ins_'::text)) ); @@ -110,13 +198,14 @@ CREATE TABLE public.institution ( CREATE TABLE public.integration ( id character varying DEFAULT concat('int_', public.generate_ulid()) NOT NULL, - provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, config jsonb DEFAULT '{}'::jsonb NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, + provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, org_id character varying NOT NULL, display_name character varying, end_user_access boolean, + env_name character varying GENERATED ALWAYS AS ((config ->> 'envName'::text)) STORED, CONSTRAINT integration_id_prefix_check CHECK (starts_with((id)::text, 'int_'::text)) ); @@ -143,43 +232,26 @@ CREATE TABLE public.pipeline ( destination_id character varying, destination_state jsonb DEFAULT '{}'::jsonb NOT NULL, link_options jsonb DEFAULT '[]'::jsonb NOT NULL, - last_sync_started_at timestamp with time zone, - last_sync_completed_at timestamp with time zone, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, + last_sync_started_at timestamp with time zone, + last_sync_completed_at timestamp with time zone, CONSTRAINT pipeline_id_prefix_check CHECK (starts_with((id)::text, 'pipe_'::text)) ); --- --- Name: raw_account; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.raw_account ( - id character varying DEFAULT concat('acct_', public.generate_ulid()) NOT NULL, - provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, - source_id character varying, - standard jsonb DEFAULT '{}'::jsonb NOT NULL, - external jsonb DEFAULT '{}'::jsonb NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL, - updated_at timestamp with time zone DEFAULT now() NOT NULL, - end_user_id character varying, - CONSTRAINT raw_account_id_prefix_check CHECK (starts_with((id)::text, 'acct_'::text)) -); - - -- -- Name: raw_commodity; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE public.raw_commodity ( id character varying DEFAULT concat('comm_', public.generate_ulid()) NOT NULL, - provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, source_id character varying, standard jsonb DEFAULT '{}'::jsonb NOT NULL, external jsonb DEFAULT '{}'::jsonb NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, + provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, end_user_id character varying, CONSTRAINT raw_commodity_id_prefix_check CHECK (starts_with((id)::text, 'comm_'::text)) ); @@ -191,12 +263,12 @@ CREATE TABLE public.raw_commodity ( CREATE TABLE public.raw_transaction ( id character varying DEFAULT concat('txn_', public.generate_ulid()) NOT NULL, - provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, source_id character varying, standard jsonb DEFAULT '{}'::jsonb NOT NULL, external jsonb DEFAULT '{}'::jsonb NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, + provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, end_user_id character varying, CONSTRAINT raw_transaction_id_prefix_check CHECK (starts_with((id)::text, 'txn_'::text)) ); @@ -208,7 +280,6 @@ CREATE TABLE public.raw_transaction ( CREATE TABLE public.resource ( id character varying DEFAULT concat('reso', public.generate_ulid()) NOT NULL, - provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, end_user_id character varying, integration_id character varying, institution_id character varying, @@ -216,11 +287,61 @@ CREATE TABLE public.resource ( settings jsonb DEFAULT '{}'::jsonb NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, + provider_name character varying GENERATED ALWAYS AS (split_part((id)::text, '_'::text, 2)) STORED NOT NULL, display_name character varying, CONSTRAINT resource_id_prefix_check CHECK (starts_with((id)::text, 'reso'::text)) ); +-- +-- Name: transaction; Type: VIEW; Schema: public; Owner: - +-- + +CREATE VIEW public.transaction WITH (security_invoker='true') AS + SELECT raw_transaction.end_user_id, + raw_transaction.id, + (raw_transaction.standard ->> 'date'::text) AS date, + (raw_transaction.standard ->> 'description'::text) AS description, + (raw_transaction.standard ->> 'payee'::text) AS payee, + ((raw_transaction.standard #> '{postingsMap,main,amount,quantity}'::text[]))::double precision AS amount_quantity, + (raw_transaction.standard #>> '{postingsMap,main,amount,unit}'::text[]) AS amount_unit, + (raw_transaction.standard #>> '{postingsMap,main,accountId}'::text[]) AS account_id, + (raw_transaction.standard ->> 'externalCategory'::text) AS external_category, + (raw_transaction.standard ->> 'notes'::text) AS notes, + ((raw_transaction.standard -> 'postingsMap'::text))::public.graphql_json AS splits, + (raw_transaction.external)::public.graphql_json AS external, + raw_transaction.provider_name, + raw_transaction.updated_at, + raw_transaction.created_at + FROM public.raw_transaction; + + +-- +-- Name: transaction_split; Type: VIEW; Schema: public; Owner: - +-- + +CREATE VIEW public.transaction_split WITH (security_invoker='true') AS + SELECT raw_transaction.end_user_id, + raw_transaction.id AS transaction_id, + s.key, + ((s.value #> '{amount,quantity}'::text[]))::double precision AS amount_quantity, + (s.value #>> '{amount,unit}'::text[]) AS amount_unit, + (s.value ->> 'accountId'::text) AS account_id, + (s.value)::public.graphql_json AS data, + raw_transaction.updated_at, + raw_transaction.created_at + FROM public.raw_transaction, + LATERAL jsonb_each((raw_transaction.standard -> 'postingsMap'::text)) s(key, value); + + +-- +-- Name: _migrations _migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public._migrations + ADD CONSTRAINT _migrations_pkey PRIMARY KEY (name); + + -- -- Name: migrations migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -499,7 +620,7 @@ CREATE INDEX transaction_updated_at ON public.raw_transaction USING btree (updat -- Name: resource on-resource-write; Type: TRIGGER; Schema: public; Owner: - -- -CREATE TRIGGER "on-resource-write" AFTER INSERT OR DELETE OR UPDATE ON public.resource FOR EACH ROW EXECUTE FUNCTION supabase_functions.http_request('https://app.venice.is/api/webhook/database', 'POST', '{"Content-type":"application/json"}', '{}', '1000'); +CREATE TRIGGER "on-resource-write" AFTER INSERT OR DELETE OR UPDATE ON public.resource FOR EACH ROW EXECUTE FUNCTION supabase_functions.http_request('https://alka.ngrok.io/api/webhook/database', 'POST', '{"Content-type":"application/json"}', '{}', '1000'); -- @@ -703,21 +824,22 @@ ALTER TABLE public.resource ENABLE ROW LEVEL SECURITY; -- Name: SCHEMA public; Type: ACL; Schema: -; Owner: - -- -GRANT USAGE ON SCHEMA public TO postgres; +REVOKE USAGE ON SCHEMA public FROM PUBLIC; GRANT USAGE ON SCHEMA public TO anon; GRANT USAGE ON SCHEMA public TO authenticated; GRANT USAGE ON SCHEMA public TO service_role; +GRANT USAGE ON SCHEMA public TO tonyx; GRANT USAGE ON SCHEMA public TO end_user; GRANT USAGE ON SCHEMA public TO org; -- --- Name: FUNCTION archive_completed_jobs(); Type: ACL; Schema: public; Owner: - +-- Name: FUNCTION format_relative_date(date timestamp with time zone); Type: ACL; Schema: public; Owner: - -- -GRANT ALL ON FUNCTION public.archive_completed_jobs() TO anon; -GRANT ALL ON FUNCTION public.archive_completed_jobs() TO authenticated; -GRANT ALL ON FUNCTION public.archive_completed_jobs() TO service_role; +GRANT ALL ON FUNCTION public.format_relative_date(date timestamp with time zone) TO anon; +GRANT ALL ON FUNCTION public.format_relative_date(date timestamp with time zone) TO authenticated; +GRANT ALL ON FUNCTION public.format_relative_date(date timestamp with time zone) TO service_role; -- @@ -729,6 +851,15 @@ GRANT ALL ON FUNCTION public.generate_ulid() TO authenticated; GRANT ALL ON FUNCTION public.generate_ulid() TO service_role; +-- +-- Name: FUNCTION jsonb_array_to_text_array(_js jsonb); Type: ACL; Schema: public; Owner: - +-- + +GRANT ALL ON FUNCTION public.jsonb_array_to_text_array(_js jsonb) TO anon; +GRANT ALL ON FUNCTION public.jsonb_array_to_text_array(_js jsonb) TO authenticated; +GRANT ALL ON FUNCTION public.jsonb_array_to_text_array(_js jsonb) TO service_role; + + -- -- Name: FUNCTION jwt_end_user_id(); Type: ACL; Schema: public; Owner: - -- @@ -756,6 +887,36 @@ GRANT ALL ON FUNCTION public.jwt_sub() TO authenticated; GRANT ALL ON FUNCTION public.jwt_sub() TO service_role; +-- +-- Name: TABLE _migrations; Type: ACL; Schema: public; Owner: - +-- + +GRANT ALL ON TABLE public._migrations TO anon; +GRANT ALL ON TABLE public._migrations TO authenticated; +GRANT ALL ON TABLE public._migrations TO service_role; + + +-- +-- Name: TABLE raw_account; Type: ACL; Schema: public; Owner: - +-- + +GRANT ALL ON TABLE public.raw_account TO anon; +GRANT ALL ON TABLE public.raw_account TO authenticated; +GRANT ALL ON TABLE public.raw_account TO service_role; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.raw_account TO tonyx; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.raw_account TO org; + + +-- +-- Name: TABLE account; Type: ACL; Schema: public; Owner: - +-- + +GRANT ALL ON TABLE public.account TO anon; +GRANT ALL ON TABLE public.account TO authenticated; +GRANT ALL ON TABLE public.account TO service_role; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.account TO org; + + -- -- Name: TABLE institution; Type: ACL; Schema: public; Owner: - -- @@ -763,9 +924,9 @@ GRANT ALL ON FUNCTION public.jwt_sub() TO service_role; GRANT ALL ON TABLE public.institution TO anon; GRANT ALL ON TABLE public.institution TO authenticated; GRANT ALL ON TABLE public.institution TO service_role; -GRANT SELECT ON TABLE public.institution TO analytics; -GRANT SELECT ON TABLE public.institution TO end_user; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.institution TO tonyx; GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.institution TO org; +GRANT SELECT ON TABLE public.institution TO end_user; -- @@ -775,7 +936,7 @@ GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.institution TO org; GRANT ALL ON TABLE public.integration TO anon; GRANT ALL ON TABLE public.integration TO authenticated; GRANT ALL ON TABLE public.integration TO service_role; -GRANT SELECT ON TABLE public.integration TO analytics; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.integration TO tonyx; GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.integration TO org; @@ -798,7 +959,7 @@ GRANT SELECT(org_id) ON TABLE public.integration TO end_user; -- GRANT ALL ON TABLE public.migrations TO service_role; -GRANT SELECT ON TABLE public.migrations TO analytics; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.migrations TO tonyx; GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.migrations TO org; @@ -809,22 +970,11 @@ GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.migrations TO org; GRANT ALL ON TABLE public.pipeline TO anon; GRANT ALL ON TABLE public.pipeline TO authenticated; GRANT ALL ON TABLE public.pipeline TO service_role; -GRANT SELECT ON TABLE public.pipeline TO analytics; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.pipeline TO tonyx; GRANT SELECT ON TABLE public.pipeline TO end_user; GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.pipeline TO org; --- --- Name: TABLE raw_account; Type: ACL; Schema: public; Owner: - --- - -GRANT ALL ON TABLE public.raw_account TO anon; -GRANT ALL ON TABLE public.raw_account TO authenticated; -GRANT ALL ON TABLE public.raw_account TO service_role; -GRANT SELECT ON TABLE public.raw_account TO analytics; -GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.raw_account TO org; - - -- -- Name: TABLE raw_commodity; Type: ACL; Schema: public; Owner: - -- @@ -832,7 +982,7 @@ GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.raw_account TO org; GRANT ALL ON TABLE public.raw_commodity TO anon; GRANT ALL ON TABLE public.raw_commodity TO authenticated; GRANT ALL ON TABLE public.raw_commodity TO service_role; -GRANT SELECT ON TABLE public.raw_commodity TO analytics; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.raw_commodity TO tonyx; GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.raw_commodity TO org; @@ -843,7 +993,7 @@ GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.raw_commodity TO org; GRANT ALL ON TABLE public.raw_transaction TO anon; GRANT ALL ON TABLE public.raw_transaction TO authenticated; GRANT ALL ON TABLE public.raw_transaction TO service_role; -GRANT SELECT ON TABLE public.raw_transaction TO analytics; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.raw_transaction TO tonyx; GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.raw_transaction TO org; @@ -854,7 +1004,7 @@ GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.raw_transaction TO org; GRANT ALL ON TABLE public.resource TO anon; GRANT ALL ON TABLE public.resource TO authenticated; GRANT ALL ON TABLE public.resource TO service_role; -GRANT SELECT ON TABLE public.resource TO analytics; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.resource TO tonyx; GRANT SELECT,DELETE ON TABLE public.resource TO end_user; GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.resource TO org; @@ -866,6 +1016,26 @@ GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.resource TO org; GRANT UPDATE(display_name) ON TABLE public.resource TO end_user; +-- +-- Name: TABLE transaction; Type: ACL; Schema: public; Owner: - +-- + +GRANT ALL ON TABLE public.transaction TO anon; +GRANT ALL ON TABLE public.transaction TO authenticated; +GRANT ALL ON TABLE public.transaction TO service_role; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.transaction TO org; + + +-- +-- Name: TABLE transaction_split; Type: ACL; Schema: public; Owner: - +-- + +GRANT ALL ON TABLE public.transaction_split TO anon; +GRANT ALL ON TABLE public.transaction_split TO authenticated; +GRANT ALL ON TABLE public.transaction_split TO service_role; +GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE public.transaction_split TO org; + + -- -- Name: DEFAULT PRIVILEGES FOR SEQUENCES; Type: DEFAULT ACL; Schema: public; Owner: - -- @@ -914,7 +1084,6 @@ ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public GRANT ALL ON TABLES ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public GRANT ALL ON TABLES TO anon; ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public GRANT ALL ON TABLES TO authenticated; ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public GRANT ALL ON TABLES TO service_role; -ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public GRANT SELECT ON TABLES TO analytics; -- diff --git a/integrations/integration-postgres/makePostgresMetaService.ts b/integrations/integration-postgres/makePostgresMetaService.ts index 169f132e..398c96d4 100644 --- a/integrations/integration-postgres/makePostgresMetaService.ts +++ b/integrations/integration-postgres/makePostgresMetaService.ts @@ -3,7 +3,6 @@ import type {TransactionFunction} from 'slonik/dist/src/types' import type { EndUserResultRow, - Id, MetaService, MetaTable, Viewer, @@ -144,11 +143,11 @@ export const makePostgresMetaService = zFunction( pool.any(sql`SELECT * FROM pipeline ${where}`), ) }, - listIntegrationIds: ({id} = {}) => { + listIntegrationInfos: ({id} = {}) => { const {runQueries, sql} = _getDeps(opts) return runQueries((pool) => - pool.anyFirst( - sql`SELECT id FROM integration ${ + pool.any( + sql`SELECT id, env_name, display_name FROM integration ${ id ? sql`WHERE id = ${id}` : sql`` }`, ), diff --git a/packages/cdk-core/NoopMetaService.ts b/packages/cdk-core/NoopMetaService.ts index e9a993f9..c91286fe 100644 --- a/packages/cdk-core/NoopMetaService.ts +++ b/packages/cdk-core/NoopMetaService.ts @@ -20,5 +20,5 @@ export const noopMetaService: MetaService = { searchEndUsers: async () => [], searchInstitutions: async () => [], findPipelines: async () => [], - listIntegrationIds: async () => [], + listIntegrationInfos: async () => [], } diff --git a/packages/cdk-core/meta.types.ts b/packages/cdk-core/meta.types.ts index c16206a1..3625b140 100644 --- a/packages/cdk-core/meta.types.ts +++ b/packages/cdk-core/meta.types.ts @@ -80,6 +80,8 @@ const zBase = z.object({ export const zRaw = { integration: zBase.extend({ id: zId('int'), + /** This is a generated column, which is not the most flexible. Maybe we need some kind of mapStandardIntegration method? */ + envName: z.string().nullish(), providerName: z.string(), config: zJsonObject.nullish(), endUserAccess: z diff --git a/packages/cdk-core/metaService.ts b/packages/cdk-core/metaService.ts index ad485771..1183bdf5 100644 --- a/packages/cdk-core/metaService.ts +++ b/packages/cdk-core/metaService.ts @@ -57,7 +57,11 @@ export interface MetaService { secondsSinceLastSync?: number }) => Promise> /** Id is used to check RLS policy right now for end user */ - listIntegrationIds: (opts?: { - id?: Id['int'] - }) => Promise> + listIntegrationInfos: (opts?: {id?: Id['int']}) => Promise< + ReadonlyArray<{ + id: Id['int'] + envName?: string | null + displayName?: string | null + }> + > } diff --git a/packages/engine-backend/contextHelpers.ts b/packages/engine-backend/contextHelpers.ts index c057c23c..1b207e11 100644 --- a/packages/engine-backend/contextHelpers.ts +++ b/packages/engine-backend/contextHelpers.ts @@ -145,7 +145,7 @@ export function getContextHelpers({ } const getIntegrationInfoOrFail = (id: Id['int']) => - metaService.listIntegrationIds({id}).then((ints) => { + metaService.listIntegrationInfos({id}).then((ints) => { if (!ints[0]) { throw new TRPCError({code: 'NOT_FOUND'}) } @@ -187,8 +187,8 @@ export function getContextHelpers({ const getResourceExpandedOrFail = (id: Id['reso']) => getResourceOrFail(id).then(async (reso) => { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const integration = await getIntegrationOrFail(reso.integrationId!) + + const integration = await getIntegrationOrFail(reso.integrationId) const settings: {} = integration.provider.def.resourceSettings?.parse( reso.settings, ) diff --git a/packages/engine-backend/router/protectedRouter.ts b/packages/engine-backend/router/protectedRouter.ts index be98d268..cb9a7b3d 100644 --- a/packages/engine-backend/router/protectedRouter.ts +++ b/packages/engine-backend/router/protectedRouter.ts @@ -147,15 +147,34 @@ export const protectedRouter = trpc.router({ }) }), listIntegrationInfos: protectedProcedure + .meta({openapi: {method: 'GET', path: '/integration_infos'}}) .input(z.object({type: z.enum(['source', 'destination']).nullish()})) + .output( + z.array( + zRaw.integration + .pick({ + id: true, + envName: true, + displayName: true, + providerName: true, + }) + .extend({ + isSource: z.boolean(), + isDestination: z.boolean(), + }), + ), + ) .query(async ({input: {type}, ctx}) => { - const intIds = await ctx.helpers.metaService.listIntegrationIds() - return intIds - .map((id) => { + const intInfos = await ctx.helpers.metaService.listIntegrationInfos() + + return intInfos + .map(({id, envName, displayName}) => { const provider = ctx.providerMap[extractId(id)[1]] return provider ? { id, + envName, + displayName, providerName: provider.name, isSource: !!provider.sourceSync, isDestination: !!provider.destinationSync, diff --git a/packages/engine-frontend/VeniceConnect.tsx b/packages/engine-frontend/VeniceConnect.tsx index ec021ee5..702d6dcc 100644 --- a/packages/engine-frontend/VeniceConnect.tsx +++ b/packages/engine-frontend/VeniceConnect.tsx @@ -36,7 +36,7 @@ import { DropdownMenuLabel, DropdownMenuSeparator, DropdownMenuTrigger, - ProviderCard, + IntegrationCard, ResourceCard, SchemaForm, useToast, @@ -112,7 +112,6 @@ export function VeniceConnectButton({ // based on calls to useQuery so it doesn't need to be separately handled again on the client... export function VeniceConnect(props: VeniceConnectProps) { const listIntegrationsRes = _trpcReact.listIntegrationInfos.useQuery({}) - const integrationIds = (listIntegrationsRes.data ?? []).map(({id}) => id) const catalogRes = _trpcReact.getIntegrationCatalog.useQuery() if (!listIntegrationsRes.data || !catalogRes.data) { @@ -120,13 +119,14 @@ export function VeniceConnect(props: VeniceConnectProps) { } return ( <_VeniceConnect - integrationIds={integrationIds} + integrationInfos={listIntegrationsRes.data ?? []} catalog={catalogRes.data} {...props} /> ) } +type IntegrationInfos = RouterOutput['listIntegrationInfos'] type Catalog = RouterOutput['getIntegrationCatalog'] type ProviderMeta = Catalog[string] @@ -142,10 +142,10 @@ export function _VeniceConnect({ onEvent, showExisting, className, - integrationIds, + integrationInfos, ...uiProps }: VeniceConnectProps & { - integrationIds: Array + integrationInfos: IntegrationInfos catalog: Catalog }) { const nangoPublicKey = @@ -168,13 +168,13 @@ export function _VeniceConnect({ {enabled: showExisting}, ) - const integrations = integrationIds - .map((id) => { + const integrations = integrationInfos + .map(({id, ...info}) => { const provider = catalog[extractProviderName(id)] if (!provider) { console.warn('Missing provider for integration', id) } - return provider ? {id, provider} : null + return provider ? {...info, id, provider} : null }) .filter((i): i is NonNullable => !!i) const integrationById = R.mapToObj(integrations, (i) => [i.id, i]) @@ -208,8 +208,8 @@ export function _VeniceConnect({ // Do we actually need this here or can this go inside a ConnectCard somehow? const connectFnMap = R.pipe( - integrationIds, - R.map(extractProviderName), + integrationInfos, + R.map((intInfo) => extractProviderName(intInfo.id)), R.uniq, R.mapToObj((providerName: string) => { let fn = clientIntegrations[providerName]?.useConnectHook?.({openDialog}) @@ -282,10 +282,11 @@ export function _VeniceConnect({ ))} {/* Add new */} {integrations.map((int) => ( - // TODO: Make use of integrationCard rather than ProviderCard - // once we allow for mapStandardIntegration such that integration labels - // can show up properly (e.g. sandbox, production labels) - + - + ))} ) diff --git a/packages/ui/domain-components/ProviderCard.tsx b/packages/ui/domain-components/ProviderCard.tsx index bebab951..6a994d1a 100644 --- a/packages/ui/domain-components/ProviderCard.tsx +++ b/packages/ui/domain-components/ProviderCard.tsx @@ -101,19 +101,13 @@ export const IntegrationCard = ({ id: Id['int'] providerName: string config?: Record | null + envName?: string | null } }) => ( ) From 5b26b4b05e43f72e7855b48af87fbb9a8faf1490 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Mon, 23 Oct 2023 08:52:14 -0700 Subject: [PATCH 57/80] Allow limiting list of integrations by integrationId on connect page --- apps/web/app/connect/ConnectPage.tsx | 14 ++++++++++++-- apps/web/app/connect/page.tsx | 2 +- packages/cdk-core/metaService.ts | 2 +- packages/engine-backend/router/protectedRouter.ts | 11 ++++++++--- packages/engine-frontend/VeniceConnect.tsx | 6 +++++- 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/apps/web/app/connect/ConnectPage.tsx b/apps/web/app/connect/ConnectPage.tsx index 6ab734f3..ba8d93cf 100644 --- a/apps/web/app/connect/ConnectPage.tsx +++ b/apps/web/app/connect/ConnectPage.tsx @@ -1,12 +1,22 @@ 'use client' import {clientIntegrations} from '@usevenice/app-config/integrations/integrations.client' +import type {VeniceConnectProps} from '@usevenice/engine-frontend' import {VeniceConnect} from '@usevenice/engine-frontend' /** * Only reason this file exists is because we cannot pass clientIntegrations directly * from a server component because it contains function references (i.e. useConnectHook) */ -export default function ConnectPage() { - return +export default function ConnectPage( + props: Omit, +) { + return ( + + ) } diff --git a/apps/web/app/connect/page.tsx b/apps/web/app/connect/page.tsx index aeb538c4..64d2cca8 100644 --- a/apps/web/app/connect/page.tsx +++ b/apps/web/app/connect/page.tsx @@ -64,7 +64,7 @@ export default async function ConnectPageContainer({ - + diff --git a/packages/cdk-core/metaService.ts b/packages/cdk-core/metaService.ts index 1183bdf5..b7608f6e 100644 --- a/packages/cdk-core/metaService.ts +++ b/packages/cdk-core/metaService.ts @@ -57,7 +57,7 @@ export interface MetaService { secondsSinceLastSync?: number }) => Promise> /** Id is used to check RLS policy right now for end user */ - listIntegrationInfos: (opts?: {id?: Id['int']}) => Promise< + listIntegrationInfos: (opts?: {id?: Id['int'] | null}) => Promise< ReadonlyArray<{ id: Id['int'] envName?: string | null diff --git a/packages/engine-backend/router/protectedRouter.ts b/packages/engine-backend/router/protectedRouter.ts index cb9a7b3d..6f886f76 100644 --- a/packages/engine-backend/router/protectedRouter.ts +++ b/packages/engine-backend/router/protectedRouter.ts @@ -148,7 +148,12 @@ export const protectedRouter = trpc.router({ }), listIntegrationInfos: protectedProcedure .meta({openapi: {method: 'GET', path: '/integration_infos'}}) - .input(z.object({type: z.enum(['source', 'destination']).nullish()})) + .input( + z.object({ + type: z.enum(['source', 'destination']).nullish(), + id: zId('int').nullish(), + }), + ) .output( z.array( zRaw.integration @@ -164,8 +169,8 @@ export const protectedRouter = trpc.router({ }), ), ) - .query(async ({input: {type}, ctx}) => { - const intInfos = await ctx.helpers.metaService.listIntegrationInfos() + .query(async ({input: {type, id}, ctx}) => { + const intInfos = await ctx.helpers.metaService.listIntegrationInfos({id}) return intInfos .map(({id, envName, displayName}) => { diff --git a/packages/engine-frontend/VeniceConnect.tsx b/packages/engine-frontend/VeniceConnect.tsx index 702d6dcc..41308f47 100644 --- a/packages/engine-frontend/VeniceConnect.tsx +++ b/packages/engine-frontend/VeniceConnect.tsx @@ -53,6 +53,8 @@ export interface VeniceConnectProps extends UIPropsNoChildren { showExisting?: boolean clientIntegrations: Record onEvent?: (event: {type: ConnectEventType; intId: Id['int']}) => void + /** Only connect to this integration */ + integrationId?: Id['int'] | null } type UseConnectScope = Parameters[0] @@ -111,7 +113,9 @@ export function VeniceConnectButton({ // Also it would be nice if there was an easy way to automatically prefetch on the server side // based on calls to useQuery so it doesn't need to be separately handled again on the client... export function VeniceConnect(props: VeniceConnectProps) { - const listIntegrationsRes = _trpcReact.listIntegrationInfos.useQuery({}) + const listIntegrationsRes = _trpcReact.listIntegrationInfos.useQuery({ + id: props.integrationId, + }) const catalogRes = _trpcReact.getIntegrationCatalog.useQuery() if (!listIntegrationsRes.data || !catalogRes.data) { From 11c21d77cbe5befb3fe0aa731bbc080b0dcf17bc Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Mon, 23 Oct 2023 17:22:45 -0700 Subject: [PATCH 58/80] Intercept Nango redirects to implement single integration oauth without needing popup --- apps/web/app/connect/ConnectPage.tsx | 1 - apps/web/app/connect/page.tsx | 42 +++++++-- apps/web/app/oauth/callback/CallbackPage.tsx | 20 ++++ apps/web/app/oauth/callback/page.tsx | 48 ++++++++++ apps/web/lib-server/server-helpers.ts | 20 ++-- apps/web/middleware.ts | 8 +- packages/cdk-core/oauth/NangoClient.spec.ts | 88 ++++++++++++++++++ packages/cdk-core/oauth/NangoClient.ts | 93 ++++++++++++++++++- packages/engine-backend/router/adminRouter.ts | 6 +- packages/util/http/makeHttpClient.ts | 17 ++-- packages/util/http/makeOpenApiClient.ts | 2 + 11 files changed, 320 insertions(+), 25 deletions(-) create mode 100644 apps/web/app/oauth/callback/CallbackPage.tsx create mode 100644 apps/web/app/oauth/callback/page.tsx create mode 100644 packages/cdk-core/oauth/NangoClient.spec.ts diff --git a/apps/web/app/connect/ConnectPage.tsx b/apps/web/app/connect/ConnectPage.tsx index ba8d93cf..61111429 100644 --- a/apps/web/app/connect/ConnectPage.tsx +++ b/apps/web/app/connect/ConnectPage.tsx @@ -15,7 +15,6 @@ export default function ConnectPage( ) diff --git a/apps/web/app/connect/page.tsx b/apps/web/app/connect/page.tsx index 64d2cca8..56f3ee8e 100644 --- a/apps/web/app/connect/page.tsx +++ b/apps/web/app/connect/page.tsx @@ -1,8 +1,19 @@ import {clerkClient} from '@clerk/nextjs' import Image from 'next/image' +import {redirect} from 'next/navigation' -import {getViewerId} from '@usevenice/cdk-core' +import {getServerUrl} from '@usevenice/app-config/constants' +import {env} from '@usevenice/app-config/env' +import {defIntegrations} from '@usevenice/app-config/integrations/integrations.def' +import type {IntegrationDef} from '@usevenice/cdk-core' +import { + extractProviderName, + getViewerId, + makeId, + makeNangoClient, +} from '@usevenice/cdk-core' import {zConnectPageParams} from '@usevenice/engine-backend/router/adminRouter' +import {joinPath, makeUlid} from '@usevenice/util' import {ClientRoot} from '@/components/ClientRoot' import {SuperHydrate} from '@/components/SuperHydrate' @@ -32,9 +43,9 @@ export default async function ConnectPageContainer({ // @see https://github.com/vercel/next.js/issues/43704 searchParams: Record }) { - const params = zConnectPageParams.parse(searchParams) + const {token, ...params} = zConnectPageParams.parse(searchParams) const {ssg, getDehydratedState, viewer} = await createServerComponentHelpers({ - searchParams: {_token: params.token}, + searchParams: {_token: token}, }) if (viewer.role !== 'end_user') { return ( @@ -42,10 +53,29 @@ export default async function ConnectPageContainer({ ) } + // Special case when we are handling a single oauth integration + if (params.integrationId) { + const providerName = extractProviderName(params.integrationId) + const intDef = defIntegrations[ + providerName as keyof typeof defIntegrations + ] as IntegrationDef + + if (intDef.metadata?.nangoProvider) { + const nango = makeNangoClient({secretKey: env.NANGO_SECRET_KEY}) + const url = await nango.getOauthConnectUrl({ + public_key: env.NEXT_PUBLIC_NANGO_PUBLIC_KEY, + connection_id: makeId('reso', providerName, makeUlid()), + provider_config_key: params.integrationId, + redirect_uri: joinPath(getServerUrl(null), '/oauth/callback'), + }) + return redirect(url) + } + } + const [org] = await Promise.all([ clerkClient.organizations.getOrganization({organizationId: viewer.orgId}), - ssg.listIntegrationInfos.prefetch({}), - ssg.listConnections.prefetch({}), + ssg.listIntegrationInfos.prefetch({id: params.integrationId}), + params.showExisting ? ssg.listConnections.prefetch({}) : Promise.resolve(), ]) return ( @@ -64,7 +94,7 @@ export default async function ConnectPageContainer({ - + diff --git a/apps/web/app/oauth/callback/CallbackPage.tsx b/apps/web/app/oauth/callback/CallbackPage.tsx new file mode 100644 index 00000000..f7b5c7b3 --- /dev/null +++ b/apps/web/app/oauth/callback/CallbackPage.tsx @@ -0,0 +1,20 @@ +'use client' + +import React from 'react' + +export function CallbackPage(props: {res: any; autoClose?: boolean}) { + React.useEffect(() => { + if (props.autoClose) { + setTimeout(() => { + window.close() + }, 300) + } + }, []) + return ( +
+ 
{JSON.stringify(props.res, null, 4)}
+ You may now close this window if it does not automatically close + +
+ ) +} diff --git a/apps/web/app/oauth/callback/page.tsx b/apps/web/app/oauth/callback/page.tsx new file mode 100644 index 00000000..4d1c5f78 --- /dev/null +++ b/apps/web/app/oauth/callback/page.tsx @@ -0,0 +1,48 @@ +import '@usevenice/app-config/register.node' + +import {env} from '@usevenice/app-config/env' +import {makeNangoClient} from '@usevenice/cdk-core' + +import {serverSideHelpersFromViewer} from '@/lib-server' + +import {CallbackPage} from './CallbackPage' + +export const metadata = { + title: 'Venice Oauth Callback', +} + +/** + * Workaround for searchParams being empty on production. Will ahve to check + * @see https://github.com/vercel/next.js/issues/43077#issuecomment-1383742153 + */ +export const dynamic = 'force-dynamic' + +/** https://beta.nextjs.org/docs/api-reference/file-conventions/page#searchparams-optional */ +export default async function OAuthCallback({ + searchParams, +}: { + // Only accessible in PageComponent rather than layout component + // @see https://github.com/vercel/next.js/issues/43704 + searchParams: Record +}) { + const nango = makeNangoClient({secretKey: env.NANGO_SECRET_KEY}) + const ret = await nango.doOauthCallback(searchParams) + + if (ret.eventType === 'AUTHORIZATION_SUCEEDED') { + const {caller} = serverSideHelpersFromViewer({role: 'system'}) + + try { + await caller.postConnect([ret.data, ret.data.providerConfigKey, {}]) + } catch (err) { + // TODO: Better handle failure here. + return ( + + ) + } + } + + // How do we do redirect here? + return +} diff --git a/apps/web/lib-server/server-helpers.ts b/apps/web/lib-server/server-helpers.ts index 25b3a0c8..5c8ca8f5 100644 --- a/apps/web/lib-server/server-helpers.ts +++ b/apps/web/lib-server/server-helpers.ts @@ -44,14 +44,11 @@ type NextContext = | URLSearchParams } -export async function createSSRHelpers(context: NextContext) { - // TODO: Remove this once we fully migrate off next.js 12 routing - await import('@usevenice/app-config/register.node') - +export function serverSideHelpersFromViewer(viewer: Viewer) { + const ctx = contextFactory.fromViewer(viewer) const queryClient = new QueryClient() - const viewer = await serverGetViewer(context) - const ctx = contextFactory.fromViewer(viewer) + const caller = flatRouter.createCaller(ctx) const ssg = createServerSideHelpers({ queryClient, @@ -59,10 +56,21 @@ export async function createSSRHelpers(context: NextContext) { ctx, // transformer: superjson, }) + return {ssg, caller, ctx, queryClient} +} + +export async function createSSRHelpers(context: NextContext) { + // TODO: Remove this once we fully migrate off next.js 12 routing + await import('@usevenice/app-config/register.node') + + const viewer = await serverGetViewer(context) + const {ssg, ctx, queryClient, caller} = serverSideHelpersFromViewer(viewer) + return { viewer, ssg, ctx, + caller, getDehydratedState: () => superjson.serialize(dehydrate(queryClient)), /** @deprecated */ getPageProps: (): PageProps => ({ diff --git a/apps/web/middleware.ts b/apps/web/middleware.ts index fbf8e506..d36731d6 100644 --- a/apps/web/middleware.ts +++ b/apps/web/middleware.ts @@ -12,6 +12,12 @@ export const config = { * - public folder * - public folder * - connect (Venice connect, which has separate auth logic) + * - debug page + * - oauth routes */ - matcher: ['/((?!.*\\..*|_next|connect|hello).*)', '/', '/(api|trpc)(.*)'], + matcher: [ + '/((?!.*\\..*|_next|connect|debug|oauth).*)', + '/', + '/(api|trpc)(.*)', + ], } diff --git a/packages/cdk-core/oauth/NangoClient.spec.ts b/packages/cdk-core/oauth/NangoClient.spec.ts new file mode 100644 index 00000000..66219bcb --- /dev/null +++ b/packages/cdk-core/oauth/NangoClient.spec.ts @@ -0,0 +1,88 @@ +import {parseNangoOauthCallbackPage} from './NangoClient' + +test('oauth callback success parsing', () => { + const result = parseNangoOauthCallbackPage(` + + + + + Authorization callback + + + + + + + `) + expect(result).toEqual({ + eventType: 'AUTHORIZATION_SUCEEDED', + data: { + providerConfigKey: 'int_qbo_01HCGK0H4FVS1949G5KCKGG2KJ', + connectionId: 'reso_qbo_01HDFCAW620Q6GQXQ5VEHH82EF', + }, + }) +}) + +test('oauth callback error parsing', () => { + const result = parseNangoOauthCallbackPage(` + + + + + Authorization callback + + + + + + + `) + expect(result).toEqual({ + eventType: 'AUTHORIZATION_FAILED', + data: { + authErrorType: 'unknown_err', + authErrorDesc: 'Unkown error during the Oauth flow. {', + }, + }) +}) diff --git a/packages/cdk-core/oauth/NangoClient.ts b/packages/cdk-core/oauth/NangoClient.ts index 305ea83d..995bb702 100644 --- a/packages/cdk-core/oauth/NangoClient.ts +++ b/packages/cdk-core/oauth/NangoClient.ts @@ -273,12 +273,101 @@ export const zNangoConfig = z.object({ secretKey: z.string(), }) +export const NANGO_API_HOST = 'https://api.nango.dev' + export function makeNangoClient(config: z.infer) { const client = makeOpenApiClient>({ - baseUrl: 'https://api.nango.dev', + baseUrl: NANGO_API_HOST, auth: {bearerToken: config.secretKey}, }) - return client + + return { + ...client, + getOauthConnectUrl: async ({ + redirect_uri, + ...params + }: {redirect_uri?: string} & z.infer) => { + const res = await client._fetch(buildNangoConnectUrl(params), { + redirect: 'manual', + }) + const location = res.headers.get('location') + if (res.status !== 302 || !location) { + throw new Error('Missing redirect from nango /oauth/connect response') + } + const url = new URL(location) + if (redirect_uri) { + url.searchParams.set('redirect_uri', redirect_uri) + } + return url.toString() + }, + doOauthCallback: async ( + params: Record, + ) => { + const url = new URL('/oauth/callback', NANGO_API_HOST) + for (const [key, value] of Object.entries(params)) { + const arr = Array.isArray(value) ? value : value != null ? [value] : [] + for (const v of arr) { + url.searchParams.append(key, v) + } + } + const res = await client._fetch(url.toString(), {redirect: 'manual'}) + const htmlBody = await res.text() + return parseNangoOauthCallbackPage(htmlBody) + }, + } } +makeNangoClient.client = () => {} + export type NangoClient = ReturnType + +export const zNangoOauthConnectParams = z + .object({ + provider_config_key: z.string(), + connection_id: z.string(), + public_key: z.string(), + }) + .passthrough() + +export function buildNangoConnectUrl({ + provider_config_key, + ...params +}: z.infer) { + // http://localhost:3000/connect?displayName=Spendoso&integrationId=int_qbo&token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTgwODM2NjgsInJvbGUiOiJlbmRfdXNlciIsInN1YiI6Im9yZ18yUEk1VU42ZDRVMTdlOHpYaTFLdDloYzg4dnMvc3BlbmRvc28tdGVzdGVyIiwiZW5kX3VzZXJfaWQiOiJzcGVuZG9zby10ZXN0ZXIiLCJvcmdfaWQiOiJvcmdfMlBJNVVONmQ0VTE3ZTh6WGkxS3Q5aGM4OHZzIiwiaWF0IjoxNjk4MDgwMDY4fQ.GNFK71PloX0LkWQ3RCaWN6KCENSQtiXwvopfQk6ymB4 + const url = new URL(`/oauth/connect/${provider_config_key}`, NANGO_API_HOST) + for (const [key, value] of Object.entries(params)) { + url.searchParams.set(key, `${value}`) + } + return url.toString() +} + +export const zNangoOauthCallbackMessage = z.discriminatedUnion('eventType', [ + z.object({ + eventType: z.literal('AUTHORIZATION_SUCEEDED'), + data: z.object({providerConfigKey: z.string(), connectionId: z.string()}), + }), + z.object({ + eventType: z.literal('AUTHORIZATION_FAILED'), + data: z.object({authErrorDesc: z.string(), authErrorType: z.string()}), + }), +]) + +export function parseNangoOauthCallbackPage(html: string) { + const parseStrVar = (name: string) => + html + .match(new RegExp(`${name.replace('.', '.')} = (?:'|\`|")(.*)`))?.[1] + ?.replace(/('|`|");?$/, '') + + const eventType = parseStrVar('message.eventType') + + const authErrorType = parseStrVar('window.authErrorType') + const authErrorDesc = parseStrVar('window.authErrorDesc') + + const providerConfigKey = parseStrVar('window.providerConfigKey') + const connectionId = parseStrVar('window.connectionId') + + return zNangoOauthCallbackMessage.parse({ + eventType, + data: {providerConfigKey, connectionId, authErrorDesc, authErrorType}, + }) +} diff --git a/packages/engine-backend/router/adminRouter.ts b/packages/engine-backend/router/adminRouter.ts index db29067b..26ad1ef4 100644 --- a/packages/engine-backend/router/adminRouter.ts +++ b/packages/engine-backend/router/adminRouter.ts @@ -39,7 +39,9 @@ export const zConnectPageParams = z.object({ 'Where to send user to after connect / if they press back button', ), /** Launch the integration right away */ - integrationId: z.string().nullish(), + integrationId: zId('int').nullish(), + /** Whether to show existing resources */ + showExisting: z.coerce.boolean().optional().default(true), }) /** @@ -186,7 +188,7 @@ export const adminRouter = trpc.router({ ) const url = new URL('/connect', ctx.apiUrl) // `/` will start from the root hostname itself for (const [key, value] of Object.entries({...params, token})) { - url.searchParams.set(key, value ?? '') + url.searchParams.set(key, `${value ?? ''}`) } return {url: url.toString()} }), diff --git a/packages/util/http/makeHttpClient.ts b/packages/util/http/makeHttpClient.ts index db4a750c..bf93b1bd 100644 --- a/packages/util/http/makeHttpClient.ts +++ b/packages/util/http/makeHttpClient.ts @@ -1,10 +1,11 @@ -import {getDefaultProxyAgent, HTTPError} from './http-utils' -import z from 'zod' import * as R from 'remeda' -import {safeJSONParse} from '../json-utils' +import z from 'zod' + import {defineProxyFn} from '../di-utils' +import {safeJSONParse} from '../json-utils' import {joinPath} from '../url-utils' import {stringifyQueryParams} from '../url-utils' +import {getDefaultProxyAgent, HTTPError} from './http-utils' const zHttpMethod = z.enum([ 'get', @@ -60,6 +61,10 @@ export function makeHttpClient(options: HttpClientOptions) { ...defaults } = options + const _fetch: typeof fetch = (url, options) => + // Node fetch specific option... Noop on other platforms. + fetch(url, {...({agent} as RequestInit), ...options}) + const agent = getDefaultProxyAgent() // TODO: Perform runtime validation on input... so we don't get body @@ -101,9 +106,7 @@ export function makeHttpClient(options: HttpClientOptions) { // NOTE: Implement proxyAgent as a middleware // This way we can transparently use reverse proxies also in addition to forward proxy // as well as just simple in-app logging. - return fetch(url, { - // @ts-expect-error Node fetch specific option... Noop on other platforms. - agent, + return _fetch(url, { ...defaults, method, headers, @@ -133,7 +136,7 @@ export function makeHttpClient(options: HttpClientOptions) { request(method.toUpperCase() as Uppercase, path, input), ]) - return {...methods, request} + return {...methods, request, _fetch} } // TODO: build url from utils? diff --git a/packages/util/http/makeOpenApiClient.ts b/packages/util/http/makeOpenApiClient.ts index 68893ea0..eb9eb67e 100644 --- a/packages/util/http/makeOpenApiClient.ts +++ b/packages/util/http/makeOpenApiClient.ts @@ -1,5 +1,6 @@ import type {Get} from 'type-fest' import type {z} from 'zod' + import type { HttpClientOptions, HTTPMethod, @@ -79,6 +80,7 @@ export function makeOpenApiClient( path: Path, input: Get, ) => Promise> + _fetch: NonNullable } } From df561b52ecea95a91b255ab08df6a781ff2b99db Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Mon, 23 Oct 2023 19:04:50 -0700 Subject: [PATCH 59/80] Default to port 4000 to avoid conflict --- apps/web/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/package.json b/apps/web/package.json index d7c0e7d6..34c7faf3 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -8,7 +8,7 @@ "build:next": "next build", "codegen:sdk": "openapi-typescript http://localhost:3000/api/openapi --output ./sdk/venice.gen.ts", "codegen:supabase": "supabase gen types typescript --linked --schema public > ./supabase/supabase.gen.ts", - "dev": "next dev", + "dev": "next dev -p 4000", "start": "next start" }, "dependencies": { From 4931b5d45d4b1036b863342ab7690a5fb5d40f69 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Mon, 23 Oct 2023 22:40:45 -0700 Subject: [PATCH 60/80] Send message to popup opener for connection result --- .vscode/settings.json | 5 ++- apps/web/app/oauth/callback/CallbackPage.tsx | 25 ++++++++++++- apps/web/app/oauth/callback/page.tsx | 39 ++++++++++++-------- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 71402804..7bba08ef 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -78,5 +78,8 @@ "tailwindCSS.experimental.configFile": "apps/web/tailwind.config.ts", "typescript.format.insertSpaceAfterOpeningAndBeforeClosingNonemptyBraces": false, "typescript.tsdk": "./node_modules/typescript/lib", - "typescript.enablePromptUseWorkspaceTsdk": true + "typescript.enablePromptUseWorkspaceTsdk": true, + "workbench.colorCustomizations": { + "titleBar.activeBackground": "#0C4B33" + } } diff --git a/apps/web/app/oauth/callback/CallbackPage.tsx b/apps/web/app/oauth/callback/CallbackPage.tsx index f7b5c7b3..17dc9eb4 100644 --- a/apps/web/app/oauth/callback/CallbackPage.tsx +++ b/apps/web/app/oauth/callback/CallbackPage.tsx @@ -2,17 +2,38 @@ import React from 'react' -export function CallbackPage(props: {res: any; autoClose?: boolean}) { +import {z} from '@usevenice/util' + +import {zId} from '@/../../packages/cdk-core' + +export const zFrameMessage = z.discriminatedUnion('type', [ + z.object({ + type: z.literal('SUCCESS'), + data: z.object({resourceId: zId('reso')}), + }), + z.object({ + type: z.literal('ERROR'), + data: z.object({code: z.string(), message: z.string()}), + }), +]) + +export type FrameMessage = z.infer + +export function CallbackPage(props: {msg: FrameMessage; autoClose?: boolean}) { React.useEffect(() => { + const opener = window.opener as Window | null + opener?.postMessage(props.msg, '*') + if (props.autoClose) { setTimeout(() => { window.close() }, 300) } + // eslint-disable-next-line react-hooks/exhaustive-deps }, []) return (
- 
{JSON.stringify(props.res, null, 4)}
+ 
{JSON.stringify(props.msg, null, 4)}
You may now close this window if it does not automatically close
diff --git a/apps/web/app/oauth/callback/page.tsx b/apps/web/app/oauth/callback/page.tsx index 4d1c5f78..bdf2af48 100644 --- a/apps/web/app/oauth/callback/page.tsx +++ b/apps/web/app/oauth/callback/page.tsx @@ -1,10 +1,12 @@ import '@usevenice/app-config/register.node' import {env} from '@usevenice/app-config/env' -import {makeNangoClient} from '@usevenice/cdk-core' +import type {Id} from '@usevenice/cdk-core'; +import { makeNangoClient} from '@usevenice/cdk-core' import {serverSideHelpersFromViewer} from '@/lib-server' +import type { FrameMessage} from './CallbackPage'; import {CallbackPage} from './CallbackPage' export const metadata = { @@ -26,23 +28,30 @@ export default async function OAuthCallback({ searchParams: Record }) { const nango = makeNangoClient({secretKey: env.NANGO_SECRET_KEY}) - const ret = await nango.doOauthCallback(searchParams) - - if (ret.eventType === 'AUTHORIZATION_SUCEEDED') { - const {caller} = serverSideHelpersFromViewer({role: 'system'}) - + const res = await nango.doOauthCallback(searchParams) + + const msg = await (async (): Promise => { + if (res.eventType !== 'AUTHORIZATION_SUCEEDED') { + return { + type: 'ERROR', + data: {code: res.data.authErrorType, message: res.data.authErrorDesc}, + } + } try { - await caller.postConnect([ret.data, ret.data.providerConfigKey, {}]) + const {caller} = serverSideHelpersFromViewer({role: 'system'}) + await caller.postConnect([res.data, res.data.providerConfigKey, {}]) + return { + type: 'SUCCESS', + data: {resourceId: res.data.connectionId as Id['reso']}, + } } catch (err) { - // TODO: Better handle failure here. - return ( - - ) + return { + type: 'ERROR', + data: {code: 'INTERNAL_SERVER_ERROR', message: `${err}`}, + } } - } + })() // How do we do redirect here? - return + return } From 93816e774c5c31c5454b4f54020569bb9569be0e Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Mon, 23 Oct 2023 23:00:13 -0700 Subject: [PATCH 61/80] /oauth/callback to just /connect/callback --- .../web/app/{oauth => connect}/callback/CallbackPage.tsx | 3 +-- apps/web/app/{oauth => connect}/callback/page.tsx | 6 +++--- apps/web/app/connect/page.tsx | 6 +++--- apps/web/middleware.ts | 9 ++------- 4 files changed, 9 insertions(+), 15 deletions(-) rename apps/web/app/{oauth => connect}/callback/CallbackPage.tsx (95%) rename apps/web/app/{oauth => connect}/callback/page.tsx (91%) diff --git a/apps/web/app/oauth/callback/CallbackPage.tsx b/apps/web/app/connect/callback/CallbackPage.tsx similarity index 95% rename from apps/web/app/oauth/callback/CallbackPage.tsx rename to apps/web/app/connect/callback/CallbackPage.tsx index 17dc9eb4..ca01f857 100644 --- a/apps/web/app/oauth/callback/CallbackPage.tsx +++ b/apps/web/app/connect/callback/CallbackPage.tsx @@ -2,10 +2,9 @@ import React from 'react' +import {zId} from '@usevenice/cdk-core' import {z} from '@usevenice/util' -import {zId} from '@/../../packages/cdk-core' - export const zFrameMessage = z.discriminatedUnion('type', [ z.object({ type: z.literal('SUCCESS'), diff --git a/apps/web/app/oauth/callback/page.tsx b/apps/web/app/connect/callback/page.tsx similarity index 91% rename from apps/web/app/oauth/callback/page.tsx rename to apps/web/app/connect/callback/page.tsx index bdf2af48..746f2e87 100644 --- a/apps/web/app/oauth/callback/page.tsx +++ b/apps/web/app/connect/callback/page.tsx @@ -1,12 +1,12 @@ import '@usevenice/app-config/register.node' import {env} from '@usevenice/app-config/env' -import type {Id} from '@usevenice/cdk-core'; -import { makeNangoClient} from '@usevenice/cdk-core' +import type {Id} from '@usevenice/cdk-core' +import {makeNangoClient} from '@usevenice/cdk-core' import {serverSideHelpersFromViewer} from '@/lib-server' -import type { FrameMessage} from './CallbackPage'; +import type {FrameMessage} from './CallbackPage' import {CallbackPage} from './CallbackPage' export const metadata = { diff --git a/apps/web/app/connect/page.tsx b/apps/web/app/connect/page.tsx index 56f3ee8e..f77cbe75 100644 --- a/apps/web/app/connect/page.tsx +++ b/apps/web/app/connect/page.tsx @@ -2,7 +2,6 @@ import {clerkClient} from '@clerk/nextjs' import Image from 'next/image' import {redirect} from 'next/navigation' -import {getServerUrl} from '@usevenice/app-config/constants' import {env} from '@usevenice/app-config/env' import {defIntegrations} from '@usevenice/app-config/integrations/integrations.def' import type {IntegrationDef} from '@usevenice/cdk-core' @@ -13,7 +12,7 @@ import { makeNangoClient, } from '@usevenice/cdk-core' import {zConnectPageParams} from '@usevenice/engine-backend/router/adminRouter' -import {joinPath, makeUlid} from '@usevenice/util' +import {makeUlid} from '@usevenice/util' import {ClientRoot} from '@/components/ClientRoot' import {SuperHydrate} from '@/components/SuperHydrate' @@ -66,7 +65,8 @@ export default async function ConnectPageContainer({ public_key: env.NEXT_PUBLIC_NANGO_PUBLIC_KEY, connection_id: makeId('reso', providerName, makeUlid()), provider_config_key: params.integrationId, - redirect_uri: joinPath(getServerUrl(null), '/oauth/callback'), + // Consider using hookdeck so we can work with any number of urls + // redirect_uri: joinPath(getServerUrl(null), '/connect/callback'), }) return redirect(url) } diff --git a/apps/web/middleware.ts b/apps/web/middleware.ts index d36731d6..6f092d6b 100644 --- a/apps/web/middleware.ts +++ b/apps/web/middleware.ts @@ -11,13 +11,8 @@ export const config = { * - favicon.ico (favicon file) * - public folder * - public folder - * - connect (Venice connect, which has separate auth logic) + * - connect (Venice connect, oauth, which has separate auth logic) * - debug page - * - oauth routes */ - matcher: [ - '/((?!.*\\..*|_next|connect|debug|oauth).*)', - '/', - '/(api|trpc)(.*)', - ], + matcher: ['/((?!.*\\..*|_next|connect|debug).*)', '/', '/(api|trpc)(.*)'], } From a995695b906ac89ae802e5928fdc8f47f9c7f3d4 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Mon, 23 Oct 2023 23:06:24 -0700 Subject: [PATCH 62/80] Filtering by integrationId --- .../integration-postgres/makePostgresMetaService.ts | 3 ++- packages/cdk-core/metaService.ts | 2 ++ packages/engine-backend/router/protectedRouter.ts | 9 ++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/integrations/integration-postgres/makePostgresMetaService.ts b/integrations/integration-postgres/makePostgresMetaService.ts index 398c96d4..9cfdb9f0 100644 --- a/integrations/integration-postgres/makePostgresMetaService.ts +++ b/integrations/integration-postgres/makePostgresMetaService.ts @@ -167,11 +167,12 @@ function metaTable>( // TODO: Convert case from snake_case to camelCase return { - list: ({ids, endUserId, keywords, ...rest}) => + list: ({ids, endUserId, integrationId, keywords, ...rest}) => runQueries((pool) => { const conditions = R.compact([ ids && sql`id = ANY(${sql.array(ids, 'varchar')})`, endUserId && sql`end_user_id = ${endUserId}`, + integrationId && sql`integration_id = ${integrationId}`, // Temp solution, shall use fts and make this work for any table... keywords && tableName === 'institution' && diff --git a/packages/cdk-core/metaService.ts b/packages/cdk-core/metaService.ts index b7608f6e..c76c1db5 100644 --- a/packages/cdk-core/metaService.ts +++ b/packages/cdk-core/metaService.ts @@ -12,6 +12,8 @@ export interface MetaTable< ids?: TID[] /** Maybe remove this? not applicable everywhere */ endUserId?: EndUserId | null + /** Maybe remove this? not applicable everywhere */ + integrationId?: Id['int'] | null /** Used for search */ keywords?: string | null /** Pagination, not necessarily supported */ diff --git a/packages/engine-backend/router/protectedRouter.ts b/packages/engine-backend/router/protectedRouter.ts index 6f886f76..99db6bd3 100644 --- a/packages/engine-backend/router/protectedRouter.ts +++ b/packages/engine-backend/router/protectedRouter.ts @@ -37,7 +37,14 @@ export const protectedRouter = trpc.router({ }), listResources: protectedProcedure .meta({openapi: {method: 'GET', path: '/resources'}}) - .input(zListParams.extend({endUserId: zEndUserId.optional()}).optional()) + .input( + zListParams + .extend({ + endUserId: zEndUserId.nullish(), + integrationId: zId('int').nullish(), + }) + .optional(), + ) .output(z.array(zRaw.resource)) .query(async ({input = {}, ctx}) => { const resources = await ctx.helpers.metaService.tables.resource.list( From 222f155a069853a9b591445514013dccc3e6f9bb Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Mon, 23 Oct 2023 23:37:32 -0700 Subject: [PATCH 63/80] Display spinner instead of JSON debug --- .../{CallbackPage.tsx => CallbackEffect.tsx} | 14 ++++++-------- apps/web/app/connect/callback/page.tsx | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 12 deletions(-) rename apps/web/app/connect/callback/{CallbackPage.tsx => CallbackEffect.tsx} (70%) diff --git a/apps/web/app/connect/callback/CallbackPage.tsx b/apps/web/app/connect/callback/CallbackEffect.tsx similarity index 70% rename from apps/web/app/connect/callback/CallbackPage.tsx rename to apps/web/app/connect/callback/CallbackEffect.tsx index ca01f857..5539a022 100644 --- a/apps/web/app/connect/callback/CallbackPage.tsx +++ b/apps/web/app/connect/callback/CallbackEffect.tsx @@ -18,7 +18,10 @@ export const zFrameMessage = z.discriminatedUnion('type', [ export type FrameMessage = z.infer -export function CallbackPage(props: {msg: FrameMessage; autoClose?: boolean}) { +export function CallbackEffect(props: { + msg: FrameMessage + autoClose?: boolean +}) { React.useEffect(() => { const opener = window.opener as Window | null opener?.postMessage(props.msg, '*') @@ -30,11 +33,6 @@ export function CallbackPage(props: {msg: FrameMessage; autoClose?: boolean}) { } // eslint-disable-next-line react-hooks/exhaustive-deps }, []) - return ( -
- 
{JSON.stringify(props.msg, null, 4)}
- You may now close this window if it does not automatically close - -
- ) + + return null } diff --git a/apps/web/app/connect/callback/page.tsx b/apps/web/app/connect/callback/page.tsx index 746f2e87..b9aa444d 100644 --- a/apps/web/app/connect/callback/page.tsx +++ b/apps/web/app/connect/callback/page.tsx @@ -1,13 +1,16 @@ import '@usevenice/app-config/register.node' +import {Loader2} from 'lucide-react' + import {env} from '@usevenice/app-config/env' import type {Id} from '@usevenice/cdk-core' import {makeNangoClient} from '@usevenice/cdk-core' +import {FullScreenCenter} from '@/components/FullScreenCenter' import {serverSideHelpersFromViewer} from '@/lib-server' -import type {FrameMessage} from './CallbackPage' -import {CallbackPage} from './CallbackPage' +import type {FrameMessage} from './CallbackEffect' +import {CallbackEffect} from './CallbackEffect' export const metadata = { title: 'Venice Oauth Callback', @@ -20,7 +23,7 @@ export const metadata = { export const dynamic = 'force-dynamic' /** https://beta.nextjs.org/docs/api-reference/file-conventions/page#searchparams-optional */ -export default async function OAuthCallback({ +export default async function ConnectCallback({ searchParams, }: { // Only accessible in PageComponent rather than layout component @@ -53,5 +56,10 @@ export default async function OAuthCallback({ })() // How do we do redirect here? - return + return ( + + + + + ) } From 79a79693622da77e14227212a59b653d5c1036e0 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Tue, 24 Oct 2023 00:21:25 -0700 Subject: [PATCH 64/80] Test out both tsc and tsup for building lib --- apps/web/tsconfig.json | 6 ++- packages/connect/.gitignore | 3 ++ packages/connect/package.json | 13 ++++- packages/connect/{ => src}/common.ts | 0 packages/connect/{ => src}/index.ts | 0 packages/connect/{ => src}/react.tsx | 1 + packages/connect/tsconfig.json | 22 ++++++++ packages/connect/tsup.config.ts | 10 ++++ pnpm-lock.yaml | 81 +++++++++++++++++++++++++--- 9 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 packages/connect/.gitignore rename packages/connect/{ => src}/common.ts (100%) rename packages/connect/{ => src}/index.ts (100%) rename packages/connect/{ => src}/react.tsx (99%) create mode 100644 packages/connect/tsconfig.json create mode 100644 packages/connect/tsup.config.ts diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index b65cfc27..28d1b013 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -16,9 +16,11 @@ "strictNullChecks": true, "plugins": [{"name": "next"}], "paths": { - "@/*": ["./*"] + "@/*": ["./*"], + // Override because we don't want to use ./dist unnecessarily + "@usevenice/connect": ["../../packages/connect/src/index.ts"] } }, - "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", "../../packages/ui/command/command-fns.spec.ts"], + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], "exclude": ["node_modules"] } diff --git a/packages/connect/.gitignore b/packages/connect/.gitignore new file mode 100644 index 00000000..ca8786d3 --- /dev/null +++ b/packages/connect/.gitignore @@ -0,0 +1,3 @@ + +dist/ +build/ diff --git a/packages/connect/package.json b/packages/connect/package.json index 8c45b0e6..7260246f 100644 --- a/packages/connect/package.json +++ b/packages/connect/package.json @@ -3,10 +3,19 @@ "version": "0.0.0", "private": true, "sideEffects": false, - "module": "./index.ts", + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "scripts": { + "build": "tsc -p ./tsconfig.json", + "build:tsup": "tsup", + "clean": "rm -rf ./dist" + }, "dependencies": {}, "devDependencies": { - "@types/react": "*" + "@types/react": "*", + "tsup": "7.2.0", + "typescript": "*" }, "peerDependencies": { "react": "*" diff --git a/packages/connect/common.ts b/packages/connect/src/common.ts similarity index 100% rename from packages/connect/common.ts rename to packages/connect/src/common.ts diff --git a/packages/connect/index.ts b/packages/connect/src/index.ts similarity index 100% rename from packages/connect/index.ts rename to packages/connect/src/index.ts diff --git a/packages/connect/react.tsx b/packages/connect/src/react.tsx similarity index 99% rename from packages/connect/react.tsx rename to packages/connect/src/react.tsx index 8df0a052..7d4dac26 100644 --- a/packages/connect/react.tsx +++ b/packages/connect/src/react.tsx @@ -1,4 +1,5 @@ import React from 'react' + import type {GetIFrameProps} from './common' import {getIFrameUrl} from './common' diff --git a/packages/connect/tsconfig.json b/packages/connect/tsconfig.json new file mode 100644 index 00000000..6f253a83 --- /dev/null +++ b/packages/connect/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "@tsconfig/strictest/tsconfig.json", + "compilerOptions": { + "target": "ES2020", + "lib": ["DOM", "DOM.Iterable", "ES2021"], + "allowJs": true, + "checkJs": false, + "exactOptionalPropertyTypes": false, + "importsNotUsedAsValues": "remove", + "esModuleInterop": true, + "module": "CommonJS", + "moduleResolution": "Node", + "resolveJsonModule": true, + "jsx": "react-jsx", + "jsxImportSource": "react", + "outDir": "./dist", + "declaration": true, + "sourceMap": true + }, + "include": ["env.d.ts", "src/index.ts", "src/common.ts"], + "exclude": ["node_modules"] +} diff --git a/packages/connect/tsup.config.ts b/packages/connect/tsup.config.ts new file mode 100644 index 00000000..d8a428e0 --- /dev/null +++ b/packages/connect/tsup.config.ts @@ -0,0 +1,10 @@ +import {defineConfig} from 'tsup' + +export default defineConfig({ + entry: ['src/index.ts', 'src/common.ts'], + format: ['cjs', 'esm'], // Build for commonJS and ESmodules + dts: true, // Generate declaration file (.d.ts) + splitting: false, + sourcemap: true, + clean: true, +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 568189c0..6639217f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1307,6 +1307,12 @@ importers: '@types/react': specifier: 18.0.27 version: 18.0.27 + tsup: + specifier: 7.2.0 + version: 7.2.0(typescript@5.0.4) + typescript: + specifier: '*' + version: 5.0.4 packages/engine-backend: dependencies: @@ -7768,6 +7774,16 @@ packages: engines: {node: '>=6'} dev: true + /bundle-require@4.0.2(esbuild@0.17.5): + resolution: {integrity: sha512-jwzPOChofl67PSTW2SGubV9HBQAhhR2i6nskiOThauo9dzwDUgOWQScFVaJkjEfYX+UXiD+LEx8EblQMc2wIag==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + peerDependencies: + esbuild: 0.17.5 + dependencies: + esbuild: 0.17.5 + load-tsconfig: 0.2.5 + dev: true + /busboy@1.6.0: resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} engines: {node: '>=10.16.0'} @@ -7781,7 +7797,6 @@ packages: /cac@6.7.12(patch_hash=nsfmzcycabp3f3q7kw2uyer77u): resolution: {integrity: sha512-rM7E2ygtMkJqD9c7WnFU6fruFcN3xe4FM5yUmgxhZzIKJk4uHl9U/fhwdajGFQbQuv43FAUo1Fe8gX/oIKDeSA==} engines: {node: '>=8'} - dev: false patched: true /cac@6.7.14: @@ -12114,6 +12129,11 @@ packages: react: 18.2.0 dev: false + /joycon@3.1.1: + resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} + engines: {node: '>=10'} + dev: true + /js-cookie@2.2.1: resolution: {integrity: sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ==} dev: false @@ -12546,6 +12566,11 @@ packages: strip-bom: 3.0.0 dev: true + /load-tsconfig@0.2.5: + resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: true + /loader-runner@4.3.0: resolution: {integrity: sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==} engines: {node: '>=6.11.5'} @@ -12648,7 +12673,6 @@ packages: /lodash.sortby@4.7.0: resolution: {integrity: sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==} - dev: false /lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} @@ -15680,6 +15704,14 @@ packages: fsevents: 2.3.2 dev: false + /rollup@3.29.4: + resolution: {integrity: sha512-oWzmBZwvYrU0iJHtDmhsm662rC15FRXmcjCk1xD771dFDx5jJ02ufAQQTn0etB2emNk4J9EZg/yWKpsn9BWGRw==} + engines: {node: '>=14.18.0', npm: '>=8.0.0'} + hasBin: true + optionalDependencies: + fsevents: 2.3.2 + dev: true + /rrweb-snapshot@1.1.14: resolution: {integrity: sha512-eP5pirNjP5+GewQfcOQY4uBiDnpqxNRc65yKPW0eSoU1XamDfc4M8oqpXGMyUyvLyxFDB0q0+DChuxxiU2FXBQ==} dev: false @@ -16266,7 +16298,6 @@ packages: engines: {node: '>= 8'} dependencies: whatwg-url: 7.1.0 - dev: false /sourcemap-codec@1.4.8: resolution: {integrity: sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==} @@ -17123,7 +17154,6 @@ packages: resolution: {integrity: sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==} dependencies: punycode: 2.1.1 - dev: false /tr46@3.0.0: resolution: {integrity: sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==} @@ -17132,6 +17162,11 @@ packages: punycode: 2.1.1 dev: false + /tree-kill@1.2.2: + resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} + hasBin: true + dev: true + /trough@1.0.5: resolution: {integrity: sha512-rvuRbTarPXmMb79SmzEp8aqXNKcK+y0XaB298IXueQ8I2PsrATcPBCSPyK/dDNa2iWOhKlfNnOjdAOTBU/nkFA==} dev: false @@ -17197,6 +17232,42 @@ packages: /tslib@2.4.1: resolution: {integrity: sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==} + /tsup@7.2.0(typescript@5.0.4): + resolution: {integrity: sha512-vDHlczXbgUvY3rWvqFEbSqmC1L7woozbzngMqTtL2PGBODTtWlRwGDDawhvWzr5c1QjKe4OAKqJGfE1xeXUvtQ==} + engines: {node: '>=16.14'} + hasBin: true + peerDependencies: + '@swc/core': ^1 + postcss: ^8.4.12 + typescript: '>=4.1.0' + peerDependenciesMeta: + '@swc/core': + optional: true + postcss: + optional: true + typescript: + optional: true + dependencies: + bundle-require: 4.0.2(esbuild@0.17.5) + cac: 6.7.12(patch_hash=nsfmzcycabp3f3q7kw2uyer77u) + chokidar: 3.5.3 + debug: 4.3.4 + esbuild: 0.17.5 + execa: 5.1.1 + globby: 11.1.0 + joycon: 3.1.1 + postcss-load-config: 4.0.1(postcss@8.4.23) + resolve-from: 5.0.0 + rollup: 3.29.4 + source-map: 0.8.0-beta.0 + sucrase: 3.32.0 + tree-kill: 1.2.2 + typescript: 5.0.4 + transitivePeerDependencies: + - supports-color + - ts-node + dev: true + /tsutils@3.21.0(typescript@5.0.4): resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} engines: {node: '>= 6'} @@ -17821,7 +17892,6 @@ packages: /webidl-conversions@4.0.2: resolution: {integrity: sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==} - dev: false /webidl-conversions@7.0.0: resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} @@ -17927,7 +17997,6 @@ packages: lodash.sortby: 4.7.0 tr46: 1.0.1 webidl-conversions: 4.0.2 - dev: false /whence@2.0.0: resolution: {integrity: sha512-exmM13v2lg8juBbfS2tao/alV68jyryPXS+jf29NBNGLzE2hRgmzvQFQGX5CxNfH4Ag9qRqd6gGpXTH2JxqKHg==} From e3ae7add9c0f59f8c2c5395b8e2564f48f1d4b4c Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Tue, 24 Oct 2023 01:19:13 -0700 Subject: [PATCH 65/80] @usevenice/connect 0.0.1 published! for working POC --- .../app/connect/callback/CallbackEffect.tsx | 16 +---- apps/web/app/connect/callback/page.tsx | 2 +- apps/web/tsconfig.json | 2 +- packages/connect/.npmignore | 2 + packages/connect/package.json | 11 ++-- packages/connect/src/api.ts | 44 +++++++++++++ packages/connect/src/common.ts | 18 ++++- .../src/{react.tsx => embed-react.tsx} | 0 packages/connect/src/index.ts | 4 +- packages/connect/src/popup.ts | 66 +++++++++++++++++++ pnpm-lock.yaml | 3 + tsconfig.json | 5 +- 12 files changed, 149 insertions(+), 24 deletions(-) create mode 100644 packages/connect/.npmignore create mode 100644 packages/connect/src/api.ts rename packages/connect/src/{react.tsx => embed-react.tsx} (100%) create mode 100644 packages/connect/src/popup.ts diff --git a/apps/web/app/connect/callback/CallbackEffect.tsx b/apps/web/app/connect/callback/CallbackEffect.tsx index 5539a022..1521cf05 100644 --- a/apps/web/app/connect/callback/CallbackEffect.tsx +++ b/apps/web/app/connect/callback/CallbackEffect.tsx @@ -2,21 +2,7 @@ import React from 'react' -import {zId} from '@usevenice/cdk-core' -import {z} from '@usevenice/util' - -export const zFrameMessage = z.discriminatedUnion('type', [ - z.object({ - type: z.literal('SUCCESS'), - data: z.object({resourceId: zId('reso')}), - }), - z.object({ - type: z.literal('ERROR'), - data: z.object({code: z.string(), message: z.string()}), - }), -]) - -export type FrameMessage = z.infer +import type {FrameMessage} from '@usevenice/connect/common' export function CallbackEffect(props: { msg: FrameMessage diff --git a/apps/web/app/connect/callback/page.tsx b/apps/web/app/connect/callback/page.tsx index b9aa444d..67d717f4 100644 --- a/apps/web/app/connect/callback/page.tsx +++ b/apps/web/app/connect/callback/page.tsx @@ -5,11 +5,11 @@ import {Loader2} from 'lucide-react' import {env} from '@usevenice/app-config/env' import type {Id} from '@usevenice/cdk-core' import {makeNangoClient} from '@usevenice/cdk-core' +import type {FrameMessage} from '@usevenice/connect' import {FullScreenCenter} from '@/components/FullScreenCenter' import {serverSideHelpersFromViewer} from '@/lib-server' -import type {FrameMessage} from './CallbackEffect' import {CallbackEffect} from './CallbackEffect' export const metadata = { diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index 28d1b013..3e1ba3cc 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -18,7 +18,7 @@ "paths": { "@/*": ["./*"], // Override because we don't want to use ./dist unnecessarily - "@usevenice/connect": ["../../packages/connect/src/index.ts"] + "@usevenice/connect/*": ["../../packages/connect/src/*"] } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], diff --git a/packages/connect/.npmignore b/packages/connect/.npmignore new file mode 100644 index 00000000..b091b9ae --- /dev/null +++ b/packages/connect/.npmignore @@ -0,0 +1,2 @@ +tsup.config.ts +tsconfig.json diff --git a/packages/connect/package.json b/packages/connect/package.json index 7260246f..76582224 100644 --- a/packages/connect/package.json +++ b/packages/connect/package.json @@ -1,7 +1,6 @@ { "name": "@usevenice/connect", - "version": "0.0.0", - "private": true, + "version": "0.0.1", "sideEffects": false, "main": "./dist/index.js", "module": "./dist/index.mjs", @@ -9,9 +8,13 @@ "scripts": { "build": "tsc -p ./tsconfig.json", "build:tsup": "tsup", - "clean": "rm -rf ./dist" + "clean": "rm -rf ./dist", + "prepack": "pnpm run build", + "publish": "pnpm publish --no-git-checks --access public" + }, + "dependencies": { + "zod": "3.21.4" }, - "dependencies": {}, "devDependencies": { "@types/react": "*", "tsup": "7.2.0", diff --git a/packages/connect/src/api.ts b/packages/connect/src/api.ts new file mode 100644 index 00000000..86b8e26f --- /dev/null +++ b/packages/connect/src/api.ts @@ -0,0 +1,44 @@ +import {z} from 'zod' + +import {defaultVeniceHost} from './common' + +export const zVeniceClientConfig = z.object({ + // Support end-user token based auth. + apiKey: z.string(), + apiHost: z.string().default(defaultVeniceHost), +}) +export type VeniceClientConfig = z.infer + +export function makeVeniceClient({apiKey, ...config}: VeniceClientConfig) { + const apiBase = new URL('/api/openapi/', config.apiHost).toString() + const headers = {'x-apikey': apiKey, 'Content-Type': 'application/json'} + + return { + listResources: async (opts: {integrationId: string}) => { + const url = new URL('resources', apiBase) + url.searchParams.set('integrationId', opts.integrationId) + const res = await fetch(url.toString(), {method: 'GET', headers}) + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const data = await res.json() + return data as Array<{id: string}> + }, + // Maybe we should allow end user to create magic link for themselves too so that + // this operation can be done client side instead of server side? + adminCreateMagicLink: async (opts: { + integrationId: string + endUserId: string + }) => { + const url = new URL('magic-link', apiBase) + // url.searchParams.set("integrationId", opts.integrationId); + + const res = await fetch(url.toString(), { + method: 'POST', + body: JSON.stringify({...opts}, null, 4), + headers, + }) + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const data = await res.json() + return data as {url: string} + }, + } +} diff --git a/packages/connect/src/common.ts b/packages/connect/src/common.ts index fc553843..4e853962 100644 --- a/packages/connect/src/common.ts +++ b/packages/connect/src/common.ts @@ -1,10 +1,26 @@ +import {z} from 'zod' + +export const zFrameMessage = z.discriminatedUnion('type', [ + z.object({ + type: z.literal('SUCCESS'), + data: z.object({resourceId: z.string()}), // Need to better type resourceId + }), + z.object({ + type: z.literal('ERROR'), + data: z.object({code: z.string(), message: z.string()}), + }), +]) +export type FrameMessage = z.infer + +export const defaultVeniceHost = 'https://app.venice.is' + export interface GetIFrameProps { deploymentUrl?: string params?: {token?: string; displayName?: string} } export const getIFrameUrl = ({ - deploymentUrl = 'https://app-staging.venice.is/', + deploymentUrl = defaultVeniceHost, params = {}, }: GetIFrameProps) => { const url = new URL('/connect', deploymentUrl) diff --git a/packages/connect/src/react.tsx b/packages/connect/src/embed-react.tsx similarity index 100% rename from packages/connect/src/react.tsx rename to packages/connect/src/embed-react.tsx diff --git a/packages/connect/src/index.ts b/packages/connect/src/index.ts index 722143f4..0b8111cc 100644 --- a/packages/connect/src/index.ts +++ b/packages/connect/src/index.ts @@ -1,4 +1,6 @@ // codegen:start {preset: barrel, include: "./{*.{ts,tsx},*/index.{ts,tsx}}", exclude: "./**/*.{d,spec,test,fixture,gen,node}.{ts,tsx}"} +export * from './api' export * from './common' -export * from './react' +export * from './embed-react' +export * from './popup' // codegen:end diff --git a/packages/connect/src/popup.ts b/packages/connect/src/popup.ts new file mode 100644 index 00000000..70fc4349 --- /dev/null +++ b/packages/connect/src/popup.ts @@ -0,0 +1,66 @@ +import type {FrameMessage} from './common' +import {zFrameMessage} from './common' + +export const VeniceFrontend = { + openMagicLink: async ({url}: {url: string}) => { + const features = { + ...popupLayout(500, 600), + scrollbars: 'yes', + resizable: 'yes', + status: 'no', + toolbar: 'no', + location: 'no', + copyhistory: 'no', + menubar: 'no', + directories: 'no', + popup: 'true', + } + const popup = window.open(url, '_blank', featuresToString(features)) + + return new Promise['data']>( + (resolve, reject) => { + const listener: Parameters< + typeof window.addEventListener<'message'> + >[1] = (event) => { + const res = zFrameMessage.safeParse(event.data) + if (!res.success) { + console.warn('Ignoring invalid message from popup', event.data) + return + } + window.removeEventListener('message', listener) + popup?.close() + if (res.data.type === 'SUCCESS') { + resolve(res.data.data) + } else { + reject(new Error(`${res.data.data.code}: ${res.data.data.message}`)) + } + } + window.addEventListener('message', listener) + }, + ) + }, +} + +function popupLayout(expectedWidth: number, expectedHeight: number) { + const screenWidth = window.screen.width + const screenHeight = window.screen.height + const left = screenWidth / 2 - expectedWidth / 2 + const top = screenHeight / 2 - expectedHeight / 2 + + const width = Math.min(expectedWidth, screenWidth) + const height = Math.min(expectedHeight, screenHeight) + + return { + left: Math.max(left, 0), + top: Math.max(top, 0), + width, + height, + } +} + +/** Helper for window.open() */ +function featuresToString(features: Record) { + return Object.entries(features) + .map(([key, value]) => `${key}=${value}`) + .join(',') +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6639217f..901300d8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1303,6 +1303,9 @@ importers: react: specifier: 18.2.0 version: 18.2.0 + zod: + specifier: 3.21.4 + version: 3.21.4(patch_hash=bzwjzhue3hmpww5lnv24u5k2ru) devDependencies: '@types/react': specifier: 18.0.27 diff --git a/tsconfig.json b/tsconfig.json index 6dde5f1e..119bc94c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,7 +19,10 @@ // In practice no module outside apps/web should use the `@/` alias ever // In addition to opening the door to invalid imports this also prevents us // from two separate next.js apps in the same monorepo which might be relevant for future - "paths": {"@/*": ["./apps/web/*"]} + "paths": { + "@/*": ["./apps/web/*"], + "@usevenice/connect/*": ["./packages/connect/src/*"] + } }, "include": ["env.d.ts", "**/*.ts", "**/*.tsx"], "exclude": ["node_modules"] From 24373127180eeaf1a657f2188e40609f936be9d4 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 25 Oct 2023 12:17:55 -0700 Subject: [PATCH 66/80] feat: Allowing endUser to create magicLink and token for themselves --- .../(authenticated)/end-users/page.tsx | 2 +- .../(authenticated)/magic-link/page.tsx | 6 +- apps/web/app/connect/page.tsx | 2 +- packages/engine-backend/router/adminRouter.ts | 86 +------------ .../engine-backend/router/endUserRouter.ts | 113 +++++++++++++++++- 5 files changed, 118 insertions(+), 91 deletions(-) diff --git a/apps/web/app/(admin)/(authenticated)/end-users/page.tsx b/apps/web/app/(admin)/(authenticated)/end-users/page.tsx index 51f2dcd0..f5d57491 100644 --- a/apps/web/app/(admin)/(authenticated)/end-users/page.tsx +++ b/apps/web/app/(admin)/(authenticated)/end-users/page.tsx @@ -43,7 +43,7 @@ type EndUser = RouterOutput['adminSearchEndUsers'][number] function EndUserMenu({endUser}: {endUser: EndUser}) { const {toast} = useToast() - const createMagicLink = _trpcReact.adminCreateMagicLink.useMutation({}) + const createMagicLink = _trpcReact.createMagicLink.useMutation({}) return ( diff --git a/apps/web/app/(admin)/(authenticated)/magic-link/page.tsx b/apps/web/app/(admin)/(authenticated)/magic-link/page.tsx index d911d56d..5ca61de1 100644 --- a/apps/web/app/(admin)/(authenticated)/magic-link/page.tsx +++ b/apps/web/app/(admin)/(authenticated)/magic-link/page.tsx @@ -1,6 +1,6 @@ 'use client' -import {adminRouterSchema} from '@usevenice/engine-backend/router/adminRouter' +import {endUserRouterSchema} from '@usevenice/engine-backend/router/endUserRouter' import {_trpcReact} from '@usevenice/engine-frontend' import {SchemaForm, useToast} from '@usevenice/ui' @@ -9,7 +9,7 @@ import {copyToClipboard} from '@/lib-client/copyToClipboard' export default function MagicLinkPage() { const {toast} = useToast() - const createMagicLink = _trpcReact.adminCreateMagicLink.useMutation({ + const createMagicLink = _trpcReact.createMagicLink.useMutation({ onError: (err) => { toast({ title: 'Error creating magic link', @@ -23,7 +23,7 @@ export default function MagicLinkPage() {

Magic link

{ createMagicLink.mutate(values, { diff --git a/apps/web/app/connect/page.tsx b/apps/web/app/connect/page.tsx index f77cbe75..75d0b3e3 100644 --- a/apps/web/app/connect/page.tsx +++ b/apps/web/app/connect/page.tsx @@ -11,7 +11,7 @@ import { makeId, makeNangoClient, } from '@usevenice/cdk-core' -import {zConnectPageParams} from '@usevenice/engine-backend/router/adminRouter' +import {zConnectPageParams} from '@usevenice/engine-backend/router/endUserRouter' import {makeUlid} from '@usevenice/util' import {ClientRoot} from '@/components/ClientRoot' diff --git a/packages/engine-backend/router/adminRouter.ts b/packages/engine-backend/router/adminRouter.ts index 26ad1ef4..06735f0d 100644 --- a/packages/engine-backend/router/adminRouter.ts +++ b/packages/engine-backend/router/adminRouter.ts @@ -7,7 +7,6 @@ import { makeOauthIntegrationServer, oauthBaseSchema, sync, - zEndUserId, zId, zRaw, } from '@usevenice/cdk-core' @@ -17,52 +16,6 @@ import {adminProcedure, trpc} from './_base' export {type inferProcedureInput} from '@trpc/server' -export const zConnectTokenPayload = z.object({ - endUserId: zEndUserId.describe( - 'Anything that uniquely identifies the end user that you will be sending the magic link to', - ), - validityInSeconds: z - .number() - .default(3600) - .describe( - 'How long the magic link will be valid for (in seconds) before it expires', - ), -}) - -export const zConnectPageParams = z.object({ - token: z.string(), - displayName: z.string().nullish().describe('What to call user by'), - redirectUrl: z - .string() - .nullish() - .describe( - 'Where to send user to after connect / if they press back button', - ), - /** Launch the integration right away */ - integrationId: zId('int').nullish(), - /** Whether to show existing resources */ - showExisting: z.coerce.boolean().optional().default(true), -}) - -/** - * Workaround to be able to re-use the schema on the frontend for now - * @see https://github.com/trpc/trpc/issues/4295 - * - * Though if we can FULLY automate the generate of forms perhaps this wouldn't actually be necessary? - * We will have to make sure though that the router themselves do not have any side effect imports - * and all server-specific logic would be part of context. - * But then again client side bundle size would still be a concern - * as we'd be sending server side code unnecessarily to client still - * unless of course we transform zod -> jsonschema and send that to the client only - * via a trpc schema endpoint (with server side rendering of course) - */ -export const adminRouterSchema = { - adminCreateConnectToken: {input: zConnectTokenPayload}, - adminCreateMagicLink: { - input: zConnectTokenPayload.merge(zConnectPageParams.omit({token: true})), - }, -} satisfies Record - export const adminRouter = trpc.router({ adminListIntegrations: adminProcedure .meta({openapi: {method: 'GET', path: '/integrations'}}) @@ -154,44 +107,7 @@ export const adminRouter = trpc.router({ } return ctx.helpers.metaService.tables.integration.delete(intId) }), - adminCreateConnectToken: adminProcedure - .meta({openapi: {method: 'POST', path: '/connect-token'}}) - .input(adminRouterSchema.adminCreateConnectToken.input) - .output(z.string()) - .mutation(({input: {endUserId, validityInSeconds}, ctx}) => { - // Figure out a better way to share code here... - if (!('orgId' in ctx.viewer) || !ctx.viewer.orgId) { - throw new TRPCError({ - code: 'BAD_REQUEST', - message: 'Current viewer missing orgId to create token', - }) - } - return ctx.jwt.signViewer( - {role: 'end_user', endUserId, orgId: ctx.viewer.orgId}, - {validityInSeconds}, - ) - }), - adminCreateMagicLink: adminProcedure - .meta({openapi: {method: 'POST', path: '/magic-link'}}) - .input(adminRouterSchema.adminCreateMagicLink.input) - .output(z.object({url: z.string()})) - .mutation(({input: {endUserId, validityInSeconds, ...params}, ctx}) => { - if (!('orgId' in ctx.viewer) || !ctx.viewer.orgId) { - throw new TRPCError({ - code: 'BAD_REQUEST', - message: 'Current viewer missing orgId to create token', - }) - } - const token = ctx.jwt.signViewer( - {role: 'end_user', endUserId, orgId: ctx.viewer.orgId}, - {validityInSeconds}, - ) - const url = new URL('/connect', ctx.apiUrl) // `/` will start from the root hostname itself - for (const [key, value] of Object.entries({...params, token})) { - url.searchParams.set(key, `${value ?? ''}`) - } - return {url: url.toString()} - }), + adminSearchEndUsers: adminProcedure .input(z.object({keywords: z.string().trim().nullish()}).optional()) .query(async ({input: {keywords} = {}, ctx}) => diff --git a/packages/engine-backend/router/endUserRouter.ts b/packages/engine-backend/router/endUserRouter.ts index 4b3c3744..d7e96646 100644 --- a/packages/engine-backend/router/endUserRouter.ts +++ b/packages/engine-backend/router/endUserRouter.ts @@ -1,8 +1,16 @@ -import type {OauthBaseTypes, ResourceUpdate} from '@usevenice/cdk-core' +import {TRPCError} from '@trpc/server' + +import type { + EndUserId, + OauthBaseTypes, + ResourceUpdate, + Viewer, +} from '@usevenice/cdk-core' import { makeId, makeOauthIntegrationServer, zConnectOptions, + zEndUserId, zId, zPostConnectOptions, zRaw, @@ -15,8 +23,111 @@ import {protectedProcedure, trpc} from './_base' export {type inferProcedureInput} from '@trpc/server' +export const zConnectTokenPayload = z.object({ + endUserId: zEndUserId + .nullish() + .describe( + 'Anything that uniquely identifies the end user that you will be sending the magic link to', + ), + validityInSeconds: z + .number() + .default(3600) + .describe( + 'How long the magic link will be valid for (in seconds) before it expires', + ), +}) + +export const zConnectPageParams = z.object({ + token: z.string(), + displayName: z.string().nullish().describe('What to call user by'), + redirectUrl: z + .string() + .nullish() + .describe( + 'Where to send user to after connect / if they press back button', + ), + /** Launch the integration right away */ + integrationId: zId('int').nullish(), + /** Whether to show existing resources */ + showExisting: z.coerce.boolean().optional().default(true), +}) + +/** + * Workaround to be able to re-use the schema on the frontend for now + * @see https://github.com/trpc/trpc/issues/4295 + * + * Though if we can FULLY automate the generate of forms perhaps this wouldn't actually be necessary? + * We will have to make sure though that the router themselves do not have any side effect imports + * and all server-specific logic would be part of context. + * But then again client side bundle size would still be a concern + * as we'd be sending server side code unnecessarily to client still + * unless of course we transform zod -> jsonschema and send that to the client only + * via a trpc schema endpoint (with server side rendering of course) + */ +export const endUserRouterSchema = { + createConnectToken: {input: zConnectTokenPayload}, + createMagicLink: { + input: zConnectTokenPayload.merge(zConnectPageParams.omit({token: true})), + }, +} satisfies Record + +// MARK: - Helpers + +function asEndUser( + viewer: Viewer, + input: {endUserId?: EndUserId | null}, +): Viewer<'end_user'> { + // Figure out a better way to share code here... + if (!('orgId' in viewer) || !viewer.orgId) { + throw new TRPCError({ + code: 'BAD_REQUEST', + message: 'Current viewer missing orgId to create token', + }) + } + if (viewer.role === 'end_user' && input.endUserId !== viewer.endUserId) { + throw new TRPCError({ + code: 'FORBIDDEN', + message: 'Current viewer cannot create token for other end user', + }) + } + const endUserId = + viewer.role === 'end_user' ? viewer.endUserId : input.endUserId + if (!endUserId) { + throw new TRPCError({ + code: 'BAD_REQUEST', + message: 'Either call as an endUser or psas endUserId explicitly', + }) + } + + return {role: 'end_user', endUserId, orgId: viewer.orgId} +} + +// MARK: - Endpoints + /** TODO: Modify this so that admin user can execute it... not just endUser */ export const endUserRouter = trpc.router({ + createConnectToken: protectedProcedure + .meta({openapi: {method: 'POST', path: '/connect-token'}}) + .input(endUserRouterSchema.createConnectToken.input) + .output(z.string()) + .mutation(({input: {validityInSeconds, ...input}, ctx}) => + ctx.jwt.signViewer(asEndUser(ctx.viewer, input), {validityInSeconds}), + ), + createMagicLink: protectedProcedure + .meta({openapi: {method: 'POST', path: '/magic-link'}}) + .input(endUserRouterSchema.createMagicLink.input) + .output(z.object({url: z.string()})) + .mutation(({input: {endUserId, validityInSeconds, ...params}, ctx}) => { + const token = ctx.jwt.signViewer(asEndUser(ctx.viewer, {endUserId}), { + validityInSeconds, + }) + const url = new URL('/connect', ctx.apiUrl) // `/` will start from the root hostname itself + for (const [key, value] of Object.entries({...params, token})) { + url.searchParams.set(key, `${value ?? ''}`) + } + return {url: url.toString()} + }), + // MARK: - Connect preConnect: protectedProcedure .input(z.tuple([zId('int'), zConnectOptions, z.unknown()])) From 6531efd188fdb90fc0cf5b2243eba35564079e91 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 26 Oct 2023 13:38:29 -0700 Subject: [PATCH 67/80] Debugging --- .github/workflows/validate-workflow.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/validate-workflow.yml b/.github/workflows/validate-workflow.yml index 2ef9437e..b64f1f44 100644 --- a/.github/workflows/validate-workflow.yml +++ b/.github/workflows/validate-workflow.yml @@ -67,6 +67,9 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Setup upterm session + uses: lhotari/action-upterm@v1 + - name: Run type checks run: pnpm run typecheck From 8aa32d8c72c2f7a99d07d880bbd28eeb6356e902 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 26 Oct 2023 14:09:19 -0700 Subject: [PATCH 68/80] Fix build --- .github/workflows/validate-workflow.yml | 10 +++++----- apps/web/tsconfig.json | 1 + tsconfig.json | 2 ++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validate-workflow.yml b/.github/workflows/validate-workflow.yml index b64f1f44..1214cfcf 100644 --- a/.github/workflows/validate-workflow.yml +++ b/.github/workflows/validate-workflow.yml @@ -36,9 +36,6 @@ jobs: - name: Checkout uses: actions/checkout@v3 - # - name: Set up tmate session - # uses: mxschmitt/action-tmate@v2 - - name: Install Node.js uses: actions/setup-node@v3 with: @@ -67,8 +64,11 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Setup upterm session - uses: lhotari/action-upterm@v1 + # - name: Set up tmate session + # uses: mxschmitt/action-tmate@v2 + + # - name: Setup upterm session + # uses: lhotari/action-upterm@v1 - name: Run type checks run: pnpm run typecheck diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index 3e1ba3cc..c5fe7e45 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -18,6 +18,7 @@ "paths": { "@/*": ["./*"], // Override because we don't want to use ./dist unnecessarily + "@usevenice/connect": ["../../packages/connect/src/index.ts"], "@usevenice/connect/*": ["../../packages/connect/src/*"] } }, diff --git a/tsconfig.json b/tsconfig.json index 119bc94c..5bf00ea7 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,6 +21,8 @@ // from two separate next.js apps in the same monorepo which might be relevant for future "paths": { "@/*": ["./apps/web/*"], + // Override because we don't want to use ./dist unnecessarily + "@usevenice/connect": ["./packages/connect/src/index.ts"], "@usevenice/connect/*": ["./packages/connect/src/*"] } }, From cb2569600744c7d1062f663e406dcb16cb31f82b Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 26 Oct 2023 18:19:26 -0700 Subject: [PATCH 69/80] feat: Experimental new mapper that allows for inline documentation --- packages/cdk-core/verticals/new-mapper.ts | 68 +++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 packages/cdk-core/verticals/new-mapper.ts diff --git a/packages/cdk-core/verticals/new-mapper.ts b/packages/cdk-core/verticals/new-mapper.ts new file mode 100644 index 00000000..13a2c90a --- /dev/null +++ b/packages/cdk-core/verticals/new-mapper.ts @@ -0,0 +1,68 @@ +import prettier from 'prettier' + +import {z} from '@usevenice/util' + +const Vendor = z.object({ + name: z.string(), + url: z.string(), + id: z.string(), +}) + +const QBOVendor = z.object({ + title: z.string(), + qboID: z.number(), +}) + +type ExtractKeyOfValueType = keyof { + [k in keyof T as T[k] extends V ? k : never]: T[k] +} + +interface Constant { + value: T +} +const constant = (t: T): Constant => ({value: t}) + +export function mapper< + ZInputSchema extends z.ZodTypeAny, + ZOutputSchema extends z.ZodTypeAny, + TOut extends z.infer = z.infer, + TIn extends z.infer = z.infer, +>( + zExt: ZInputSchema, + zCom: ZOutputSchema, + mapping: { + [k in keyof TOut]: + | ExtractKeyOfValueType + | Constant + | ((ext: TIn) => TOut[k]) + }, +) { + return { + inputSchema: zCom, + outputSchema: zExt, + mapping, + } +} + +const qboVendorMap = mapper(QBOVendor, Vendor, { + id: 'title', + name: constant('qbooooo'), + url: (vendor) => `${vendor.qboID}`, +}).mapping + +Object.entries(qboVendorMap).forEach(([k, v]) => { + console.log( + k, + ':', + typeof v === 'function' + ? prettier.format(v.toString(), { + arrowParens: 'avoid', + parser: 'typescript', + singleQuote: true, + semi: false, + }) + : typeof v === 'string' + ? `.${v}` + : `${v.value}`, + ) +}) From a30eb7f8289413b104f7fe22b04aaa8c58b45b7c Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 26 Oct 2023 18:23:43 -0700 Subject: [PATCH 70/80] Maybe printing out the fn itself is better? --- packages/cdk-core/verticals/new-mapper.ts | 25 +++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/cdk-core/verticals/new-mapper.ts b/packages/cdk-core/verticals/new-mapper.ts index 13a2c90a..df4be3a4 100644 --- a/packages/cdk-core/verticals/new-mapper.ts +++ b/packages/cdk-core/verticals/new-mapper.ts @@ -50,19 +50,32 @@ const qboVendorMap = mapper(QBOVendor, Vendor, { url: (vendor) => `${vendor.qboID}`, }).mapping +const mapper2 = (vendor: z.infer) => ({ + id: vendor.title, + name: 'qbooooo', + url: `${vendor.qboID}`, +}) + +function prettify(code: string) { + return prettier.format(code, { + arrowParens: 'avoid', + parser: 'typescript', + singleQuote: true, + semi: false, + printWidth: 30, + }) +} + Object.entries(qboVendorMap).forEach(([k, v]) => { console.log( k, ':', typeof v === 'function' - ? prettier.format(v.toString(), { - arrowParens: 'avoid', - parser: 'typescript', - singleQuote: true, - semi: false, - }) + ? prettify(v.toString()) : typeof v === 'string' ? `.${v}` : `${v.value}`, ) }) + +console.log(prettify(mapper2.toString())) From 562d5cbb936ef28aa1134b3af3d6c378de00c2c2 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 26 Oct 2023 18:37:50 -0700 Subject: [PATCH 71/80] Getter infra --- packages/cdk-core/verticals/new-mapper.ts | 29 +++++++++++++---------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/cdk-core/verticals/new-mapper.ts b/packages/cdk-core/verticals/new-mapper.ts index df4be3a4..01e45910 100644 --- a/packages/cdk-core/verticals/new-mapper.ts +++ b/packages/cdk-core/verticals/new-mapper.ts @@ -13,14 +13,17 @@ const QBOVendor = z.object({ qboID: z.number(), }) -type ExtractKeyOfValueType = keyof { - [k in keyof T as T[k] extends V ? k : never]: T[k] -} +type ExtractKeyOfValueType = Extract< + keyof { + [k in keyof T as T[k] extends V ? k : never]: T[k] + }, + string +> -interface Constant { - value: T +interface Getter { + keypath: T } -const constant = (t: T): Constant => ({value: t}) +const get = (keypath: T): Getter => ({keypath}) export function mapper< ZInputSchema extends z.ZodTypeAny, @@ -32,8 +35,8 @@ export function mapper< zCom: ZOutputSchema, mapping: { [k in keyof TOut]: - | ExtractKeyOfValueType - | Constant + | TOut[k] // Constant + | Getter> | ((ext: TIn) => TOut[k]) }, ) { @@ -45,8 +48,8 @@ export function mapper< } const qboVendorMap = mapper(QBOVendor, Vendor, { - id: 'title', - name: constant('qbooooo'), + id: get('title'), + name: 'qboooo', url: (vendor) => `${vendor.qboID}`, }).mapping @@ -72,9 +75,9 @@ Object.entries(qboVendorMap).forEach(([k, v]) => { ':', typeof v === 'function' ? prettify(v.toString()) - : typeof v === 'string' - ? `.${v}` - : `${v.value}`, + : typeof v === 'object' && 'keypath' in v + ? `.${v.keypath}` + : JSON.stringify(v), ) }) From cff15c086c829fb409df2b229a470b1efa7ae45f Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 27 Oct 2023 11:24:51 -0700 Subject: [PATCH 72/80] Store the endUser token in connectSession cookie and pass into oauth postConnect --- .../app/connect/callback/CallbackEffect.tsx | 6 +- apps/web/app/connect/callback/page.tsx | 63 ++++++++++++--- apps/web/app/connect/init/route.ts | 80 +++++++++++++++++++ apps/web/app/connect/page.tsx | 3 +- apps/web/package.json | 1 + .../engine-backend/router/endUserRouter.ts | 2 +- pnpm-lock.yaml | 15 ++++ 7 files changed, 154 insertions(+), 16 deletions(-) create mode 100644 apps/web/app/connect/init/route.ts diff --git a/apps/web/app/connect/callback/CallbackEffect.tsx b/apps/web/app/connect/callback/CallbackEffect.tsx index 1521cf05..ef6d741e 100644 --- a/apps/web/app/connect/callback/CallbackEffect.tsx +++ b/apps/web/app/connect/callback/CallbackEffect.tsx @@ -1,5 +1,6 @@ 'use client' +import {Loader2} from 'lucide-react' import React from 'react' import type {FrameMessage} from '@usevenice/connect/common' @@ -8,8 +9,9 @@ export function CallbackEffect(props: { msg: FrameMessage autoClose?: boolean }) { + const opener = window.opener as Window | null + React.useEffect(() => { - const opener = window.opener as Window | null opener?.postMessage(props.msg, '*') if (props.autoClose) { @@ -20,5 +22,5 @@ export function CallbackEffect(props: { // eslint-disable-next-line react-hooks/exhaustive-deps }, []) - return null + return opener ? : null } diff --git a/apps/web/app/connect/callback/page.tsx b/apps/web/app/connect/callback/page.tsx index 67d717f4..eb91cc6b 100644 --- a/apps/web/app/connect/callback/page.tsx +++ b/apps/web/app/connect/callback/page.tsx @@ -1,7 +1,8 @@ import '@usevenice/app-config/register.node' -import {Loader2} from 'lucide-react' +import {cookies} from 'next/headers' +import {kAccessToken} from '@usevenice/app-config/constants' import {env} from '@usevenice/app-config/env' import type {Id} from '@usevenice/cdk-core' import {makeNangoClient} from '@usevenice/cdk-core' @@ -9,7 +10,9 @@ import type {FrameMessage} from '@usevenice/connect' import {FullScreenCenter} from '@/components/FullScreenCenter' import {serverSideHelpersFromViewer} from '@/lib-server' +import {serverComponentGetViewer} from '@/lib-server/server-component-helpers' +import {kConnectSession, zConnectSession} from '../init/route' import {CallbackEffect} from './CallbackEffect' export const metadata = { @@ -30,18 +33,47 @@ export default async function ConnectCallback({ // @see https://github.com/vercel/next.js/issues/43704 searchParams: Record }) { - const nango = makeNangoClient({secretKey: env.NANGO_SECRET_KEY}) - const res = await nango.doOauthCallback(searchParams) - const msg = await (async (): Promise => { - if (res.eventType !== 'AUTHORIZATION_SUCEEDED') { - return { - type: 'ERROR', - data: {code: res.data.authErrorType, message: res.data.authErrorDesc}, - } - } try { - const {caller} = serverSideHelpersFromViewer({role: 'system'}) + const cookie = cookies().get(kConnectSession) + if (!cookie) { + return { + type: 'ERROR', + data: {code: 'BAD_REQUEST', message: 'No session found'}, + } + } + const session = zConnectSession.parse(JSON.parse(cookie.value)) + const viewer = await serverComponentGetViewer({ + searchParams: {[kAccessToken]: session.token}, + }) + + const nango = makeNangoClient({secretKey: env.NANGO_SECRET_KEY}) + const res = await nango.doOauthCallback(searchParams) + + if (res.eventType !== 'AUTHORIZATION_SUCEEDED') { + return { + type: 'ERROR', + data: {code: res.data.authErrorType, message: res.data.authErrorDesc}, + } + } + + const resourceId = res.data.connectionId as Id['reso'] + if (session.resourceId !== resourceId) { + console.warn('Revoking due to unmatched resourceId') + await nango.delete('/connection/{connection_id}', { + path: {connection_id: res.data.connectionId}, + query: {provider_config_key: res.data.providerConfigKey}, + }) + return { + type: 'ERROR', + data: { + code: 'FORBIDDEN', + message: `Session resourceId (${session.resourceId}) not matching connecte resourceId ${resourceId}`, + }, + } + } + + const {caller} = serverSideHelpersFromViewer(viewer) await caller.postConnect([res.data, res.data.providerConfigKey, {}]) return { type: 'SUCCESS', @@ -55,10 +87,17 @@ export default async function ConnectCallback({ } })() + console.log('[oauth] callback result', msg) + // How do we do redirect here? return ( - + {msg.type} + + {msg.type === 'ERROR' + ? `[${msg.data.code}] ${msg.data.message}` + : msg.data.resourceId} + ) diff --git a/apps/web/app/connect/init/route.ts b/apps/web/app/connect/init/route.ts new file mode 100644 index 00000000..039ae06c --- /dev/null +++ b/apps/web/app/connect/init/route.ts @@ -0,0 +1,80 @@ +import {setCookie} from 'cookies-next' +import type {NextRequest} from 'next/server' +import {NextResponse} from 'next/server' + +import {kAccessToken} from '@usevenice/app-config/constants' +import {env} from '@usevenice/app-config/env' +import type {IntegrationDef} from '@usevenice/cdk-core' +import { + extractProviderName, + getViewerId, + makeId, + makeNangoClient, + zId, +} from '@usevenice/cdk-core' +import {zConnectPageParams} from '@usevenice/engine-backend/router/endUserRouter' +import {makeUlid, z} from '@usevenice/util' + +import {defIntegrations} from '@/../app-config/integrations/integrations.def' +import {withErrorHandler} from '@/lib-server' +import {serverComponentGetViewer} from '@/lib-server/server-component-helpers' + +export const kConnectSession = 'connect-session' + +type ConnectSession = z.infer +export const zConnectSession = z.object({ + token: z.string(), + resourceId: zId('reso'), +}) + +export async function GET(req: NextRequest) { + return withErrorHandler(async () => { + const params = zConnectPageParams + .required({integrationId: true}) + .parse(Object.fromEntries(req.nextUrl.searchParams.entries())) + + const viewer = await serverComponentGetViewer({ + searchParams: {[kAccessToken]: params.token}, + }) + + if (viewer.role !== 'end_user') { + throw new Error( + `Authenticated user only. Your role is ${getViewerId(viewer)}`, + ) + } + if (!params.integrationId) { + throw new Error('Missing integrationId') + } + + const providerName = extractProviderName(params.integrationId) + const intDef = defIntegrations[ + providerName as keyof typeof defIntegrations + ] as IntegrationDef + + if (!intDef.metadata?.nangoProvider) { + throw new Error('/connect/init is only supported for oauth providers') + } + + const nango = makeNangoClient({secretKey: env.NANGO_SECRET_KEY}) + const resourceId = makeId('reso', providerName, makeUlid()) + const url = await nango.getOauthConnectUrl({ + public_key: env.NEXT_PUBLIC_NANGO_PUBLIC_KEY, + connection_id: resourceId, + provider_config_key: params.integrationId, + // Consider using hookdeck so we can work with any number of urls + // redirect_uri: joinPath(getServerUrl(null), '/connect/callback'), + }) + const res = NextResponse.redirect(url) + setCookie( + kConnectSession, + JSON.stringify({ + resourceId, + token: params.token, + } satisfies ConnectSession), + // https://datatracker.ietf.org/doc/html/draft-ietf-httpbis-rfc6265bis-03#section-4.1.2.7%7Cthe + // Need sameSite to be lax in order for this to work + {req, res, secure: true, sameSite: 'lax', maxAge: 3600}, // Expire cookie so we clean ourselves up + ) + return res + }) +} diff --git a/apps/web/app/connect/page.tsx b/apps/web/app/connect/page.tsx index 75d0b3e3..32977794 100644 --- a/apps/web/app/connect/page.tsx +++ b/apps/web/app/connect/page.tsx @@ -2,6 +2,7 @@ import {clerkClient} from '@clerk/nextjs' import Image from 'next/image' import {redirect} from 'next/navigation' +import {kAccessToken} from '@usevenice/app-config/constants' import {env} from '@usevenice/app-config/env' import {defIntegrations} from '@usevenice/app-config/integrations/integrations.def' import type {IntegrationDef} from '@usevenice/cdk-core' @@ -44,7 +45,7 @@ export default async function ConnectPageContainer({ }) { const {token, ...params} = zConnectPageParams.parse(searchParams) const {ssg, getDehydratedState, viewer} = await createServerComponentHelpers({ - searchParams: {_token: token}, + searchParams: {[kAccessToken]: token}, }) if (viewer.role !== 'end_user') { return ( diff --git a/apps/web/package.json b/apps/web/package.json index 34c7faf3..c82d2bd4 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -32,6 +32,7 @@ "@usevenice/ui": "workspace:*", "@usevenice/util": "workspace:*", "commandbar": "1.7.3", + "cookies-next": "4.0.0", "graphiql": "^2.2.0", "graphql": "^16.6.0", "graphql-ws": "^5.11.3", diff --git a/packages/engine-backend/router/endUserRouter.ts b/packages/engine-backend/router/endUserRouter.ts index d7e96646..179f4260 100644 --- a/packages/engine-backend/router/endUserRouter.ts +++ b/packages/engine-backend/router/endUserRouter.ts @@ -121,7 +121,7 @@ export const endUserRouter = trpc.router({ const token = ctx.jwt.signViewer(asEndUser(ctx.viewer, {endUserId}), { validityInSeconds, }) - const url = new URL('/connect', ctx.apiUrl) // `/` will start from the root hostname itself + const url = new URL('/connect/init', ctx.apiUrl) // `/` will start from the root hostname itself for (const [key, value] of Object.entries({...params, token})) { url.searchParams.set(key, `${value ?? ''}`) } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 901300d8..beceb904 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -542,6 +542,9 @@ importers: commandbar: specifier: 1.7.3 version: 1.7.3(react@18.2.0) + cookies-next: + specifier: 4.0.0 + version: 4.0.0 graphiql: specifier: ^2.2.0 version: 2.2.0(@codemirror/language@6.0.0)(@types/node@18.11.18)(@types/react@18.0.27)(graphql-ws@5.11.3)(graphql@16.6.0)(react-dom@18.2.0)(react-is@17.0.2)(react@18.2.0) @@ -6245,6 +6248,10 @@ packages: '@types/node': 18.11.18 dev: false + /@types/cookie@0.4.1: + resolution: {integrity: sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q==} + dev: false + /@types/cookies@0.7.7: resolution: {integrity: sha512-h7BcvPUogWbKCzBR2lY4oqaZbO3jXZksexYJVFvkrFeLgbZjQkU4x8pRq6eg2MHXQhY0McQdqmmsxRWlVAHooA==} dependencies: @@ -8367,6 +8374,14 @@ packages: engines: {node: '>= 0.6'} dev: false + /cookies-next@4.0.0: + resolution: {integrity: sha512-3TyzeltFCGgdOlVOVTPClSq+YV9ZCdOyA3aHRZv9f5aSgg7EyI4NSvXFOCgzT/xIxeHR4Rz8/z5Tdo9oPqaVpA==} + dependencies: + '@types/cookie': 0.4.1 + '@types/node': 16.18.6 + cookie: 0.4.2 + dev: false + /copy-anything@3.0.2: resolution: {integrity: sha512-CzATjGXzUQ0EvuvgOCI6A4BGOo2bcVx8B+eC2nF862iv9fopnPQwlrbACakNCHRIJbCSBj+J/9JeDf60k64MkA==} engines: {node: '>=12.13'} From 8d84c8d3200ac4453f353c1be51bff2fbc97ca63 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Fri, 27 Oct 2023 12:05:59 -0700 Subject: [PATCH 73/80] Removing separate /connect/init endpoint by using client side setCookie and redirect --- apps/web/app/connect/SetCookieAndRedirect.tsx | 26 ++++++ apps/web/app/connect/callback/page.tsx | 3 +- apps/web/app/connect/init/route.ts | 80 ------------------- apps/web/app/connect/page.tsx | 34 +++++++- .../engine-backend/router/endUserRouter.ts | 2 +- 5 files changed, 59 insertions(+), 86 deletions(-) create mode 100644 apps/web/app/connect/SetCookieAndRedirect.tsx delete mode 100644 apps/web/app/connect/init/route.ts diff --git a/apps/web/app/connect/SetCookieAndRedirect.tsx b/apps/web/app/connect/SetCookieAndRedirect.tsx new file mode 100644 index 00000000..ab676781 --- /dev/null +++ b/apps/web/app/connect/SetCookieAndRedirect.tsx @@ -0,0 +1,26 @@ +'use client' + +import {setCookie} from 'cookies-next' +import type {OptionsType} from 'cookies-next/lib/types' +import React from 'react' + +/** + * Workaround for inability to set cookie in server components + * @see https://github.com/vercel/next.js/discussions/49843 + */ +export function SetCookieAndRedirect(props: { + cookies: Array<{key: string; value: string; options: OptionsType}> + redirectUrl: string +}) { + React.useEffect(() => { + for (const {key, value, options} of props.cookies) { + setCookie(key, value, options) + } + + // no need for NextRouter since we are doing a full page redirect + window.location.href = props.redirectUrl + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + return null +} diff --git a/apps/web/app/connect/callback/page.tsx b/apps/web/app/connect/callback/page.tsx index eb91cc6b..78084664 100644 --- a/apps/web/app/connect/callback/page.tsx +++ b/apps/web/app/connect/callback/page.tsx @@ -12,7 +12,7 @@ import {FullScreenCenter} from '@/components/FullScreenCenter' import {serverSideHelpersFromViewer} from '@/lib-server' import {serverComponentGetViewer} from '@/lib-server/server-component-helpers' -import {kConnectSession, zConnectSession} from '../init/route' +import {kConnectSession, zConnectSession} from '../page' import {CallbackEffect} from './CallbackEffect' export const metadata = { @@ -35,6 +35,7 @@ export default async function ConnectCallback({ }) { const msg = await (async (): Promise => { try { + // TODO: Can we use cookies-next to read cookie in this environment? const cookie = cookies().get(kConnectSession) if (!cookie) { return { diff --git a/apps/web/app/connect/init/route.ts b/apps/web/app/connect/init/route.ts deleted file mode 100644 index 039ae06c..00000000 --- a/apps/web/app/connect/init/route.ts +++ /dev/null @@ -1,80 +0,0 @@ -import {setCookie} from 'cookies-next' -import type {NextRequest} from 'next/server' -import {NextResponse} from 'next/server' - -import {kAccessToken} from '@usevenice/app-config/constants' -import {env} from '@usevenice/app-config/env' -import type {IntegrationDef} from '@usevenice/cdk-core' -import { - extractProviderName, - getViewerId, - makeId, - makeNangoClient, - zId, -} from '@usevenice/cdk-core' -import {zConnectPageParams} from '@usevenice/engine-backend/router/endUserRouter' -import {makeUlid, z} from '@usevenice/util' - -import {defIntegrations} from '@/../app-config/integrations/integrations.def' -import {withErrorHandler} from '@/lib-server' -import {serverComponentGetViewer} from '@/lib-server/server-component-helpers' - -export const kConnectSession = 'connect-session' - -type ConnectSession = z.infer -export const zConnectSession = z.object({ - token: z.string(), - resourceId: zId('reso'), -}) - -export async function GET(req: NextRequest) { - return withErrorHandler(async () => { - const params = zConnectPageParams - .required({integrationId: true}) - .parse(Object.fromEntries(req.nextUrl.searchParams.entries())) - - const viewer = await serverComponentGetViewer({ - searchParams: {[kAccessToken]: params.token}, - }) - - if (viewer.role !== 'end_user') { - throw new Error( - `Authenticated user only. Your role is ${getViewerId(viewer)}`, - ) - } - if (!params.integrationId) { - throw new Error('Missing integrationId') - } - - const providerName = extractProviderName(params.integrationId) - const intDef = defIntegrations[ - providerName as keyof typeof defIntegrations - ] as IntegrationDef - - if (!intDef.metadata?.nangoProvider) { - throw new Error('/connect/init is only supported for oauth providers') - } - - const nango = makeNangoClient({secretKey: env.NANGO_SECRET_KEY}) - const resourceId = makeId('reso', providerName, makeUlid()) - const url = await nango.getOauthConnectUrl({ - public_key: env.NEXT_PUBLIC_NANGO_PUBLIC_KEY, - connection_id: resourceId, - provider_config_key: params.integrationId, - // Consider using hookdeck so we can work with any number of urls - // redirect_uri: joinPath(getServerUrl(null), '/connect/callback'), - }) - const res = NextResponse.redirect(url) - setCookie( - kConnectSession, - JSON.stringify({ - resourceId, - token: params.token, - } satisfies ConnectSession), - // https://datatracker.ietf.org/doc/html/draft-ietf-httpbis-rfc6265bis-03#section-4.1.2.7%7Cthe - // Need sameSite to be lax in order for this to work - {req, res, secure: true, sameSite: 'lax', maxAge: 3600}, // Expire cookie so we clean ourselves up - ) - return res - }) -} diff --git a/apps/web/app/connect/page.tsx b/apps/web/app/connect/page.tsx index 32977794..bcf077a0 100644 --- a/apps/web/app/connect/page.tsx +++ b/apps/web/app/connect/page.tsx @@ -1,6 +1,5 @@ import {clerkClient} from '@clerk/nextjs' import Image from 'next/image' -import {redirect} from 'next/navigation' import {kAccessToken} from '@usevenice/app-config/constants' import {env} from '@usevenice/app-config/env' @@ -11,15 +10,25 @@ import { getViewerId, makeId, makeNangoClient, + zId, } from '@usevenice/cdk-core' import {zConnectPageParams} from '@usevenice/engine-backend/router/endUserRouter' -import {makeUlid} from '@usevenice/util' +import {makeUlid, z} from '@usevenice/util' import {ClientRoot} from '@/components/ClientRoot' import {SuperHydrate} from '@/components/SuperHydrate' import {createServerComponentHelpers} from '@/lib-server/server-component-helpers' import ConnectPage from './ConnectPage' +import {SetCookieAndRedirect} from './SetCookieAndRedirect' + +export const kConnectSession = 'connect-session' + +type ConnectSession = z.infer +export const zConnectSession = z.object({ + token: z.string(), + resourceId: zId('reso'), +}) export const metadata = { title: 'Venice Connect', @@ -62,14 +71,31 @@ export default async function ConnectPageContainer({ if (intDef.metadata?.nangoProvider) { const nango = makeNangoClient({secretKey: env.NANGO_SECRET_KEY}) + const resourceId = makeId('reso', providerName, makeUlid()) const url = await nango.getOauthConnectUrl({ public_key: env.NEXT_PUBLIC_NANGO_PUBLIC_KEY, - connection_id: makeId('reso', providerName, makeUlid()), + connection_id: resourceId, provider_config_key: params.integrationId, // Consider using hookdeck so we can work with any number of urls // redirect_uri: joinPath(getServerUrl(null), '/connect/callback'), }) - return redirect(url) + return ( + + ) } } diff --git a/packages/engine-backend/router/endUserRouter.ts b/packages/engine-backend/router/endUserRouter.ts index 179f4260..d7e96646 100644 --- a/packages/engine-backend/router/endUserRouter.ts +++ b/packages/engine-backend/router/endUserRouter.ts @@ -121,7 +121,7 @@ export const endUserRouter = trpc.router({ const token = ctx.jwt.signViewer(asEndUser(ctx.viewer, {endUserId}), { validityInSeconds, }) - const url = new URL('/connect/init', ctx.apiUrl) // `/` will start from the root hostname itself + const url = new URL('/connect', ctx.apiUrl) // `/` will start from the root hostname itself for (const [key, value] of Object.entries({...params, token})) { url.searchParams.set(key, `${value ?? ''}`) } From 1275c6d2e3a8ebc0361b15a9ab503daa829e24ba Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 29 Oct 2023 14:11:38 -0700 Subject: [PATCH 74/80] feat: Allow using connectToken with sdk --- packages/connect/src/api.ts | 36 +++++++++++++++---- .../engine-backend/router/endUserRouter.ts | 10 +++--- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/packages/connect/src/api.ts b/packages/connect/src/api.ts index 86b8e26f..38e75768 100644 --- a/packages/connect/src/api.ts +++ b/packages/connect/src/api.ts @@ -3,15 +3,25 @@ import {z} from 'zod' import {defaultVeniceHost} from './common' export const zVeniceClientConfig = z.object({ - // Support end-user token based auth. - apiKey: z.string(), + /** Service account auth */ + apiKey: z.string().nullable(), + /** User auth, should pass in one or the other */ + accessToken: z.string().nullable(), apiHost: z.string().default(defaultVeniceHost), }) export type VeniceClientConfig = z.infer -export function makeVeniceClient({apiKey, ...config}: VeniceClientConfig) { +export function makeVeniceClient({ + apiKey, + accessToken, + ...config +}: VeniceClientConfig) { const apiBase = new URL('/api/openapi/', config.apiHost).toString() - const headers = {'x-apikey': apiKey, 'Content-Type': 'application/json'} + const headers = { + 'Content-Type': 'application/json', + ...(apiKey ? {'x-apikey': apiKey} : {}), + ...(accessToken ? {Authorization: `Bearer ${accessToken}`} : {}), + } return { listResources: async (opts: {integrationId: string}) => { @@ -24,13 +34,11 @@ export function makeVeniceClient({apiKey, ...config}: VeniceClientConfig) { }, // Maybe we should allow end user to create magic link for themselves too so that // this operation can be done client side instead of server side? - adminCreateMagicLink: async (opts: { + createMagicLink: async (opts: { integrationId: string endUserId: string }) => { const url = new URL('magic-link', apiBase) - // url.searchParams.set("integrationId", opts.integrationId); - const res = await fetch(url.toString(), { method: 'POST', body: JSON.stringify({...opts}, null, 4), @@ -40,5 +48,19 @@ export function makeVeniceClient({apiKey, ...config}: VeniceClientConfig) { const data = await res.json() return data as {url: string} }, + createConnectToken: async (opts: { + integrationId: string + endUserId: string + }) => { + const url = new URL('connect-token', apiBase) + const res = await fetch(url.toString(), { + method: 'POST', + body: JSON.stringify({...opts}, null, 4), + headers, + }) + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const data = await res.json() + return data as {token: string} + }, } } diff --git a/packages/engine-backend/router/endUserRouter.ts b/packages/engine-backend/router/endUserRouter.ts index d7e96646..6d7b5277 100644 --- a/packages/engine-backend/router/endUserRouter.ts +++ b/packages/engine-backend/router/endUserRouter.ts @@ -109,10 +109,12 @@ export const endUserRouter = trpc.router({ createConnectToken: protectedProcedure .meta({openapi: {method: 'POST', path: '/connect-token'}}) .input(endUserRouterSchema.createConnectToken.input) - .output(z.string()) - .mutation(({input: {validityInSeconds, ...input}, ctx}) => - ctx.jwt.signViewer(asEndUser(ctx.viewer, input), {validityInSeconds}), - ), + .output(z.object({token: z.string()})) + .mutation(({input: {validityInSeconds, ...input}, ctx}) => ({ + token: ctx.jwt.signViewer(asEndUser(ctx.viewer, input), { + validityInSeconds, + }), + })), createMagicLink: protectedProcedure .meta({openapi: {method: 'POST', path: '/magic-link'}}) .input(endUserRouterSchema.createMagicLink.input) From 4bcbd1aa855744b21c08afe3bf1fcb97a4f80818 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 29 Oct 2023 14:39:50 -0700 Subject: [PATCH 75/80] Enable specify integration by providerName during connect --- .../integrations/integrations.def.ts | 9 ++++++ apps/web/app/connect/page.tsx | 29 ++++++++++++++++--- ...9_2125_integration_provider_name_grant.sql | 1 + .../makePostgresMetaService.ts | 10 +++++-- packages/cdk-core/metaService.ts | 5 +++- .../engine-backend/router/endUserRouter.ts | 2 ++ .../engine-backend/router/protectedRouter.ts | 8 +++-- packages/engine-frontend/VeniceConnect.tsx | 2 ++ 8 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 apps/web/migrations/2023-10-29_2125_integration_provider_name_grant.sql diff --git a/apps/app-config/integrations/integrations.def.ts b/apps/app-config/integrations/integrations.def.ts index c6077f97..7337f140 100644 --- a/apps/app-config/integrations/integrations.def.ts +++ b/apps/app-config/integrations/integrations.def.ts @@ -27,6 +27,8 @@ import {default as integrationVenmo} from '@usevenice/integration-venmo/def' import {default as integrationWebhook} from '@usevenice/integration-webhook/def' import {default as integrationWise} from '@usevenice/integration-wise/def' import {default as integrationYodlee} from '@usevenice/integration-yodlee/def' +import type {NonEmptyArray} from '@usevenice/util'; +import { z} from '@usevenice/util' export const defIntegrations = { airtable: integrationAirtable, @@ -58,3 +60,10 @@ export const defIntegrations = { wise: integrationWise, yodlee: integrationYodlee, } + +// TODO: make sure to generate this +export const zProviderName = z.enum( + Object.keys(defIntegrations) as NonEmptyArray, +) + +export type ProviderName = z.infer diff --git a/apps/web/app/connect/page.tsx b/apps/web/app/connect/page.tsx index bcf077a0..17a126ba 100644 --- a/apps/web/app/connect/page.tsx +++ b/apps/web/app/connect/page.tsx @@ -62,9 +62,26 @@ export default async function ConnectPageContainer({ ) } + // Implement shorthand for specifying only integrationId by providerName + let integrationId = params.integrationId + if (!integrationId && params.providerName) { + const ints = await ssg.listIntegrationInfos.fetch({ + providerName: params.providerName, + }) + if (ints.length === 1 && ints[0]?.id) { + integrationId = ints[0]?.id + } else if (ints.length < 1) { + return
No integration for {params.providerName} configured
+ } else if (ints.length > 1) { + console.warn( + `${ints.length} integrations found for ${params.providerName}`, + ) + } + } + // Special case when we are handling a single oauth integration - if (params.integrationId) { - const providerName = extractProviderName(params.integrationId) + if (integrationId) { + const providerName = extractProviderName(integrationId) const intDef = defIntegrations[ providerName as keyof typeof defIntegrations ] as IntegrationDef @@ -75,7 +92,7 @@ export default async function ConnectPageContainer({ const url = await nango.getOauthConnectUrl({ public_key: env.NEXT_PUBLIC_NANGO_PUBLIC_KEY, connection_id: resourceId, - provider_config_key: params.integrationId, + provider_config_key: integrationId, // Consider using hookdeck so we can work with any number of urls // redirect_uri: joinPath(getServerUrl(null), '/connect/callback'), }) @@ -101,7 +118,11 @@ export default async function ConnectPageContainer({ const [org] = await Promise.all([ clerkClient.organizations.getOrganization({organizationId: viewer.orgId}), - ssg.listIntegrationInfos.prefetch({id: params.integrationId}), + // Switch to using react suspense / server fetch for this instead of prefetch + ssg.listIntegrationInfos.prefetch({ + id: integrationId, + providerName: params.providerName, + }), params.showExisting ? ssg.listConnections.prefetch({}) : Promise.resolve(), ]) diff --git a/apps/web/migrations/2023-10-29_2125_integration_provider_name_grant.sql b/apps/web/migrations/2023-10-29_2125_integration_provider_name_grant.sql new file mode 100644 index 00000000..2b768249 --- /dev/null +++ b/apps/web/migrations/2023-10-29_2125_integration_provider_name_grant.sql @@ -0,0 +1 @@ +GRANT SELECT(provider_name) ON TABLE public.integration TO end_user; diff --git a/integrations/integration-postgres/makePostgresMetaService.ts b/integrations/integration-postgres/makePostgresMetaService.ts index 9cfdb9f0..d4dd3681 100644 --- a/integrations/integration-postgres/makePostgresMetaService.ts +++ b/integrations/integration-postgres/makePostgresMetaService.ts @@ -143,12 +143,18 @@ export const makePostgresMetaService = zFunction( pool.any(sql`SELECT * FROM pipeline ${where}`), ) }, - listIntegrationInfos: ({id} = {}) => { + listIntegrationInfos: ({id, providerName} = {}) => { const {runQueries, sql} = _getDeps(opts) return runQueries((pool) => pool.any( sql`SELECT id, env_name, display_name FROM integration ${ - id ? sql`WHERE id = ${id}` : sql`` + id && providerName + ? sql`WHERE id = ${id} AND provider_name = ${providerName}` + : id + ? sql`WHERE id = ${id}` + : providerName + ? sql`WHERE provider_name = ${providerName}` + : sql`` }`, ), ) diff --git a/packages/cdk-core/metaService.ts b/packages/cdk-core/metaService.ts index c76c1db5..796c6233 100644 --- a/packages/cdk-core/metaService.ts +++ b/packages/cdk-core/metaService.ts @@ -59,7 +59,10 @@ export interface MetaService { secondsSinceLastSync?: number }) => Promise> /** Id is used to check RLS policy right now for end user */ - listIntegrationInfos: (opts?: {id?: Id['int'] | null}) => Promise< + listIntegrationInfos: (opts?: { + id?: Id['int'] | null + providerName?: string | null + }) => Promise< ReadonlyArray<{ id: Id['int'] envName?: string | null diff --git a/packages/engine-backend/router/endUserRouter.ts b/packages/engine-backend/router/endUserRouter.ts index 6d7b5277..f086bb00 100644 --- a/packages/engine-backend/router/endUserRouter.ts +++ b/packages/engine-backend/router/endUserRouter.ts @@ -46,6 +46,8 @@ export const zConnectPageParams = z.object({ .describe( 'Where to send user to after connect / if they press back button', ), + // TODO: How to make sure we actually have a typed api here and can use zProviderName + providerName: z.string().nullish().describe('Which provider to use'), /** Launch the integration right away */ integrationId: zId('int').nullish(), /** Whether to show existing resources */ diff --git a/packages/engine-backend/router/protectedRouter.ts b/packages/engine-backend/router/protectedRouter.ts index 99db6bd3..762ffc00 100644 --- a/packages/engine-backend/router/protectedRouter.ts +++ b/packages/engine-backend/router/protectedRouter.ts @@ -159,6 +159,7 @@ export const protectedRouter = trpc.router({ z.object({ type: z.enum(['source', 'destination']).nullish(), id: zId('int').nullish(), + providerName: z.string().nullish(), }), ) .output( @@ -176,8 +177,11 @@ export const protectedRouter = trpc.router({ }), ), ) - .query(async ({input: {type, id}, ctx}) => { - const intInfos = await ctx.helpers.metaService.listIntegrationInfos({id}) + .query(async ({input: {type, id, providerName}, ctx}) => { + const intInfos = await ctx.helpers.metaService.listIntegrationInfos({ + id, + providerName, + }) return intInfos .map(({id, envName, displayName}) => { diff --git a/packages/engine-frontend/VeniceConnect.tsx b/packages/engine-frontend/VeniceConnect.tsx index 41308f47..bff15fe4 100644 --- a/packages/engine-frontend/VeniceConnect.tsx +++ b/packages/engine-frontend/VeniceConnect.tsx @@ -55,6 +55,7 @@ export interface VeniceConnectProps extends UIPropsNoChildren { onEvent?: (event: {type: ConnectEventType; intId: Id['int']}) => void /** Only connect to this integration */ integrationId?: Id['int'] | null + providerName?: string | null } type UseConnectScope = Parameters[0] @@ -115,6 +116,7 @@ export function VeniceConnectButton({ export function VeniceConnect(props: VeniceConnectProps) { const listIntegrationsRes = _trpcReact.listIntegrationInfos.useQuery({ id: props.integrationId, + providerName: props.providerName, }) const catalogRes = _trpcReact.getIntegrationCatalog.useQuery() From 299e4fd909b77b55af28225e318590f5a966ca12 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 29 Oct 2023 14:44:37 -0700 Subject: [PATCH 76/80] Update the API to be able to take in providerName --- .../makePostgresMetaService.ts | 3 ++- packages/cdk-core/metaService.ts | 2 ++ packages/connect/src/api.ts | 18 ++++++++++++++---- .../engine-backend/router/protectedRouter.ts | 1 + 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/integrations/integration-postgres/makePostgresMetaService.ts b/integrations/integration-postgres/makePostgresMetaService.ts index d4dd3681..88f23f53 100644 --- a/integrations/integration-postgres/makePostgresMetaService.ts +++ b/integrations/integration-postgres/makePostgresMetaService.ts @@ -173,12 +173,13 @@ function metaTable>( // TODO: Convert case from snake_case to camelCase return { - list: ({ids, endUserId, integrationId, keywords, ...rest}) => + list: ({ids, endUserId, integrationId, providerName, keywords, ...rest}) => runQueries((pool) => { const conditions = R.compact([ ids && sql`id = ANY(${sql.array(ids, 'varchar')})`, endUserId && sql`end_user_id = ${endUserId}`, integrationId && sql`integration_id = ${integrationId}`, + providerName && sql`provider_name = ${providerName}`, // Temp solution, shall use fts and make this work for any table... keywords && tableName === 'institution' && diff --git a/packages/cdk-core/metaService.ts b/packages/cdk-core/metaService.ts index 796c6233..c3c42c8d 100644 --- a/packages/cdk-core/metaService.ts +++ b/packages/cdk-core/metaService.ts @@ -14,6 +14,8 @@ export interface MetaTable< endUserId?: EndUserId | null /** Maybe remove this? not applicable everywhere */ integrationId?: Id['int'] | null + /** Maybe remove this? not applicable everywhere */ + providerName?: string | null /** Used for search */ keywords?: string | null /** Pagination, not necessarily supported */ diff --git a/packages/connect/src/api.ts b/packages/connect/src/api.ts index 38e75768..8a2b6729 100644 --- a/packages/connect/src/api.ts +++ b/packages/connect/src/api.ts @@ -24,9 +24,17 @@ export function makeVeniceClient({ } return { - listResources: async (opts: {integrationId: string}) => { + listResources: async (opts: { + integrationId?: string + providerName?: string + }) => { const url = new URL('resources', apiBase) - url.searchParams.set('integrationId', opts.integrationId) + if (opts.integrationId) { + url.searchParams.set('integrationId', opts.integrationId) + } + if (opts.providerName) { + url.searchParams.set('providerName', opts.providerName) + } const res = await fetch(url.toString(), {method: 'GET', headers}) // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const data = await res.json() @@ -35,7 +43,8 @@ export function makeVeniceClient({ // Maybe we should allow end user to create magic link for themselves too so that // this operation can be done client side instead of server side? createMagicLink: async (opts: { - integrationId: string + integrationId?: string + providerName?: string endUserId: string }) => { const url = new URL('magic-link', apiBase) @@ -49,7 +58,8 @@ export function makeVeniceClient({ return data as {url: string} }, createConnectToken: async (opts: { - integrationId: string + integrationId?: string + providerName?: string endUserId: string }) => { const url = new URL('connect-token', apiBase) diff --git a/packages/engine-backend/router/protectedRouter.ts b/packages/engine-backend/router/protectedRouter.ts index 762ffc00..9ba4f0b6 100644 --- a/packages/engine-backend/router/protectedRouter.ts +++ b/packages/engine-backend/router/protectedRouter.ts @@ -42,6 +42,7 @@ export const protectedRouter = trpc.router({ .extend({ endUserId: zEndUserId.nullish(), integrationId: zId('int').nullish(), + providerName: z.string().nullish(), }) .optional(), ) From e8c390881dc64ea23fe49b67aecd40f859cb267a Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 29 Oct 2023 14:50:38 -0700 Subject: [PATCH 77/80] Publish new version of connect --- packages/connect/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/connect/package.json b/packages/connect/package.json index 76582224..b8c49c04 100644 --- a/packages/connect/package.json +++ b/packages/connect/package.json @@ -1,6 +1,6 @@ { "name": "@usevenice/connect", - "version": "0.0.1", + "version": "0.0.3", "sideEffects": false, "main": "./dist/index.js", "module": "./dist/index.mjs", From cfa9a805f757f23d9ca4ca1d3bff1b7951f54417 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 29 Oct 2023 15:19:49 -0700 Subject: [PATCH 78/80] Update SDK --- packages/connect/package.json | 4 ++-- packages/connect/src/api.ts | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/connect/package.json b/packages/connect/package.json index b8c49c04..ac0363a4 100644 --- a/packages/connect/package.json +++ b/packages/connect/package.json @@ -1,6 +1,6 @@ { "name": "@usevenice/connect", - "version": "0.0.3", + "version": "0.0.4", "sideEffects": false, "main": "./dist/index.js", "module": "./dist/index.mjs", @@ -10,7 +10,7 @@ "build:tsup": "tsup", "clean": "rm -rf ./dist", "prepack": "pnpm run build", - "publish": "pnpm publish --no-git-checks --access public" + "pub": "pnpm publish --no-git-checks --access public" }, "dependencies": { "zod": "3.21.4" diff --git a/packages/connect/src/api.ts b/packages/connect/src/api.ts index 8a2b6729..fafc250a 100644 --- a/packages/connect/src/api.ts +++ b/packages/connect/src/api.ts @@ -4,9 +4,9 @@ import {defaultVeniceHost} from './common' export const zVeniceClientConfig = z.object({ /** Service account auth */ - apiKey: z.string().nullable(), + apiKey: z.string().nullish(), /** User auth, should pass in one or the other */ - accessToken: z.string().nullable(), + accessToken: z.string().nullish(), apiHost: z.string().default(defaultVeniceHost), }) export type VeniceClientConfig = z.infer @@ -45,7 +45,8 @@ export function makeVeniceClient({ createMagicLink: async (opts: { integrationId?: string providerName?: string - endUserId: string + endUserId?: string + validityInSeconds?: number }) => { const url = new URL('magic-link', apiBase) const res = await fetch(url.toString(), { @@ -58,9 +59,8 @@ export function makeVeniceClient({ return data as {url: string} }, createConnectToken: async (opts: { - integrationId?: string - providerName?: string - endUserId: string + endUserId?: string + validityInSeconds?: number }) => { const url = new URL('connect-token', apiBase) const res = await fetch(url.toString(), { From 99a4574ad25f01527d44ae99523c6519c1117064 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 29 Oct 2023 17:57:05 -0700 Subject: [PATCH 79/80] fix: endUser createToken --- packages/engine-backend/router/endUserRouter.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/engine-backend/router/endUserRouter.ts b/packages/engine-backend/router/endUserRouter.ts index f086bb00..b7a8bf5c 100644 --- a/packages/engine-backend/router/endUserRouter.ts +++ b/packages/engine-backend/router/endUserRouter.ts @@ -86,7 +86,11 @@ function asEndUser( message: 'Current viewer missing orgId to create token', }) } - if (viewer.role === 'end_user' && input.endUserId !== viewer.endUserId) { + if ( + viewer.role === 'end_user' && + input.endUserId && + input.endUserId !== viewer.endUserId + ) { throw new TRPCError({ code: 'FORBIDDEN', message: 'Current viewer cannot create token for other end user', From e2a037a0a6b997009e7d758c4ddc25aaa3cd8b6b Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sun, 29 Oct 2023 18:43:20 -0700 Subject: [PATCH 80/80] Display the current env --- apps/app-config/env.ts | 2 ++ apps/web/app/layout.tsx | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/app-config/env.ts b/apps/app-config/env.ts index ab51d6bc..d35edab2 100644 --- a/apps/app-config/env.ts +++ b/apps/app-config/env.ts @@ -20,6 +20,7 @@ Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent t INNGEST_EVENT_KEY: z.string(), INNGEST_SIGNING_KEY: z.string(), NANGO_SECRET_KEY: z.string(), + VERCEL_ENV: z.enum(['production', 'preview', 'development']), }, client: { NEXT_PUBLIC_SUPABASE_URL: z.string(), @@ -35,6 +36,7 @@ Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent t NEXT_PUBLIC_COMMANDBAR_ORG_ID: z.string().optional(), }, runtimeEnv: overrideFromLocalStorage({ + VERCEL_ENV: process.env['VERCEL_ENV'], CLERK_SECRET_KEY: process.env['CLERK_SECRET_KEY'], INNGEST_EVENT_KEY: process.env['INNGEST_EVENT_KEY'], INNGEST_SIGNING_KEY: process.env['INNGEST_SIGNING_KEY'], diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx index 576d228a..fdcd373d 100644 --- a/apps/web/app/layout.tsx +++ b/apps/web/app/layout.tsx @@ -1,7 +1,11 @@ import './global.css' +import {env} from '@usevenice/app-config/env' + export const metadata = { - title: 'Venice — Financial data, fast.', + title: `${ + env.VERCEL_ENV === 'production' ? '' : `[${env.VERCEL_ENV}] ` + }Venice — Financial data, fast.`, icons: [{url: '/favicon.svg', type: 'image/svg+xml'}], }