From 9ef6cbd566a817a541590fb73534d6b902898103 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 3 Dec 2024 16:39:53 +0000 Subject: [PATCH 01/13] basic 3.0 metrics --- .../backend-core/src/events/publishers/index.ts | 1 + .../backend-core/src/events/publishers/rowAction.ts | 13 +++++++++++++ packages/backend-core/src/events/publishers/view.ts | 7 ++++--- .../server/src/api/controllers/rowAction/crud.ts | 3 +++ packages/server/src/api/controllers/view/viewsV2.ts | 4 ++++ packages/types/src/sdk/events/index.ts | 1 + packages/types/src/sdk/events/rowAction.ts | 6 ++++++ packages/types/src/sdk/events/view.ts | 7 +++++-- 8 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 packages/backend-core/src/events/publishers/rowAction.ts create mode 100644 packages/types/src/sdk/events/rowAction.ts diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts index 9c92b80499e..aaec62f9795 100644 --- a/packages/backend-core/src/events/publishers/index.ts +++ b/packages/backend-core/src/events/publishers/index.ts @@ -23,3 +23,4 @@ export { default as plugin } from "./plugin" export { default as backup } from "./backup" export { default as environmentVariable } from "./environmentVariable" export { default as auditLog } from "./auditLog" +export { default as rowAction } from "./rowAction" diff --git a/packages/backend-core/src/events/publishers/rowAction.ts b/packages/backend-core/src/events/publishers/rowAction.ts new file mode 100644 index 00000000000..5adfd22d2d4 --- /dev/null +++ b/packages/backend-core/src/events/publishers/rowAction.ts @@ -0,0 +1,13 @@ +import { publishEvent } from "../events" +import { Event, RowActionCreatedEvent } from "@budibase/types" + +async function created( + rowAction: RowActionCreatedEvent, + timestamp?: string | number +) { + await publishEvent(Event.TABLE_CREATED, rowAction, timestamp) +} + +export default { + created, +} diff --git a/packages/backend-core/src/events/publishers/view.ts b/packages/backend-core/src/events/publishers/view.ts index ccbf960b04e..0ec8f67485e 100644 --- a/packages/backend-core/src/events/publishers/view.ts +++ b/packages/backend-core/src/events/publishers/view.ts @@ -11,7 +11,7 @@ import { ViewFilterDeletedEvent, ViewFilterUpdatedEvent, ViewUpdatedEvent, - View, + ViewV2, ViewCalculation, Table, TableExportFormat, @@ -19,9 +19,10 @@ import { /* eslint-disable */ -async function created(view: View, timestamp?: string | number) { +async function created(view: Partial, timestamp?: string | number) { const properties: ViewCreatedEvent = { - tableId: view.tableId, + name: view.name, + type: view.type, } await publishEvent(Event.VIEW_CREATED, properties, timestamp) } diff --git a/packages/server/src/api/controllers/rowAction/crud.ts b/packages/server/src/api/controllers/rowAction/crud.ts index 3baaff6fcca..525fcabcfc6 100644 --- a/packages/server/src/api/controllers/rowAction/crud.ts +++ b/packages/server/src/api/controllers/rowAction/crud.ts @@ -6,6 +6,7 @@ import { RowActionResponse, RowActionsResponse, } from "@budibase/types" +import { events } from "@budibase/backend-core" import sdk from "../../../sdk" async function getTable(ctx: Ctx) { @@ -59,6 +60,8 @@ export async function create( name: ctx.request.body.name, }) + await events.rowAction.created(createdAction) + ctx.body = { tableId, id: createdAction.id, diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 986764a697e..46c44ed5777 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -16,6 +16,7 @@ import { CountDistinctCalculationFieldMetadata, CountCalculationFieldMetadata, } from "@budibase/types" +import { events } from "@budibase/backend-core" import { builderSocket, gridSocket } from "../../../websockets" import { helpers } from "@budibase/shared-core" @@ -149,6 +150,9 @@ export async function create(ctx: Ctx) { primaryDisplay: view.primaryDisplay, } const result = await sdk.views.create(tableId, parsedView) + + await events.view.created(view) + ctx.status = 201 ctx.body = { data: result, diff --git a/packages/types/src/sdk/events/index.ts b/packages/types/src/sdk/events/index.ts index 043e62faa45..7a067fd2021 100644 --- a/packages/types/src/sdk/events/index.ts +++ b/packages/types/src/sdk/events/index.ts @@ -24,3 +24,4 @@ export * from "./plugin" export * from "./backup" export * from "./environmentVariable" export * from "./auditLog" +export * from "./rowAction" diff --git a/packages/types/src/sdk/events/rowAction.ts b/packages/types/src/sdk/events/rowAction.ts new file mode 100644 index 00000000000..924f11cf764 --- /dev/null +++ b/packages/types/src/sdk/events/rowAction.ts @@ -0,0 +1,6 @@ +import { BaseEvent } from "./event" + +export interface RowActionCreatedEvent extends BaseEvent { + name: string + automationId: string +} diff --git a/packages/types/src/sdk/events/view.ts b/packages/types/src/sdk/events/view.ts index 452094d2f44..84fcb0faccd 100644 --- a/packages/types/src/sdk/events/view.ts +++ b/packages/types/src/sdk/events/view.ts @@ -1,8 +1,11 @@ -import { ViewCalculation } from "../../documents" +import { ViewCalculation, ViewV2Schema, ViewV2Type } from "../../documents" import { BaseEvent, TableExportFormat } from "./event" +import { LegacyFilter, SortOrder, SortType, UISearchFilter } from "../../api" +import { SearchFilters } from "../search" export interface ViewCreatedEvent extends BaseEvent { - tableId: string + name: string + type?: ViewV2Type } export interface ViewUpdatedEvent extends BaseEvent { From 3332f2fa228e958f2a2541ed7c9715a3d2544867 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 5 Dec 2024 17:32:07 +0000 Subject: [PATCH 02/13] metrics for view joins and grouped filters --- .../processors/posthog/PosthogProcessor.ts | 4 +- .../src/events/publishers/table.ts | 33 +++++++++--- .../src/events/publishers/view.ts | 17 ++++-- .../src/api/controllers/table/external.ts | 4 +- .../server/src/api/controllers/table/index.ts | 11 +++- .../src/api/controllers/table/internal.ts | 4 +- .../server/src/api/controllers/view/views.ts | 54 +++++++------------ .../src/api/controllers/view/viewsV2.ts | 11 ++-- .../functions/backfill/app/tables.ts | 2 +- .../src/sdk/app/tables/external/index.ts | 2 +- .../src/sdk/app/tables/internal/index.ts | 2 +- packages/server/src/sdk/app/views/external.ts | 4 +- packages/server/src/sdk/app/views/index.ts | 5 +- packages/server/src/sdk/app/views/internal.ts | 4 +- packages/types/src/sdk/events/table.ts | 3 ++ packages/types/src/sdk/events/view.ts | 7 +-- 16 files changed, 100 insertions(+), 67 deletions(-) diff --git a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts index 12d2bb7e2c5..687fe542565 100644 --- a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts @@ -13,8 +13,8 @@ const EXCLUDED_EVENTS: Event[] = [ Event.ROLE_UPDATED, Event.DATASOURCE_UPDATED, Event.QUERY_UPDATED, - Event.TABLE_UPDATED, - Event.VIEW_UPDATED, + // Event.TABLE_UPDATED, + // Event.VIEW_UPDATED, Event.VIEW_FILTER_UPDATED, Event.VIEW_CALCULATION_UPDATED, Event.AUTOMATION_TRIGGER_UPDATED, diff --git a/packages/backend-core/src/events/publishers/table.ts b/packages/backend-core/src/events/publishers/table.ts index dc3200291a2..de765e2cdda 100644 --- a/packages/backend-core/src/events/publishers/table.ts +++ b/packages/backend-core/src/events/publishers/table.ts @@ -1,13 +1,14 @@ import { publishEvent } from "../events" import { Event, - TableExportFormat, + FieldType, Table, TableCreatedEvent, - TableUpdatedEvent, TableDeletedEvent, TableExportedEvent, + TableExportFormat, TableImportedEvent, + TableUpdatedEvent, } from "@budibase/types" async function created(table: Table, timestamp?: string | number) { @@ -20,14 +21,34 @@ async function created(table: Table, timestamp?: string | number) { await publishEvent(Event.TABLE_CREATED, properties, timestamp) } -async function updated(table: Table) { +async function updated(oldTable: Table, newTable: Table) { + // only publish the event if it has fields we are interested in + let defaultValues, aiColumn + + // check that new fields have been added + for (const key in newTable.schema) { + if (!oldTable.schema[key]) { + const newColumn = newTable.schema[key] + if ("default" in newColumn) { + defaultValues = true + } + if (newColumn.type === FieldType.AI) { + aiColumn = newColumn.operation + } + } + } + const properties: TableUpdatedEvent = { - tableId: table._id as string, + tableId: newTable._id as string, + defaultValues, + aiColumn, audited: { - name: table.name, + name: newTable.name, }, } - await publishEvent(Event.TABLE_UPDATED, properties) + if (defaultValues || aiColumn) { + await publishEvent(Event.TABLE_UPDATED, properties) + } } async function deleted(table: Table) { diff --git a/packages/backend-core/src/events/publishers/view.ts b/packages/backend-core/src/events/publishers/view.ts index 0ec8f67485e..6d107d58f3b 100644 --- a/packages/backend-core/src/events/publishers/view.ts +++ b/packages/backend-core/src/events/publishers/view.ts @@ -11,6 +11,7 @@ import { ViewFilterDeletedEvent, ViewFilterUpdatedEvent, ViewUpdatedEvent, + View, ViewV2, ViewCalculation, Table, @@ -19,17 +20,27 @@ import { /* eslint-disable */ -async function created(view: Partial, timestamp?: string | number) { +async function created(view: ViewV2, timestamp?: string | number) { const properties: ViewCreatedEvent = { name: view.name, type: view.type, + tableId: view.tableId, } await publishEvent(Event.VIEW_CREATED, properties, timestamp) } -async function updated(view: View) { +async function updated(newView: ViewV2) { + // // check whether any of the fields are different + let viewJoins = 0 + for (const key in newView.schema) { + if (newView.schema[key]?.columns) { + viewJoins += Object.keys(newView.schema[key]?.columns).length + } + } const properties: ViewUpdatedEvent = { - tableId: view.tableId, + tableId: newView.tableId, + groupedFilters: newView.queryUI?.groups?.length || 0, + viewJoins, } await publishEvent(Event.VIEW_UPDATED, properties) } diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 6f09bf4a61c..d3f5ef99f61 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -45,13 +45,13 @@ export async function updateTable( inputs.created = true } try { - const { datasource, table } = await sdk.tables.external.save( + const { datasource, oldTable, table } = await sdk.tables.external.save( datasourceId!, inputs, { tableId, renaming } ) builderSocket?.emitDatasourceUpdate(ctx, datasource) - return table + return { table, oldTable } } catch (err: any) { if (err instanceof Error) { ctx.throw(400, err.message) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 77c1f3923ae..0b0c9531e93 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -119,8 +119,15 @@ export async function save(ctx: UserCtx) { await events.table.created(savedTable) } else { const api = pickApi({ table }) - savedTable = await api.updateTable(ctx, renaming) - await events.table.updated(savedTable) + const { table: updatedTable, oldTable } = await api.updateTable( + ctx, + renaming + ) + savedTable = updatedTable + + if (oldTable) { + await events.table.updated(oldTable, savedTable) + } } if (renaming) { await sdk.views.renameLinkedViews(savedTable, renaming) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 40ce5e279d2..67c4ec100c5 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -30,14 +30,14 @@ export async function updateTable( } try { - const { table } = await sdk.tables.internal.save(tableToSave, { + const { table, oldTable } = await sdk.tables.internal.save(tableToSave, { userId: ctx.user._id, rowsToImport: rows, tableId: ctx.request.body._id, renaming, }) - return table + return { table, oldTable } } catch (err: any) { if (err instanceof Error) { ctx.throw(400, err.message) diff --git a/packages/server/src/api/controllers/view/views.ts b/packages/server/src/api/controllers/view/views.ts index b1f1f6c1542..f1aa219aca5 100644 --- a/packages/server/src/api/controllers/view/views.ts +++ b/packages/server/src/api/controllers/view/views.ts @@ -60,35 +60,31 @@ export async function save(ctx: Ctx) { existingTable.views[viewName] = existingTable.views[originalName] } await db.put(table) - await handleViewEvents( - existingTable.views[viewName] as View, - table.views[viewName] - ) ctx.body = table.views[viewName] builderSocket?.emitTableUpdate(ctx, table) } -export async function calculationEvents(existingView: View, newView: View) { - const existingCalculation = existingView && existingView.calculation - const newCalculation = newView && newView.calculation - - if (existingCalculation && !newCalculation) { - await events.view.calculationDeleted(existingView) - } - - if (!existingCalculation && newCalculation) { - await events.view.calculationCreated(newView) - } - - if ( - existingCalculation && - newCalculation && - existingCalculation !== newCalculation - ) { - await events.view.calculationUpdated(newView) - } -} +// export async function calculationEvents(existingView: View, newView: View) { +// const existingCalculation = existingView && existingView.calculation +// const newCalculation = newView && newView.calculation +// +// if (existingCalculation && !newCalculation) { +// await events.view.calculationDeleted(existingView) +// } +// +// if (!existingCalculation && newCalculation) { +// await events.view.calculationCreated(newView) +// } +// +// if ( +// existingCalculation && +// newCalculation && +// existingCalculation !== newCalculation +// ) { +// await events.view.calculationUpdated(newView) +// } +// } export async function filterEvents(existingView: View, newView: View) { const hasExistingFilters = !!( @@ -115,16 +111,6 @@ export async function filterEvents(existingView: View, newView: View) { } } -async function handleViewEvents(existingView: View, newView: View) { - if (!existingView) { - await events.view.created(newView) - } else { - await events.view.updated(newView) - } - await calculationEvents(existingView, newView) - await filterEvents(existingView, newView) -} - export async function destroy(ctx: Ctx) { const db = context.getAppDB() const viewName = decodeURIComponent(ctx.params.viewName) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 46c44ed5777..38983a978ef 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -151,7 +151,7 @@ export async function create(ctx: Ctx) { } const result = await sdk.views.create(tableId, parsedView) - await events.view.created(view) + await events.view.created(result) ctx.status = 201 ctx.body = { @@ -190,10 +190,11 @@ export async function update(ctx: Ctx) { primaryDisplay: view.primaryDisplay, } - const result = await sdk.views.update(tableId, parsedView) - ctx.body = { - data: result, - } + const { view: result } = await sdk.views.update(tableId, parsedView) + + await events.view.updated(result) + + ctx.body = { data: result } const table = await sdk.tables.getTable(tableId) builderSocket?.emitTableUpdate(ctx, table) diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index 081b81ede5e..c6e46173dc9 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -14,7 +14,7 @@ export const backfill = async (appDb: Database, timestamp: string | number) => { continue } - await events.view.created(view, timestamp) + // await events.view.created(view, timestamp) if (view.calculation) { await events.view.calculationCreated(view, timestamp) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 941d193b947..a7a15cac227 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -282,7 +282,7 @@ export async function save( tableToSave.sql = true } - return { datasource: updatedDatasource, table: tableToSave } + return { datasource: updatedDatasource, table: tableToSave, oldTable } } export async function destroy(datasourceId: string, table: Table) { diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index fbcbed03dc8..5b9f346e934 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -171,7 +171,7 @@ export async function save( } // has to run after, make sure it has _id await runStaticFormulaChecks(table, { oldTable, deletion: false }) - return { table } + return { table, oldTable } } export async function destroy(table: Table) { diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts index bee153a9107..9016a3bd619 100644 --- a/packages/server/src/sdk/app/views/external.ts +++ b/packages/server/src/sdk/app/views/external.ts @@ -63,7 +63,7 @@ export async function create( export async function update( tableId: string, view: Readonly -): Promise { +): Promise<{ view: ViewV2; existingView: ViewV2 }> { const db = context.getAppDB() const { datasourceId, tableName } = breakExternalTableId(tableId) @@ -87,7 +87,7 @@ export async function update( delete views[existingView.name] views[view.name] = view await db.put(ds) - return view + return { view, existingView } } export async function remove(viewId: string): Promise { diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 58537c96adc..f483ebc0bc3 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -315,7 +315,10 @@ export async function create( return view } -export async function update(tableId: string, view: ViewV2): Promise { +export async function update( + tableId: string, + view: ViewV2 +): Promise<{ view: ViewV2; existingView: ViewV2 }> { await guardViewSchema(tableId, view) return pickApi(tableId).update(tableId, view) diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts index 63807bcfd44..ec152abe547 100644 --- a/packages/server/src/sdk/app/views/internal.ts +++ b/packages/server/src/sdk/app/views/internal.ts @@ -54,7 +54,7 @@ export async function create( export async function update( tableId: string, view: Readonly -): Promise { +): Promise<{ view: ViewV2; existingView: ViewV2 }> { const db = context.getAppDB() const table = await sdk.tables.getTable(tableId) table.views ??= {} @@ -76,7 +76,7 @@ export async function update( delete table.views[existingView.name] table.views[view.name] = view await db.put(table) - return view + return { view, existingView } } export async function remove(viewId: string): Promise { diff --git a/packages/types/src/sdk/events/table.ts b/packages/types/src/sdk/events/table.ts index 8df2a957966..4a5880b1db9 100644 --- a/packages/types/src/sdk/events/table.ts +++ b/packages/types/src/sdk/events/table.ts @@ -1,4 +1,5 @@ import { BaseEvent, TableExportFormat } from "./event" +import { AIOperationEnum } from "../ai" export interface TableCreatedEvent extends BaseEvent { tableId: string @@ -9,6 +10,8 @@ export interface TableCreatedEvent extends BaseEvent { export interface TableUpdatedEvent extends BaseEvent { tableId: string + defaultValues: boolean | undefined + aiColumn: AIOperationEnum | undefined audited: { name: string } diff --git a/packages/types/src/sdk/events/view.ts b/packages/types/src/sdk/events/view.ts index 84fcb0faccd..c73a591a34d 100644 --- a/packages/types/src/sdk/events/view.ts +++ b/packages/types/src/sdk/events/view.ts @@ -1,15 +1,16 @@ -import { ViewCalculation, ViewV2Schema, ViewV2Type } from "../../documents" +import { ViewCalculation, ViewV2Type } from "../../documents" import { BaseEvent, TableExportFormat } from "./event" -import { LegacyFilter, SortOrder, SortType, UISearchFilter } from "../../api" -import { SearchFilters } from "../search" export interface ViewCreatedEvent extends BaseEvent { name: string type?: ViewV2Type + tableId: string } export interface ViewUpdatedEvent extends BaseEvent { tableId: string + groupedFilters: number + viewJoins: number } export interface ViewDeletedEvent extends BaseEvent { From 8f165d5b684c564514f17b4c42cd9a51ada82d4b Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 6 Dec 2024 17:02:49 +0000 Subject: [PATCH 03/13] types --- .../processors/posthog/PosthogProcessor.ts | 2 -- .../src/events/publishers/view.ts | 1 - .../server/src/api/controllers/view/views.ts | 21 ------------------- .../functions/backfill/app/tables.ts | 18 ---------------- packages/server/src/sdk/app/views/external.ts | 4 ++-- packages/server/src/sdk/app/views/internal.ts | 2 +- 6 files changed, 3 insertions(+), 45 deletions(-) diff --git a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts index 687fe542565..4700b47be24 100644 --- a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts @@ -13,8 +13,6 @@ const EXCLUDED_EVENTS: Event[] = [ Event.ROLE_UPDATED, Event.DATASOURCE_UPDATED, Event.QUERY_UPDATED, - // Event.TABLE_UPDATED, - // Event.VIEW_UPDATED, Event.VIEW_FILTER_UPDATED, Event.VIEW_CALCULATION_UPDATED, Event.AUTOMATION_TRIGGER_UPDATED, diff --git a/packages/backend-core/src/events/publishers/view.ts b/packages/backend-core/src/events/publishers/view.ts index 6d107d58f3b..3097ae4cfae 100644 --- a/packages/backend-core/src/events/publishers/view.ts +++ b/packages/backend-core/src/events/publishers/view.ts @@ -30,7 +30,6 @@ async function created(view: ViewV2, timestamp?: string | number) { } async function updated(newView: ViewV2) { - // // check whether any of the fields are different let viewJoins = 0 for (const key in newView.schema) { if (newView.schema[key]?.columns) { diff --git a/packages/server/src/api/controllers/view/views.ts b/packages/server/src/api/controllers/view/views.ts index f1aa219aca5..9fdd808d382 100644 --- a/packages/server/src/api/controllers/view/views.ts +++ b/packages/server/src/api/controllers/view/views.ts @@ -65,27 +65,6 @@ export async function save(ctx: Ctx) { builderSocket?.emitTableUpdate(ctx, table) } -// export async function calculationEvents(existingView: View, newView: View) { -// const existingCalculation = existingView && existingView.calculation -// const newCalculation = newView && newView.calculation -// -// if (existingCalculation && !newCalculation) { -// await events.view.calculationDeleted(existingView) -// } -// -// if (!existingCalculation && newCalculation) { -// await events.view.calculationCreated(newView) -// } -// -// if ( -// existingCalculation && -// newCalculation && -// existingCalculation !== newCalculation -// ) { -// await events.view.calculationUpdated(newView) -// } -// } - export async function filterEvents(existingView: View, newView: View) { const hasExistingFilters = !!( existingView && diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index c6e46173dc9..e8437bd5298 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -7,24 +7,6 @@ export const backfill = async (appDb: Database, timestamp: string | number) => { for (const table of tables) { await events.table.created(table, timestamp) - - if (table.views) { - for (const view of Object.values(table.views)) { - if (sdk.views.isV2(view)) { - continue - } - - // await events.view.created(view, timestamp) - - if (view.calculation) { - await events.view.calculationCreated(view, timestamp) - } - - if (view.filters?.length) { - await events.view.filterCreated(view, timestamp) - } - } - } } return tables.length diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts index 9016a3bd619..65e0ff410d0 100644 --- a/packages/server/src/sdk/app/views/external.ts +++ b/packages/server/src/sdk/app/views/external.ts @@ -63,7 +63,7 @@ export async function create( export async function update( tableId: string, view: Readonly -): Promise<{ view: ViewV2; existingView: ViewV2 }> { +): Promise<{ view: Readonly; existingView: ViewV2 }> { const db = context.getAppDB() const { datasourceId, tableName } = breakExternalTableId(tableId) @@ -87,7 +87,7 @@ export async function update( delete views[existingView.name] views[view.name] = view await db.put(ds) - return { view, existingView } + return { view, existingView } as { view: ViewV2; existingView: ViewV2 } } export async function remove(viewId: string): Promise { diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts index ec152abe547..4f7abad357f 100644 --- a/packages/server/src/sdk/app/views/internal.ts +++ b/packages/server/src/sdk/app/views/internal.ts @@ -76,7 +76,7 @@ export async function update( delete table.views[existingView.name] table.views[view.name] = view await db.put(table) - return { view, existingView } + return { view, existingView } as { view: ViewV2; existingView: ViewV2 } } export async function remove(viewId: string): Promise { From 492d5a37f28a8ed88eae084f8897d9c50f0baf94 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 6 Dec 2024 19:44:35 +0000 Subject: [PATCH 04/13] fix table tests --- packages/server/src/api/routes/tests/table.spec.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 8556a598c6b..393fa8260c1 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -247,6 +247,8 @@ if (descriptions.length) { }, }, }, + views: {}, + sql: true }) ) @@ -254,9 +256,8 @@ if (descriptions.length) { ...table, name: generator.guid(), }) - expect(events.table.updated).toHaveBeenCalledTimes(1) - expect(events.table.updated).toHaveBeenCalledWith(updatedTable) + expect(events.table.updated).toHaveBeenCalledWith(table, updatedTable) }) it("updates all the row fields for a table when a schema key is renamed", async () => { From fb5f32da23d72857649bb452df6a77920208bf1d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 6 Dec 2024 20:08:42 +0000 Subject: [PATCH 05/13] remove view events from v1 tests --- .../server/src/api/routes/tests/view.spec.ts | 51 ------------------- 1 file changed, 51 deletions(-) diff --git a/packages/server/src/api/routes/tests/view.spec.ts b/packages/server/src/api/routes/tests/view.spec.ts index 57b589e79df..2217116f2d2 100644 --- a/packages/server/src/api/routes/tests/view.spec.ts +++ b/packages/server/src/api/routes/tests/view.spec.ts @@ -84,14 +84,6 @@ describe("/views", () => { const view = await saveView({ calculation: ViewCalculation.COUNT }) expect(view.tableId).toBe(table._id) - expect(events.view.created).toHaveBeenCalledTimes(1) - expect(events.view.updated).not.toHaveBeenCalled() - expect(events.view.calculationCreated).toHaveBeenCalledTimes(1) - expect(events.view.calculationUpdated).not.toHaveBeenCalled() - expect(events.view.calculationDeleted).not.toHaveBeenCalled() - expect(events.view.filterCreated).not.toHaveBeenCalled() - expect(events.view.filterUpdated).not.toHaveBeenCalled() - expect(events.view.filterDeleted).not.toHaveBeenCalled() }) it("creates a view with a filter", async () => { @@ -109,14 +101,6 @@ describe("/views", () => { }) expect(view.tableId).toBe(table._id) - expect(events.view.created).toHaveBeenCalledTimes(1) - expect(events.view.updated).not.toHaveBeenCalled() - expect(events.view.calculationCreated).not.toHaveBeenCalled() - expect(events.view.calculationUpdated).not.toHaveBeenCalled() - expect(events.view.calculationDeleted).not.toHaveBeenCalled() - expect(events.view.filterCreated).toHaveBeenCalledTimes(1) - expect(events.view.filterUpdated).not.toHaveBeenCalled() - expect(events.view.filterDeleted).not.toHaveBeenCalled() }) it("updates the table row with the new view metadata", async () => { @@ -166,13 +150,6 @@ describe("/views", () => { await saveView() expect(events.view.created).not.toHaveBeenCalled() - expect(events.view.updated).toHaveBeenCalledTimes(1) - expect(events.view.calculationCreated).not.toHaveBeenCalled() - expect(events.view.calculationUpdated).not.toHaveBeenCalled() - expect(events.view.calculationDeleted).not.toHaveBeenCalled() - expect(events.view.filterCreated).not.toHaveBeenCalled() - expect(events.view.filterUpdated).not.toHaveBeenCalled() - expect(events.view.filterDeleted).not.toHaveBeenCalled() }) it("updates a view calculation", async () => { @@ -182,13 +159,6 @@ describe("/views", () => { await saveView({ calculation: ViewCalculation.COUNT }) expect(events.view.created).not.toHaveBeenCalled() - expect(events.view.updated).toHaveBeenCalledTimes(1) - expect(events.view.calculationCreated).not.toHaveBeenCalled() - expect(events.view.calculationUpdated).toHaveBeenCalledTimes(1) - expect(events.view.calculationDeleted).not.toHaveBeenCalled() - expect(events.view.filterCreated).not.toHaveBeenCalled() - expect(events.view.filterUpdated).not.toHaveBeenCalled() - expect(events.view.filterDeleted).not.toHaveBeenCalled() }) it("deletes a view calculation", async () => { @@ -198,13 +168,6 @@ describe("/views", () => { await saveView({ calculation: undefined }) expect(events.view.created).not.toHaveBeenCalled() - expect(events.view.updated).toHaveBeenCalledTimes(1) - expect(events.view.calculationCreated).not.toHaveBeenCalled() - expect(events.view.calculationUpdated).not.toHaveBeenCalled() - expect(events.view.calculationDeleted).toHaveBeenCalledTimes(1) - expect(events.view.filterCreated).not.toHaveBeenCalled() - expect(events.view.filterUpdated).not.toHaveBeenCalled() - expect(events.view.filterDeleted).not.toHaveBeenCalled() }) it("updates a view filter", async () => { @@ -230,13 +193,6 @@ describe("/views", () => { }) expect(events.view.created).not.toHaveBeenCalled() - expect(events.view.updated).toHaveBeenCalledTimes(1) - expect(events.view.calculationCreated).not.toHaveBeenCalled() - expect(events.view.calculationUpdated).not.toHaveBeenCalled() - expect(events.view.calculationDeleted).not.toHaveBeenCalled() - expect(events.view.filterCreated).not.toHaveBeenCalled() - expect(events.view.filterUpdated).toHaveBeenCalledTimes(1) - expect(events.view.filterDeleted).not.toHaveBeenCalled() }) it("deletes a view filter", async () => { @@ -254,13 +210,6 @@ describe("/views", () => { await saveView({ filters: [] }) expect(events.view.created).not.toHaveBeenCalled() - expect(events.view.updated).toHaveBeenCalledTimes(1) - expect(events.view.calculationCreated).not.toHaveBeenCalled() - expect(events.view.calculationUpdated).not.toHaveBeenCalled() - expect(events.view.calculationDeleted).not.toHaveBeenCalled() - expect(events.view.filterCreated).not.toHaveBeenCalled() - expect(events.view.filterUpdated).not.toHaveBeenCalled() - expect(events.view.filterDeleted).toHaveBeenCalledTimes(1) }) }) From 41c1632d60f8551bf8f1638856e42a47cce7f004 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 6 Dec 2024 20:09:18 +0000 Subject: [PATCH 06/13] lint --- packages/server/src/api/routes/tests/table.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 393fa8260c1..39b6a68eaf8 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -248,7 +248,7 @@ if (descriptions.length) { }, }, views: {}, - sql: true + sql: true, }) ) From 917c23529546299a4b519a0d12e3aea2eea764d8 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Sun, 8 Dec 2024 23:18:07 +0000 Subject: [PATCH 07/13] send individual events for view calcs and joins --- .../processors/posthog/PosthogProcessor.ts | 1 + .../src/events/publishers/rowAction.ts | 2 +- .../src/events/publishers/view.ts | 61 ++++++++++++------- .../tests/core/utilities/mocks/events.ts | 1 + .../server/src/api/controllers/view/views.ts | 25 -------- .../src/api/controllers/view/viewsV2.ts | 58 +++++++++++++++++- .../src/api/routes/tests/viewV2.spec.ts | 9 ++- packages/types/src/sdk/events/event.ts | 8 +++ packages/types/src/sdk/events/view.ts | 12 ++-- 9 files changed, 124 insertions(+), 53 deletions(-) diff --git a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts index 4700b47be24..e8e8f35d6da 100644 --- a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts @@ -13,6 +13,7 @@ const EXCLUDED_EVENTS: Event[] = [ Event.ROLE_UPDATED, Event.DATASOURCE_UPDATED, Event.QUERY_UPDATED, + Event.VIEW_UPDATED, Event.VIEW_FILTER_UPDATED, Event.VIEW_CALCULATION_UPDATED, Event.AUTOMATION_TRIGGER_UPDATED, diff --git a/packages/backend-core/src/events/publishers/rowAction.ts b/packages/backend-core/src/events/publishers/rowAction.ts index 5adfd22d2d4..eac35cc4892 100644 --- a/packages/backend-core/src/events/publishers/rowAction.ts +++ b/packages/backend-core/src/events/publishers/rowAction.ts @@ -5,7 +5,7 @@ async function created( rowAction: RowActionCreatedEvent, timestamp?: string | number ) { - await publishEvent(Event.TABLE_CREATED, rowAction, timestamp) + await publishEvent(Event.ROW_ACTION_CREATED, rowAction, timestamp) } export default { diff --git a/packages/backend-core/src/events/publishers/view.ts b/packages/backend-core/src/events/publishers/view.ts index 3097ae4cfae..3ce24e9a0a0 100644 --- a/packages/backend-core/src/events/publishers/view.ts +++ b/packages/backend-core/src/events/publishers/view.ts @@ -1,6 +1,11 @@ import { publishEvent } from "../events" import { + CalculationType, Event, + Table, + TableExportFormat, + View, + ViewCalculation, ViewCalculationCreatedEvent, ViewCalculationDeletedEvent, ViewCalculationUpdatedEvent, @@ -11,11 +16,8 @@ import { ViewFilterDeletedEvent, ViewFilterUpdatedEvent, ViewUpdatedEvent, - View, ViewV2, - ViewCalculation, - Table, - TableExportFormat, + ViewJoinCreatedEvent, } from "@budibase/types" /* eslint-disable */ @@ -29,17 +31,9 @@ async function created(view: ViewV2, timestamp?: string | number) { await publishEvent(Event.VIEW_CREATED, properties, timestamp) } -async function updated(newView: ViewV2) { - let viewJoins = 0 - for (const key in newView.schema) { - if (newView.schema[key]?.columns) { - viewJoins += Object.keys(newView.schema[key]?.columns).length - } - } +async function updated(view: ViewV2) { const properties: ViewUpdatedEvent = { - tableId: newView.tableId, - groupedFilters: newView.queryUI?.groups?.length || 0, - viewJoins, + tableId: view.tableId, } await publishEvent(Event.VIEW_UPDATED, properties) } @@ -59,16 +53,27 @@ async function exported(table: Table, format: TableExportFormat) { await publishEvent(Event.VIEW_EXPORTED, properties) } -async function filterCreated(view: View, timestamp?: string | number) { +async function filterCreated( + { tableId, filterGroups }: { tableId: string; filterGroups: number }, + timestamp?: string | number +) { const properties: ViewFilterCreatedEvent = { - tableId: view.tableId, + tableId, + filterGroups, } await publishEvent(Event.VIEW_FILTER_CREATED, properties, timestamp) } -async function filterUpdated(view: View) { +async function filterUpdated({ + tableId, + filterGroups, +}: { + tableId: string + filterGroups: number +}) { const properties: ViewFilterUpdatedEvent = { - tableId: view.tableId, + tableId: tableId, + filterGroups, } await publishEvent(Event.VIEW_FILTER_UPDATED, properties) } @@ -80,10 +85,16 @@ async function filterDeleted(view: View) { await publishEvent(Event.VIEW_FILTER_DELETED, properties) } -async function calculationCreated(view: View, timestamp?: string | number) { +async function calculationCreated( + { + tableId, + calculationType, + }: { tableId: string; calculationType: CalculationType }, + timestamp?: string | number +) { const properties: ViewCalculationCreatedEvent = { - tableId: view.tableId, - calculation: view.calculation as ViewCalculation, + tableId, + calculation: calculationType, } await publishEvent(Event.VIEW_CALCULATION_CREATED, properties, timestamp) } @@ -104,6 +115,13 @@ async function calculationDeleted(existingView: View) { await publishEvent(Event.VIEW_CALCULATION_DELETED, properties) } +async function viewJoinCreated(tableId: any, timestamp?: string | number) { + const properties: ViewJoinCreatedEvent = { + tableId, + } + await publishEvent(Event.VIEW_JOIN_CREATED, properties, timestamp) +} + export default { created, updated, @@ -115,4 +133,5 @@ export default { calculationCreated, calculationUpdated, calculationDeleted, + viewJoinCreated, } diff --git a/packages/backend-core/tests/core/utilities/mocks/events.ts b/packages/backend-core/tests/core/utilities/mocks/events.ts index 96f351de106..433986352e0 100644 --- a/packages/backend-core/tests/core/utilities/mocks/events.ts +++ b/packages/backend-core/tests/core/utilities/mocks/events.ts @@ -117,6 +117,7 @@ beforeAll(async () => { jest.spyOn(events.view, "calculationCreated") jest.spyOn(events.view, "calculationUpdated") jest.spyOn(events.view, "calculationDeleted") + jest.spyOn(events.view, "viewJoinCreated") jest.spyOn(events.plugin, "init") jest.spyOn(events.plugin, "imported") diff --git a/packages/server/src/api/controllers/view/views.ts b/packages/server/src/api/controllers/view/views.ts index 9fdd808d382..0626f4e55c9 100644 --- a/packages/server/src/api/controllers/view/views.ts +++ b/packages/server/src/api/controllers/view/views.ts @@ -65,31 +65,6 @@ export async function save(ctx: Ctx) { builderSocket?.emitTableUpdate(ctx, table) } -export async function filterEvents(existingView: View, newView: View) { - const hasExistingFilters = !!( - existingView && - existingView.filters && - existingView.filters.length - ) - const hasNewFilters = !!(newView && newView.filters && newView.filters.length) - - if (hasExistingFilters && !hasNewFilters) { - await events.view.filterDeleted(newView) - } - - if (!hasExistingFilters && hasNewFilters) { - await events.view.filterCreated(newView) - } - - if ( - hasExistingFilters && - hasNewFilters && - !isEqual(existingView.filters, newView.filters) - ) { - await events.view.filterUpdated(newView) - } -} - export async function destroy(ctx: Ctx) { const db = context.getAppDB() const viewName = decodeURIComponent(ctx.params.viewName) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 497a880fab9..e611df0de3d 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -16,10 +16,14 @@ import { CountCalculationFieldMetadata, CreateViewResponse, UpdateViewResponse, + View, + Event, } from "@budibase/types" import { events } from "@budibase/backend-core" import { builderSocket, gridSocket } from "../../../websockets" import { helpers } from "@budibase/shared-core" +import isEqual from "lodash/isEqual" +import { publishEvent } from "@budibase/backend-core/src/events" function stripUnknownFields( field: ViewFieldMetadata @@ -164,6 +168,54 @@ export async function create(ctx: Ctx) { gridSocket?.emitViewUpdate(ctx, result) } +async function handleViewEvents(existingView: ViewV2, view: ViewV2) { + // Grouped filters + if (view.queryUI?.groups) { + const filterGroups = view.queryUI?.groups?.length || 0 + const properties = { filterGroups, tableId: view.tableId } + if (!existingView?.queryUI) { + await publishEvent(Event.VIEW_FILTER_CREATED, properties) + await events.view.filterCreated(properties) + } else { + if ( + filterGroups > + ((existingView && existingView?.queryUI?.groups?.length) || 0) + ) { + await events.view.filterUpdated(properties) + } + } + } + + // if new columns in the view + for (const key in view.schema) { + if (!existingView?.schema?.[key]) { + const newColumn = view.schema[key] + + // view calculations + // @ts-expect-error non calculation types just won't have the calculationType field + const calculationType = newColumn.calculationType + if (calculationType) { + // Send the event + await events.view.calculationCreated({ + calculationType, + tableId: view.tableId, + }) + } + + // view joins + if (newColumn.columns) { + for (const column in newColumn?.columns) { + // if the new column is visible and it wasn't before + if (!existingView?.schema?.[key].columns?.[column].visible) { + // new view join exposing a column + await events.view.viewJoinCreated({ tableId: view.tableId }) + } + } + } + } + } +} + export async function update(ctx: Ctx) { const view = ctx.request.body @@ -191,8 +243,12 @@ export async function update(ctx: Ctx) { primaryDisplay: view.primaryDisplay, } - const { view: result } = await sdk.views.update(tableId, parsedView) + const { view: result, existingView } = await sdk.views.update( + tableId, + parsedView + ) + await handleViewEvents(existingView, result) await events.view.updated(result) ctx.body = { data: result } diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 23ae7c79d3f..9dfe565ceea 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -42,7 +42,7 @@ import { } from "../../../integrations/tests/utils" import merge from "lodash/merge" import { quotas } from "@budibase/pro" -import { db, roles, context } from "@budibase/backend-core" +import { db, roles, context, events } from "@budibase/backend-core" const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) @@ -129,6 +129,7 @@ if (descriptions.length) { id: expect.stringMatching(new RegExp(`${table._id!}_`)), version: 2, }) + expect(events.view.created).toHaveBeenCalledTimes(1) }) it("can persist views with all fields", async () => { @@ -195,6 +196,7 @@ if (descriptions.length) { } expect(res).toEqual(expected) + expect(events.view.created).toHaveBeenCalledTimes(1) }) it("can create a view with just a query field, no queryUI, for backwards compatibility", async () => { @@ -224,6 +226,7 @@ if (descriptions.length) { }, } const res = await config.api.viewV2.create(newView) + expect(events.view.created).toHaveBeenCalledTimes(1) const expected: ViewV2 = { ...newView, @@ -283,6 +286,7 @@ if (descriptions.length) { } const createdView = await config.api.viewV2.create(newView) + expect(events.view.created).toHaveBeenCalledTimes(1) expect(createdView).toEqual({ ...newView, @@ -990,6 +994,7 @@ if (descriptions.length) { expect((await config.api.table.get(tableId)).views).toEqual({ [view.name]: expected, }) + expect(events.view.updated).toHaveBeenCalledTimes(1) }) it("can update all fields", async () => { @@ -1621,6 +1626,7 @@ if (descriptions.length) { field: "age", } await config.api.viewV2.update(view) + expect(events.view.calculationCreated).toHaveBeenCalledTimes(1) const { rows } = await config.api.row.search(view.id) expect(rows).toHaveLength(2) @@ -2154,6 +2160,7 @@ if (descriptions.length) { }), }) ) + expect(events.view.viewJoinCreated).not.toBeCalled() }) it("does not rename columns with the same name but from other tables", async () => { diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index 242b182decc..23c0eb0cbd0 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -118,6 +118,7 @@ export enum Event { VIEW_CALCULATION_CREATED = "view:calculation:created", VIEW_CALCULATION_UPDATED = "view:calculation:updated", VIEW_CALCULATION_DELETED = "view:calculation:deleted", + VIEW_JOIN_CREATED = "view:join:created", // ROWS ROWS_CREATED = "rows:created", @@ -192,6 +193,9 @@ export enum Event { // AUDIT LOG AUDIT_LOGS_FILTERED = "audit_log:filtered", AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded", + + // ROW ACTION + ROW_ACTION_CREATED = "row_action:created", } export const UserGroupSyncEvents: Event[] = [ @@ -376,6 +380,7 @@ export const AuditedEventFriendlyName: Record = { [Event.VIEW_CALCULATION_CREATED]: undefined, [Event.VIEW_CALCULATION_UPDATED]: undefined, [Event.VIEW_CALCULATION_DELETED]: undefined, + [Event.VIEW_JOIN_CREATED]: undefined, // SERVED - NOT AUDITED [Event.SERVED_BUILDER]: undefined, @@ -395,6 +400,9 @@ export const AuditedEventFriendlyName: Record = { // AUDIT LOG - NOT AUDITED [Event.AUDIT_LOGS_FILTERED]: undefined, [Event.AUDIT_LOGS_DOWNLOADED]: undefined, + + // ROW ACTIONS - NOT AUDITED + [Event.ROW_ACTION_CREATED]: undefined, } // properties added at the final stage of the event pipeline diff --git a/packages/types/src/sdk/events/view.ts b/packages/types/src/sdk/events/view.ts index c73a591a34d..0ea153ad9d6 100644 --- a/packages/types/src/sdk/events/view.ts +++ b/packages/types/src/sdk/events/view.ts @@ -1,4 +1,4 @@ -import { ViewCalculation, ViewV2Type } from "../../documents" +import { CalculationType, ViewCalculation, ViewV2Type } from "../../documents" import { BaseEvent, TableExportFormat } from "./event" export interface ViewCreatedEvent extends BaseEvent { @@ -9,8 +9,6 @@ export interface ViewCreatedEvent extends BaseEvent { export interface ViewUpdatedEvent extends BaseEvent { tableId: string - groupedFilters: number - viewJoins: number } export interface ViewDeletedEvent extends BaseEvent { @@ -24,10 +22,12 @@ export interface ViewExportedEvent extends BaseEvent { export interface ViewFilterCreatedEvent extends BaseEvent { tableId: string + filterGroups: number } export interface ViewFilterUpdatedEvent extends BaseEvent { tableId: string + filterGroups: number } export interface ViewFilterDeletedEvent extends BaseEvent { @@ -36,7 +36,7 @@ export interface ViewFilterDeletedEvent extends BaseEvent { export interface ViewCalculationCreatedEvent extends BaseEvent { tableId: string - calculation: ViewCalculation + calculation: CalculationType } export interface ViewCalculationUpdatedEvent extends BaseEvent { @@ -48,3 +48,7 @@ export interface ViewCalculationDeletedEvent extends BaseEvent { tableId: string calculation: ViewCalculation } + +export interface ViewJoinCreatedEvent extends BaseEvent { + tableId: string +} From ec84cf1a56ffe87a6f23fc9a74f3900c52e6ced1 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Sun, 8 Dec 2024 23:50:22 +0000 Subject: [PATCH 08/13] lint --- .../server/src/api/controllers/view/views.ts | 2 - .../src/api/controllers/view/viewsV2.ts | 57 ++++++++------- .../server/src/api/routes/tests/table.spec.ts | 1 + .../src/api/routes/tests/viewV2.spec.ts | 70 ++++++++++++++----- 4 files changed, 80 insertions(+), 50 deletions(-) diff --git a/packages/server/src/api/controllers/view/views.ts b/packages/server/src/api/controllers/view/views.ts index 0626f4e55c9..bc734c56570 100644 --- a/packages/server/src/api/controllers/view/views.ts +++ b/packages/server/src/api/controllers/view/views.ts @@ -19,8 +19,6 @@ import { builderSocket } from "../../../websockets" const cloneDeep = require("lodash/cloneDeep") -import isEqual from "lodash/isEqual" - export async function fetch(ctx: Ctx) { ctx.body = await getViews() } diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index e611df0de3d..2751a812349 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -16,14 +16,10 @@ import { CountCalculationFieldMetadata, CreateViewResponse, UpdateViewResponse, - View, - Event, } from "@budibase/types" import { events } from "@budibase/backend-core" import { builderSocket, gridSocket } from "../../../websockets" import { helpers } from "@budibase/shared-core" -import isEqual from "lodash/isEqual" -import { publishEvent } from "@budibase/backend-core/src/events" function stripUnknownFields( field: ViewFieldMetadata @@ -168,48 +164,51 @@ export async function create(ctx: Ctx) { gridSocket?.emitViewUpdate(ctx, result) } +async function handleViewFilterEvents(existingView: ViewV2, view: ViewV2) { + const filterGroups = view.queryUI?.groups?.length || 0 + const properties = { filterGroups, tableId: view.tableId } + if (!existingView?.queryUI) { + await events.view.filterCreated(properties) + } else { + if ( + filterGroups > + ((existingView && existingView?.queryUI?.groups?.length) || 0) + ) { + await events.view.filterUpdated(properties) + } + } +} + async function handleViewEvents(existingView: ViewV2, view: ViewV2) { // Grouped filters if (view.queryUI?.groups) { - const filterGroups = view.queryUI?.groups?.length || 0 - const properties = { filterGroups, tableId: view.tableId } - if (!existingView?.queryUI) { - await publishEvent(Event.VIEW_FILTER_CREATED, properties) - await events.view.filterCreated(properties) - } else { - if ( - filterGroups > - ((existingView && existingView?.queryUI?.groups?.length) || 0) - ) { - await events.view.filterUpdated(properties) - } - } + await handleViewFilterEvents(existingView, view) } // if new columns in the view for (const key in view.schema) { if (!existingView?.schema?.[key]) { - const newColumn = view.schema[key] - // view calculations // @ts-expect-error non calculation types just won't have the calculationType field - const calculationType = newColumn.calculationType + const calculationType = view.schema[key].calculationType if (calculationType) { - // Send the event await events.view.calculationCreated({ calculationType, tableId: view.tableId, }) } + } - // view joins - if (newColumn.columns) { - for (const column in newColumn?.columns) { - // if the new column is visible and it wasn't before - if (!existingView?.schema?.[key].columns?.[column].visible) { - // new view join exposing a column - await events.view.viewJoinCreated({ tableId: view.tableId }) - } + // view joins + if (view.schema[key].columns) { + for (const column in view.schema[key]?.columns) { + // if the new column is visible and it wasn't before + if ( + !existingView?.schema?.[key].columns?.[column].visible && + view.schema?.[key].columns?.[column].visible + ) { + // new view join exposing a column + await events.view.viewJoinCreated({ tableId: view.tableId }) } } } diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 39b6a68eaf8..e47181e21f3 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -247,6 +247,7 @@ if (descriptions.length) { }, }, }, + primary: ["_id"], views: {}, sql: true, }) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 9dfe565ceea..739603bb3ce 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1,39 +1,39 @@ import { + ArrayOperator, + BasicOperator, + BBReferenceFieldSubType, + CalculationType, CreateViewRequest, Datasource, + EmptyFilterOption, FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, + JsonFieldSubType, + JsonTypes, + LegacyFilter, + NumericCalculationFieldMetadata, PermissionLevel, QuotaUsageType, + RelationshipType, + RenameColumn, Row, SaveTableRequest, + SearchFilters, + SearchResponse, + SearchViewRowRequest, SortOrder, SortType, StaticQuotaName, Table, + TableSchema, TableSourceType, + UILogicalOperator, + UISearchFilter, UpdateViewRequest, ViewV2, - SearchResponse, - BasicOperator, - CalculationType, - RelationshipType, - TableSchema, - RenameColumn, - BBReferenceFieldSubType, - NumericCalculationFieldMetadata, ViewV2Schema, ViewV2Type, - JsonTypes, - EmptyFilterOption, - JsonFieldSubType, - UISearchFilter, - LegacyFilter, - SearchViewRowRequest, - ArrayOperator, - UILogicalOperator, - SearchFilters, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { @@ -42,7 +42,7 @@ import { } from "../../../integrations/tests/utils" import merge from "lodash/merge" import { quotas } from "@budibase/pro" -import { db, roles, context, events } from "@budibase/backend-core" +import { context, db, events, roles } from "@budibase/backend-core" const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) @@ -1360,6 +1360,8 @@ if (descriptions.length) { }, }) + expect(events.view.filterCreated).toHaveBeenCalledTimes(1) + updatedView = await config.api.viewV2.get(view.id) expected = { onEmptyFilter: EmptyFilterOption.RETURN_ALL, @@ -2160,7 +2162,7 @@ if (descriptions.length) { }), }) ) - expect(events.view.viewJoinCreated).not.toBeCalled() + expect(events.view.viewJoinCreated).not.toHaveBeenCalled() }) it("does not rename columns with the same name but from other tables", async () => { @@ -2233,6 +2235,36 @@ if (descriptions.length) { ) }) + it("handles events for changing column visibility from default false", async () => { + let auxTable = await createAuxTable() + let aux2Table = await createAuxTable() + + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + { name: "aux2", tableId: aux2Table._id!, fk: "fk_aux2" }, + ]) + + const view = await createView(table._id!, { + aux: { + visible: true, + columns: { + name: { visible: false, readonly: true }, + }, + }, + aux2: { + visible: true, + columns: { + name: { visible: false, readonly: true }, + }, + }, + }) + + // @ts-expect-error column exists above + view.schema.aux2.columns.name.visible = true + await config.api.viewV2.update(view) + expect(events.view.viewJoinCreated).toHaveBeenCalledTimes(1) + }) + it("updates all views references", async () => { let auxTable = await createAuxTable() From 1d0b7d83e7b8f46e06f7ec77da039ffe32f6a07d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 9 Dec 2024 08:45:36 +0000 Subject: [PATCH 09/13] view test --- packages/server/src/api/routes/tests/view.spec.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/view.spec.ts b/packages/server/src/api/routes/tests/view.spec.ts index 2217116f2d2..e1968d28993 100644 --- a/packages/server/src/api/routes/tests/view.spec.ts +++ b/packages/server/src/api/routes/tests/view.spec.ts @@ -73,11 +73,6 @@ describe("/views", () => { } describe("create", () => { - it("returns a success message when the view is successfully created", async () => { - await saveView() - expect(events.view.created).toHaveBeenCalledTimes(1) - }) - it("creates a view with a calculation", async () => { jest.clearAllMocks() From bca9bd27dbeb8666dd589d49f387da4455f6ccc8 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 9 Dec 2024 10:17:17 +0000 Subject: [PATCH 10/13] backfill tests --- packages/server/src/migrations/tests/index.spec.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts index 6b3f3314ba8..3a23d8f0119 100644 --- a/packages/server/src/migrations/tests/index.spec.ts +++ b/packages/server/src/migrations/tests/index.spec.ts @@ -73,16 +73,12 @@ describe("migrations", () => { expect(events.query.created).toHaveBeenCalledTimes(2) expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation) expect(events.table.created).toHaveBeenCalledTimes(3) - expect(events.view.created).toHaveBeenCalledTimes(2) - expect(events.view.calculationCreated).toHaveBeenCalledTimes(1) - expect(events.view.filterCreated).toHaveBeenCalledTimes(1) - expect(events.screen.created).toHaveBeenCalledTimes(2) expect(events.backfill.appSucceeded).toHaveBeenCalledTimes(2) // to make sure caching is working as expected expect( events.processors.analyticsProcessor.processEvent - ).toHaveBeenCalledTimes(24) // Addtion of of the events above + ).toHaveBeenCalledTimes(20) // Addition of of the events above }) }) }) From b6fcdf301dbb352cd00dba25434ee894af7231fc Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 9 Dec 2024 11:59:04 +0000 Subject: [PATCH 11/13] PR comments --- .../src/events/publishers/table.ts | 2 +- .../src/api/controllers/view/viewsV2.ts | 33 ++++++++----------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/packages/backend-core/src/events/publishers/table.ts b/packages/backend-core/src/events/publishers/table.ts index de765e2cdda..77a2c3e1a4b 100644 --- a/packages/backend-core/src/events/publishers/table.ts +++ b/packages/backend-core/src/events/publishers/table.ts @@ -29,7 +29,7 @@ async function updated(oldTable: Table, newTable: Table) { for (const key in newTable.schema) { if (!oldTable.schema[key]) { const newColumn = newTable.schema[key] - if ("default" in newColumn) { + if ("default" in newColumn && newColumn.default != null) { defaultValues = true } if (newColumn.type === FieldType.AI) { diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 2751a812349..66a61940964 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -187,29 +187,22 @@ async function handleViewEvents(existingView: ViewV2, view: ViewV2) { // if new columns in the view for (const key in view.schema) { - if (!existingView?.schema?.[key]) { - // view calculations - // @ts-expect-error non calculation types just won't have the calculationType field - const calculationType = view.schema[key].calculationType - if (calculationType) { - await events.view.calculationCreated({ - calculationType, - tableId: view.tableId, - }) - } + if ("calculationType" in view.schema[key] && !existingView?.schema?.[key]) { + await events.view.calculationCreated({ + calculationType: view.schema[key].calculationType, + tableId: view.tableId, + }) } // view joins - if (view.schema[key].columns) { - for (const column in view.schema[key]?.columns) { - // if the new column is visible and it wasn't before - if ( - !existingView?.schema?.[key].columns?.[column].visible && - view.schema?.[key].columns?.[column].visible - ) { - // new view join exposing a column - await events.view.viewJoinCreated({ tableId: view.tableId }) - } + for (const column in view.schema[key]?.columns ?? []) { + // if the new column is visible and it wasn't before + if ( + !existingView?.schema?.[key].columns?.[column].visible && + view.schema?.[key].columns?.[column].visible + ) { + // new view join exposing a column + await events.view.viewJoinCreated({ tableId: view.tableId }) } } } From e5453cc766db170affd212cc61dac0697f7ca9e0 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 9 Dec 2024 13:45:11 +0000 Subject: [PATCH 12/13] only fire view updated events when 2 or more groups are applied --- .../src/api/controllers/view/viewsV2.ts | 14 +++---- .../src/api/routes/tests/viewV2.spec.ts | 41 ++++++++++++++++++- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 66a61940964..4c65d673e87 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -167,15 +167,11 @@ export async function create(ctx: Ctx) { async function handleViewFilterEvents(existingView: ViewV2, view: ViewV2) { const filterGroups = view.queryUI?.groups?.length || 0 const properties = { filterGroups, tableId: view.tableId } - if (!existingView?.queryUI) { - await events.view.filterCreated(properties) - } else { - if ( - filterGroups > - ((existingView && existingView?.queryUI?.groups?.length) || 0) - ) { - await events.view.filterUpdated(properties) - } + if ( + filterGroups >= 2 && + ((existingView && existingView?.queryUI?.groups?.length) || 0) + ) { + await events.view.filterUpdated(properties) } } diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 739603bb3ce..244a0a23ebe 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -997,6 +997,45 @@ if (descriptions.length) { expect(events.view.updated).toHaveBeenCalledTimes(1) }) + it("handles view grouped filter events", async () => { + view.queryUI = { + logicalOperator: UILogicalOperator.ALL, + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + groups: [ + { + logicalOperator: UILogicalOperator.ALL, + filters: [ + { + operator: BasicOperator.EQUAL, + field: "newField", + value: "newValue", + }, + ], + }, + ], + } + await config.api.viewV2.update(view) + expect(events.view.filterUpdated).not.toHaveBeenCalled() + + // @ts-ignore + view.queryUI.groups.push({ + logicalOperator: UILogicalOperator.ALL, + filters: [ + { + operator: BasicOperator.EQUAL, + field: "otherField", + value: "otherValue", + }, + ], + }) + + await config.api.viewV2.update(view) + expect(events.view.filterUpdated).toHaveBeenCalledWith({ + filterGroups: 2, + tableId: view.tableId, + }) + }) + it("can update all fields", async () => { const tableId = table._id! @@ -1360,8 +1399,6 @@ if (descriptions.length) { }, }) - expect(events.view.filterCreated).toHaveBeenCalledTimes(1) - updatedView = await config.api.viewV2.get(view.id) expected = { onEmptyFilter: EmptyFilterOption.RETURN_ALL, From 7d5ca8449fb5b98ec74e2966b633fb1514077865 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 9 Dec 2024 14:01:47 +0000 Subject: [PATCH 13/13] only fire for 2 grouped filters --- .../src/events/processors/posthog/PosthogProcessor.ts | 1 - packages/server/src/api/controllers/view/viewsV2.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts index e8e8f35d6da..6c45da09e6c 100644 --- a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts @@ -14,7 +14,6 @@ const EXCLUDED_EVENTS: Event[] = [ Event.DATASOURCE_UPDATED, Event.QUERY_UPDATED, Event.VIEW_UPDATED, - Event.VIEW_FILTER_UPDATED, Event.VIEW_CALCULATION_UPDATED, Event.AUTOMATION_TRIGGER_UPDATED, Event.USER_GROUP_UPDATED, diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 4c65d673e87..da579efed72 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -169,7 +169,7 @@ async function handleViewFilterEvents(existingView: ViewV2, view: ViewV2) { const properties = { filterGroups, tableId: view.tableId } if ( filterGroups >= 2 && - ((existingView && existingView?.queryUI?.groups?.length) || 0) + filterGroups > (existingView?.queryUI?.groups?.length || 0) ) { await events.view.filterUpdated(properties) }