From 07a14fbde8adc00919eb5b7df4ccac3243495c10 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Oct 2023 17:15:14 +0200
Subject: [PATCH 01/17] Allow selecting user column type

---
 .../TableNavigator/TableDataImport.svelte     | 73 +++++++++++++++----
 1 file changed, 60 insertions(+), 13 deletions(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index 7bf2bbbbabd..6f441350064 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -10,36 +10,73 @@
   export let displayColumn = null
   export let promptUpload = false
 
-  const typeOptions = [
-    {
+  const typeOptions = {
+    [FIELDS.STRING.type]: {
       label: "Text",
       value: FIELDS.STRING.type,
+      config: {
+        type: FIELDS.STRING.type,
+        constraints: FIELDS.STRING.constraints,
+      },
     },
-    {
+    [FIELDS.NUMBER.type]: {
       label: "Number",
       value: FIELDS.NUMBER.type,
+      config: {
+        type: FIELDS.NUMBER.type,
+        constraints: FIELDS.NUMBER.constraints,
+      },
     },
-    {
+    [FIELDS.DATETIME.type]: {
       label: "Date",
       value: FIELDS.DATETIME.type,
+      config: {
+        type: FIELDS.DATETIME.type,
+        constraints: FIELDS.DATETIME.constraints,
+      },
     },
-    {
+    [FIELDS.OPTIONS.type]: {
       label: "Options",
       value: FIELDS.OPTIONS.type,
+      config: {
+        type: FIELDS.OPTIONS.type,
+        constraints: FIELDS.OPTIONS.constraints,
+      },
     },
-    {
+    [FIELDS.ARRAY.type]: {
       label: "Multi-select",
       value: FIELDS.ARRAY.type,
+      config: {
+        type: FIELDS.ARRAY.type,
+        constraints: FIELDS.ARRAY.constraints,
+      },
     },
-    {
+    [FIELDS.BARCODEQR.type]: {
       label: "Barcode/QR",
       value: FIELDS.BARCODEQR.type,
+      config: {
+        type: FIELDS.BARCODEQR.type,
+        constraints: FIELDS.BARCODEQR.constraints,
+      },
     },
-    {
+    [FIELDS.LONGFORM.type]: {
       label: "Long Form Text",
       value: FIELDS.LONGFORM.type,
+      config: {
+        type: FIELDS.LONGFORM.type,
+        constraints: FIELDS.LONGFORM.constraints,
+      },
     },
-  ]
+    user: {
+      label: "User",
+      value: "user",
+      config: {
+        type: FIELDS.USER.type,
+        subtype: FIELDS.USER.subtype,
+        constraints: FIELDS.USER.constraints,
+      },
+    },
+  }
 
   let fileInput
   let error = null
@@ -48,6 +85,7 @@
   let validation = {}
   let validateHash = ""
   let errors = {}
+  let selectedColumnTypes = {}
 
   $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
     return validation[column]
@@ -72,6 +110,13 @@
       rows = response.rows
       schema = response.schema
       fileName = response.fileName
+      selectedColumnTypes = Object.entries(response.schema).reduce(
+        (acc, [colName, fieldConfig]) => ({
+          ...acc,
+          [colName]: fieldConfig.type,
+        }),
+        {}
+      )
     } catch (e) {
       loading = false
       error = e
@@ -98,8 +143,10 @@
   }
 
   const handleChange = (name, e) => {
-    schema[name].type = e.detail
-    schema[name].constraints = FIELDS[e.detail.toUpperCase()].constraints
+    const { config } = typeOptions[e.detail]
+    schema[name].type = config.type
+    schema[name].subtype = config.subtype
+    schema[name].constraints = config.constraints
   }
 
   const openFileUpload = (promptUpload, fileInput) => {
@@ -142,9 +189,9 @@
       <div class="field">
         <span>{column.name}</span>
         <Select
-          bind:value={column.type}
+          bind:value={selectedColumnTypes[column.name]}
           on:change={e => handleChange(name, e)}
-          options={typeOptions}
+          options={Object.values(typeOptions)}
           placeholder={null}
           getOptionLabel={option => option.label}
           getOptionValue={option => option.value}

From 24e793a191639f78dc31cff9bbbfb73bd09ccfd5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Oct 2023 18:02:21 +0200
Subject: [PATCH 02/17] Validate user column

---
 packages/backend-core/src/docIds/ids.ts       |  5 ++
 .../TableNavigator/TableDataImport.svelte     | 28 ++++++++++-
 packages/server/src/utilities/schema.ts       | 50 ++++++++++++++++++-
 3 files changed, 81 insertions(+), 2 deletions(-)

diff --git a/packages/backend-core/src/docIds/ids.ts b/packages/backend-core/src/docIds/ids.ts
index e0ac85b3df9..4c9eb713c85 100644
--- a/packages/backend-core/src/docIds/ids.ts
+++ b/packages/backend-core/src/docIds/ids.ts
@@ -45,6 +45,11 @@ export function generateGlobalUserID(id?: any) {
   return `${DocumentType.USER}${SEPARATOR}${id || newid()}`
 }
 
+const isGlobalUserIDRegex = new RegExp(`^${DocumentType.USER}${SEPARATOR}.+`)
+export function isGlobalUserID(id: string) {
+  return isGlobalUserIDRegex.test(id)
+}
+
 /**
  * Generates a new user ID based on the passed in global ID.
  * @param {string} globalId The ID of the global user.
diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index 6f441350064..82b883c4d4d 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -3,6 +3,7 @@
   import { FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
+  import { FieldType } from "@budibase/types"
 
   export let rows = []
   export let schema = {}
@@ -86,10 +87,35 @@
   let validateHash = ""
   let errors = {}
   let selectedColumnTypes = {}
+  let rawRows = []
 
   $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
     return validation[column]
   })
+
+  $: {
+    rows = []
+
+    for (const row of rawRows) {
+      const castedRow = { ...row }
+      for (const fieldSchema of Object.values(schema || {})) {
+        if (fieldSchema.type === FieldType.BB_REFERENCE) {
+          try {
+            castedRow[fieldSchema.name] = JSON.parse(
+              row[fieldSchema.name].replace(/'/g, '"')
+            )
+          } catch {
+            console.warn("Not a valid json", {
+              field: fieldSchema.name,
+              data: row[fieldSchema.name],
+            })
+          }
+        }
+      }
+      rows.push(castedRow)
+    }
+  }
+
   $: {
     // binding in consumer is causing double renders here
     const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
@@ -107,7 +133,7 @@
 
     try {
       const response = await parseFile(e)
-      rows = response.rows
+      rawRows = response.rows
       schema = response.schema
       fileName = response.fileName
       selectedColumnTypes = Object.entries(response.schema).reduce(
diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts
index 5ced82d8cf4..e66bc97c8f9 100644
--- a/packages/server/src/utilities/schema.ts
+++ b/packages/server/src/utilities/schema.ts
@@ -1,9 +1,12 @@
+import { FieldSubtype } from "@budibase/types"
 import { FieldTypes } from "../constants"
-import { ValidColumnNameRegex } from "@budibase/shared-core"
+import { ValidColumnNameRegex, utils } from "@budibase/shared-core"
+import { db } from "@budibase/backend-core"
 
 interface SchemaColumn {
   readonly name: string
   readonly type: FieldTypes
+  readonly subtype: FieldSubtype
   readonly autocolumn?: boolean
   readonly constraints?: {
     presence: boolean
@@ -77,8 +80,14 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
   rows.forEach(row => {
     Object.entries(row).forEach(([columnName, columnData]) => {
       const columnType = schema[columnName]?.type
+      const columnSubtype = schema[columnName]?.subtype
       const isAutoColumn = schema[columnName]?.autocolumn
 
+      // If the column had an invalid value we don't want to override it
+      if (results.schemaValidation[columnName] === false) {
+        return
+      }
+
       // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
       if (typeof columnType !== "string") {
         results.invalidColumns.push(columnName)
@@ -112,6 +121,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
         isNaN(new Date(columnData).getTime())
       ) {
         results.schemaValidation[columnName] = false
+      } else if (
+        columnType === FieldTypes.BB_REFERENCE &&
+        !isValidBBReference(columnData, columnSubtype)
+      ) {
+        results.schemaValidation[columnName] = false
       } else {
         results.schemaValidation[columnName] = true
       }
@@ -155,3 +169,37 @@ export function parse(rows: Rows, schema: Schema): Rows {
     return parsedRow
   })
 }
+
+function isValidBBReference(
+  columnData: any,
+  columnSubtype: FieldSubtype
+): boolean {
+  switch (columnSubtype) {
+    case FieldSubtype.USER:
+    case FieldSubtype.USERS:
+      if (!columnData) {
+        // Empty columns are valid by default
+        return true
+      }
+
+      if (!Array.isArray(columnData)) {
+        // It must be an array field
+        return false
+      }
+
+      if (columnSubtype === FieldSubtype.USER && columnData.length > 1) {
+        return false
+      }
+
+      for (const d of columnData) {
+        if (!db.isGlobalUserID(d._id)) {
+          return false
+        }
+      }
+
+      return true
+
+    default:
+      throw utils.unreachable(columnSubtype)
+  }
+}

From 9f742db9bdd44abfd03243e12603656188596461 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Oct 2023 18:03:16 +0200
Subject: [PATCH 03/17] Allow users

---
 .../backend/TableNavigator/TableDataImport.svelte        | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index 82b883c4d4d..7bc53ad07bc 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -77,6 +77,15 @@
         constraints: FIELDS.USER.constraints,
       },
     },
+    users: {
+      label: "Users",
+      value: "users",
+      config: {
+        type: FIELDS.USERS.type,
+        subtype: FIELDS.USERS.subtype,
+        constraints: FIELDS.USERS.constraints,
+      },
+    },
   }
 
   let fileInput

From 1b96fd7b9ad7a233f7d763119ae99aef461c2058 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 10:47:43 +0200
Subject: [PATCH 04/17] Expect BB ref data as csv strings

---
 .../server/src/api/controllers/view/exporters.ts    |  4 ++++
 packages/server/src/utilities/schema.ts             | 13 ++++++++++---
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts
index e707cad0cce..b04d139e741 100644
--- a/packages/server/src/api/controllers/view/exporters.ts
+++ b/packages/server/src/api/controllers/view/exporters.ts
@@ -43,3 +43,7 @@ export enum Format {
 export function isFormat(format: any): format is Format {
   return Object.values(Format).includes(format as Format)
 }
+
+export function parseCsvExport(value: string) {
+  return JSON.parse(value?.replace(/'/g, '"'))
+}
diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts
index e66bc97c8f9..52adc792988 100644
--- a/packages/server/src/utilities/schema.ts
+++ b/packages/server/src/utilities/schema.ts
@@ -2,6 +2,7 @@ import { FieldSubtype } from "@budibase/types"
 import { FieldTypes } from "../constants"
 import { ValidColumnNameRegex, utils } from "@budibase/shared-core"
 import { db } from "@budibase/backend-core"
+import { parseCsvExport } from "../api/controllers/view/exporters"
 
 interface SchemaColumn {
   readonly name: string
@@ -182,16 +183,22 @@ function isValidBBReference(
         return true
       }
 
-      if (!Array.isArray(columnData)) {
+      if (typeof columnData !== "string") {
+        return false
+      }
+
+      const castedData = parseCsvExport(columnData)
+
+      if (!Array.isArray(castedData)) {
         // It must be an array field
         return false
       }
 
-      if (columnSubtype === FieldSubtype.USER && columnData.length > 1) {
+      if (columnSubtype === FieldSubtype.USER && castedData.length > 1) {
         return false
       }
 
-      for (const d of columnData) {
+      for (const d of castedData) {
         if (!db.isGlobalUserID(d._id)) {
           return false
         }

From 902b1c99ce1db9512873e8ad2e7240d3454e7125 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 10:47:49 +0200
Subject: [PATCH 05/17] Return only id and primary display

---
 .../server/src/utilities/rowProcessor/bbReferenceProcessor.ts  | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
index c7b8998bad5..c126530b8db 100644
--- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
+++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
@@ -82,9 +82,6 @@ export async function processOutputBBReferences(
       return users.map(u => ({
         _id: u._id,
         primaryDisplay: u.email,
-        email: u.email,
-        firstName: u.firstName,
-        lastName: u.lastName,
       }))
 
     default:

From a0c6141506f3edf3e289a8f458920b652e62fd7f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 10:47:56 +0200
Subject: [PATCH 06/17] Don't cast on the fe

---
 .../TableNavigator/TableDataImport.svelte     | 26 +------------------
 1 file changed, 1 insertion(+), 25 deletions(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index 7bc53ad07bc..a4794c18369 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -96,35 +96,11 @@
   let validateHash = ""
   let errors = {}
   let selectedColumnTypes = {}
-  let rawRows = []
 
   $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
     return validation[column]
   })
 
-  $: {
-    rows = []
-
-    for (const row of rawRows) {
-      const castedRow = { ...row }
-      for (const fieldSchema of Object.values(schema || {})) {
-        if (fieldSchema.type === FieldType.BB_REFERENCE) {
-          try {
-            castedRow[fieldSchema.name] = JSON.parse(
-              row[fieldSchema.name].replace(/'/g, '"')
-            )
-          } catch {
-            console.warn("Not a valid json", {
-              field: fieldSchema.name,
-              data: row[fieldSchema.name],
-            })
-          }
-        }
-      }
-      rows.push(castedRow)
-    }
-  }
-
   $: {
     // binding in consumer is causing double renders here
     const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
@@ -142,7 +118,7 @@
 
     try {
       const response = await parseFile(e)
-      rawRows = response.rows
+      rows = response.rows
       schema = response.schema
       fileName = response.fileName
       selectedColumnTypes = Object.entries(response.schema).reduce(

From 605563a34e9e7be71adb8b4393f1a5178df96bc0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 11:58:45 +0200
Subject: [PATCH 07/17] Enable importing bbrefs as string csv

---
 packages/server/src/api/routes/tests/table.spec.ts | 9 +++++++--
 packages/server/src/utilities/schema.ts            | 2 ++
 packages/types/src/api/web/app/table.ts            | 5 ++++-
 packages/types/src/documents/app/table/table.ts    | 1 -
 4 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index f56c6e4e44c..aabb64b7639 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -1,6 +1,11 @@
 import { generator } from "@budibase/backend-core/tests"
 import { events, context } from "@budibase/backend-core"
-import { FieldType, Table, ViewCalculation } from "@budibase/types"
+import {
+  FieldType,
+  SaveTableRequest,
+  Table,
+  ViewCalculation,
+} from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 const { basicTable } = setup.structures
@@ -47,7 +52,7 @@ describe("/tables", () => {
     })
 
     it("creates a table via data import", async () => {
-      const table = basicTable()
+      const table: SaveTableRequest = basicTable()
       table.rows = [{ name: "test-name", description: "test-desc" }]
 
       const res = await createTable(table)
diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts
index 52adc792988..97e67e92a7e 100644
--- a/packages/server/src/utilities/schema.ts
+++ b/packages/server/src/utilities/schema.ts
@@ -162,6 +162,8 @@ export function parse(rows: Rows, schema: Schema): Rows {
         parsedRow[columnName] = columnData
           ? new Date(columnData).toISOString()
           : columnData
+      } else if (columnType === FieldTypes.BB_REFERENCE) {
+        parsedRow[columnName] = columnData && parseCsvExport(columnData)
       } else {
         parsedRow[columnName] = columnData
       }
diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts
index ff288811c9b..0885dac72a8 100644
--- a/packages/types/src/api/web/app/table.ts
+++ b/packages/types/src/api/web/app/table.ts
@@ -1,4 +1,5 @@
 import {
+  Row,
   Table,
   TableRequest,
   TableSchema,
@@ -18,6 +19,8 @@ export interface TableResponse extends Table {
 
 export type FetchTablesResponse = TableResponse[]
 
-export interface SaveTableRequest extends TableRequest {}
+export interface SaveTableRequest extends TableRequest {
+  rows?: Row[]
+}
 
 export type SaveTableResponse = Table
diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts
index 76b2c587b2d..5174ec608fc 100644
--- a/packages/types/src/documents/app/table/table.ts
+++ b/packages/types/src/documents/app/table/table.ts
@@ -15,7 +15,6 @@ export interface Table extends Document {
   constrained?: string[]
   sql?: boolean
   indexes?: { [key: string]: any }
-  rows?: { [key: string]: any }
   created?: boolean
   rowHeight?: number
 }

From 471462d3befb21f2bdee46e75600f468af33e74c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 12:58:51 +0200
Subject: [PATCH 08/17] Import internal

---
 .../server/src/api/controllers/table/external.ts |  8 ++++----
 .../server/src/api/controllers/table/index.ts    |  3 ++-
 .../server/src/api/controllers/table/internal.ts |  3 ++-
 .../server/src/api/controllers/table/utils.ts    | 16 +++++++++++-----
 packages/types/src/api/web/app/table.ts          |  5 +++++
 5 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts
index 327904666de..8917bcaab1f 100644
--- a/packages/server/src/api/controllers/table/external.ts
+++ b/packages/server/src/api/controllers/table/external.ts
@@ -15,7 +15,7 @@ import { handleRequest } from "../row/external"
 import { context, events } from "@budibase/backend-core"
 import { isRows, isSchema, parse } from "../../../utilities/schema"
 import {
-  AutoReason,
+  BulkImportRequest,
   Datasource,
   FieldSchema,
   Operation,
@@ -374,10 +374,10 @@ export async function destroy(ctx: UserCtx) {
   return tableToDelete
 }
 
-export async function bulkImport(ctx: UserCtx) {
+export async function bulkImport(ctx: UserCtx<BulkImportRequest>) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
-  const { rows }: { rows: unknown } = ctx.request.body
-  const schema: unknown = table.schema
+  const { rows } = ctx.request.body
+  const schema = table.schema
 
   if (!rows || !isRows(rows) || !isSchema(schema)) {
     ctx.throw(400, "Provided data import information is invalid.")
diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index e7c6ae57b07..e1956e33c04 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -8,6 +8,7 @@ import {
 import { isExternalTable, isSQL } from "../../../integrations/utils"
 import { events } from "@budibase/backend-core"
 import {
+  BulkImportRequest,
   FetchTablesResponse,
   SaveTableRequest,
   SaveTableResponse,
@@ -97,7 +98,7 @@ export async function destroy(ctx: UserCtx) {
   builderSocket?.emitTableDeletion(ctx, deletedTable)
 }
 
-export async function bulkImport(ctx: UserCtx) {
+export async function bulkImport(ctx: UserCtx<BulkImportRequest>) {
   const tableId = ctx.params.tableId
   await pickApi({ tableId }).bulkImport(ctx)
   // right now we don't trigger anything for bulk import because it
diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts
index e468848c572..999ea35d373 100644
--- a/packages/server/src/api/controllers/table/internal.ts
+++ b/packages/server/src/api/controllers/table/internal.ts
@@ -10,6 +10,7 @@ import {
 } from "../../../utilities/rowProcessor"
 import { runStaticFormulaChecks } from "./bulkFormula"
 import {
+  BulkImportRequest,
   RenameColumn,
   SaveTableRequest,
   SaveTableResponse,
@@ -206,7 +207,7 @@ export async function destroy(ctx: any) {
   return tableToDelete
 }
 
-export async function bulkImport(ctx: any) {
+export async function bulkImport(ctx: UserCtx<BulkImportRequest>) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
   const { rows, identifierFields } = ctx.request.body
   await handleDataImport(ctx.user, table, rows, identifierFields)
diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index e0d564db2ae..bf64bc32abf 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -20,7 +20,13 @@ import viewTemplate from "../view/viewBuilder"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
 import { events, context } from "@budibase/backend-core"
-import { ContextUser, Datasource, SourceName, Table } from "@budibase/types"
+import {
+  ContextUser,
+  Datasource,
+  Row,
+  SourceName,
+  Table,
+} from "@budibase/types"
 
 export async function clearColumns(table: any, columnNames: any) {
   const db = context.getAppDB()
@@ -144,12 +150,12 @@ export async function importToRows(
 }
 
 export async function handleDataImport(
-  user: any,
-  table: any,
-  rows: any,
+  user: ContextUser,
+  table: Table,
+  rows: Row[],
   identifierFields: Array<string> = []
 ) {
-  const schema: unknown = table.schema
+  const schema = table.schema
 
   if (!rows || !isRows(rows) || !isSchema(schema)) {
     return table
diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts
index 0885dac72a8..8fb0297a9ea 100644
--- a/packages/types/src/api/web/app/table.ts
+++ b/packages/types/src/api/web/app/table.ts
@@ -24,3 +24,8 @@ export interface SaveTableRequest extends TableRequest {
 }
 
 export type SaveTableResponse = Table
+
+export interface BulkImportRequest {
+  rows: Row[]
+  identifierFields?: Array<string>
+}

From b94e10dceada10bcc8edd90f0da966365e841cf4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 12:59:04 +0200
Subject: [PATCH 09/17] Fix bulk delete

---
 packages/server/src/api/controllers/row/internal.ts |  6 +++++-
 packages/server/src/utilities/rowProcessor/index.ts | 12 ++++++++++--
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index f33df4536f2..c838208a3b0 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -156,7 +156,10 @@ export async function destroy(ctx: UserCtx) {
   }
   const table = await sdk.tables.getTable(row.tableId)
   // update the row to include full relationships before deleting them
-  row = await outputProcessing(table, row, { squash: false })
+  row = await outputProcessing(table, row, {
+    squash: false,
+    skipBBReferences: true,
+  })
   // now remove the relationships
   await linkRows.updateLinks({
     eventType: linkRows.EventType.ROW_DELETE,
@@ -190,6 +193,7 @@ export async function bulkDestroy(ctx: UserCtx) {
   // they need to be the full rows (including previous relationships) for automations
   const processedRows = (await outputProcessing(table, rows, {
     squash: false,
+    skipBBReferences: true,
   })) as Row[]
 
   // remove the relationships first
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index d6aa5e682a9..7995fc25455 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -201,9 +201,14 @@ export async function inputProcessing(
 export async function outputProcessing<T extends Row[] | Row>(
   table: Table,
   rows: T,
-  opts: { squash?: boolean; preserveLinks?: boolean } = {
+  opts: {
+    squash?: boolean
+    preserveLinks?: boolean
+    skipBBReferences?: boolean
+  } = {
     squash: true,
     preserveLinks: false,
+    skipBBReferences: false,
   }
 ): Promise<T> {
   let safeRows: Row[]
@@ -230,7 +235,10 @@ export async function outputProcessing<T extends Row[] | Row>(
           attachment.url = objectStore.getAppFileUrl(attachment.key)
         })
       }
-    } else if (column.type == FieldTypes.BB_REFERENCE) {
+    } else if (
+      !opts.skipBBReferences &&
+      column.type == FieldTypes.BB_REFERENCE
+    ) {
       for (let row of enriched) {
         row[property] = await processOutputBBReferences(
           row[property],

From 809243566260012335a023fc60930fcef20afc98 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 13:37:59 +0200
Subject: [PATCH 10/17] Fix importing

---
 .../src/api/controllers/view/exporters.ts      |  4 ++--
 packages/server/src/utilities/schema.ts        | 18 +++++++++++++++++-
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts
index b04d139e741..d6caff60359 100644
--- a/packages/server/src/api/controllers/view/exporters.ts
+++ b/packages/server/src/api/controllers/view/exporters.ts
@@ -44,6 +44,6 @@ export function isFormat(format: any): format is Format {
   return Object.values(Format).includes(format as Format)
 }
 
-export function parseCsvExport(value: string) {
-  return JSON.parse(value?.replace(/'/g, '"'))
+export function parseCsvExport<T>(value: string) {
+  return JSON.parse(value?.replace(/'/g, '"')) as T
 }
diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts
index 97e67e92a7e..79c797ab002 100644
--- a/packages/server/src/utilities/schema.ts
+++ b/packages/server/src/utilities/schema.ts
@@ -153,6 +153,7 @@ export function parse(rows: Rows, schema: Schema): Rows {
       }
 
       const columnType = schema[columnName].type
+      const columnSubtype = schema[columnName].subtype
 
       if (columnType === FieldTypes.NUMBER) {
         // If provided must be a valid number
@@ -163,7 +164,22 @@ export function parse(rows: Rows, schema: Schema): Rows {
           ? new Date(columnData).toISOString()
           : columnData
       } else if (columnType === FieldTypes.BB_REFERENCE) {
-        parsedRow[columnName] = columnData && parseCsvExport(columnData)
+        const parsedValues =
+          !!columnData && parseCsvExport<{ _id: string }[]>(columnData)
+        if (!parsedValues) {
+          parsedRow[columnName] = undefined
+        } else {
+          switch (columnSubtype) {
+            case FieldSubtype.USER:
+              parsedRow[columnName] = parsedValues[0]?._id
+              break
+            case FieldSubtype.USERS:
+              parsedRow[columnName] = parsedValues.map(u => u._id)
+              break
+            default:
+              utils.unreachable(columnSubtype)
+          }
+        }
       } else {
         parsedRow[columnName] = columnData
       }

From d0141d5811cb8c7b6506932ba15fb95162a3d216 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 13:50:16 +0200
Subject: [PATCH 11/17] Display user/users fields

---
 .../TableNavigator/ExistingTableDataImport.svelte     | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
index 6ea2d771e94..43751ad9443 100644
--- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte
@@ -49,6 +49,15 @@
       label: "Long Form Text",
       value: FIELDS.LONGFORM.type,
     },
+
+    {
+      label: "User",
+      value: `${FIELDS.USER.type}${FIELDS.USER.subtype}`,
+    },
+    {
+      label: "Users",
+      value: `${FIELDS.USERS.type}${FIELDS.USERS.subtype}`,
+    },
   ]
 
   $: {
@@ -143,7 +152,7 @@
       <div class="field">
         <span>{name}</span>
         <Select
-          value={schema[name]?.type}
+          value={`${schema[name]?.type}${schema[name]?.subtype || ""}`}
           options={typeOptions}
           placeholder={null}
           getOptionLabel={option => option.label}

From 5108d2277c60f52b817277e9b3babfbac303d678 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 13:52:16 +0200
Subject: [PATCH 12/17] Lint

---
 .../src/components/backend/TableNavigator/TableDataImport.svelte | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index a4794c18369..7294a43dd68 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -3,7 +3,6 @@
   import { FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
-  import { FieldType } from "@budibase/types"
 
   export let rows = []
   export let schema = {}

From 4bcc44954dcddb6f755e55cc6e5ef920270b0273 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 14:07:12 +0200
Subject: [PATCH 13/17] Fix tests

---
 packages/server/src/api/routes/tests/row.spec.ts | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 4dcc9b3c8c8..186326964c0 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1578,9 +1578,6 @@ describe.each([
       () => config.createUser(),
       (row: Row) => ({
         _id: row._id,
-        email: row.email,
-        firstName: row.firstName,
-        lastName: row.lastName,
         primaryDisplay: row.email,
       }),
     ],

From 8588c5275d4253be3dcc5cb3bc786beb3fef63eb Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 14:20:26 +0200
Subject: [PATCH 14/17] Clean

---
 packages/server/src/utilities/schema.ts | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts
index 79c797ab002..4e1b726cfdf 100644
--- a/packages/server/src/utilities/schema.ts
+++ b/packages/server/src/utilities/schema.ts
@@ -196,33 +196,22 @@ function isValidBBReference(
   switch (columnSubtype) {
     case FieldSubtype.USER:
     case FieldSubtype.USERS:
-      if (!columnData) {
-        // Empty columns are valid by default
-        return true
-      }
-
       if (typeof columnData !== "string") {
         return false
       }
-
-      const castedData = parseCsvExport(columnData)
-
-      if (!Array.isArray(castedData)) {
-        // It must be an array field
+      const userArray = parseCsvExport<{ _id: string }[]>(columnData)
+      if (columnSubtype === FieldSubtype.USER && userArray.length > 1) {
         return false
       }
 
-      if (columnSubtype === FieldSubtype.USER && castedData.length > 1) {
+      if (!Array.isArray(columnData)) {
         return false
       }
 
-      for (const d of castedData) {
-        if (!db.isGlobalUserID(d._id)) {
-          return false
-        }
-      }
-
-      return true
+      const constainsWrongId = userArray.find(
+        user => !db.isGlobalUserID(user._id)
+      )
+      return !constainsWrongId
 
     default:
       throw utils.unreachable(columnSubtype)

From 6e0ad038d7de27ffe643d72ff45b4231fe522f5b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 14:39:55 +0200
Subject: [PATCH 15/17] Fix order

---
 packages/server/src/utilities/schema.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts
index 4e1b726cfdf..04c96431834 100644
--- a/packages/server/src/utilities/schema.ts
+++ b/packages/server/src/utilities/schema.ts
@@ -200,11 +200,11 @@ function isValidBBReference(
         return false
       }
       const userArray = parseCsvExport<{ _id: string }[]>(columnData)
-      if (columnSubtype === FieldSubtype.USER && userArray.length > 1) {
+      if (!Array.isArray(columnData)) {
         return false
       }
 
-      if (!Array.isArray(columnData)) {
+      if (columnSubtype === FieldSubtype.USER && userArray.length > 1) {
         return false
       }
 

From ef983a2486531ab706d51c6742ecea419388e422 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 15:35:00 +0200
Subject: [PATCH 16/17] Fix tests

---
 .../rowProcessor/tests/bbReferenceProcessor.spec.ts         | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
index b6174861d4d..10d339f6b4c 100644
--- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
@@ -180,9 +180,6 @@ describe("bbReferenceProcessor", () => {
           {
             _id: user._id,
             primaryDisplay: user.email,
-            email: user.email,
-            firstName: user.firstName,
-            lastName: user.lastName,
           },
         ])
         expect(cacheGetUsersSpy).toBeCalledTimes(1)
@@ -207,9 +204,6 @@ describe("bbReferenceProcessor", () => {
             [user1, user2].map(u => ({
               _id: u._id,
               primaryDisplay: u.email,
-              email: u.email,
-              firstName: u.firstName,
-              lastName: u.lastName,
             }))
           )
         )

From 9a5dd716149c0fdfc0309ba56764765ff0adcfe4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Oct 2023 15:39:05 +0200
Subject: [PATCH 17/17] Fix

---
 packages/server/src/utilities/schema.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts
index 04c96431834..40e4152b632 100644
--- a/packages/server/src/utilities/schema.ts
+++ b/packages/server/src/utilities/schema.ts
@@ -200,7 +200,7 @@ function isValidBBReference(
         return false
       }
       const userArray = parseCsvExport<{ _id: string }[]>(columnData)
-      if (!Array.isArray(columnData)) {
+      if (!Array.isArray(userArray)) {
         return false
       }