From 9733ba5f95eb49c55be6dbbc36020b74d6d0e505 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 7 Aug 2024 18:04:07 +0100
Subject: [PATCH] Allowing incorrectly setup column schemas to still function
 as part of search - requires further investigation as to how this happens,
 but search should still work.

---
 .../src/api/controllers/row/utils/basic.ts    |  3 +-
 .../src/api/routes/tests/search.spec.ts       | 29 +++++++++++++++++++
 .../server/src/integrations/utils/utils.ts    |  7 +++--
 .../src/sdk/app/rows/search/internal/sqs.ts   |  5 ++--
 4 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts
index 883ba5a8065..f28f6504229 100644
--- a/packages/server/src/api/controllers/row/utils/basic.ts
+++ b/packages/server/src/api/controllers/row/utils/basic.ts
@@ -71,8 +71,7 @@ export function basicProcessing({
 }): Row {
   const thisRow: Row = {}
   // filter the row down to what is actually the row (not joined)
-  for (let field of Object.values(table.schema)) {
-    const fieldName = field.name
+  for (let fieldName of Object.keys(table.schema)) {
     let value = extractFieldValue({
       row,
       tableName: table.name,
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index d8c2a4a257a..6967f7bd6e5 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2353,6 +2353,35 @@ describe.each([
       })
     })
 
+  describe("Invalid column definitions", () => {
+    beforeAll(async () => {
+      // need to create an invalid table - means ignoring typescript
+      table = await createTable({
+        // @ts-ignore
+        invalid: {
+          type: FieldType.STRING,
+        },
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+        },
+      })
+      await createRows([
+        { name: "foo", invalid: "id1" },
+        { name: "bar", invalid: "id2" },
+      ])
+    })
+
+    it("can get rows with all table data", async () => {
+      await expectSearch({
+        query: {},
+      }).toContain([
+        { name: "foo", invalid: "id1" },
+        { name: "bar", invalid: "id2" },
+      ])
+    })
+  })
+
   describe.each(["data_name_test", "name_data_test", "name_test_data_"])(
     "special (%s) case",
     column => {
diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts
index 84742626c1c..6d0e4b838f0 100644
--- a/packages/server/src/integrations/utils/utils.ts
+++ b/packages/server/src/integrations/utils/utils.ts
@@ -15,6 +15,7 @@ import { helpers, utils } from "@budibase/shared-core"
 import { pipeline } from "stream/promises"
 import tmp from "tmp"
 import fs from "fs"
+import { merge } from "lodash"
 
 type PrimitiveTypes =
   | FieldType.STRING
@@ -291,10 +292,12 @@ function copyExistingPropsOver(
         const fetchedColumnDefinition: FieldSchema | undefined =
           table.schema[key]
         table.schema[key] = {
-          ...existingTableSchema[key],
+          // merge the properties - anything missing will be filled in, old definition preferred
+          ...merge(fetchedColumnDefinition, existingTableSchema[key]),
+          // always take externalType and autocolumn from the fetched definition
           externalType:
             existingTableSchema[key].externalType ||
-            table.schema[key]?.externalType,
+            fetchedColumnDefinition?.externalType,
           autocolumn: fetchedColumnDefinition?.autocolumn,
         } as FieldSchema
         // check constraints which can be fetched from the DB (they could be updated)
diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index 66ec905c619..4bfa8f8fa52 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -73,13 +73,14 @@ function buildInternalFieldList(
   fieldList = fieldList.concat(
     PROTECTED_INTERNAL_COLUMNS.map(col => `${table._id}.${col}`)
   )
-  for (let col of Object.values(table.schema)) {
+  for (let key of Object.keys(table.schema)) {
+    const col = table.schema[key]
     const isRelationship = col.type === FieldType.LINK
     if (!opts?.relationships && isRelationship) {
       continue
     }
     if (!isRelationship) {
-      fieldList.push(`${table._id}.${mapToUserColumn(col.name)}`)
+      fieldList.push(`${table._id}.${mapToUserColumn(key)}`)
     } else {
       const linkCol = col as RelationshipFieldMetadata
       const relatedTable = tables.find(table => table._id === linkCol.tableId)