Honor table and view schema on query

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -192,10 +192,10 @@ export function buildSqlFieldList(
   function extractRealFields(table: Table, existing: string[] = []) {
     return Object.entries(table.schema)
       .filter(
-        column =>
-          column[1].type !== FieldType.LINK &&
-          column[1].type !== FieldType.FORMULA &&
-          !existing.find((field: string) => field === column[0])
+        ([columnName, column]) =>
+          column.type !== FieldType.LINK &&
+          column.type !== FieldType.FORMULA &&
+          !existing.find((field: string) => field === columnName)
       )
       .map(column => `${table.name}.${column[0]}`)
   }

packages/server/src/api/routes/tests/row.spec.ts

@@ -1664,7 +1664,7 @@ describe.each([
   isInternal &&
     describe("attachments and signatures", () => {
       const coreAttachmentEnrichment = async (
-        schema: any,
+        schema: TableSchema,
         field: string,
         attachmentCfg: string | string[]
       ) => {
@@ -1691,7 +1691,7 @@ describe.each([
 
         await withEnv({ SELF_HOSTED: "true" }, async () => {
           return context.doInAppContext(config.getAppId(), async () => {
-            const enriched: Row[] = await outputProcessing(table, [row])
+            const enriched: Row[] = await outputProcessing(testTable, [row])
             const [targetRow] = enriched
             const attachmentEntries = Array.isArray(targetRow[field])
               ? targetRow[field]

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -30,6 +30,7 @@ import {
   withEnv as withCoreEnv,
   setEnv as setCoreEnv,
 } from "@budibase/backend-core"
+import sdk from "../../../sdk"
 
 describe.each([
   ["lucene", undefined],
@@ -120,6 +121,7 @@ describe.each([
   })
 
   beforeEach(() => {
+    jest.clearAllMocks()
     mocks.licenses.useCloudFree()
   })
 
@@ -1602,6 +1604,28 @@ describe.each([
           })
           expect(response.rows).toHaveLength(0)
         })
+
+      it("queries the row api passing the view fields only", async () => {
+        const searchSpy = jest.spyOn(sdk.rows, "search")
+
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          schema: {
+            id: { visible: true },
+            one: { visible: false },
+          },
+        })
+
+        await config.api.viewV2.search(view.id, { query: {} })
+        expect(searchSpy).toHaveBeenCalledTimes(1)
+
+        expect(searchSpy).toHaveBeenCalledWith(
+          expect.objectContaining({
+            fields: ["id"],
+          })
+        )
+      })
     })
 
     describe("permissions", () => {

packages/server/src/sdk/app/rows/queryUtils.ts

@@ -0,0 +1,102 @@
+import { db } from "@budibase/backend-core"
+import {
+  FieldType,
+  isLogicalSearchOperator,
+  SearchFilters,
+  Table,
+} from "@budibase/types"
+import { cloneDeep } from "lodash/fp"
+import sdk from "../../../sdk"
+
+export const removeInvalidFilters = (
+  filters: SearchFilters,
+  validFields: string[]
+) => {
+  const result = cloneDeep(filters)
+
+  validFields = validFields.map(f => f.toLowerCase())
+  for (const filterKey of Object.keys(result) as (keyof SearchFilters)[]) {
+    if (typeof result[filterKey] !== "object") {
+      continue
+    }
+    if (isLogicalSearchOperator(filterKey)) {
+      const resultingConditions: SearchFilters[] = []
+      for (const condition of result[filterKey].conditions) {
+        const resultingCondition = removeInvalidFilters(condition, validFields)
+        if (Object.keys(resultingCondition).length) {
+          resultingConditions.push(resultingCondition)
+        }
+      }
+      if (resultingConditions.length) {
+        result[filterKey].conditions = resultingConditions
+      } else {
+        delete result[filterKey]
+      }
+      continue
+    }
+
+    const filter = result[filterKey]
+    for (const columnKey of Object.keys(filter)) {
+      const possibleKeys = [columnKey, db.removeKeyNumbering(columnKey)].map(
+        c => c.toLowerCase()
+      )
+      if (!validFields.some(f => possibleKeys.includes(f.toLowerCase()))) {
+        delete filter[columnKey]
+      }
+    }
+    if (!Object.keys(filter).length) {
+      delete result[filterKey]
+    }
+  }
+
+  return result
+}
+
+export const getQueryableFields = async (
+  fields: string[],
+  table: Table
+): Promise<string[]> => {
+  const extractTableFields = async (
+    table: Table,
+    allowedFields: string[],
+    fromTables: string[]
+  ): Promise<string[]> => {
+    const result = []
+    for (const field of Object.keys(table.schema).filter(
+      f => allowedFields.includes(f) && table.schema[f].visible !== false
+    )) {
+      const subSchema = table.schema[field]
+      if (subSchema.type === FieldType.LINK) {
+        if (fromTables.includes(subSchema.tableId)) {
+          // avoid circular loops
+          continue
+        }
+        const relatedTable = await sdk.tables.getTable(subSchema.tableId)
+        const relatedFields = await extractTableFields(
+          relatedTable,
+          Object.keys(relatedTable.schema),
+          [...fromTables, subSchema.tableId]
+        )
+
+        result.push(
+          ...relatedFields.flatMap(f => [
+            `${subSchema.name}.${f}`,
+            // should be able to filter by relationship using table name
+            `${relatedTable.name}.${f}`,
+          ])
+        )
+      } else {
+        result.push(field)
+      }
+    }
+    return result
+  }
+
+  const result = [
+    "_id", // Querying by _id is always allowed, even if it's never part of the schema
+  ]
+
+  result.push(...(await extractTableFields(table, fields, [table._id!])))
+
+  return result
+}

packages/server/src/sdk/app/rows/search.ts

@@ -14,6 +14,7 @@ import sdk from "../../index"
 import { searchInputMapping } from "./search/utils"
 import { db as dbCore } from "@budibase/backend-core"
 import tracer from "dd-trace"
+import { getQueryableFields, removeInvalidFilters } from "./queryUtils"
 
 export { isValidFilter } from "../../../integrations/utils"
 
@@ -73,6 +74,18 @@ export async function search(
     const table = await sdk.tables.getTable(options.tableId)
     options = searchInputMapping(table, options)
 
+    if (options.query) {
+      const tableFields = Object.keys(table.schema).filter(
+        f => table.schema[f].visible !== false
+      )
+
+      const queriableFields = await getQueryableFields(
+        options.fields?.filter(f => tableFields.includes(f)) ?? tableFields,
+        table
+      )
+      options.query = removeInvalidFilters(options.query, queriableFields)
+    }
+
     let result: SearchResponse<Row>
     if (isExternalTable) {
       span?.addTags({ searchType: "external" })

packages/server/src/sdk/app/rows/search/tests/search.spec.ts

@@ -1,4 +1,11 @@
-import { Datasource, FieldType, Row, Table } from "@budibase/types"
+import {
+  AutoColumnFieldMetadata,
+  AutoFieldSubType,
+  Datasource,
+  FieldType,
+  NumberFieldMetadata,
+  Table,
+} from "@budibase/types"
 
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
 import { search } from "../../../../../sdk/app/rows/search"
@@ -32,7 +39,6 @@ describe.each([
   let envCleanup: (() => void) | undefined
   let datasource: Datasource | undefined
   let table: Table
-  let rows: Row[]
 
   beforeAll(async () => {
     await withCoreEnv({ SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, () =>
@@ -51,16 +57,28 @@ describe.each([
         datasource: await dsProvider,
       })
     }
+  })
+
+  beforeEach(async () => {
+    const idFieldSchema: NumberFieldMetadata | AutoColumnFieldMetadata =
+      isInternal
+        ? {
+            name: "id",
+            type: FieldType.AUTO,
+            subtype: AutoFieldSubType.AUTO_ID,
+            autocolumn: true,
+          }
+        : {
+            name: "id",
+            type: FieldType.NUMBER,
+            autocolumn: true,
+          }
 
     table = await config.api.table.save(
       tableForDatasource(datasource, {
         primary: ["id"],
         schema: {
-          id: {
-            name: "id",
-            type: FieldType.NUMBER,
-            autocolumn: true,
-          },
+          id: idFieldSchema,
           name: {
             name: "name",
             type: FieldType.STRING,
@@ -81,16 +99,13 @@ describe.each([
       })
     )
 
-    rows = []
     for (let i = 0; i < 10; i++) {
-      rows.push(
-        await config.api.row.save(table._id!, {
-          name: generator.first(),
-          surname: generator.last(),
-          age: generator.age(),
-          address: generator.address(),
-        })
-      )
+      await config.api.row.save(table._id!, {
+        name: generator.first(),
+        surname: generator.last(),
+        age: generator.age(),
+        address: generator.address(),
+      })
     }
   })
 
@@ -138,4 +153,100 @@ describe.each([
         )
       })
     })
+
+  it("does not allow accessing hidden fields", async () => {
+    await config.doInContext(config.appId, async () => {
+      await config.api.table.save({
+        ...table,
+        schema: {
+          ...table.schema,
+          name: {
+            ...table.schema.name,
+            visible: true,
+          },
+          age: {
+            ...table.schema.age,
+            visible: false,
+          },
+        },
+      })
+      const result = await search({
+        tableId: table._id!,
+        query: {},
+      })
+      expect(result.rows).toHaveLength(10)
+      for (const row of result.rows) {
+        const keys = Object.keys(row)
+        expect(keys).toContain("name")
+        expect(keys).toContain("surname")
+        expect(keys).toContain("address")
+        expect(keys).not.toContain("age")
+      }
+    })
+  })
+
+  it("does not allow accessing hidden fields even if requested", async () => {
+    await config.doInContext(config.appId, async () => {
+      await config.api.table.save({
+        ...table,
+        schema: {
+          ...table.schema,
+          name: {
+            ...table.schema.name,
+            visible: true,
+          },
+          age: {
+            ...table.schema.age,
+            visible: false,
+          },
+        },
+      })
+      const result = await search({
+        tableId: table._id!,
+        query: {},
+        fields: ["name", "age"],
+      })
+      expect(result.rows).toHaveLength(10)
+      for (const row of result.rows) {
+        const keys = Object.keys(row)
+        expect(keys).toContain("name")
+        expect(keys).not.toContain("age")
+        expect(keys).not.toContain("surname")
+        expect(keys).not.toContain("address")
+      }
+    })
+  })
+
+  !isLucene &&
+    it.each([
+      [["id", "name", "age"], 3],
+      [["name", "age"], 10],
+    ])(
+      "cannot query by non search fields (fields: %s)",
+      async (queryFields, expectedRows) => {
+        await config.doInContext(config.appId, async () => {
+          const { rows } = await search({
+            tableId: table._id!,
+            query: {
+              $or: {
+                conditions: [
+                  {
+                    $and: {
+                      conditions: [
+                        { range: { id: { low: 2, high: 4 } } },
+                        { range: { id: { low: 3, high: 5 } } },
+                      ],
+                    },
+                  },
+                  { equal: { id: 7 } },
+                ],
+              },
+            },
+            fields: queryFields,
+          })
+
+          expect(rows).toHaveLength(expectedRows)
+        })
+      }
+    )
 })