Merge branch 'develop' into feature/creators-count

Budibase · Oct 16, 2023 · 53219a6 · 53219a6
2 parents a855689 + 495a582
commit 53219a6
Show file tree

Hide file tree

Showing 265 changed files with 6,330 additions and 5,686 deletions.
diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
@@ -20,18 +20,12 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
   USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
+  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - name: Maximize build space
-        uses: easimon/maximize-build-space@master
-        with:
-          root-reserve-mb: 35000
-          swap-size-mb: 1024
-          remove-android: 'true'
-          remove-dotnet: 'true'
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
@@ -121,7 +115,43 @@ jobs:
           name: codecov-umbrella
           verbose: true
 
-  test-services:
+  test-worker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo and submodules
+        uses: actions/checkout@v3
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
+      - name: Checkout repo only
+        uses: actions/checkout@v3
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
+
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+          cache: "yarn"
+      - run: yarn --frozen-lockfile
+      - name: Test worker
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/worker
+          fi
+
+      - uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
+          name: codecov-umbrella
+          verbose: true
+
+  test-server:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo and submodules
@@ -143,12 +173,12 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - name: Test worker and server
+      - name: Test server
         run: |
           if ${{ env.USE_NX_AFFECTED }}; then
-            yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+            yarn test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn test --scope=@budibase/worker --scope=@budibase/server
+            yarn test --scope=@budibase/server
           fi
 
       - uses: codecov/codecov-action@v3
@@ -234,18 +264,23 @@ jobs:
 
           if [[ $branch == "master" ]]; then
             base_commit=$(git rev-parse origin/master)
-          else
+          elif [[ $branch == "develop" ]]; then
             base_commit=$(git rev-parse origin/develop)
           fi
 
-          echo "target_branch=$branch"
-          echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
-          echo "pro_commit=$pro_commit"
-          echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
-          echo "base_commit=$base_commit"
-          echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
+          if [[ ! -z $base_commit ]]; then
+            echo "target_branch=$branch"
+            echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
+            echo "pro_commit=$pro_commit"
+            echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
+            echo "base_commit=$base_commit"
+            echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
+          else
+            echo "Nothing to do - branch to branch merge."
+          fi
 
-      - name: Check submodule merged to develop
+      - name: Check submodule merged to base branch
+        if: ${{ steps.get_pro_commits.outputs.base_commit != '' }}
         uses: actions/github-script@v4
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -254,9 +289,9 @@ jobs:
             const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}';
 
             if (submoduleCommit !== baseCommit) {
-              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}"" branch.');
+              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.');
               console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md')
               process.exit(1);
             } else {
               console.log('All good, the submodule had been merged and setup correctly!')
-            }
+            }
diff --git a/.github/workflows/check_unreleased_changes.yml b/.github/workflows/check_unreleased_changes.yml
diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml
@@ -0,0 +1,21 @@
+name: close-featurebranch
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - develop
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_BRANCH: ${{ github.head_ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
+        with:
+          repository: budibase/budibase-deploys
+          event: featurebranch-qa-close
+          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}
diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml
@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - develop
+      - master
 
 jobs:
   release:
@@ -12,7 +13,8 @@ jobs:
       - uses: actions/checkout@v3
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
-            BRANCH: ${{ github.head_ref }}
+          PAYLOAD_BRANCH: ${{ github.head_ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
@@ -138,6 +138,8 @@ To develop the Budibase platform you'll need [Docker](https://www.docker.com/) a
 
 `yarn setup` will check that all necessary components are installed and setup the repo for usage.
 
+If you have access to the `@budibase/pro` submodule then please follow the Pro section of this guide before running the above command.
+
 ##### Manual method
 
 The following commands can be executed to manually get Budibase up and running (assuming Docker/Docker Compose has been installed).
@@ -146,6 +148,8 @@ The following commands can be executed to manually get Budibase up and running (
 
 `yarn build` will build all budibase packages.
 
+If you have access to the `@budibase/pro` submodule then please follow the Pro section of this guide before running the above commands.
+
 #### 4. Running
 
 To run the budibase server and builder in dev mode (i.e. with live reloading):

diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf
@@ -55,7 +55,7 @@ http {
     set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
     set $csp_object "object-src 'none'";
     set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibase.qa https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
+    set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
     set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
     set $csp_frame "frame-src 'self' https:";
     set $csp_img "img-src http: https: data: blob:";

diff --git a/lerna.json b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.12-alpha.3",
+  "version": "2.11.27-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

diff --git a/package.json b/package.json
@@ -5,7 +5,7 @@
     "@esbuild-plugins/tsconfig-paths": "^0.1.2",
     "@nx/js": "16.4.3",
     "@rollup/plugin-json": "^4.0.2",
-    "@typescript-eslint/parser": "5.45.0",
+    "@typescript-eslint/parser": "6.7.2",
     "esbuild": "^0.18.17",
     "esbuild-node-externals": "^1.8.0",
     "eslint": "^8.44.0",
@@ -21,8 +21,8 @@
     "prettier-plugin-svelte": "^2.3.0",
     "rimraf": "^3.0.2",
     "rollup-plugin-replace": "^2.2.0",
-    "svelte": "^3.38.2",
-    "typescript": "4.7.3",
+    "svelte": "3.49.0",
+    "typescript": "5.2.2",
     "@babel/core": "^7.22.5",
     "@babel/eslint-parser": "^7.22.5",
     "@babel/preset-env": "^7.22.5",
@@ -74,7 +74,6 @@
     "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
     "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb",
     "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",
-    "build:docs": "lerna run --stream build:docs",
     "release:helm": "node scripts/releaseHelmChart",
     "env:multi:enable": "lerna run --stream env:multi:enable",
     "env:multi:disable": "lerna run --stream env:multi:disable",

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
@@ -33,17 +33,14 @@
     "bull": "4.10.1",
     "correlation-id": "4.0.0",
     "dotenv": "16.0.1",
-    "emitter-listener": "1.1.2",
     "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.0",
     "koa-passport": "4.1.4",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
-    "lodash.isarguments": "3.1.0",
     "node-fetch": "2.6.7",
     "passport-google-oauth": "2.0.0",
-    "passport-jwt": "4.0.0",
     "passport-local": "1.0.0",
     "passport-oauth2-refresh": "^2.1.0",
     "pino": "8.11.0",
@@ -59,14 +56,13 @@
     "uuid": "8.3.2"
   },
   "devDependencies": {
-    "@jest/test-sequencer": "29.6.2",
     "@shopify/jest-koa-mocks": "5.1.1",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
-    "@types/jest": "29.5.3",
-    "@types/koa": "2.13.4",
+    "@types/cookies": "0.7.8",
+    "@types/jest": "29.5.5",
     "@types/lodash": "4.14.180",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",
@@ -80,14 +76,10 @@
     "jest": "29.6.2",
     "jest-environment-node": "29.6.2",
     "jest-serial-runner": "1.2.1",
-    "koa": "2.13.4",
-    "nodemon": "2.0.16",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "timekeeper": "2.2.0",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
-    "typescript": "4.7.3"
+    "typescript": "5.2.2"
   },
   "nx": {
     "targets": {

diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts
@@ -120,7 +120,7 @@ export async function getUsers(
 ): Promise<{ users: User[]; notFoundIds?: string[] }> {
   const client = await redis.getUserClient()
   // try cache
-  let usersFromCache = await client.bulkGet(userIds)
+  let usersFromCache = await client.bulkGet<User>(userIds)
   const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
   const users = Object.values(usersFromCache)
   let notFoundIds

diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts
@@ -1,5 +1,10 @@
 import { prefixed, DocumentType } from "@budibase/types"
-export { SEPARATOR, UNICODE_MAX, DocumentType } from "@budibase/types"
+export {
+  SEPARATOR,
+  UNICODE_MAX,
+  DocumentType,
+  InternalTable,
+} from "@budibase/types"
 
 /**
  * Can be used to create a few different forms of querying a view.
@@ -18,7 +23,7 @@ export enum ViewName {
   ROUTING = "screen_routes",
   AUTOMATION_LOGS = "automation_logs",
   ACCOUNT_BY_EMAIL = "account_by_email",
-  PLATFORM_USERS_LOWERCASE = "platform_users_lowercase",
+  PLATFORM_USERS_LOWERCASE = "platform_users_lowercase_2",
   USER_BY_GROUP = "user_by_group",
   APP_BACKUP_BY_TRIGGER = "by_trigger",
 }
@@ -30,10 +35,6 @@ export const DeprecatedViews = {
   ],
 }
 
-export enum InternalTable {
-  USER_METADATA = "ta_users",
-}
-
 export const StaticDatabases = {
   GLOBAL: {
     name: "global-db",

diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts
@@ -190,6 +190,10 @@ export const createPlatformUserView = async () => {
     if (doc.tenantId) {
       emit(doc._id.toLowerCase(), doc._id)
     }
+
+    if (doc.ssoId) {
+      emit(doc.ssoId, doc._id)
+    }
   }`
   await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
 }

diff --git a/packages/backend-core/src/docIds/ids.ts b/packages/backend-core/src/docIds/ids.ts
@@ -45,6 +45,11 @@ export function generateGlobalUserID(id?: any) {
   return `${DocumentType.USER}${SEPARATOR}${id || newid()}`
 }
 
+const isGlobalUserIDRegex = new RegExp(`^${DocumentType.USER}${SEPARATOR}.+`)
+export function isGlobalUserID(id: string) {
+  return isGlobalUserIDRegex.test(id)
+}
+
 /**
  * Generates a new user ID based on the passed in global ID.
  * @param {string} globalId The ID of the global user.