feat(server): run tests in multi region db mode

.circleci/config.yml

@@ -16,7 +16,7 @@ workflows:
                 - main
                 - hotfix*
 
-      - test-server:
+      - test-server: &test-server-job-definition
           context:
             - speckle-server-licensing
             - stripe-integration
@@ -32,6 +32,8 @@ workflows:
           requires:
             - docker-publish-postgres-container
 
+      - test-server-multiregion: *test-server-job-definition
+
       - test-frontend-2:
           filters: *filters-allow-all
 
@@ -190,6 +192,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-frontend:
@@ -205,6 +208,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-frontend-2:
@@ -220,6 +224,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-webhooks:
@@ -235,6 +240,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-file-imports:
@@ -250,6 +256,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-previews:
@@ -265,6 +272,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-test-container:
@@ -280,6 +288,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-postgres-container:
@@ -301,6 +310,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-docker-compose-ingress:
@@ -316,6 +326,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - publish-helm-chart:
@@ -356,6 +367,7 @@ workflows:
             - get-version
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-ui-components
             - test-frontend-2
             - test-viewer
@@ -573,6 +585,48 @@ jobs:
       FF_GATEKEEPER_MODULE_ENABLED: 'false'
       FF_BILLING_INTEGRATION_ENABLED: 'false'
 
+  test-server-multiregion:
+    <<: *test-server-job
+    docker:
+      - image: cimg/node:18.19.0
+      - image: cimg/redis:7.2.4
+      - image: 'speckle/speckle-postgres'
+        environment:
+          POSTGRES_DB: speckle2_test
+          POSTGRES_PASSWORD: speckle
+          POSTGRES_USER: speckle
+        command: -c 'max_connections=1000'
+      - image: 'speckle/speckle-postgres'
+        environment:
+          POSTGRES_DB: speckle2_test
+          POSTGRES_PASSWORD: speckle
+          POSTGRES_USER: speckle
+        command: -c 'max_connections=1000' -c 'port=5433'
+      - image: 'minio/minio'
+        command: server /data --console-address ":9001"
+    environment:
+      # Same as test-server:
+      NODE_ENV: test
+      DATABASE_URL: 'postgres://speckle:[email protected]:5432/speckle2_test'
+      PGDATABASE: speckle2_test
+      POSTGRES_MAX_CONNECTIONS_SERVER: 20
+      PGUSER: speckle
+      SESSION_SECRET: 'keyboard cat'
+      STRATEGY_LOCAL: 'true'
+      CANONICAL_URL: 'http://127.0.0.1:3000'
+      S3_ENDPOINT: 'http://127.0.0.1:9000'
+      S3_ACCESS_KEY: 'minioadmin'
+      S3_SECRET_KEY: 'minioadmin'
+      S3_BUCKET: 'speckle-server'
+      S3_CREATE_BUCKET: 'true'
+      REDIS_URL: 'redis://127.0.0.1:6379'
+      S3_REGION: '' # optional, defaults to 'us-east-1'
+      AUTOMATE_ENCRYPTION_KEYS_PATH: 'test/assets/automate/encryptionKeys.json'
+      FF_BILLING_INTEGRATION_ENABLED: 'true'
+      # These are the only 2 different env keys:
+      MULTI_REGION_CONFIG_PATH: '../../.circleci/multiregion.test-ci.json'
+      RUN_TESTS_IN_MULTIREGION_MODE: true
+
   test-frontend-2:
     docker: &docker-node-browsers-image
       - image: cimg/node:18.19.0-browsers

.circleci/multiregion.test-ci.json

@@ -0,0 +1,14 @@
+{
+  "main": {
+    "postgres": {
+      "connectionUri": "postgresql://speckle:[email protected]:5432/speckle2_test"
+    }
+  },
+  "regions": {
+    "region1": {
+      "postgres": {
+        "connectionUri": "postgresql://speckle:[email protected]:5433/speckle2_test"
+      }
+    }
+  }
+}

.gitignore

@@ -74,4 +74,5 @@ redis-data/
 .tshy-build
 
 # Server
-multiregion.json
+multiregion.json
+multiregion.test.json

package.json

@@ -21,6 +21,7 @@
     "dev:docker": "docker compose -f ./docker-compose-deps.yml",
     "dev:docker:up": "docker compose -f ./docker-compose-deps.yml up -d",
     "dev:docker:down": "docker compose -f ./docker-compose-deps.yml down",
+    "dev:docker:restart": "yarn dev:docker:down && yarn dev:docker:up",
     "dev:kind:up": "ctlptl apply --filename ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:down": "ctlptl delete -f ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:helm:up": "yarn dev:kind:up && tilt up --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server",

packages/server/.env-example

@@ -147,3 +147,8 @@ OIDC_CLIENT_SECRET="gLb9IEutYQ0npyvA8iHxPsObY3duGB0w"
 # OTEL_TRACE_URL=""
 # OTEL_TRACE_KEY=""
 # OTEL_TRACE_VALUE=""
+
+############################################################
+# Multi region settings
+############################################################
+MULTI_REGION_CONFIG_PATH="multiregion.json"

packages/server/.env.test-example

@@ -4,4 +4,6 @@
 
 PORT=0
 POSTGRES_URL=postgres://speckle:[email protected]/speckle2_test
-POSTGRES_USER=''
+POSTGRES_USER=''
+MULTI_REGION_CONFIG_PATH="multiregion.test.json"
+#RUN_TESTS_IN_MULTIREGION_MODE=true

packages/server/.mocharc.js

@@ -14,7 +14,7 @@ const ignore = [
 /** @type {import("mocha").MochaOptions} */
 const config = {
   spec: ['modules/**/*.spec.js', 'modules/**/*.spec.ts', 'logging/**/*.spec.js'],
-  require: ['ts-node/register', 'test/hooks.js'],
+  require: ['ts-node/register', 'test/hooks.ts'],
   ...(ignore.length ? { ignore } : {}),
   slow: 0,
   timeout: '150000',

packages/server/modules/cli/commands/db/migrate/latest.ts

@@ -1,13 +1,28 @@
 import knex from '@/db/knex'
 import { logger } from '@/logging/logging'
+import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
+import { mochaHooks } from '@/test/hooks'
 import { CommandModule } from 'yargs'
 
 const command: CommandModule = {
   command: 'latest',
   describe: 'Run all migrations that have not yet been run',
   async handler() {
     logger.info('Running latest migration...')
-    await knex.migrate.latest()
+
+    // In tests we want different logic - just run beforeAll
+    if (isTestEnv()) {
+      // Run before hooks, to properly initialize everything
+      await (mochaHooks.beforeAll as () => Promise<void>)()
+    } else {
+      const regionDbs = await getRegisteredRegionClients()
+      const dbs = [knex, ...Object.values(regionDbs)]
+      for (const db of dbs) {
+        await db.migrate.latest()
+      }
+    }
+
     logger.info('Completed running migration')
   }
 }

packages/server/modules/cli/commands/db/migrate/rollback.ts

@@ -1,13 +1,30 @@
 import knex from '@/db/knex'
 import { logger } from '@/logging/logging'
+import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
+import { mochaHooks, resetPubSubFactory } from '@/test/hooks'
 import { CommandModule } from 'yargs'
 
 const command: CommandModule = {
   command: 'rollback',
   describe: 'Roll back all migrations',
   async handler() {
     logger.info('Rolling back migrations...')
-    await knex.migrate.rollback(undefined, true)
+
+    if (isTestEnv()) {
+      // Run before hooks, to properly initialize everything first
+      await (mochaHooks.beforeAll as () => Promise<void>)()
+    }
+
+    const regionDbs = await getRegisteredRegionClients()
+    const dbs = [knex, ...Object.values(regionDbs)]
+
+    for (const db of dbs) {
+      const resetPubSub = resetPubSubFactory({ db })
+      await resetPubSub()
+      await db.migrate.rollback(undefined, true)
+    }
+
     logger.info('Completed rolling back migrations')
   }
 }

packages/server/modules/comments/tests/comments.spec.js

@@ -1290,7 +1290,7 @@ describe('Comments @comments', () => {
 
       before(async () => {
         // Truncate comments
-        truncateTables([Comments.name])
+        await truncateTables([Comments.name])
 
         // Create a single comment with a blob
         const createCommentResult = await createComment({

packages/server/modules/fileuploads/tests/fileuploads.integration.spec.ts

@@ -142,7 +142,7 @@ describe('FileUploads @fileuploads', () => {
   let existingCanonicalUrl: string
   let existingPort: string
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  let sendRequest: (token: string, query: unknown) => Promise<any>
+  let sendRequest: (token: string, query: string | object) => Promise<any>
   let serverAddress: string
   let serverPort: string