From ee16b083ba59951b8a3fb47931de07f5955adf80 Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Fri, 8 Nov 2024 18:03:13 +0200
Subject: [PATCH 01/12] feat(server): run tests in multiregion mode if
 RUN_TESTS_IN_MULTIREGION_MODE=true

---
 .gitignore                                    |   3 +-
 package.json                                  |   1 +
 packages/server/.env-example                  |   5 +
 packages/server/.env.test-example             |   4 +-
 packages/server/.mocharc.js                   |   2 +-
 .../modules/cli/commands/db/migrate/latest.ts |  17 +-
 .../cli/commands/db/migrate/rollback.ts       |  19 +-
 .../server/modules/multiregion/dbSelector.ts  |  63 +++--
 packages/server/modules/multiregion/index.ts  |  11 +-
 .../modules/multiregion/regionConfig.ts       |   3 +-
 .../modules/shared/helpers/envHelper.ts       |   3 +
 packages/server/multiregion.test.example.json |  16 ++
 packages/server/package.json                  |   2 +
 packages/server/test/hooks.js                 | 119 --------
 packages/server/test/hooks.ts                 | 256 ++++++++++++++++++
 15 files changed, 374 insertions(+), 150 deletions(-)
 create mode 100644 packages/server/multiregion.test.example.json
 delete mode 100644 packages/server/test/hooks.js
 create mode 100644 packages/server/test/hooks.ts

diff --git a/.gitignore b/.gitignore
index c090a9b77a..a1dfce7e9a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -74,4 +74,5 @@ redis-data/
 .tshy-build
 
 # Server
-multiregion.json
\ No newline at end of file
+multiregion.json
+multiregion.test.json
\ No newline at end of file
diff --git a/package.json b/package.json
index f70ce271db..911f0a3344 100644
--- a/package.json
+++ b/package.json
@@ -21,6 +21,7 @@
     "dev:docker": "docker compose -f ./docker-compose-deps.yml",
     "dev:docker:up": "docker compose -f ./docker-compose-deps.yml up -d",
     "dev:docker:down": "docker compose -f ./docker-compose-deps.yml down",
+    "dev:docker:restart": "yarn dev:docker:down && yarn dev:docker:up",
     "dev:kind:up": "ctlptl apply --filename ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:down": "ctlptl delete -f ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:helm:up": "yarn dev:kind:up && tilt up --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server",
diff --git a/packages/server/.env-example b/packages/server/.env-example
index f6104277aa..2982b08b9e 100644
--- a/packages/server/.env-example
+++ b/packages/server/.env-example
@@ -147,3 +147,8 @@ OIDC_CLIENT_SECRET="gLb9IEutYQ0npyvA8iHxPsObY3duGB0w"
 # OTEL_TRACE_URL=""
 # OTEL_TRACE_KEY=""
 # OTEL_TRACE_VALUE=""
+
+############################################################
+# Multi region settings
+############################################################
+MULTI_REGION_CONFIG_PATH="multiregion.json"
\ No newline at end of file
diff --git a/packages/server/.env.test-example b/packages/server/.env.test-example
index 0662ea0b7f..47dd87c134 100644
--- a/packages/server/.env.test-example
+++ b/packages/server/.env.test-example
@@ -4,4 +4,6 @@
 
 PORT=0
 POSTGRES_URL=postgres://speckle:speckle@127.0.0.1/speckle2_test
-POSTGRES_USER=''
\ No newline at end of file
+POSTGRES_USER=''
+MULTI_REGION_CONFIG_PATH="multiregion.test.json"
+#RUN_TESTS_IN_MULTIREGION_MODE=true
\ No newline at end of file
diff --git a/packages/server/.mocharc.js b/packages/server/.mocharc.js
index 9ca96c989c..5ce9d9f539 100644
--- a/packages/server/.mocharc.js
+++ b/packages/server/.mocharc.js
@@ -14,7 +14,7 @@ const ignore = [
 /** @type {import("mocha").MochaOptions} */
 const config = {
   spec: ['modules/**/*.spec.js', 'modules/**/*.spec.ts', 'logging/**/*.spec.js'],
-  require: ['ts-node/register', 'test/hooks.js'],
+  require: ['ts-node/register', 'test/hooks.ts'],
   ...(ignore.length ? { ignore } : {}),
   slow: 0,
   timeout: '150000',
diff --git a/packages/server/modules/cli/commands/db/migrate/latest.ts b/packages/server/modules/cli/commands/db/migrate/latest.ts
index 5b73ea855d..2686fd53ce 100644
--- a/packages/server/modules/cli/commands/db/migrate/latest.ts
+++ b/packages/server/modules/cli/commands/db/migrate/latest.ts
@@ -1,5 +1,8 @@
 import knex from '@/db/knex'
 import { logger } from '@/logging/logging'
+import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
+import { mochaHooks } from '@/test/hooks'
 import { CommandModule } from 'yargs'
 
 const command: CommandModule = {
@@ -7,7 +10,19 @@ const command: CommandModule = {
   describe: 'Run all migrations that have not yet been run',
   async handler() {
     logger.info('Running latest migration...')
-    await knex.migrate.latest()
+
+    // In tests we want different logic - just run beforeAll
+    if (isTestEnv()) {
+      // Run before hooks, to properly initialize everything
+      await (mochaHooks.beforeAll as () => Promise<void>)()
+    } else {
+      const regionDbs = await getRegisteredRegionClients()
+      const dbs = [knex, ...Object.values(regionDbs)]
+      for (const db of dbs) {
+        await db.migrate.latest()
+      }
+    }
+
     logger.info('Completed running migration')
   }
 }
diff --git a/packages/server/modules/cli/commands/db/migrate/rollback.ts b/packages/server/modules/cli/commands/db/migrate/rollback.ts
index 9b5563bf55..4d86c0ddcd 100644
--- a/packages/server/modules/cli/commands/db/migrate/rollback.ts
+++ b/packages/server/modules/cli/commands/db/migrate/rollback.ts
@@ -1,5 +1,8 @@
 import knex from '@/db/knex'
 import { logger } from '@/logging/logging'
+import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
+import { mochaHooks, resetPubSubFactory } from '@/test/hooks'
 import { CommandModule } from 'yargs'
 
 const command: CommandModule = {
@@ -7,7 +10,21 @@ const command: CommandModule = {
   describe: 'Roll back all migrations',
   async handler() {
     logger.info('Rolling back migrations...')
-    await knex.migrate.rollback(undefined, true)
+
+    if (isTestEnv()) {
+      // Run before hooks, to properly initialize everything first
+      await (mochaHooks.beforeAll as () => Promise<void>)()
+    }
+
+    const regionDbs = await getRegisteredRegionClients()
+    const dbs = [knex, ...Object.values(regionDbs)]
+
+    for (const db of dbs) {
+      const resetPubSub = resetPubSubFactory({ db })
+      await resetPubSub()
+      await db.migrate.rollback(undefined, true)
+    }
+
     logger.info('Completed rolling back migrations')
   }
 }
diff --git a/packages/server/modules/multiregion/dbSelector.ts b/packages/server/modules/multiregion/dbSelector.ts
index 32f8c122ae..84dcb7fa96 100644
--- a/packages/server/modules/multiregion/dbSelector.ts
+++ b/packages/server/modules/multiregion/dbSelector.ts
@@ -24,9 +24,16 @@ import {
 } from '@/modules/multiregion/regionConfig'
 import { RegionServerConfig } from '@/modules/multiregion/domain/types'
 import { MaybeNullOrUndefined } from '@speckle/shared'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
 
 let getter: GetProjectDb | undefined = undefined
 
+/**
+ * All dbs share the list of pubs/subs, so we need to make sure the test db uses their own.
+ * As long as there's only 1 test db per instance, it should be fine
+ */
+const createPubSubName = (name: string): string => (isTestEnv() ? `test_${name}` : name)
+
 export const getRegionDb: GetRegionDb = async ({ regionKey }) => {
   const getRegion = getRegionFactory({ db })
   const regionClients = await getRegisteredRegionClients()
@@ -87,11 +94,16 @@ export const getProjectDbClient: GetProjectDb = async ({ projectId }) => {
 type RegionClients = Record<string, Knex>
 let registeredRegionClients: RegionClients | undefined = undefined
 
-const initializeRegisteredRegionClients = async (): Promise<RegionClients> => {
+/**
+ * Idempotently initialize registered region (in db) Knex clients
+ */
+export const initializeRegisteredRegionClients = async (): Promise<RegionClients> => {
   const configuredRegions = await getRegionsFactory({ db })()
-  const regionConfigs = await getAvailableRegionConfig()
+  if (!configuredRegions.length) return {}
 
-  return Object.fromEntries(
+  // init knex clients
+  const regionConfigs = await getAvailableRegionConfig()
+  const ret = Object.fromEntries(
     configuredRegions.map((region) => {
       if (!(region.key in regionConfigs))
         throw new MisconfiguredEnvironmentError(
@@ -100,6 +112,17 @@ const initializeRegisteredRegionClients = async (): Promise<RegionClients> => {
       return [region.key, configureKnexClient(regionConfigs[region.key]).public]
     })
   )
+
+  // run migrations
+  await Promise.all(Object.values(ret).map((db) => db.migrate.latest()))
+
+  // (re-)set up pub-sub, if needed
+  await Promise.all(
+    Object.keys(ret).map((regionKey) => initializeRegion({ regionKey }))
+  )
+
+  registeredRegionClients = ret
+  return ret
 }
 
 const configureKnexClient = (
@@ -126,11 +149,10 @@ export const getRegisteredRegionClients = async (): Promise<RegionClients> => {
   return registeredRegionClients
 }
 
+/**
+ * Idempotently initialize region
+ */
 export const initializeRegion: InitializeRegion = async ({ regionKey }) => {
-  const knownClients = await getRegisteredRegionClients()
-  if (regionKey in knownClients)
-    throw new Error(`Region ${regionKey} is already initialized`)
-
   const regionConfigs = await getAvailableRegionConfig()
   if (!(regionKey in regionConfigs))
     throw new Error(`RegionKey ${regionKey} not available in config`)
@@ -138,7 +160,6 @@ export const initializeRegion: InitializeRegion = async ({ regionKey }) => {
   const newRegionConfig = regionConfigs[regionKey]
   const regionDb = configureKnexClient(newRegionConfig)
   await regionDb.public.migrate.latest()
-  // TODO, set up pub-sub shit
 
   const mainDbConfig = await getMainRegionConfig()
   const mainDb = configureKnexClient(mainDbConfig)
@@ -158,8 +179,12 @@ export const initializeRegion: InitializeRegion = async ({ regionKey }) => {
     regionName: regionKey,
     sslmode
   })
-  // pushing to the singleton object here
-  knownClients[regionKey] = regionDb.public
+
+  // pushing to the singleton object here, its only not available
+  // if this is being triggered from init, and in that case its gonna be set after anyway
+  if (registeredRegionClients) {
+    registeredRegionClients[regionKey] = regionDb.public
+  }
 }
 
 interface ReplicationArgs {
@@ -175,9 +200,11 @@ const setUpUserReplication = async ({
   sslmode,
   regionName
 }: ReplicationArgs): Promise<void> => {
-  // TODO: ensure its created...
+  const subName = createPubSubName(`userssub_${regionName}`)
+  const pubName = createPubSubName('userspub')
+
   try {
-    await from.public.raw('CREATE PUBLICATION userspub FOR TABLE users;')
+    await from.public.raw(`CREATE PUBLICATION ${pubName} FOR TABLE users;`)
   } catch (err) {
     if (!(err instanceof Error)) throw err
     if (!err.message.includes('already exists')) throw err
@@ -190,11 +217,10 @@ const setUpUserReplication = async ({
   )
   const port = fromUrl.port ? fromUrl.port : '5432'
   const fromDbName = fromUrl.pathname.replace('/', '')
-  const subName = `userssub_${regionName}`
   const rawSqeel = `SELECT * FROM aiven_extras.pg_create_subscription(
     '${subName}',
     'dbname=${fromDbName} host=${fromUrl.hostname} port=${port} sslmode=${sslmode} user=${fromUrl.username} password=${fromUrl.password}',
-    'userspub', 
+    '${pubName}', 
     '${subName}',
     TRUE,
     TRUE
@@ -214,9 +240,11 @@ const setUpProjectReplication = async ({
   regionName,
   sslmode
 }: ReplicationArgs): Promise<void> => {
-  // TODO: ensure its created...
+  const subName = createPubSubName(`projectsub_${regionName}`)
+  const pubName = createPubSubName('projectpub')
+
   try {
-    await from.public.raw('CREATE PUBLICATION projectpub FOR TABLE streams;')
+    await from.public.raw(`CREATE PUBLICATION ${pubName} FOR TABLE streams;`)
   } catch (err) {
     if (!(err instanceof Error)) throw err
     if (!err.message.includes('already exists')) throw err
@@ -229,11 +257,10 @@ const setUpProjectReplication = async ({
   )
   const port = fromUrl.port ? fromUrl.port : '5432'
   const fromDbName = fromUrl.pathname.replace('/', '')
-  const subName = `projectsub_${regionName}`
   const rawSqeel = `SELECT * FROM aiven_extras.pg_create_subscription(
     '${subName}',
     'dbname=${fromDbName} host=${fromUrl.hostname} port=${port} sslmode=${sslmode} user=${fromUrl.username} password=${fromUrl.password}',
-    'projectpub', 
+    '${pubName}', 
     '${subName}',
     TRUE,
     TRUE
diff --git a/packages/server/modules/multiregion/index.ts b/packages/server/modules/multiregion/index.ts
index f5957245d2..ca3d2c7ff5 100644
--- a/packages/server/modules/multiregion/index.ts
+++ b/packages/server/modules/multiregion/index.ts
@@ -1,5 +1,5 @@
 import { moduleLogger } from '@/logging/logging'
-import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { initializeRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
 import { isMultiRegionEnabled } from '@/modules/multiregion/helpers'
 import { SpeckleModule } from '@/modules/shared/helpers/typeHelper'
 
@@ -11,12 +11,9 @@ const multiRegion: SpeckleModule = {
     }
 
     moduleLogger.info('🌍 Init multiRegion module')
-    // this should have all the builtin checks to make sure all regions are working
-    // and no regions are missing
-    const regionClients = await getRegisteredRegionClients()
-    moduleLogger.info('Migrating region databases')
-    await Promise.all(Object.values(regionClients).map((db) => db.migrate.latest()))
-    moduleLogger.info('Migrations done')
+
+    // Init registered region clients
+    await initializeRegisteredRegionClients()
   }
 }
 
diff --git a/packages/server/modules/multiregion/regionConfig.ts b/packages/server/modules/multiregion/regionConfig.ts
index 2b8e250d29..8d08942eac 100644
--- a/packages/server/modules/multiregion/regionConfig.ts
+++ b/packages/server/modules/multiregion/regionConfig.ts
@@ -17,9 +17,10 @@ import { isMultiRegionEnabled } from '@/modules/multiregion/helpers'
 let multiRegionConfig: Optional<AllRegionsConfig> = undefined
 
 const getAllRegionsConfig = async (): Promise<AllRegionsConfig> => {
-  if (isDevOrTestEnv() && !isMultiRegionEnabled())
+  if (isDevOrTestEnv() && !isMultiRegionEnabled()) {
     // this should throw somehow
     return { main: { postgres: { connectionUri: '' } }, regions: {} }
+  }
   if (multiRegionConfig) return multiRegionConfig
 
   const relativePath = getMultiRegionConfigPath()
diff --git a/packages/server/modules/shared/helpers/envHelper.ts b/packages/server/modules/shared/helpers/envHelper.ts
index b08683e024..fc220a8135 100644
--- a/packages/server/modules/shared/helpers/envHelper.ts
+++ b/packages/server/modules/shared/helpers/envHelper.ts
@@ -418,3 +418,6 @@ export function getOtelHeaderValue() {
 export function getMultiRegionConfigPath() {
   return getStringFromEnv('MULTI_REGION_CONFIG_PATH')
 }
+
+export const shouldRunTestsInMultiregionMode = () =>
+  getBooleanFromEnv('RUN_TESTS_IN_MULTIREGION_MODE')
diff --git a/packages/server/multiregion.test.example.json b/packages/server/multiregion.test.example.json
new file mode 100644
index 0000000000..d3693edd9f
--- /dev/null
+++ b/packages/server/multiregion.test.example.json
@@ -0,0 +1,16 @@
+{
+  "main": {
+    "postgres": {
+      "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5432/speckle2_test",
+      "privateConnectionUri": "postgresql://speckle:speckle@postgres:5432/speckle2_test"
+    }
+  },
+  "regions": {
+    "region1": {
+      "postgres": {
+        "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5401/speckle2_test",
+        "privateConnectionUri": "postgresql://speckle:speckle@postgres-region1:5432/speckle2_test"
+      }
+    }
+  }
+}
diff --git a/packages/server/package.json b/packages/server/package.json
index 60cf07f84b..cd0c38ffdf 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -23,6 +23,7 @@
     "dev:clean": "yarn build:clean && yarn dev",
     "dev:server:test": "cross-env DISABLE_NOTIFICATIONS_CONSUMPTION=true NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true node ./bin/ts-www",
     "test": "cross-env NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true mocha",
+    "test:multiregion": "cross-env RUN_TESTS_IN_MULTIREGION_MODE=true yarn test",
     "test:coverage": "cross-env NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true nyc --reporter lcov mocha",
     "test:report": "yarn test:coverage -- --reporter mocha-junit-reporter --reporter-options mochaFile=reports/test-results.xml",
     "lint": "yarn lint:tsc && yarn lint:eslint",
@@ -30,6 +31,7 @@
     "lint:tsc": "tsc --noEmit",
     "lint:eslint": "eslint .",
     "cli": "cross-env LOG_LEVEL=debug LOG_PRETTY=true NODE_ENV=development ts-node ./modules/cli/index.ts",
+    "cli:test": "cross-env LOG_LEVEL=debug LOG_PRETTY=true NODE_ENV=test ts-node ./modules/cli/index.ts",
     "cli:download:commit": "cross-env LOG_PRETTY=true LOG_LEVEL=debug yarn cli download commit",
     "migrate": "yarn cli db migrate",
     "migrate:test": "cross-env NODE_ENV=test ts-node ./modules/cli/index.js db migrate",
diff --git a/packages/server/test/hooks.js b/packages/server/test/hooks.js
deleted file mode 100644
index 9ac7bda20e..0000000000
--- a/packages/server/test/hooks.js
+++ /dev/null
@@ -1,119 +0,0 @@
-require('../bootstrap')
-
-// Register global mocks as early as possible
-require('@/test/mocks/global')
-
-const chai = require('chai')
-const chaiAsPromised = require('chai-as-promised')
-const chaiHttp = require('chai-http')
-const deepEqualInAnyOrder = require('deep-equal-in-any-order')
-const { knex } = require(`@/db/knex`)
-const { init, startHttp, shutdown } = require(`@/app`)
-const { default: graphqlChaiPlugin } = require('@/test/plugins/graphql')
-const { logger } = require('@/logging/logging')
-const { once } = require('events')
-
-// Register chai plugins
-chai.use(chaiAsPromised)
-chai.use(chaiHttp)
-chai.use(deepEqualInAnyOrder)
-chai.use(graphqlChaiPlugin)
-
-const unlock = async () => {
-  const exists = await knex.schema.hasTable('knex_migrations_lock')
-  if (exists) {
-    await knex('knex_migrations_lock').update('is_locked', '0')
-  }
-}
-
-exports.truncateTables = async (tableNames) => {
-  if (!tableNames?.length) {
-    //why is server config only created once!????
-    // because its done in a migration, to not override existing configs
-    const protectedTables = ['server_config']
-    // const protectedTables = [ 'server_config', 'user_roles', 'scopes', 'server_acl' ]
-    tableNames = (
-      await knex('pg_tables')
-        .select('tablename')
-        .where({ schemaname: 'public' })
-        .whereRaw("tablename not like '%knex%'")
-        .whereNotIn('tablename', protectedTables)
-    ).map((table) => table.tablename)
-    if (!tableNames.length) return // Nothing to truncate
-
-    // We're deleting everything, so lets turn off triggers to avoid deadlocks/slowdowns
-    await knex.transaction(async (trx) => {
-      await trx.raw(`
-        -- Disable triggers and foreign key constraints for this session
-        SET session_replication_role = replica;
-        
-        truncate table ${tableNames.join(',')};
-
-        -- Re-enable triggers and foreign key constraints
-        SET session_replication_role = DEFAULT;
-      `)
-    })
-  } else {
-    await knex.raw(`truncate table ${tableNames.join(',')} cascade`)
-  }
-}
-
-/**
- * @param {import('http').Server} server
- * @param {import('express').Express} app
- */
-const initializeTestServer = async (server, app) => {
-  await startHttp(server, app, 0)
-
-  await once(app, 'appStarted')
-  const port = server.address().port
-  const serverAddress = `http://127.0.0.1:${port}`
-  const wsAddress = `ws://127.0.0.1:${port}`
-  return {
-    server,
-    serverAddress,
-    serverPort: port,
-    wsAddress,
-    sendRequest(auth, obj) {
-      return (
-        chai
-          .request(serverAddress)
-          .post('/graphql')
-          // if you set the header to null, the actual header in the req will be
-          // a string -> 'null'
-          // this is now treated as an invalid token, and gets forbidden
-          // switching to an empty string token
-          .set('Authorization', auth || '')
-          .send(obj)
-      )
-    }
-  }
-}
-
-exports.mochaHooks = {
-  beforeAll: async () => {
-    logger.info('running before all')
-    await unlock()
-    await exports.truncateTables()
-    await knex.migrate.rollback()
-    await knex.migrate.latest()
-    await init()
-  },
-  afterAll: async () => {
-    logger.info('running after all')
-    await unlock()
-    await shutdown()
-  }
-}
-
-exports.buildApp = async () => {
-  const { app, graphqlServer, server } = await init()
-  return { app, graphqlServer, server }
-}
-
-exports.beforeEachContext = async () => {
-  await exports.truncateTables()
-  return await exports.buildApp()
-}
-
-exports.initializeTestServer = initializeTestServer
diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts
new file mode 100644
index 0000000000..f5a9d45be6
--- /dev/null
+++ b/packages/server/test/hooks.ts
@@ -0,0 +1,256 @@
+// eslint-disable-next-line no-restricted-imports
+import '../bootstrap'
+
+// Register global mocks as early as possible
+import '@/test/mocks/global'
+
+import chai from 'chai'
+import chaiAsPromised from 'chai-as-promised'
+import chaiHttp from 'chai-http'
+import deepEqualInAnyOrder from 'deep-equal-in-any-order'
+import { knex as mainDb } from '@/db/knex'
+import { init, startHttp, shutdown } from '@/app'
+import graphqlChaiPlugin from '@/test/plugins/graphql'
+import { logger } from '@/logging/logging'
+import { once } from 'events'
+import type http from 'http'
+import type express from 'express'
+import type net from 'net'
+import { MaybeAsync, MaybeNullOrUndefined } from '@speckle/shared'
+import type mocha from 'mocha'
+import { shouldRunTestsInMultiregionMode } from '@/modules/shared/helpers/envHelper'
+import {
+  getAvailableRegionKeysFactory,
+  getFreeRegionKeysFactory
+} from '@/modules/multiregion/services/config'
+import { getAvailableRegionConfig } from '@/modules/multiregion/regionConfig'
+import { createAndValidateNewRegionFactory } from '@/modules/multiregion/services/management'
+import {
+  getRegionFactory,
+  getRegionsFactory,
+  storeRegionFactory
+} from '@/modules/multiregion/repositories'
+import {
+  getRegisteredRegionClients,
+  initializeRegion
+} from '@/modules/multiregion/dbSelector'
+import { Knex } from 'knex'
+
+// why is server config only created once!????
+// because its done in a migration, to not override existing configs
+// similarly wiping regions will break multi region setup
+const protectedTables = ['server_config', 'regions']
+let regionClients: Record<string, Knex> = {}
+
+// Register chai plugins
+chai.use(chaiAsPromised)
+chai.use(chaiHttp)
+chai.use(deepEqualInAnyOrder)
+chai.use(graphqlChaiPlugin)
+
+const inEachDb = async (fn: (db: Knex) => MaybeAsync<void>) => {
+  await fn(mainDb)
+  for (const regionClient of Object.values(regionClients)) {
+    await fn(regionClient)
+  }
+}
+
+const setupMultiregionMode = async () => {
+  const db = mainDb
+  const getAvailableRegionKeys = getAvailableRegionKeysFactory({
+    getAvailableRegionConfig
+  })
+  const regionKeys = await getAvailableRegionKeys()
+
+  // Create DB region entries for each key
+  const createRegion = createAndValidateNewRegionFactory({
+    getFreeRegionKeys: getFreeRegionKeysFactory({
+      getAvailableRegionKeys,
+      getRegions: getRegionsFactory({ db })
+    }),
+    getRegion: getRegionFactory({ db }),
+    storeRegion: storeRegionFactory({ db }),
+    initializeRegion
+  })
+  for (const regionKey of regionKeys) {
+    await createRegion({
+      region: {
+        key: regionKey,
+        name: regionKey,
+        description: 'Auto created test region'
+      }
+    })
+  }
+
+  // Store active region clients
+  regionClients = await getRegisteredRegionClients()
+
+  // Reset each DB client (re-run all migrations and setup)
+  for (const [, regionClient] of Object.entries(regionClients)) {
+    const reset = resetSchemaFactory({ db: regionClient })
+    await reset()
+  }
+}
+
+const unlockFactory = (deps: { db: Knex }) => async () => {
+  const exists = await deps.db.schema.hasTable('knex_migrations_lock')
+  if (exists) {
+    await deps.db('knex_migrations_lock').update('is_locked', '0')
+  }
+}
+
+export const resetPubSubFactory = (deps: { db: Knex }) => async () => {
+  if (!shouldRunTestsInMultiregionMode()) {
+    return { drop: async () => {}, reenable: async () => {} }
+  }
+
+  const subscriptions = (await deps.db.raw(
+    `SELECT subname, subconninfo, subpublications, subslotname FROM aiven_extras.pg_list_all_subscriptions() WHERE subname ILIKE 'test_%';`
+  )) as {
+    rows: Array<{
+      subname: string
+      subconninfo: string
+      subpublications: string[]
+      subslotname: string
+    }>
+  }
+  const publications = (await deps.db.raw(
+    `SELECT pubname FROM pg_publication WHERE pubname ILIKE 'test_%';`
+  )) as {
+    rows: Array<{ pubname: string }>
+  }
+
+  // Drop all subs
+  for (const sub of subscriptions.rows) {
+    await deps.db.raw(`
+          SELECT * FROM aiven_extras.pg_alter_subscription_disable('${sub.subname}');
+          SELECT * FROM aiven_extras.pg_drop_subscription('${sub.subname}');
+          SELECT * FROM aiven_extras.dblink_slot_create_or_drop('${sub.subconninfo}', '${sub.subslotname}', 'drop');
+        `)
+  }
+
+  // Drop all pubs
+  for (const pub of publications.rows) {
+    await deps.db.raw(`DROP PUBLICATION ${pub.pubname};`)
+  }
+}
+
+const truncateTablesFactory = (deps: { db: Knex }) => async (tableNames?: string[]) => {
+  if (!tableNames?.length) {
+    tableNames = (
+      await deps
+        .db('pg_tables')
+        .select('tablename')
+        .where({ schemaname: 'public' })
+        .whereRaw("tablename not like '%knex%'")
+        .whereNotIn('tablename', protectedTables)
+    ).map((table: { tablename: string }) => table.tablename)
+    if (!tableNames.length) return // Nothing to truncate
+
+    // We're deleting everything, so lets turn off triggers to avoid deadlocks/slowdowns
+    await deps.db.transaction(async (trx) => {
+      await trx.raw(`
+        -- Disable triggers and foreign key constraints for this session
+        SET session_replication_role = replica;
+
+        truncate table ${tableNames?.join(',') || ''};
+
+        -- Re-enable triggers and foreign key constraints
+        SET session_replication_role = DEFAULT;
+      `)
+    })
+  } else {
+    await deps.db.raw(`truncate table ${tableNames.join(',')} cascade`)
+  }
+}
+
+const resetSchemaFactory = (deps: { db: Knex }) => async () => {
+  const resetPubSub = resetPubSubFactory(deps)
+
+  await unlockFactory(deps)()
+  await resetPubSub()
+
+  // Reset schema
+  await deps.db.migrate.rollback()
+  await deps.db.migrate.latest()
+}
+
+export const truncateTables = async (tableNames?: string[]) => {
+  const dbs = [mainDb, ...Object.values(regionClients)]
+
+  // First reset pubsubs
+  for (const db of dbs) {
+    const resetPubSub = resetPubSubFactory({ db })
+    await resetPubSub()
+  }
+
+  // Now truncate
+  for (const db of dbs) {
+    const truncate = truncateTablesFactory({ db })
+    await truncate(tableNames)
+  }
+}
+
+export const initializeTestServer = async (
+  server: http.Server,
+  app: express.Express
+) => {
+  await startHttp(server, app, 0)
+
+  await once(app, 'appStarted')
+  const port = (server.address() as net.AddressInfo).port
+  const serverAddress = `http://127.0.0.1:${port}`
+  const wsAddress = `ws://127.0.0.1:${port}`
+  return {
+    server,
+    serverAddress,
+    serverPort: port,
+    wsAddress,
+    sendRequest(auth: MaybeNullOrUndefined<string>, obj: string | object) {
+      return (
+        chai
+          .request(serverAddress)
+          .post('/graphql')
+          // if you set the header to null, the actual header in the req will be
+          // a string -> 'null'
+          // this is now treated as an invalid token, and gets forbidden
+          // switching to an empty string token
+          .set('Authorization', auth || '')
+          .send(obj)
+      )
+    }
+  }
+}
+
+export const mochaHooks: mocha.RootHookObject = {
+  beforeAll: async () => {
+    logger.info('running before all')
+
+    // Init main db
+    const reset = resetSchemaFactory({ db: mainDb })
+    await reset()
+
+    // Init (or cleanup) multi-region mode
+    await setupMultiregionMode()
+
+    // Init app
+    await init()
+  },
+  afterAll: async () => {
+    logger.info('running after all')
+    await inEachDb(async (db) => {
+      await unlockFactory({ db })()
+    })
+    await shutdown()
+  }
+}
+
+export const buildApp = async () => {
+  const { app, graphqlServer, server } = await init()
+  return { app, graphqlServer, server }
+}
+
+export const beforeEachContext = async () => {
+  await truncateTables()
+  return await buildApp()
+}

From d02ca824c36b1a3de9f237469999e60145a6b2eb Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Mon, 11 Nov 2024 14:51:54 +0200
Subject: [PATCH 02/12] test fixes

---
 .../server/modules/comments/tests/comments.spec.js |  2 +-
 .../server/modules/workspaces/domain/operations.ts | 14 +++++++++++++-
 .../tests/integration/repositories.spec.ts         |  4 ++--
 .../workspaces/tests/integration/sso.graph.spec.ts |  2 +-
 .../workspaces/tests/integration/sso.spec.ts       |  2 +-
 5 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/packages/server/modules/comments/tests/comments.spec.js b/packages/server/modules/comments/tests/comments.spec.js
index 2399407bf5..59dcd56dfe 100644
--- a/packages/server/modules/comments/tests/comments.spec.js
+++ b/packages/server/modules/comments/tests/comments.spec.js
@@ -1298,7 +1298,7 @@ describe('Comments @comments', () => {
 
       before(async () => {
         // Truncate comments
-        truncateTables([Comments.name])
+        await truncateTables([Comments.name])
 
         // Create a single comment with a blob
         const createCommentResult = await createComment({
diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts
index 7ab953fdee..e96a065753 100644
--- a/packages/server/modules/workspaces/domain/operations.ts
+++ b/packages/server/modules/workspaces/domain/operations.ts
@@ -12,6 +12,7 @@ import { EventBusPayloads } from '@/modules/shared/services/eventBus'
 import {
   MaybeNullOrUndefined,
   Nullable,
+  NullableKeysToOptional,
   Optional,
   PartialNullable,
   StreamRoles,
@@ -22,11 +23,22 @@ import { WorkspaceTeam } from '@/modules/workspaces/domain/types'
 import { Stream } from '@/modules/core/domain/streams/types'
 import { TokenResourceIdentifier } from '@/modules/core/domain/tokens/types'
 import { ServerRegion } from '@/modules/multiregion/domain/types'
+import { SetOptional } from 'type-fest'
 
 /** Workspace */
 
 type UpsertWorkspaceArgs = {
-  workspace: Omit<Workspace, 'domains'>
+  workspace: Omit<
+    SetOptional<
+      NullableKeysToOptional<Workspace>,
+      | 'domainBasedMembershipProtectionEnabled'
+      | 'discoverabilityEnabled'
+      | 'defaultLogoIndex'
+      | 'defaultProjectRole'
+      | 'slug'
+    >,
+    'domains'
+  >
 }
 
 export type UpsertWorkspace = (args: UpsertWorkspaceArgs) => Promise<void>
diff --git a/packages/server/modules/workspaces/tests/integration/repositories.spec.ts b/packages/server/modules/workspaces/tests/integration/repositories.spec.ts
index e83834ff27..4e6b2201b2 100644
--- a/packages/server/modules/workspaces/tests/integration/repositories.spec.ts
+++ b/packages/server/modules/workspaces/tests/integration/repositories.spec.ts
@@ -161,7 +161,7 @@ describe('Workspace repositories', () => {
       })
 
       afterEach(async () => {
-        truncateTables(['workspaces'])
+        await truncateTables(['workspaces'])
       })
 
       it('returns all workspace members', async () => {
@@ -209,7 +209,7 @@ describe('Workspace repositories', () => {
       })
 
       afterEach(async () => {
-        truncateTables(['workspaces'])
+        await truncateTables(['workspaces'])
       })
 
       it('limits search results to specified workspace', async () => {
diff --git a/packages/server/modules/workspaces/tests/integration/sso.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/sso.graph.spec.ts
index 0e90db119c..cdb4d76cbe 100644
--- a/packages/server/modules/workspaces/tests/integration/sso.graph.spec.ts
+++ b/packages/server/modules/workspaces/tests/integration/sso.graph.spec.ts
@@ -117,7 +117,7 @@ describe('Workspace SSO', () => {
   })
 
   afterEach(async () => {
-    truncateTables(['user_sso_sessions'])
+    await truncateTables(['user_sso_sessions'])
   })
 
   describe('given a workspace with SSO configured', () => {
diff --git a/packages/server/modules/workspaces/tests/integration/sso.spec.ts b/packages/server/modules/workspaces/tests/integration/sso.spec.ts
index 8d77da647d..99f6fe6d54 100644
--- a/packages/server/modules/workspaces/tests/integration/sso.spec.ts
+++ b/packages/server/modules/workspaces/tests/integration/sso.spec.ts
@@ -251,7 +251,7 @@ describe('Workspace SSO repositories', () => {
     })
 
     afterEach(async () => {
-      truncateTables(['user_sso_sessions'])
+      await truncateTables(['user_sso_sessions'])
     })
 
     it('returns an empty array if there are no sessions', async () => {

From 9abf5b0a7779147c4fb4bf432306f03b448ec6ba Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Mon, 11 Nov 2024 15:38:05 +0200
Subject: [PATCH 03/12] more adjustments & fixes

---
 .../intergration/repositories/projectRegion.spec.ts |  5 +++++
 .../tests/integration/regions.graph.spec.ts         |  8 +++++---
 packages/server/test/hooks.ts                       | 13 +++++++++++++
 packages/server/test/speckle-helpers/regions.ts     | 11 +++++++++++
 4 files changed, 34 insertions(+), 3 deletions(-)
 create mode 100644 packages/server/test/speckle-helpers/regions.ts

diff --git a/packages/server/modules/multiregion/tests/intergration/repositories/projectRegion.spec.ts b/packages/server/modules/multiregion/tests/intergration/repositories/projectRegion.spec.ts
index 7db90fe899..56ad593723 100644
--- a/packages/server/modules/multiregion/tests/intergration/repositories/projectRegion.spec.ts
+++ b/packages/server/modules/multiregion/tests/intergration/repositories/projectRegion.spec.ts
@@ -10,8 +10,13 @@ import { createInmemoryRedisClient } from '@/test/redisHelper'
 import { createStreamFactory } from '@/modules/core/repositories/streams'
 import { db } from '@/db/knex'
 import { storeRegionFactory } from '@/modules/multiregion/repositories'
+import { truncateRegionsSafely } from '@/test/speckle-helpers/regions'
 
 describe('projectRegion repositories @multiregion', () => {
+  after(async () => {
+    await truncateRegionsSafely()
+  })
+
   describe('inMemoryKeyStoreFactory creates an object, which', () => {
     const { getRegionKey, writeRegion } = inMemoryRegionKeyStoreFactory()
     it('returns undefined if projectId is not in the cache', () => {
diff --git a/packages/server/modules/workspaces/tests/integration/regions.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/regions.graph.spec.ts
index d4132d1f5f..18456ec886 100644
--- a/packages/server/modules/workspaces/tests/integration/regions.graph.spec.ts
+++ b/packages/server/modules/workspaces/tests/integration/regions.graph.spec.ts
@@ -13,8 +13,9 @@ import {
   SetWorkspaceDefaultRegionDocument
 } from '@/test/graphql/generated/graphql'
 import { testApolloServer, TestApolloServer } from '@/test/graphqlHelper'
-import { beforeEachContext } from '@/test/hooks'
+import { beforeEachContext, getRegionKeys } from '@/test/hooks'
 import { MultiRegionDbSelectorMock } from '@/test/mocks/global'
+import { truncateRegionsSafely } from '@/test/speckle-helpers/regions'
 import { Roles } from '@speckle/shared'
 import { expect } from 'chai'
 
@@ -71,8 +72,9 @@ describe('Workspace regions GQL', () => {
     apollo = await testApolloServer({ authUserId: me.id })
   })
 
-  after(() => {
+  after(async () => {
     MultiRegionDbSelectorMock.resetMockedFunctions()
+    await truncateRegionsSafely()
   })
 
   describe('when listing', () => {
@@ -95,7 +97,7 @@ describe('Workspace regions GQL', () => {
       expect(res).to.not.haveGraphQLErrors()
       expect(
         res.data?.workspace.availableRegions.map((r) => r.key)
-      ).to.deep.equalInAnyOrder([region1Key, region2Key])
+      ).to.deep.equalInAnyOrder([region1Key, region2Key, ...getRegionKeys()])
     })
   })
 
diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts
index f5a9d45be6..a356ce4887 100644
--- a/packages/server/test/hooks.ts
+++ b/packages/server/test/hooks.ts
@@ -28,6 +28,7 @@ import { createAndValidateNewRegionFactory } from '@/modules/multiregion/service
 import {
   getRegionFactory,
   getRegionsFactory,
+  Regions,
   storeRegionFactory
 } from '@/modules/multiregion/repositories'
 import {
@@ -90,6 +91,12 @@ const setupMultiregionMode = async () => {
     const reset = resetSchemaFactory({ db: regionClient })
     await reset()
   }
+
+  // If not in multi region mode, delete region entries
+  // we only needed them to reset schemas
+  if (!shouldRunTestsInMultiregionMode()) {
+    await truncateTables([Regions.name])
+  }
 }
 
 const unlockFactory = (deps: { db: Knex }) => async () => {
@@ -99,6 +106,8 @@ const unlockFactory = (deps: { db: Knex }) => async () => {
   }
 }
 
+export const getRegionKeys = () => Object.keys(regionClients)
+
 export const resetPubSubFactory = (deps: { db: Knex }) => async () => {
   if (!shouldRunTestsInMultiregionMode()) {
     return { drop: async () => {}, reenable: async () => {} }
@@ -224,6 +233,10 @@ export const initializeTestServer = async (
 
 export const mochaHooks: mocha.RootHookObject = {
   beforeAll: async () => {
+    if (shouldRunTestsInMultiregionMode()) {
+      console.log('Running tests in multi-region mode...')
+    }
+
     logger.info('running before all')
 
     // Init main db
diff --git a/packages/server/test/speckle-helpers/regions.ts b/packages/server/test/speckle-helpers/regions.ts
new file mode 100644
index 0000000000..33efbecc9e
--- /dev/null
+++ b/packages/server/test/speckle-helpers/regions.ts
@@ -0,0 +1,11 @@
+import { db } from '@/db/knex'
+import { Regions } from '@/modules/multiregion/repositories'
+import { getRegionKeys } from '@/test/hooks'
+
+/**
+ * Delete all regions entries that are not part of the main multi region mode
+ */
+export const truncateRegionsSafely = async () => {
+  const regionKeys = getRegionKeys()
+  await db(Regions.name).whereNotIn(Regions.col.key, regionKeys).delete()
+}

From 2d0aea90cfc5425ce0a21bc489c907624090f89d Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Mon, 11 Nov 2024 16:00:37 +0200
Subject: [PATCH 04/12] linter fixes

---
 .../tests/fileuploads.integration.spec.ts         |  2 +-
 .../modules/workspaces/domain/operations.ts       |  2 +-
 .../tests/unit/services/management.spec.ts        | 15 +++++++--------
 packages/server/test/hooks.ts                     |  2 +-
 4 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/packages/server/modules/fileuploads/tests/fileuploads.integration.spec.ts b/packages/server/modules/fileuploads/tests/fileuploads.integration.spec.ts
index bb219862dc..2256439d0a 100644
--- a/packages/server/modules/fileuploads/tests/fileuploads.integration.spec.ts
+++ b/packages/server/modules/fileuploads/tests/fileuploads.integration.spec.ts
@@ -142,7 +142,7 @@ describe('FileUploads @fileuploads', () => {
   let existingCanonicalUrl: string
   let existingPort: string
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  let sendRequest: (token: string, query: unknown) => Promise<any>
+  let sendRequest: (token: string, query: string | object) => Promise<any>
   let serverAddress: string
   let serverPort: string
 
diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts
index e96a065753..230b80ce17 100644
--- a/packages/server/modules/workspaces/domain/operations.ts
+++ b/packages/server/modules/workspaces/domain/operations.ts
@@ -27,7 +27,7 @@ import { SetOptional } from 'type-fest'
 
 /** Workspace */
 
-type UpsertWorkspaceArgs = {
+export type UpsertWorkspaceArgs = {
   workspace: Omit<
     SetOptional<
       NullableKeysToOptional<Workspace>,
diff --git a/packages/server/modules/workspaces/tests/unit/services/management.spec.ts b/packages/server/modules/workspaces/tests/unit/services/management.spec.ts
index 7a4803ed04..5d22a8f3aa 100644
--- a/packages/server/modules/workspaces/tests/unit/services/management.spec.ts
+++ b/packages/server/modules/workspaces/tests/unit/services/management.spec.ts
@@ -35,12 +35,15 @@ import {
 } from '@/modules/workspaces/errors/workspace'
 import { UserEmail } from '@/modules/core/domain/userEmails/types'
 import { merge, omit } from 'lodash'
-import { GetWorkspaceWithDomains } from '@/modules/workspaces/domain/operations'
+import {
+  GetWorkspaceWithDomains,
+  UpsertWorkspaceArgs
+} from '@/modules/workspaces/domain/operations'
 import { FindVerifiedEmailsByUserId } from '@/modules/core/domain/userEmails/operations'
 import { EventNames } from '@/modules/shared/services/eventBus'
 
 type WorkspaceTestContext = {
-  storedWorkspaces: Omit<Workspace, 'domains'>[]
+  storedWorkspaces: UpsertWorkspaceArgs['workspace'][]
   storedRoles: WorkspaceAcl[]
   eventData: {
     isCalled: boolean
@@ -63,11 +66,7 @@ const buildCreateWorkspaceWithTestContext = (
   }
 
   const deps: Parameters<typeof createWorkspaceFactory>[0] = {
-    upsertWorkspace: async ({
-      workspace
-    }: {
-      workspace: Omit<Workspace, 'domains'>
-    }) => {
+    upsertWorkspace: async ({ workspace }) => {
       context.storedWorkspaces.push(workspace)
     },
     validateSlug: async () => {},
@@ -1160,7 +1159,7 @@ describe('Workspace role services', () => {
         }
 
         let storedDomains: WorkspaceDomain | undefined = undefined
-        let storedWorkspace: Omit<Workspace, 'domains'> | undefined = undefined
+        let storedWorkspace: UpsertWorkspaceArgs['workspace'] | undefined = undefined
         let omittedEventName: EventNames | undefined = undefined
 
         const workspace: Workspace = {
diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts
index a356ce4887..bdfbc3cfc8 100644
--- a/packages/server/test/hooks.ts
+++ b/packages/server/test/hooks.ts
@@ -207,7 +207,7 @@ export const initializeTestServer = async (
   await startHttp(server, app, 0)
 
   await once(app, 'appStarted')
-  const port = (server.address() as net.AddressInfo).port
+  const port = (server.address() as net.AddressInfo).port + ''
   const serverAddress = `http://127.0.0.1:${port}`
   const wsAddress = `ws://127.0.0.1:${port}`
   return {

From 653ba5dc53a9850c4b82dfae26252859dd38138b Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Mon, 11 Nov 2024 17:05:53 +0200
Subject: [PATCH 05/12] reenable server admin region tests

---
 .../tests/e2e/serverAdmin.graph.spec.ts       | 23 +++++++++----------
 packages/server/test/mocks/global.ts          |  8 +++----
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts b/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts
index df6e9f1cb2..b4a7fb328f 100644
--- a/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts
+++ b/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts
@@ -15,11 +15,11 @@ import {
   TestApolloServer
 } from '@/test/graphqlHelper'
 import { beforeEachContext, truncateTables } from '@/test/hooks'
-import { MultiRegionConfigServiceMock } from '@/test/mocks/global'
+import { MultiRegionConfigMock, MultiRegionDbSelectorMock } from '@/test/mocks/global'
 import { Roles } from '@speckle/shared'
 import { expect } from 'chai'
 
-describe.skip('Multi Region Server Settings', () => {
+describe('Multi Region Server Settings', () => {
   let testAdminUser: BasicTestUser
   let testBasicUser: BasicTestUser
   let apollo: TestApolloServer
@@ -41,14 +41,12 @@ describe.skip('Multi Region Server Settings', () => {
   }
 
   before(async () => {
-    // Have to mock both
-    // MultiRegionConfigServiceMock.mockFunction(
-    //   'getAvailableRegionConfigsFactory',
-    //   () => async () => fakeRegionConfig
-    // )
-    MultiRegionConfigServiceMock.mockFunction(
-      'getAvailableRegionKeysFactory',
-      () => async () => Object.keys(fakeRegionConfig)
+    MultiRegionConfigMock.mockFunction(
+      'getAvailableRegionConfig',
+      async () => fakeRegionConfig
+    )
+    MultiRegionDbSelectorMock.mockFunction('initializeRegion', async () =>
+      Promise.resolve()
     )
 
     await beforeEachContext()
@@ -58,7 +56,8 @@ describe.skip('Multi Region Server Settings', () => {
   })
 
   after(() => {
-    MultiRegionConfigServiceMock.resetMockedFunctions()
+    MultiRegionConfigMock.resetMockedFunctions()
+    MultiRegionDbSelectorMock.resetMockedFunctions()
   })
 
   describe('server config', () => {
@@ -121,11 +120,11 @@ describe.skip('Multi Region Server Settings', () => {
         }
 
         const res = await createRegion(input)
+        expect(res).to.not.haveGraphQLErrors()
         expect(res.data?.serverInfoMutations.multiRegion.create).to.deep.equal({
           ...input,
           id: input.key
         })
-        expect(res).to.not.haveGraphQLErrors()
       })
 
       it("doesn't work with already used up key", async () => {
diff --git a/packages/server/test/mocks/global.ts b/packages/server/test/mocks/global.ts
index 76d136da99..d80a63e08c 100644
--- a/packages/server/test/mocks/global.ts
+++ b/packages/server/test/mocks/global.ts
@@ -12,10 +12,10 @@ export const CommentsRepositoryMock = mockRequireModule<
   typeof import('@/modules/comments/repositories/comments')
 >(['@/modules/comments/repositories/comments'])
 
-export const MultiRegionConfigServiceMock = mockRequireModule<
-  typeof import('@/modules/multiregion/services/config')
->(['@/modules/multiregion/services/config'])
-
 export const MultiRegionDbSelectorMock = mockRequireModule<
   typeof import('@/modules/multiregion/dbSelector')
 >(['@/modules/multiregion/dbSelector'])
+
+export const MultiRegionConfigMock = mockRequireModule<
+  typeof import('@/modules/multiregion/regionConfig')
+>(['@/modules/multiregion/regionConfig'])

From de4159efc9a90fed58858b748eb93fa24696ca9e Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Tue, 12 Nov 2024 12:39:36 +0200
Subject: [PATCH 06/12] set up CI test job

---
 .circleci/config.yml               | 23 +++++++++++++++++++++++
 .circleci/multiregion.test-ci.json | 14 ++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 .circleci/multiregion.test-ci.json

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 1ee94f809d..73757df7b2 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -573,6 +573,29 @@ jobs:
       FF_GATEKEEPER_MODULE_ENABLED: 'false'
       FF_BILLING_INTEGRATION_ENABLED: 'false'
 
+  test-server-multiregion:
+    <<: *test-server-job
+    docker:
+      - image: cimg/node:18.19.0
+      - image: cimg/redis:7.2.4
+      - image: 'speckle/speckle-postgres'
+        environment:
+          POSTGRES_DB: speckle2_test
+          POSTGRES_PASSWORD: speckle
+          POSTGRES_USER: speckle
+        command: -c 'max_connections=1000'
+      - image: 'speckle/speckle-postgres'
+        environment:
+          POSTGRES_DB: speckle2_test
+          POSTGRES_PASSWORD: speckle
+          POSTGRES_USER: speckle
+        command: -c 'max_connections=1000 port=5433'
+      - image: 'minio/minio'
+        command: server /data --console-address ":9001"
+    environment:
+      MULTI_REGION_CONFIG_PATH: '../../.circleci/multiregion.test-ci.json'
+      RUN_TESTS_IN_MULTIREGION_MODE: true
+
   test-frontend-2:
     docker: &docker-node-browsers-image
       - image: cimg/node:18.19.0-browsers
diff --git a/.circleci/multiregion.test-ci.json b/.circleci/multiregion.test-ci.json
new file mode 100644
index 0000000000..bbf5d3142b
--- /dev/null
+++ b/.circleci/multiregion.test-ci.json
@@ -0,0 +1,14 @@
+{
+  "main": {
+    "postgres": {
+      "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5432/speckle2_test"
+    }
+  },
+  "regions": {
+    "region1": {
+      "postgres": {
+        "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5433/speckle2_test"
+      }
+    }
+  }
+}

From 5c2504b03ced169c09f7ce4db5f1228ca6e569fe Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Tue, 12 Nov 2024 12:42:28 +0200
Subject: [PATCH 07/12] job definition fix

---
 .circleci/config.yml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 73757df7b2..1438f886e9 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -32,6 +32,11 @@ workflows:
           requires:
             - docker-publish-postgres-container
 
+      - test-server-multiregion:
+          filters: *filters-allow-all
+          requires:
+            - docker-publish-postgres-container
+
       - test-frontend-2:
           filters: *filters-allow-all
 
@@ -190,6 +195,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-frontend:
@@ -205,6 +211,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-frontend-2:
@@ -220,6 +227,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-webhooks:
@@ -235,6 +243,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-file-imports:
@@ -250,6 +259,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-previews:
@@ -265,6 +275,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-test-container:
@@ -280,6 +291,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-postgres-container:
@@ -301,6 +313,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-docker-compose-ingress:
@@ -316,6 +329,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - publish-helm-chart:
@@ -356,6 +370,7 @@ workflows:
             - get-version
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-ui-components
             - test-frontend-2
             - test-viewer

From 0cf7b04f80dd65303b7b8842f82c555e85e8b45a Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Tue, 12 Nov 2024 12:47:15 +0200
Subject: [PATCH 08/12] port fix

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 1438f886e9..02ffe11c00 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -604,7 +604,7 @@ jobs:
           POSTGRES_DB: speckle2_test
           POSTGRES_PASSWORD: speckle
           POSTGRES_USER: speckle
-        command: -c 'max_connections=1000 port=5433'
+        command: -c 'max_connections=1000' -c 'port=5433'
       - image: 'minio/minio'
         command: server /data --console-address ":9001"
     environment:

From db11ef46abcf14f986eb12c85630fd9750e25b87 Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Tue, 12 Nov 2024 12:55:32 +0200
Subject: [PATCH 09/12] env fix

---
 .circleci/config.yml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 02ffe11c00..3f41a4a69d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -608,6 +608,25 @@ jobs:
       - image: 'minio/minio'
         command: server /data --console-address ":9001"
     environment:
+      # Same as test-server:
+      NODE_ENV: test
+      DATABASE_URL: 'postgres://speckle:speckle@127.0.0.1:5432/speckle2_test'
+      PGDATABASE: speckle2_test
+      POSTGRES_MAX_CONNECTIONS_SERVER: 20
+      PGUSER: speckle
+      SESSION_SECRET: 'keyboard cat'
+      STRATEGY_LOCAL: 'true'
+      CANONICAL_URL: 'http://127.0.0.1:3000'
+      S3_ENDPOINT: 'http://127.0.0.1:9000'
+      S3_ACCESS_KEY: 'minioadmin'
+      S3_SECRET_KEY: 'minioadmin'
+      S3_BUCKET: 'speckle-server'
+      S3_CREATE_BUCKET: 'true'
+      REDIS_URL: 'redis://127.0.0.1:6379'
+      S3_REGION: '' # optional, defaults to 'us-east-1'
+      AUTOMATE_ENCRYPTION_KEYS_PATH: 'test/assets/automate/encryptionKeys.json'
+      FF_BILLING_INTEGRATION_ENABLED: 'true'
+      # These are the only 2 different env keys:
       MULTI_REGION_CONFIG_PATH: '../../.circleci/multiregion.test-ci.json'
       RUN_TESTS_IN_MULTIREGION_MODE: true
 

From 8087f3a22338708d613ad9d17a00bc4b250fd704 Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Tue, 12 Nov 2024 13:17:28 +0200
Subject: [PATCH 10/12] hopefully fixing aiven extras issue

---
 packages/server/test/hooks.ts | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts
index bdfbc3cfc8..0a0ce1a97c 100644
--- a/packages/server/test/hooks.ts
+++ b/packages/server/test/hooks.ts
@@ -56,6 +56,10 @@ const inEachDb = async (fn: (db: Knex) => MaybeAsync<void>) => {
   }
 }
 
+const ensureAivenExtrasFactory = (deps: { db: Knex }) => async () => {
+  await deps.db.raw('CREATE EXTENSION IF NOT EXISTS "aiven_extras";')
+}
+
 const setupMultiregionMode = async () => {
   const db = mainDb
   const getAvailableRegionKeys = getAvailableRegionKeysFactory({
@@ -113,6 +117,9 @@ export const resetPubSubFactory = (deps: { db: Knex }) => async () => {
     return { drop: async () => {}, reenable: async () => {} }
   }
 
+  const ensureAivenExtras = ensureAivenExtrasFactory(deps)
+  await ensureAivenExtras()
+
   const subscriptions = (await deps.db.raw(
     `SELECT subname, subconninfo, subpublications, subslotname FROM aiven_extras.pg_list_all_subscriptions() WHERE subname ILIKE 'test_%';`
   )) as {

From 24ed4d6f4f1407ee71f17f95d070924942f65656 Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Tue, 12 Nov 2024 13:36:56 +0200
Subject: [PATCH 11/12] add missing ctx

---
 .circleci/config.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3f41a4a69d..916839de96 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -16,7 +16,7 @@ workflows:
                 - main
                 - hotfix*
 
-      - test-server:
+      - test-server: &test-server-job
           context:
             - speckle-server-licensing
             - stripe-integration
@@ -32,10 +32,7 @@ workflows:
           requires:
             - docker-publish-postgres-container
 
-      - test-server-multiregion:
-          filters: *filters-allow-all
-          requires:
-            - docker-publish-postgres-container
+      - test-server-multiregion: *test-server-job
 
       - test-frontend-2:
           filters: *filters-allow-all

From f46f47bc68900018637d3d8a4d9903ac51234479 Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabis94@live.com>
Date: Tue, 12 Nov 2024 13:50:10 +0200
Subject: [PATCH 12/12] minor circleci cleanup

---
 .circleci/config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 916839de96..b672554095 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -16,7 +16,7 @@ workflows:
                 - main
                 - hotfix*
 
-      - test-server: &test-server-job
+      - test-server: &test-server-job-definition
           context:
             - speckle-server-licensing
             - stripe-integration
@@ -32,7 +32,7 @@ workflows:
           requires:
             - docker-publish-postgres-container
 
-      - test-server-multiregion: *test-server-job
+      - test-server-multiregion: *test-server-job-definition
 
       - test-frontend-2:
           filters: *filters-allow-all