Merge pull request #3481 from specklesystems/fabians/multiregion-testing3

feat(server): run tests in multi region db mode

.circleci/config.yml

@@ -16,7 +16,7 @@ workflows:
                 - main
                 - hotfix*
 
-      - test-server:
+      - test-server: &test-server-job-definition
           context:
             - speckle-server-licensing
             - stripe-integration
@@ -32,6 +32,8 @@ workflows:
           requires:
             - docker-publish-postgres-container
 
+      - test-server-multiregion: *test-server-job-definition
+
       - test-frontend-2:
           filters: *filters-allow-all
 
@@ -190,6 +192,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-frontend:
@@ -205,6 +208,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-frontend-2:
@@ -220,6 +224,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-webhooks:
@@ -235,6 +240,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-file-imports:
@@ -250,6 +256,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-previews:
@@ -265,6 +272,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-test-container:
@@ -280,6 +288,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-postgres-container:
@@ -301,6 +310,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-docker-compose-ingress:
@@ -316,6 +326,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - publish-helm-chart:
@@ -356,6 +367,7 @@ workflows:
             - get-version
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-ui-components
             - test-frontend-2
             - test-viewer
@@ -573,6 +585,48 @@ jobs:
       FF_GATEKEEPER_MODULE_ENABLED: 'false'
       FF_BILLING_INTEGRATION_ENABLED: 'false'
 
+  test-server-multiregion:
+    <<: *test-server-job
+    docker:
+      - image: cimg/node:18.19.0
+      - image: cimg/redis:7.2.4
+      - image: 'speckle/speckle-postgres'
+        environment:
+          POSTGRES_DB: speckle2_test
+          POSTGRES_PASSWORD: speckle
+          POSTGRES_USER: speckle
+        command: -c 'max_connections=1000'
+      - image: 'speckle/speckle-postgres'
+        environment:
+          POSTGRES_DB: speckle2_test
+          POSTGRES_PASSWORD: speckle
+          POSTGRES_USER: speckle
+        command: -c 'max_connections=1000' -c 'port=5433'
+      - image: 'minio/minio'
+        command: server /data --console-address ":9001"
+    environment:
+      # Same as test-server:
+      NODE_ENV: test
+      DATABASE_URL: 'postgres://speckle:speckle@127.0.0.1:5432/speckle2_test'
+      PGDATABASE: speckle2_test
+      POSTGRES_MAX_CONNECTIONS_SERVER: 20
+      PGUSER: speckle
+      SESSION_SECRET: 'keyboard cat'
+      STRATEGY_LOCAL: 'true'
+      CANONICAL_URL: 'http://127.0.0.1:3000'
+      S3_ENDPOINT: 'http://127.0.0.1:9000'
+      S3_ACCESS_KEY: 'minioadmin'
+      S3_SECRET_KEY: 'minioadmin'
+      S3_BUCKET: 'speckle-server'
+      S3_CREATE_BUCKET: 'true'
+      REDIS_URL: 'redis://127.0.0.1:6379'
+      S3_REGION: '' # optional, defaults to 'us-east-1'
+      AUTOMATE_ENCRYPTION_KEYS_PATH: 'test/assets/automate/encryptionKeys.json'
+      FF_BILLING_INTEGRATION_ENABLED: 'true'
+      # These are the only 2 different env keys:
+      MULTI_REGION_CONFIG_PATH: '../../.circleci/multiregion.test-ci.json'
+      RUN_TESTS_IN_MULTIREGION_MODE: true
+
   test-frontend-2:
     docker: &docker-node-browsers-image
       - image: cimg/node:18.19.0-browsers

.circleci/multiregion.test-ci.json

@@ -0,0 +1,14 @@
+{
+  "main": {
+    "postgres": {
+      "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5432/speckle2_test"
+    }
+  },
+  "regions": {
+    "region1": {
+      "postgres": {
+        "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5433/speckle2_test"
+      }
+    }
+  }
+}

.gitignore

@@ -74,4 +74,5 @@ redis-data/
 .tshy-build
 
 # Server
-multiregion.json
+multiregion.json
+multiregion.test.json

package.json

@@ -21,6 +21,7 @@
     "dev:docker": "docker compose -f ./docker-compose-deps.yml",
     "dev:docker:up": "docker compose -f ./docker-compose-deps.yml up -d",
     "dev:docker:down": "docker compose -f ./docker-compose-deps.yml down",
+    "dev:docker:restart": "yarn dev:docker:down && yarn dev:docker:up",
     "dev:kind:up": "ctlptl apply --filename ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:down": "ctlptl delete -f ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:helm:up": "yarn dev:kind:up && tilt up --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server",

packages/server/.env-example

@@ -147,3 +147,8 @@ OIDC_CLIENT_SECRET="gLb9IEutYQ0npyvA8iHxPsObY3duGB0w"
 # OTEL_TRACE_URL=""
 # OTEL_TRACE_KEY=""
 # OTEL_TRACE_VALUE=""
+
+############################################################
+# Multi region settings
+############################################################
+MULTI_REGION_CONFIG_PATH="multiregion.json"

packages/server/.env.test-example

@@ -4,4 +4,6 @@
 
 PORT=0
 POSTGRES_URL=postgres://speckle:speckle@127.0.0.1/speckle2_test
-POSTGRES_USER=''
+POSTGRES_USER=''
+MULTI_REGION_CONFIG_PATH="multiregion.test.json"
+#RUN_TESTS_IN_MULTIREGION_MODE=true

packages/server/.mocharc.js

@@ -14,7 +14,7 @@ const ignore = [
 /** @type {import("mocha").MochaOptions} */
 const config = {
   spec: ['modules/**/*.spec.js', 'modules/**/*.spec.ts', 'logging/**/*.spec.js'],
-  require: ['ts-node/register', 'test/hooks.js'],
+  require: ['ts-node/register', 'test/hooks.ts'],
   ...(ignore.length ? { ignore } : {}),
   slow: 0,
   timeout: '150000',

packages/server/modules/cli/commands/db/migrate/latest.ts

@@ -1,5 +1,8 @@
 import knex from '@/db/knex'
 import { logger } from '@/logging/logging'
+import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
+import { mochaHooks } from '@/test/hooks'
 import { CommandModule } from 'yargs'
 
 const command: CommandModule = {
@@ -7,7 +10,19 @@ const command: CommandModule = {
   describe: 'Run all migrations that have not yet been run',
   async handler() {
     logger.info('Running latest migration...')
-    await knex.migrate.latest()
+
+    // In tests we want different logic - just run beforeAll
+    if (isTestEnv()) {
+      // Run before hooks, to properly initialize everything
+      await (mochaHooks.beforeAll as () => Promise<void>)()
+    } else {
+      const regionDbs = await getRegisteredRegionClients()
+      const dbs = [knex, ...Object.values(regionDbs)]
+      for (const db of dbs) {
+        await db.migrate.latest()
+      }
+    }
+
     logger.info('Completed running migration')
   }
 }

packages/server/modules/cli/commands/db/migrate/rollback.ts

@@ -1,5 +1,8 @@
 import knex from '@/db/knex'
 import { logger } from '@/logging/logging'
+import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
+import { mochaHooks, resetPubSubFactory } from '@/test/hooks'
 import { CommandModule } from 'yargs'
 
 const command: CommandModule = {
@@ -7,7 +10,21 @@ const command: CommandModule = {
   describe: 'Roll back all migrations',
   async handler() {
     logger.info('Rolling back migrations...')
-    await knex.migrate.rollback(undefined, true)
+
+    if (isTestEnv()) {
+      // Run before hooks, to properly initialize everything first
+      await (mochaHooks.beforeAll as () => Promise<void>)()
+    }
+
+    const regionDbs = await getRegisteredRegionClients()
+    const dbs = [knex, ...Object.values(regionDbs)]
+
+    for (const db of dbs) {
+      const resetPubSub = resetPubSubFactory({ db })
+      await resetPubSub()
+      await db.migrate.rollback(undefined, true)
+    }
+
     logger.info('Completed rolling back migrations')
   }
 }

packages/server/modules/comments/tests/comments.spec.js

@@ -1290,7 +1290,7 @@ describe('Comments @comments', () => {
 
       before(async () => {
         // Truncate comments
-        truncateTables([Comments.name])
+        await truncateTables([Comments.name])
 
         // Create a single comment with a blob
         const createCommentResult = await createComment({

packages/server/modules/fileuploads/tests/fileuploads.integration.spec.ts

@@ -142,7 +142,7 @@ describe('FileUploads @fileuploads', () => {
   let existingCanonicalUrl: string
   let existingPort: string
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  let sendRequest: (token: string, query: unknown) => Promise<any>
+  let sendRequest: (token: string, query: string | object) => Promise<any>
   let serverAddress: string
   let serverPort: string
 

packages/server/modules/multiregion/dbSelector.ts

@@ -23,9 +23,16 @@ import {
   getMainRegionConfig
 } from '@/modules/multiregion/regionConfig'
 import { MaybeNullOrUndefined } from '@speckle/shared'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
 
 let getter: GetProjectDb | undefined = undefined
 
+/**
+ * All dbs share the list of pubs/subs, so we need to make sure the test db uses their own.
+ * As long as there's only 1 test db per instance, it should be fine
+ */
+const createPubSubName = (name: string): string => (isTestEnv() ? `test_${name}` : name)
+
 export const getRegionDb: GetRegionDb = async ({ regionKey }) => {
   const getRegion = getRegionFactory({ db })
   const regionClients = await getRegisteredRegionClients()
@@ -86,11 +93,16 @@ export const getProjectDbClient: GetProjectDb = async ({ projectId }) => {
 type RegionClients = Record<string, Knex>
 let registeredRegionClients: RegionClients | undefined = undefined
 
-const initializeRegisteredRegionClients = async (): Promise<RegionClients> => {
+/**
+ * Idempotently initialize registered region (in db) Knex clients
+ */
+export const initializeRegisteredRegionClients = async (): Promise<RegionClients> => {
   const configuredRegions = await getRegionsFactory({ db })()
-  const regionConfigs = await getAvailableRegionConfig()
+  if (!configuredRegions.length) return {}
 
-  return Object.fromEntries(
+  // init knex clients
+  const regionConfigs = await getAvailableRegionConfig()
+  const ret = Object.fromEntries(
     configuredRegions.map((region) => {
       if (!(region.key in regionConfigs))
         throw new MisconfiguredEnvironmentError(
@@ -99,6 +111,17 @@ const initializeRegisteredRegionClients = async (): Promise<RegionClients> => {
       return [region.key, configureClient(regionConfigs[region.key]).public]
     })
   )
+
+  // run migrations
+  await Promise.all(Object.values(ret).map((db) => db.migrate.latest()))
+
+  // (re-)set up pub-sub, if needed
+  await Promise.all(
+    Object.keys(ret).map((regionKey) => initializeRegion({ regionKey }))
+  )
+
+  registeredRegionClients = ret
+  return ret
 }
 
 export const getRegisteredRegionClients = async (): Promise<RegionClients> => {
@@ -110,11 +133,10 @@ export const getRegisteredRegionClients = async (): Promise<RegionClients> => {
 export const getRegisteredDbClients = async (): Promise<Knex[]> =>
   Object.values(await getRegisteredRegionClients())
 
+/**
+ * Idempotently initialize region
+ */
 export const initializeRegion: InitializeRegion = async ({ regionKey }) => {
-  const knownClients = await getRegisteredRegionClients()
-  if (regionKey in knownClients)
-    throw new Error(`Region ${regionKey} is already initialized`)
-
   const regionConfigs = await getAvailableRegionConfig()
   if (!(regionKey in regionConfigs))
     throw new Error(`RegionKey ${regionKey} not available in config`)
@@ -122,7 +144,6 @@ export const initializeRegion: InitializeRegion = async ({ regionKey }) => {
   const newRegionConfig = regionConfigs[regionKey]
   const regionDb = configureClient(newRegionConfig)
   await regionDb.public.migrate.latest()
-  // TODO, set up pub-sub shit
 
   const mainDbConfig = await getMainRegionConfig()
   const mainDb = configureClient(mainDbConfig)
@@ -142,8 +163,12 @@ export const initializeRegion: InitializeRegion = async ({ regionKey }) => {
     regionName: regionKey,
     sslmode
   })
-  // pushing to the singleton object here
-  knownClients[regionKey] = regionDb.public
+
+  // pushing to the singleton object here, its only not available
+  // if this is being triggered from init, and in that case its gonna be set after anyway
+  if (registeredRegionClients) {
+    registeredRegionClients[regionKey] = regionDb.public
+  }
 }
 
 interface ReplicationArgs {
@@ -159,9 +184,11 @@ const setUpUserReplication = async ({
   sslmode,
   regionName
 }: ReplicationArgs): Promise<void> => {
-  // TODO: ensure its created...
+  const subName = createPubSubName(`userssub_${regionName}`)
+  const pubName = createPubSubName('userspub')
+
   try {
-    await from.public.raw('CREATE PUBLICATION userspub FOR TABLE users;')
+    await from.public.raw(`CREATE PUBLICATION ${pubName} FOR TABLE users;`)
   } catch (err) {
     if (!(err instanceof Error)) throw err
     if (!err.message.includes('already exists')) throw err
@@ -174,11 +201,10 @@ const setUpUserReplication = async ({
   )
   const port = fromUrl.port ? fromUrl.port : '5432'
   const fromDbName = fromUrl.pathname.replace('/', '')
-  const subName = `userssub_${regionName}`
   const rawSqeel = `SELECT * FROM aiven_extras.pg_create_subscription(
     '${subName}',
     'dbname=${fromDbName} host=${fromUrl.hostname} port=${port} sslmode=${sslmode} user=${fromUrl.username} password=${fromUrl.password}',
-    'userspub', 
+    '${pubName}', 
     '${subName}',
     TRUE,
     TRUE
@@ -198,9 +224,11 @@ const setUpProjectReplication = async ({
   regionName,
   sslmode
 }: ReplicationArgs): Promise<void> => {
-  // TODO: ensure its created...
+  const subName = createPubSubName(`projectsub_${regionName}`)
+  const pubName = createPubSubName('projectpub')
+
   try {
-    await from.public.raw('CREATE PUBLICATION projectpub FOR TABLE streams;')
+    await from.public.raw(`CREATE PUBLICATION ${pubName} FOR TABLE streams;`)
   } catch (err) {
     if (!(err instanceof Error)) throw err
     if (!err.message.includes('already exists')) throw err
@@ -213,11 +241,10 @@ const setUpProjectReplication = async ({
   )
   const port = fromUrl.port ? fromUrl.port : '5432'
   const fromDbName = fromUrl.pathname.replace('/', '')
-  const subName = `projectsub_${regionName}`
   const rawSqeel = `SELECT * FROM aiven_extras.pg_create_subscription(
     '${subName}',
     'dbname=${fromDbName} host=${fromUrl.hostname} port=${port} sslmode=${sslmode} user=${fromUrl.username} password=${fromUrl.password}',
-    'projectpub', 
+    '${pubName}', 
     '${subName}',
     TRUE,
     TRUE

packages/server/modules/multiregion/index.ts

@@ -1,5 +1,5 @@
 import { moduleLogger } from '@/logging/logging'
-import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { initializeRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
 import { isMultiRegionEnabled } from '@/modules/multiregion/helpers'
 import { SpeckleModule } from '@/modules/shared/helpers/typeHelper'
 
@@ -11,12 +11,9 @@ const multiRegion: SpeckleModule = {
     }
 
     moduleLogger.info('🌍 Init multiRegion module')
-    // this should have all the builtin checks to make sure all regions are working
-    // and no regions are missing
-    const regionClients = await getRegisteredRegionClients()
-    moduleLogger.info('Migrating region databases')
-    await Promise.all(Object.values(regionClients).map((db) => db.migrate.latest()))
-    moduleLogger.info('Migrations done')
+
+    // Init registered region clients
+    await initializeRegisteredRegionClients()
   }
 }
 

packages/server/modules/multiregion/regionConfig.ts

@@ -17,9 +17,11 @@ import {
 let multiRegionConfig: Optional<MultiRegionConfig> = undefined
 
 const getMultiRegionConfig = async (): Promise<MultiRegionConfig> => {
-  if (isDevOrTestEnv() && !isMultiRegionEnabled())
+  if (isDevOrTestEnv() && !isMultiRegionEnabled()) {
     // this should throw somehow
     return { main: { postgres: { connectionUri: '' } }, regions: {} }
+  }
+
   if (!multiRegionConfig) {
     const relativePath = getMultiRegionConfigPath()
 

packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts

@@ -15,11 +15,11 @@ import {
   TestApolloServer
 } from '@/test/graphqlHelper'
 import { beforeEachContext, truncateTables } from '@/test/hooks'
-import { MultiRegionConfigServiceMock } from '@/test/mocks/global'
+import { MultiRegionConfigMock, MultiRegionDbSelectorMock } from '@/test/mocks/global'
 import { Roles } from '@speckle/shared'
 import { expect } from 'chai'
 
-describe.skip('Multi Region Server Settings', () => {
+describe('Multi Region Server Settings', () => {
   let testAdminUser: BasicTestUser
   let testBasicUser: BasicTestUser
   let apollo: TestApolloServer
@@ -41,14 +41,12 @@ describe.skip('Multi Region Server Settings', () => {
   }
 
   before(async () => {
-    // Have to mock both
-    // MultiRegionConfigServiceMock.mockFunction(
-    //   'getAvailableRegionConfigsFactory',
-    //   () => async () => fakeRegionConfig
-    // )
-    MultiRegionConfigServiceMock.mockFunction(
-      'getAvailableRegionKeysFactory',
-      () => async () => Object.keys(fakeRegionConfig)
+    MultiRegionConfigMock.mockFunction(
+      'getAvailableRegionConfig',
+      async () => fakeRegionConfig
+    )
+    MultiRegionDbSelectorMock.mockFunction('initializeRegion', async () =>
+      Promise.resolve()
     )
 
     await beforeEachContext()
@@ -58,7 +56,8 @@ describe.skip('Multi Region Server Settings', () => {
   })
 
   after(() => {
-    MultiRegionConfigServiceMock.resetMockedFunctions()
+    MultiRegionConfigMock.resetMockedFunctions()
+    MultiRegionDbSelectorMock.resetMockedFunctions()
   })
 
   describe('server config', () => {
@@ -121,11 +120,11 @@ describe.skip('Multi Region Server Settings', () => {
         }
 
         const res = await createRegion(input)
+        expect(res).to.not.haveGraphQLErrors()
         expect(res.data?.serverInfoMutations.multiRegion.create).to.deep.equal({
           ...input,
           id: input.key
         })
-        expect(res).to.not.haveGraphQLErrors()
       })
 
       it("doesn't work with already used up key", async () => {

packages/server/modules/multiregion/tests/intergration/repositories/projectRegion.spec.ts

@@ -10,8 +10,13 @@ import { createInmemoryRedisClient } from '@/test/redisHelper'
 import { createStreamFactory } from '@/modules/core/repositories/streams'
 import { db } from '@/db/knex'
 import { storeRegionFactory } from '@/modules/multiregion/repositories'
+import { truncateRegionsSafely } from '@/test/speckle-helpers/regions'
 
 describe('projectRegion repositories @multiregion', () => {
+  after(async () => {
+    await truncateRegionsSafely()
+  })
+
   describe('inMemoryKeyStoreFactory creates an object, which', () => {
     const { getRegionKey, writeRegion } = inMemoryRegionKeyStoreFactory()
     it('returns undefined if projectId is not in the cache', () => {

packages/server/modules/shared/helpers/envHelper.ts

@@ -418,3 +418,6 @@ export function getOtelHeaderValue() {
 export function getMultiRegionConfigPath() {
   return getStringFromEnv('MULTI_REGION_CONFIG_PATH')
 }
+
+export const shouldRunTestsInMultiregionMode = () =>
+  getBooleanFromEnv('RUN_TESTS_IN_MULTIREGION_MODE')

packages/server/modules/workspaces/domain/operations.ts

@@ -12,6 +12,7 @@ import { EventBusPayloads } from '@/modules/shared/services/eventBus'
 import {
   MaybeNullOrUndefined,
   Nullable,
+  NullableKeysToOptional,
   Optional,
   PartialNullable,
   StreamRoles,
@@ -22,11 +23,22 @@ import { WorkspaceTeam } from '@/modules/workspaces/domain/types'
 import { Stream } from '@/modules/core/domain/streams/types'
 import { TokenResourceIdentifier } from '@/modules/core/domain/tokens/types'
 import { ServerRegion } from '@/modules/multiregion/domain/types'
+import { SetOptional } from 'type-fest'
 
 /** Workspace */
 
-type UpsertWorkspaceArgs = {
-  workspace: Omit<Workspace, 'domains'>
+export type UpsertWorkspaceArgs = {
+  workspace: Omit<
+    SetOptional<
+      NullableKeysToOptional<Workspace>,
+      | 'domainBasedMembershipProtectionEnabled'
+      | 'discoverabilityEnabled'
+      | 'defaultLogoIndex'
+      | 'defaultProjectRole'
+      | 'slug'
+    >,
+    'domains'
+  >
 }
 
 export type UpsertWorkspace = (args: UpsertWorkspaceArgs) => Promise<void>

packages/server/modules/workspaces/tests/integration/regions.graph.spec.ts

@@ -13,8 +13,9 @@ import {
   SetWorkspaceDefaultRegionDocument
 } from '@/test/graphql/generated/graphql'
 import { testApolloServer, TestApolloServer } from '@/test/graphqlHelper'
-import { beforeEachContext } from '@/test/hooks'
+import { beforeEachContext, getRegionKeys } from '@/test/hooks'
 import { MultiRegionDbSelectorMock } from '@/test/mocks/global'
+import { truncateRegionsSafely } from '@/test/speckle-helpers/regions'
 import { Roles } from '@speckle/shared'
 import { expect } from 'chai'
 
@@ -71,8 +72,9 @@ describe('Workspace regions GQL', () => {
     apollo = await testApolloServer({ authUserId: me.id })
   })
 
-  after(() => {
+  after(async () => {
     MultiRegionDbSelectorMock.resetMockedFunctions()
+    await truncateRegionsSafely()
   })
 
   describe('when listing', () => {
@@ -95,7 +97,7 @@ describe('Workspace regions GQL', () => {
       expect(res).to.not.haveGraphQLErrors()
       expect(
         res.data?.workspace.availableRegions.map((r) => r.key)
-      ).to.deep.equalInAnyOrder([region1Key, region2Key])
+      ).to.deep.equalInAnyOrder([region1Key, region2Key, ...getRegionKeys()])
     })
   })
 

packages/server/modules/workspaces/tests/integration/repositories.spec.ts

@@ -161,7 +161,7 @@ describe('Workspace repositories', () => {
       })
 
       afterEach(async () => {
-        truncateTables(['workspaces'])
+        await truncateTables(['workspaces'])
       })
 
       it('returns all workspace members', async () => {
@@ -209,7 +209,7 @@ describe('Workspace repositories', () => {
       })
 
       afterEach(async () => {
-        truncateTables(['workspaces'])
+        await truncateTables(['workspaces'])
       })
 
       it('limits search results to specified workspace', async () => {

packages/server/modules/workspaces/tests/integration/sso.graph.spec.ts

@@ -117,7 +117,7 @@ describe('Workspace SSO', () => {
   })
 
   afterEach(async () => {
-    truncateTables(['user_sso_sessions'])
+    await truncateTables(['user_sso_sessions'])
   })
 
   describe('given a workspace with SSO configured', () => {

packages/server/modules/workspaces/tests/integration/sso.spec.ts

@@ -256,7 +256,7 @@ describe('Workspace SSO repositories', () => {
     })
 
     afterEach(async () => {
-      truncateTables(['user_sso_sessions'])
+      await truncateTables(['user_sso_sessions'])
     })
 
     it('returns an empty array if there are no sessions', async () => {

packages/server/modules/workspaces/tests/unit/services/management.spec.ts

@@ -35,12 +35,15 @@ import {
 } from '@/modules/workspaces/errors/workspace'
 import { UserEmail } from '@/modules/core/domain/userEmails/types'
 import { merge, omit } from 'lodash'
-import { GetWorkspaceWithDomains } from '@/modules/workspaces/domain/operations'
+import {
+  GetWorkspaceWithDomains,
+  UpsertWorkspaceArgs
+} from '@/modules/workspaces/domain/operations'
 import { FindVerifiedEmailsByUserId } from '@/modules/core/domain/userEmails/operations'
 import { EventNames } from '@/modules/shared/services/eventBus'
 
 type WorkspaceTestContext = {
-  storedWorkspaces: Omit<Workspace, 'domains'>[]
+  storedWorkspaces: UpsertWorkspaceArgs['workspace'][]
   storedRoles: WorkspaceAcl[]
   eventData: {
     isCalled: boolean
@@ -63,11 +66,7 @@ const buildCreateWorkspaceWithTestContext = (
   }
 
   const deps: Parameters<typeof createWorkspaceFactory>[0] = {
-    upsertWorkspace: async ({
-      workspace
-    }: {
-      workspace: Omit<Workspace, 'domains'>
-    }) => {
+    upsertWorkspace: async ({ workspace }) => {
       context.storedWorkspaces.push(workspace)
     },
     validateSlug: async () => {},
@@ -1160,7 +1159,7 @@ describe('Workspace role services', () => {
         }
 
         let storedDomains: WorkspaceDomain | undefined = undefined
-        let storedWorkspace: Omit<Workspace, 'domains'> | undefined = undefined
+        let storedWorkspace: UpsertWorkspaceArgs['workspace'] | undefined = undefined
         let omittedEventName: EventNames | undefined = undefined
 
         const workspace: Workspace = {

packages/server/multiregion.test.example.json

@@ -0,0 +1,16 @@
+{
+  "main": {
+    "postgres": {
+      "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5432/speckle2_test",
+      "privateConnectionUri": "postgresql://speckle:speckle@postgres:5432/speckle2_test"
+    }
+  },
+  "regions": {
+    "region1": {
+      "postgres": {
+        "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5401/speckle2_test",
+        "privateConnectionUri": "postgresql://speckle:speckle@postgres-region1:5432/speckle2_test"
+      }
+    }
+  }
+}

packages/server/package.json

@@ -23,6 +23,7 @@
     "dev:clean": "yarn build:clean && yarn dev",
     "dev:server:test": "cross-env DISABLE_NOTIFICATIONS_CONSUMPTION=true NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true node ./bin/ts-www",
     "test": "cross-env NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true mocha",
+    "test:multiregion": "cross-env RUN_TESTS_IN_MULTIREGION_MODE=true yarn test",
     "test:coverage": "cross-env NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true nyc --reporter lcov mocha",
     "test:report": "yarn test:coverage -- --reporter mocha-junit-reporter --reporter-options mochaFile=reports/test-results.xml",
     "lint": "yarn lint:tsc && yarn lint:eslint",
@@ -30,6 +31,7 @@
     "lint:tsc": "tsc --noEmit",
     "lint:eslint": "eslint .",
     "cli": "cross-env LOG_LEVEL=debug LOG_PRETTY=true NODE_ENV=development ts-node ./modules/cli/index.ts",
+    "cli:test": "cross-env LOG_LEVEL=debug LOG_PRETTY=true NODE_ENV=test ts-node ./modules/cli/index.ts",
     "cli:download:commit": "cross-env LOG_PRETTY=true LOG_LEVEL=debug yarn cli download commit",
     "migrate": "yarn cli db migrate",
     "migrate:test": "cross-env NODE_ENV=test ts-node ./modules/cli/index.js db migrate",

packages/server/test/hooks.js

@@ -1,119 +0,0 @@
-require('../bootstrap')
-
-// Register global mocks as early as possible
-require('@/test/mocks/global')
-
-const chai = require('chai')
-const chaiAsPromised = require('chai-as-promised')
-const chaiHttp = require('chai-http')
-const deepEqualInAnyOrder = require('deep-equal-in-any-order')
-const { knex } = require(`@/db/knex`)
-const { init, startHttp, shutdown } = require(`@/app`)
-const { default: graphqlChaiPlugin } = require('@/test/plugins/graphql')
-const { logger } = require('@/logging/logging')
-const { once } = require('events')
-
-// Register chai plugins
-chai.use(chaiAsPromised)
-chai.use(chaiHttp)
-chai.use(deepEqualInAnyOrder)
-chai.use(graphqlChaiPlugin)
-
-const unlock = async () => {
-  const exists = await knex.schema.hasTable('knex_migrations_lock')
-  if (exists) {
-    await knex('knex_migrations_lock').update('is_locked', '0')
-  }
-}
-
-exports.truncateTables = async (tableNames) => {
-  if (!tableNames?.length) {
-    //why is server config only created once!????
-    // because its done in a migration, to not override existing configs
-    const protectedTables = ['server_config']
-    // const protectedTables = [ 'server_config', 'user_roles', 'scopes', 'server_acl' ]
-    tableNames = (
-      await knex('pg_tables')
-        .select('tablename')
-        .where({ schemaname: 'public' })
-        .whereRaw("tablename not like '%knex%'")
-        .whereNotIn('tablename', protectedTables)
-    ).map((table) => table.tablename)
-    if (!tableNames.length) return // Nothing to truncate
-
-    // We're deleting everything, so lets turn off triggers to avoid deadlocks/slowdowns
-    await knex.transaction(async (trx) => {
-      await trx.raw(`
-        -- Disable triggers and foreign key constraints for this session
-        SET session_replication_role = replica;
-        
-        truncate table ${tableNames.join(',')};
-
-        -- Re-enable triggers and foreign key constraints
-        SET session_replication_role = DEFAULT;
-      `)
-    })
-  } else {
-    await knex.raw(`truncate table ${tableNames.join(',')} cascade`)
-  }
-}
-
-/**
- * @param {import('http').Server} server
- * @param {import('express').Express} app
- */
-const initializeTestServer = async (server, app) => {
-  await startHttp(server, app, 0)
-
-  await once(app, 'appStarted')
-  const port = server.address().port
-  const serverAddress = `http://127.0.0.1:${port}`
-  const wsAddress = `ws://127.0.0.1:${port}`
-  return {
-    server,
-    serverAddress,
-    serverPort: port,
-    wsAddress,
-    sendRequest(auth, obj) {
-      return (
-        chai
-          .request(serverAddress)
-          .post('/graphql')
-          // if you set the header to null, the actual header in the req will be
-          // a string -> 'null'
-          // this is now treated as an invalid token, and gets forbidden
-          // switching to an empty string token
-          .set('Authorization', auth || '')
-          .send(obj)
-      )
-    }
-  }
-}
-
-exports.mochaHooks = {
-  beforeAll: async () => {
-    logger.info('running before all')
-    await unlock()
-    await exports.truncateTables()
-    await knex.migrate.rollback()
-    await knex.migrate.latest()
-    await init()
-  },
-  afterAll: async () => {
-    logger.info('running after all')
-    await unlock()
-    await shutdown()
-  }
-}
-
-exports.buildApp = async () => {
-  const { app, graphqlServer, server } = await init()
-  return { app, graphqlServer, server }
-}
-
-exports.beforeEachContext = async () => {
-  await exports.truncateTables()
-  return await exports.buildApp()
-}
-
-exports.initializeTestServer = initializeTestServer

packages/server/test/hooks.ts

@@ -0,0 +1,276 @@
+// eslint-disable-next-line no-restricted-imports
+import '../bootstrap'
+
+// Register global mocks as early as possible
+import '@/test/mocks/global'
+
+import chai from 'chai'
+import chaiAsPromised from 'chai-as-promised'
+import chaiHttp from 'chai-http'
+import deepEqualInAnyOrder from 'deep-equal-in-any-order'
+import { knex as mainDb } from '@/db/knex'
+import { init, startHttp, shutdown } from '@/app'
+import graphqlChaiPlugin from '@/test/plugins/graphql'
+import { logger } from '@/logging/logging'
+import { once } from 'events'
+import type http from 'http'
+import type express from 'express'
+import type net from 'net'
+import { MaybeAsync, MaybeNullOrUndefined } from '@speckle/shared'
+import type mocha from 'mocha'
+import { shouldRunTestsInMultiregionMode } from '@/modules/shared/helpers/envHelper'
+import {
+  getAvailableRegionKeysFactory,
+  getFreeRegionKeysFactory
+} from '@/modules/multiregion/services/config'
+import { getAvailableRegionConfig } from '@/modules/multiregion/regionConfig'
+import { createAndValidateNewRegionFactory } from '@/modules/multiregion/services/management'
+import {
+  getRegionFactory,
+  getRegionsFactory,
+  Regions,
+  storeRegionFactory
+} from '@/modules/multiregion/repositories'
+import {
+  getRegisteredRegionClients,
+  initializeRegion
+} from '@/modules/multiregion/dbSelector'
+import { Knex } from 'knex'
+
+// why is server config only created once!????
+// because its done in a migration, to not override existing configs
+// similarly wiping regions will break multi region setup
+const protectedTables = ['server_config', 'regions']
+let regionClients: Record<string, Knex> = {}
+
+// Register chai plugins
+chai.use(chaiAsPromised)
+chai.use(chaiHttp)
+chai.use(deepEqualInAnyOrder)
+chai.use(graphqlChaiPlugin)
+
+const inEachDb = async (fn: (db: Knex) => MaybeAsync<void>) => {
+  await fn(mainDb)
+  for (const regionClient of Object.values(regionClients)) {
+    await fn(regionClient)
+  }
+}
+
+const ensureAivenExtrasFactory = (deps: { db: Knex }) => async () => {
+  await deps.db.raw('CREATE EXTENSION IF NOT EXISTS "aiven_extras";')
+}
+
+const setupMultiregionMode = async () => {
+  const db = mainDb
+  const getAvailableRegionKeys = getAvailableRegionKeysFactory({
+    getAvailableRegionConfig
+  })
+  const regionKeys = await getAvailableRegionKeys()
+
+  // Create DB region entries for each key
+  const createRegion = createAndValidateNewRegionFactory({
+    getFreeRegionKeys: getFreeRegionKeysFactory({
+      getAvailableRegionKeys,
+      getRegions: getRegionsFactory({ db })
+    }),
+    getRegion: getRegionFactory({ db }),
+    storeRegion: storeRegionFactory({ db }),
+    initializeRegion
+  })
+  for (const regionKey of regionKeys) {
+    await createRegion({
+      region: {
+        key: regionKey,
+        name: regionKey,
+        description: 'Auto created test region'
+      }
+    })
+  }
+
+  // Store active region clients
+  regionClients = await getRegisteredRegionClients()
+
+  // Reset each DB client (re-run all migrations and setup)
+  for (const [, regionClient] of Object.entries(regionClients)) {
+    const reset = resetSchemaFactory({ db: regionClient })
+    await reset()
+  }
+
+  // If not in multi region mode, delete region entries
+  // we only needed them to reset schemas
+  if (!shouldRunTestsInMultiregionMode()) {
+    await truncateTables([Regions.name])
+  }
+}
+
+const unlockFactory = (deps: { db: Knex }) => async () => {
+  const exists = await deps.db.schema.hasTable('knex_migrations_lock')
+  if (exists) {
+    await deps.db('knex_migrations_lock').update('is_locked', '0')
+  }
+}
+
+export const getRegionKeys = () => Object.keys(regionClients)
+
+export const resetPubSubFactory = (deps: { db: Knex }) => async () => {
+  if (!shouldRunTestsInMultiregionMode()) {
+    return { drop: async () => {}, reenable: async () => {} }
+  }
+
+  const ensureAivenExtras = ensureAivenExtrasFactory(deps)
+  await ensureAivenExtras()
+
+  const subscriptions = (await deps.db.raw(
+    `SELECT subname, subconninfo, subpublications, subslotname FROM aiven_extras.pg_list_all_subscriptions() WHERE subname ILIKE 'test_%';`
+  )) as {
+    rows: Array<{
+      subname: string
+      subconninfo: string
+      subpublications: string[]
+      subslotname: string
+    }>
+  }
+  const publications = (await deps.db.raw(
+    `SELECT pubname FROM pg_publication WHERE pubname ILIKE 'test_%';`
+  )) as {
+    rows: Array<{ pubname: string }>
+  }
+
+  // Drop all subs
+  for (const sub of subscriptions.rows) {
+    await deps.db.raw(`
+          SELECT * FROM aiven_extras.pg_alter_subscription_disable('${sub.subname}');
+          SELECT * FROM aiven_extras.pg_drop_subscription('${sub.subname}');
+          SELECT * FROM aiven_extras.dblink_slot_create_or_drop('${sub.subconninfo}', '${sub.subslotname}', 'drop');
+        `)
+  }
+
+  // Drop all pubs
+  for (const pub of publications.rows) {
+    await deps.db.raw(`DROP PUBLICATION ${pub.pubname};`)
+  }
+}
+
+const truncateTablesFactory = (deps: { db: Knex }) => async (tableNames?: string[]) => {
+  if (!tableNames?.length) {
+    tableNames = (
+      await deps
+        .db('pg_tables')
+        .select('tablename')
+        .where({ schemaname: 'public' })
+        .whereRaw("tablename not like '%knex%'")
+        .whereNotIn('tablename', protectedTables)
+    ).map((table: { tablename: string }) => table.tablename)
+    if (!tableNames.length) return // Nothing to truncate
+
+    // We're deleting everything, so lets turn off triggers to avoid deadlocks/slowdowns
+    await deps.db.transaction(async (trx) => {
+      await trx.raw(`
+        -- Disable triggers and foreign key constraints for this session
+        SET session_replication_role = replica;
+
+        truncate table ${tableNames?.join(',') || ''};
+
+        -- Re-enable triggers and foreign key constraints
+        SET session_replication_role = DEFAULT;
+      `)
+    })
+  } else {
+    await deps.db.raw(`truncate table ${tableNames.join(',')} cascade`)
+  }
+}
+
+const resetSchemaFactory = (deps: { db: Knex }) => async () => {
+  const resetPubSub = resetPubSubFactory(deps)
+
+  await unlockFactory(deps)()
+  await resetPubSub()
+
+  // Reset schema
+  await deps.db.migrate.rollback()
+  await deps.db.migrate.latest()
+}
+
+export const truncateTables = async (tableNames?: string[]) => {
+  const dbs = [mainDb, ...Object.values(regionClients)]
+
+  // First reset pubsubs
+  for (const db of dbs) {
+    const resetPubSub = resetPubSubFactory({ db })
+    await resetPubSub()
+  }
+
+  // Now truncate
+  for (const db of dbs) {
+    const truncate = truncateTablesFactory({ db })
+    await truncate(tableNames)
+  }
+}
+
+export const initializeTestServer = async (
+  server: http.Server,
+  app: express.Express
+) => {
+  await startHttp(server, app, 0)
+
+  await once(app, 'appStarted')
+  const port = (server.address() as net.AddressInfo).port + ''
+  const serverAddress = `http://127.0.0.1:${port}`
+  const wsAddress = `ws://127.0.0.1:${port}`
+  return {
+    server,
+    serverAddress,
+    serverPort: port,
+    wsAddress,
+    sendRequest(auth: MaybeNullOrUndefined<string>, obj: string | object) {
+      return (
+        chai
+          .request(serverAddress)
+          .post('/graphql')
+          // if you set the header to null, the actual header in the req will be
+          // a string -> 'null'
+          // this is now treated as an invalid token, and gets forbidden
+          // switching to an empty string token
+          .set('Authorization', auth || '')
+          .send(obj)
+      )
+    }
+  }
+}
+
+export const mochaHooks: mocha.RootHookObject = {
+  beforeAll: async () => {
+    if (shouldRunTestsInMultiregionMode()) {
+      console.log('Running tests in multi-region mode...')
+    }
+
+    logger.info('running before all')
+
+    // Init main db
+    const reset = resetSchemaFactory({ db: mainDb })
+    await reset()
+
+    // Init (or cleanup) multi-region mode
+    await setupMultiregionMode()
+
+    // Init app
+    await init()
+  },
+  afterAll: async () => {
+    logger.info('running after all')
+    await inEachDb(async (db) => {
+      await unlockFactory({ db })()
+    })
+    await shutdown()
+  }
+}
+
+export const buildApp = async () => {
+  const { app, graphqlServer, server } = await init()
+  return { app, graphqlServer, server }
+}
+
+export const beforeEachContext = async () => {
+  await truncateTables()
+  return await buildApp()
+}

packages/server/test/mocks/global.ts

@@ -12,10 +12,10 @@ export const CommentsRepositoryMock = mockRequireModule<
   typeof import('@/modules/comments/repositories/comments')
 >(['@/modules/comments/repositories/comments'])
 
-export const MultiRegionConfigServiceMock = mockRequireModule<
-  typeof import('@/modules/multiregion/services/config')
->(['@/modules/multiregion/services/config'])
-
 export const MultiRegionDbSelectorMock = mockRequireModule<
   typeof import('@/modules/multiregion/dbSelector')
 >(['@/modules/multiregion/dbSelector'])
+
+export const MultiRegionConfigMock = mockRequireModule<
+  typeof import('@/modules/multiregion/regionConfig')
+>(['@/modules/multiregion/regionConfig'])

packages/server/test/speckle-helpers/regions.ts

@@ -0,0 +1,11 @@
+import { db } from '@/db/knex'
+import { Regions } from '@/modules/multiregion/repositories'
+import { getRegionKeys } from '@/test/hooks'
+
+/**
+ * Delete all regions entries that are not part of the main multi region mode
+ */
+export const truncateRegionsSafely = async () => {
+  const regionKeys = getRegionKeys()
+  await db(Regions.name).whereNotIn(Regions.col.key, regionKeys).delete()
+}