Dim/dui3 main merge jan

.circleci/config.yml

@@ -16,7 +16,7 @@ workflows:
                 - main
                 - hotfix*
 
-      - test-server:
+      - test-server: &test-server-job-definition
           context:
             - speckle-server-licensing
             - stripe-integration
@@ -32,6 +32,8 @@ workflows:
           requires:
             - docker-publish-postgres-container
 
+      - test-server-multiregion: *test-server-job-definition
+
       - test-frontend-2:
           filters: *filters-allow-all
 
@@ -190,6 +192,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-frontend:
@@ -205,6 +208,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-frontend-2:
@@ -220,6 +224,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-webhooks:
@@ -235,6 +240,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-file-imports:
@@ -250,6 +256,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-previews:
@@ -265,6 +272,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-test-container:
@@ -280,6 +288,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-postgres-container:
@@ -301,6 +310,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-docker-compose-ingress:
@@ -316,6 +326,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - publish-helm-chart:
@@ -356,6 +367,7 @@ workflows:
             - get-version
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-ui-components
             - test-frontend-2
             - test-viewer
@@ -482,8 +494,9 @@ jobs:
       S3_CREATE_BUCKET: 'true'
       REDIS_URL: 'redis://127.0.0.1:6379'
       S3_REGION: '' # optional, defaults to 'us-east-1'
-      AUTOMATE_ENCRYPTION_KEYS_PATH: 'test/assets/automate/encryptionKeys.json'
+      ENCRYPTION_KEYS_PATH: 'test/assets/automate/encryptionKeys.json'
       FF_BILLING_INTEGRATION_ENABLED: 'true'
+      RATELIMITER_ENABLED: 'false'
     steps:
       - checkout
       - restore_cache:
@@ -531,8 +544,8 @@ jobs:
           working_directory: 'packages/server'
 
       - run:
-          name: Checking for GQL schema breakages against speckle.xyz
-          command: 'yarn rover graph check Speckle-Server@speckle-xyz --schema ./introspected-schema.graphql'
+          name: Checking for GQL schema breakages against app.speckle.systems
+          command: 'yarn rover graph check Speckle-Server@app-speckle-systems --schema ./introspected-schema.graphql'
           working_directory: 'packages/server'
 
       - run:
@@ -564,14 +577,56 @@ jobs:
       S3_CREATE_BUCKET: 'true'
       REDIS_URL: 'redis://127.0.0.1:6379'
       S3_REGION: '' # optional, defaults to 'us-east-1'
-      AUTOMATE_ENCRYPTION_KEYS_PATH: 'test/assets/automate/encryptionKeys.json'
-      FF_AUTOMATE_MODULE_ENABLED: 'false' # Disable all FFs
-      FF_WORKSPACES_MODULE_ENABLED: 'false'
-      FF_WORKSPACES_SSO_ENABLED: 'false'
-      FF_MULTIPLE_EMAILS_MODULE_ENABLED: 'false'
-      FF_GENDOAI_MODULE_ENABLED: 'false'
-      FF_GATEKEEPER_MODULE_ENABLED: 'false'
-      FF_BILLING_INTEGRATION_ENABLED: 'false'
+      ENCRYPTION_KEYS_PATH: 'test/assets/automate/encryptionKeys.json'
+      DISABLE_ALL_FFS: 'true'
+      RATELIMITER_ENABLED: 'false'
+
+  test-server-multiregion:
+    <<: *test-server-job
+    docker:
+      - image: cimg/node:18.19.0
+      - image: cimg/redis:7.2.4
+      - image: 'speckle/speckle-postgres'
+        environment:
+          POSTGRES_DB: speckle2_test
+          POSTGRES_PASSWORD: speckle
+          POSTGRES_USER: speckle
+        command: -c 'max_connections=1000' -c 'wal_level=logical'
+      - image: 'speckle/speckle-postgres'
+        environment:
+          POSTGRES_DB: speckle2_test
+          POSTGRES_PASSWORD: speckle
+          POSTGRES_USER: speckle
+        command: -c 'max_connections=1000' -c 'port=5433' -c 'wal_level=logical'
+      - image: 'minio/minio'
+        command: server /data --console-address ":9001" --address "0.0.0.0:9000"
+      - image: 'minio/minio'
+        command: server /data --console-address ":9021" --address "0.0.0.0:9020"
+    environment:
+      # Same as test-server:
+      NODE_ENV: test
+      DATABASE_URL: 'postgres://speckle:[email protected]:5432/speckle2_test'
+      PGDATABASE: speckle2_test
+      POSTGRES_MAX_CONNECTIONS_SERVER: 20
+      PGUSER: speckle
+      SESSION_SECRET: 'keyboard cat'
+      STRATEGY_LOCAL: 'true'
+      CANONICAL_URL: 'http://127.0.0.1:3000'
+      S3_ENDPOINT: 'http://127.0.0.1:9000'
+      S3_ACCESS_KEY: 'minioadmin'
+      S3_SECRET_KEY: 'minioadmin'
+      S3_BUCKET: 'speckle-server'
+      S3_CREATE_BUCKET: 'true'
+      REDIS_URL: 'redis://127.0.0.1:6379'
+      S3_REGION: '' # optional, defaults to 'us-east-1'
+      ENCRYPTION_KEYS_PATH: 'test/assets/automate/encryptionKeys.json'
+      FF_BILLING_INTEGRATION_ENABLED: 'true'
+      # These are the only different env keys:
+      MULTI_REGION_CONFIG_PATH: '../../.circleci/multiregion.test-ci.json'
+      FF_WORKSPACES_MODULE_ENABLED: 'true'
+      FF_WORKSPACES_MULTI_REGION_ENABLED: 'true'
+      RUN_TESTS_IN_MULTIREGION_MODE: true
+      RATELIMITER_ENABLED: 'false'
 
   test-frontend-2:
     docker: &docker-node-browsers-image
@@ -996,7 +1051,6 @@ jobs:
   docker-build-monitor-container:
     <<: *build-job
     environment:
-      FOLDER: utils
       SPECKLE_SERVER_PACKAGE: monitor-deployment
 
   docker-build-docker-compose-ingress:
@@ -1066,7 +1120,6 @@ jobs:
   docker-publish-monitor-container:
     <<: *publish-job
     environment:
-      FOLDER: utils
       SPECKLE_SERVER_PACKAGE: monitor-deployment
 
   docker-publish-docker-compose-ingress:

.circleci/multiregion.test-ci.json

@@ -0,0 +1,30 @@
+{
+  "main": {
+    "postgres": {
+      "connectionUri": "postgresql://speckle:[email protected]:5432/speckle2_test"
+    },
+    "blobStorage": {
+      "accessKey": "minioadmin",
+      "secretKey": "minioadmin",
+      "bucket": "speckle-server",
+      "createBucketIfNotExists": true,
+      "endpoint": "http://127.0.0.1:9000",
+      "s3Region": "us-east-1"
+    }
+  },
+  "regions": {
+    "region1": {
+      "postgres": {
+        "connectionUri": "postgresql://speckle:[email protected]:5433/speckle2_test"
+      },
+      "blobStorage": {
+        "accessKey": "minioadmin",
+        "secretKey": "minioadmin",
+        "bucket": "speckle-server",
+        "createBucketIfNotExists": true,
+        "endpoint": "http://127.0.0.1:9020",
+        "s3Region": "us-east-1"
+      }
+    }
+  }
+}

.github/workflows/preview-service-acceptance.yml

@@ -69,7 +69,7 @@ jobs:
     services:
       postgres:
         # Docker Hub image
-        image: postgres:16.4-bookworm@sha256:91f464e7ba0ad91a106c94cff079fb4384139291b8c0502fd36989cf2c788bbb
+        image: postgres:16.4-bookworm@sha256:e62fbf9d3e2b49816a32c400ed2dba83e3b361e6833e624024309c35d334b412
         env:
           POSTGRES_DB: preview_service_test
           POSTGRES_PASSWORD: preview_service_test

.gitignore

@@ -72,6 +72,12 @@ postgres-data/
 redis-data/
 
 .tshy-build
+obj/
+bin/
+!packages/monitor-deployment/bin
+!packages/preview-service/bin
+!packages/server/bin
 
 # Server
-multiregion.json
+multiregion.json
+multiregion.test.json

.prettierignore

@@ -38,4 +38,8 @@ venv
 
 storybook-static
 .tshy
-.tshy-build
+.tshy-build
+
+packages/fileimport-service/ifc-dotnet/
+packages/fileimport-service/stl/
+packages/fileimport-service/obj/

README.md

@@ -99,6 +99,14 @@ EMAIL_PORT="1025"
 
 The web portal is available at `localhost:1080` and it's listening for mail on port `1025`.
 
+### Minio (S3 storage)
+
+Default credentials are: `minioadmin:minioadmin`
+Main storage Web UI: [http://localhost:9001/](http://localhost:9001/)
+Region1 storage Web UI: [http://localhost:9021/](http://localhost:9021/)
+
+You can use the web UI to validate uploaded blobs
+
 # Contributing
 
 Please make sure you read the [contribution guidelines](https://github.com/specklesystems/speckle-server/blob/main/CONTRIBUTING.md) for an overview of the best practices we try to follow.

docker-compose-deps.yml

@@ -52,6 +52,16 @@ services:
       - '127.0.0.1:9000:9000'
       - '127.0.0.1:9001:9001'
 
+  minio-region1:
+    image: 'minio/minio'
+    command: server /data --console-address ":9001"
+    restart: always
+    volumes:
+      - minio-region1-data:/data
+    ports:
+      - '127.0.0.1:9020:9000'
+      - '127.0.0.1:9021:9001'
+
   # Local OIDC provider for testing
   keycloak:
     image: quay.io/keycloak/keycloak:25.0
@@ -127,3 +137,4 @@ volumes:
   pgadmin-data:
   redis_insight-data:
   minio-data:
+  minio-region1-data:

mise.toml

@@ -0,0 +1,2 @@
+[tools]
+node = '22'

package.json

@@ -21,6 +21,7 @@
     "dev:docker": "docker compose -f ./docker-compose-deps.yml",
     "dev:docker:up": "docker compose -f ./docker-compose-deps.yml up -d",
     "dev:docker:down": "docker compose -f ./docker-compose-deps.yml down",
+    "dev:docker:restart": "yarn dev:docker:down && yarn dev:docker:up",
     "dev:kind:up": "ctlptl apply --filename ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:down": "ctlptl delete -f ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:helm:up": "yarn dev:kind:up && tilt up --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server",