test(backend): add federation test (#14582)

* test(backend): add federation test * fix(ci): install pnpm * fix(ci): cd * fix(ci): build entire project * fix(ci): skip frontend build * fix(ci): pull submodule when checkout * chore: show log for debugging * Revert "chore: show log for debugging" This reverts commit a930964. * fix(ci): build entire project * chore: omit unused globals * refactor: use strictEqual and simplify some asserts * test: follow requests * refactor: add resolveRemoteNote function * refactor: refine resolveRemoteUser function * refactor: cache admin credentials * refactor: simplify assertion with excluded fields * refactor: use assert * test: note * chore: labeler detect federation * test: blocking * test: move * fix: use appropriate TLD * chore: shorter purge interval * fix(ci): change TLD * refactor: delete trivial comment * test(user): isCat * chore: use jest * chore: omit logs * chore: add memo * fix(ci): omit unnecessary build * test: pinning Note * fix: build daemon in container * style: indent * test(streaming): timeline * chore: rename * fix: delete role after test * refactor: resolve users by uri * fix: delete antenna after test * test: api timeline * test: Note deletion * refactor: sleep function * test: notification * style: indent * refactor: type-safe host * docs: update description * refactor: resolve function params * fix(block): wrong test name * fix: invalid type * fix: longer timeout for fire testing * test(timeline): hashtag * test(note): vote delivery * fix: wrong description * fix: hashtag channel param type * refactor: wrap basic cases * test(timeline): add homeTimeline tests * fix(timeline): correct wrong case and description * test(notification): add tests for Note * refactor(user): wrap profile consistency with describe * chore(note): add issue link * test(timeline): add test * test(user): suspension * test: emoji * refactor: fetch admin first * perf: faster tests * test(drive): sensitive flag * test(emoji): add tests * chore: ignore .config/docker.env * chore: hard-coded tester IP address * test(emoji): custom emoji are surrounded by zero width space * refactor: client and username as property * test(notification): mute * fix(notification): correct description * test(block): mention * refactor(emoji): addCustomEmoji function * fix: typo * test(note): add reaction tests * test(timeline): Note deletion * fix: unnecessary ts-expect-error * refactor: unnecessary fetch mocking * chore: add TODO comments * test(user): deletion * chore: enable --frozen-lockfile * fix(ci): copying configs * docs: update CONTRIBUTING.md * docs: fix typo * chore: set default sleep duration * fix(notification): omit flaky tests * fix(notification): correct type * test(notification): add api endpoint tests * chore: remove redundant mute test * refactor: use param client * fix: start timer after trigger * refactor: remove unnecessary any * chore: shorter timeout for checking if fired * fix(block): remove outdated comment * refactor: shorten remote user variable name * refactor(block): use existing function * refactor: file upload * docs: update description * test(user): ffVisibility * fix: `/api/signin` -> `/api/signin-flow` * test: abuse report * refactor: use existing type * refactor: extract duplicate configs to template file * fix: typo * fix: avoid conflict * refactor: change container dependency * perf: start misskey parallelly * fix: remove dependency * chore(backend): add typecheck * test: add check for #14728 * chore: enable eslint check * perf: don't start linked services when test * test(note): remote note deletion for moderation * chore: define config template * chore: write setup script * refactor: omit unnecessary conditional * refactor: clarify scope * refactor: omit type assertion * refactor: omit logs * style * refactor: redundant promise * refactor: unnecessary imports * refactor: use readable error code * refactor: cache set in signin function * refactor: optimize import
misskey-dev · Oct 15, 2024 · b990ae6 · b990ae6
1 parent 3cea890
commit b990ae6
Show file tree

Hide file tree

Showing 32 changed files with 3,154 additions and 21 deletions.
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -6,7 +6,7 @@
 'packages/backend:test':
 - any:
   - changed-files:
-    - any-glob-to-any-file: ['packages/backend/test/**/*']
+    - any-glob-to-any-file: ['packages/backend/test/**/*', 'packages/backend/test-federation/**/*']
 
 'packages/frontend':
 - any:

diff --git a/.github/workflows/test-federation.yml b/.github/workflows/test-federation.yml
@@ -0,0 +1,59 @@
+name: Test (federation)
+
+on:
+  push:
+    branches:
+      - master
+      - develop
+    paths:
+      - packages/backend/**
+      - packages/misskey-js/**
+      - .github/workflows/test-federation.yml
+  pull_request:
+    paths:
+      - packages/backend/**
+      - packages/misskey-js/**
+      - .github/workflows/test-federation.yml
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [20.16.0]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+      - name: Install FFmpeg
+        uses: FedericoCarboni/setup-ffmpeg@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/[email protected]
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'pnpm'
+      - name: Build Misskey
+        run: |
+          corepack enable && corepack prepare
+          pnpm i --frozen-lockfile
+          pnpm build
+      - name: Setup
+        run: |
+          cd packages/backend/test-federation
+          bash ./setup.sh
+          sudo chmod 644 ./certificates/*.test.key
+      - name: Start servers
+        # https://github.com/docker/compose/issues/1294#issuecomment-374847206
+        run: |
+          cd packages/backend/test-federation
+          docker compose up -d --scale tester=0
+      - name: Test
+        run: |
+          cd packages/backend/test-federation
+          docker compose run --no-deps tester
+      - name: Stop servers
+        run: |
+          cd packages/backend/test-federation
+          docker compose down
diff --git a/.gitignore b/.gitignore
@@ -37,7 +37,7 @@ coverage
 !/.config/docker_example.env
 !/.config/cypress-devcontainer.yml
 docker-compose.yml
-compose.yml
+./compose.yml
 .devcontainer/compose.yml
 !/.devcontainer/compose.yml
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -181,31 +181,45 @@ MK_DEV_PREFER=backend pnpm dev
 - HMR may not work in some environments such as Windows.
 
 ## Testing
-- Test codes are located in [`/packages/backend/test`](/packages/backend/test).
-
-### Run test
-Create a config file.
+You can run non-backend tests by executing following commands:
+```sh
+pnpm --filter frontend test
+pnpm --filter misskey-js test
 ```
+
+Backend tests require manual preparation of servers. See the next section for more on this.
+
+### Backend
+There are three types of test codes for the backend:
+- Unit tests: [`/packages/backend/test/unit`](/packages/backend/test/unit)
+- Single-server E2E tests: [`/packages/backend/test/e2e`](/packages/backend/test/e2e)
+- Multiple-server E2E tests: [`/packages/backend/test-federation`](/packages/backend/test-federation)
+
+#### Running Unit Tests or Single-server E2E Tests
+1. Create a config file:
+```sh
 cp .github/misskey/test.yml .config/
 ```
-Prepare DB/Redis for testing.
-```
-docker compose -f packages/backend/test/compose.yml up
-```
-Alternatively, prepare an empty (data can be erased) DB and edit `.config/test.yml`.
 
-Run all test.
-```
-pnpm test
+2. Start DB and Redis servers for testing:
+```sh
+docker compose -f packages/backend/test/compose.yml up
 ```
+Instead, you can prepare an empty (data can be erased) DB and edit `.config/test.yml` appropriately.
 
-#### Run specify test
+3. Run all tests:
+```sh
+pnpm --filter backend test     # unit tests
+pnpm --filter backend test:e2e # single-server E2E tests
 ```
-pnpm jest -- foo.ts
+If you want to run a specific test, run as a following command:
+```sh
+pnpm --filter backend test -- packages/backend/test/unit/activitypub.ts
+pnpm --filter backend test:e2e -- packages/backend/test/e2e/nodeinfo.ts
 ```
 
-### e2e tests
-TODO
+#### Running Multiple-server E2E Tests
+See [`/packages/backend/test-federation/README.md`](/packages/backend/test-federation/README.md).
 
 ## Environment Variable
 

diff --git a/packages/backend/eslint.config.js b/packages/backend/eslint.config.js
@@ -11,7 +11,7 @@ export default [
 		languageOptions: {
 			parserOptions: {
 				parser: tsParser,
-				project: ['./tsconfig.json', './test/tsconfig.json'],
+				project: ['./tsconfig.json', './test/tsconfig.json', './test-federation/tsconfig.json'],
 				sourceType: 'module',
 				tsconfigRootDir: import.meta.dirname,
 			},

diff --git a/packages/backend/jest.config.fed.cjs b/packages/backend/jest.config.fed.cjs
@@ -0,0 +1,13 @@
+/*
+ * For a detailed explanation regarding each configuration property and type check, visit:
+ * https://jestjs.io/docs/en/configuration.html
+ */
+
+const base = require('./jest.config.cjs');
+
+module.exports = {
+	...base,
+	testMatch: [
+		'<rootDir>/test-federation/test/**/*.test.ts',
+	],
+};
diff --git a/packages/backend/package.json b/packages/backend/package.json
@@ -19,16 +19,18 @@
 		"watch": "node ./scripts/watch.mjs",
 		"restart": "pnpm build && pnpm start",
 		"dev": "node ./scripts/dev.mjs",
-		"typecheck": "tsc --noEmit && tsc -p test --noEmit",
-		"eslint": "eslint --quiet \"src/**/*.ts\"",
+		"typecheck": "tsc --noEmit && tsc -p test --noEmit && tsc -p test-federation --noEmit",
+		"eslint": "eslint --quiet \"{src,test-federation}/**/*.ts\"",
 		"lint": "pnpm typecheck && pnpm eslint",
 		"jest": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --forceExit --config jest.config.unit.cjs",
 		"jest:e2e": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --forceExit --config jest.config.e2e.cjs",
+		"jest:fed": "node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --forceExit --config jest.config.fed.cjs",
 		"jest-and-coverage": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --coverage --forceExit --config jest.config.unit.cjs",
 		"jest-and-coverage:e2e": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --coverage --forceExit --config jest.config.e2e.cjs",
 		"jest-clear": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --clearCache",
 		"test": "pnpm jest",
 		"test:e2e": "pnpm build && pnpm build:test && pnpm jest:e2e",
+		"test:fed": "pnpm jest:fed",
 		"test-and-coverage": "pnpm jest-and-coverage",
 		"test-and-coverage:e2e": "pnpm build && pnpm build:test && pnpm jest-and-coverage:e2e",
 		"generate-api-json": "node ./scripts/generate_api_json.js"

diff --git a/packages/backend/test-federation/.config/example.conf b/packages/backend/test-federation/.config/example.conf
@@ -0,0 +1,70 @@
+# based on https://github.com/misskey-dev/misskey-hub/blob/7071f63a1c80ee35c71f0fd8a6d8722c118c7574/src/docs/admin/nginx.md
+
+# For WebSocket
+map $http_upgrade $connection_upgrade {
+	default upgrade;
+	'' close;
+}
+
+proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=cache1:16m max_size=1g inactive=720m use_temp_path=off;
+
+server {
+	listen 80;
+	listen [::]:80;
+	server_name ${HOST};
+
+	# For SSL domain validation
+	root /var/www/html;
+	location /.well-known/acme-challenge/ { allow all; }
+	location /.well-known/pki-validation/ { allow all; }
+	location / { return 301 https://$server_name$request_uri; }
+}
+
+server {
+	listen 443 ssl;
+	listen [::]:443 ssl;
+	http2 on;
+	server_name ${HOST};
+
+	ssl_session_timeout 1d;
+	ssl_session_cache shared:ssl_session_cache:10m;
+	ssl_session_tickets off;
+
+	ssl_trusted_certificate /etc/nginx/certificates/rootCA.crt;
+	ssl_certificate /etc/nginx/certificates/$server_name.crt;
+	ssl_certificate_key /etc/nginx/certificates/$server_name.key;
+
+	# SSL protocol settings
+	ssl_protocols TLSv1.2 TLSv1.3;
+	ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+	ssl_prefer_server_ciphers off;
+	ssl_stapling on;
+	ssl_stapling_verify on;
+
+	# Change to your upload limit
+	client_max_body_size 80m;
+
+	# Proxy to Node
+	location / {
+		proxy_pass http://misskey.${HOST}:3000;
+		proxy_set_header Host $host;
+		proxy_http_version 1.1;
+		proxy_redirect off;
+
+		# If it's behind another reverse proxy or CDN, remove the following.
+		proxy_set_header X-Real-IP $remote_addr;
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_set_header X-Forwarded-Proto https;
+
+		# For WebSocket
+		proxy_set_header Upgrade $http_upgrade;
+		proxy_set_header Connection $connection_upgrade;
+
+		# Cache settings
+		proxy_cache cache1;
+		proxy_cache_lock on;
+		proxy_cache_use_stale updating;
+		proxy_force_ranges on;
+		add_header X-Cache $upstream_cache_status;
+	}
+}
diff --git a/packages/backend/test-federation/.config/example.default.yml b/packages/backend/test-federation/.config/example.default.yml
@@ -0,0 +1,25 @@
+url: https://${HOST}/
+port: 3000
+db:
+  host: db.${HOST}
+  port: 5432
+  db: misskey
+  user: postgres
+  pass: postgres
+dbReplications: false
+redis:
+  host: redis.test
+  port: 6379
+id: 'aidx'
+proxyBypassHosts:
+  - api.deepl.com
+  - api-free.deepl.com
+  - www.recaptcha.net
+  - hcaptcha.com
+  - challenges.cloudflare.com
+proxyRemoteFiles: true
+signToActivityPubGet: true
+allowedPrivateNetworks: [
+  '127.0.0.1/32',
+  '172.20.0.0/16'
+]
diff --git a/packages/backend/test-federation/.config/example.docker.env b/packages/backend/test-federation/.config/example.docker.env
@@ -0,0 +1,5 @@
+NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/rootCA.crt
+POSTGRES_DB=misskey
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+MK_VERBOSE=true
diff --git a/packages/backend/test-federation/.gitignore b/packages/backend/test-federation/.gitignore
@@ -0,0 +1,6 @@
+certificates
+volumes
+.env
+docker.env
+*.test.conf
+*.test.default.yml
diff --git a/packages/backend/test-federation/README.md b/packages/backend/test-federation/README.md
@@ -0,0 +1,24 @@
+## test-federation
+Test federation between two Misskey servers: `a.test` and `b.test`.
+
+Before testing, you need to build the entire project, and change working directory to here:
+```sh
+pnpm build
+cd packages/backend/test-federation
+```
+
+First, you need to start servers by executing following commands:
+```sh
+bash ./setup.sh
+docker compose up --scale tester=0
+```
+
+Then you can run all tests by a following command:
+```sh
+docker compose run --no-deps --rm tester
+```
+
+For testing a specific file, run a following command:
+```sh
+docker compose run --no-deps --rm tester -- pnpm -F backend test:fed packages/backend/test-federation/test/user.test.ts
+```
diff --git a/packages/backend/test-federation/compose.a.yml b/packages/backend/test-federation/compose.a.yml
@@ -0,0 +1,64 @@
+services:
+  a.test:
+    extends:
+      file: ./compose.tpl.yml
+      service: nginx
+    depends_on:
+      misskey.a.test:
+        condition: service_healthy
+    networks:
+      - internal_network_a
+    volumes:
+      - type: bind
+        source: ./.config/a.test.conf
+        target: /etc/nginx/conf.d/a.test.conf
+        read_only: true
+      - type: bind
+        source: ./certificates/a.test.crt
+        target: /etc/nginx/certificates/a.test.crt
+        read_only: true
+      - type: bind
+        source: ./certificates/a.test.key
+        target: /etc/nginx/certificates/a.test.key
+        read_only: true
+
+  misskey.a.test:
+    extends:
+      file: ./compose.tpl.yml
+      service: misskey
+    depends_on:
+      db.a.test:
+        condition: service_healthy
+      redis.test:
+        condition: service_healthy
+      setup:
+        condition: service_completed_successfully
+    networks:
+      - internal_network_a
+    volumes:
+      - type: bind
+        source: ./.config/a.test.default.yml
+        target: /misskey/.config/default.yml
+        read_only: true
+
+  db.a.test:
+    extends:
+      file: ./compose.tpl.yml
+      service: db
+    networks:
+      - internal_network_a
+    volumes:
+      - type: bind
+        source: ./volumes/db.a
+        target: /var/lib/postgresql/data
+        bind:
+          create_host_path: true
+
+networks:
+  internal_network_a:
+    internal: true
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.21.0.0/16
+          ip_range: 172.21.0.0/24