Merge branch 'transfem-org:develop' into develop

.config/example.yml

@@ -124,7 +124,7 @@ redis:
 #  apiKey: ''
 #  ssl: true
 #  index: ''
-#  scope: local
+#  scope: global
 
 #   ┌───────────────┐
 #───┘ ID generation └───────────────────────────────────────────

CHANGELOG.md

@@ -21,6 +21,7 @@
 		- 最大でも黄色いエリア内にデコレーションを収めることを推奨します。
 		- 画像は512x512pxを推奨します。
 - Enhance: すでにフォローしたすべての人の返信をTLに追加できるように
+- Enhance: 未読の通知数を表示できるように
 - Enhance: ローカリゼーションの更新
 - Enhance: 依存関係の更新
 
@@ -53,6 +54,7 @@
 - Enhance: フォローしているチャンネルをフォロー解除した時（またはその逆）、タイムラインに反映される間隔を改善
 - Enhance: プロフィールの自己紹介欄のMFMが連合するようになりました
 	- 相手がMisskey v2023.11.0以降である必要があります
+- Enhance: チャンネル取得時のパフォーマンスを向上
 - Fix: リストTLに自分のフォロワー限定投稿が含まれない問題を修正
 - Fix: ローカルタイムラインに投稿者自身の投稿への返信が含まれない問題を修正
 - Fix: 自分のフォローしているユーザーの自分のフォローしていないユーザーの visibility: followers な投稿への返信がストリーミングで流れてくる問題を修正
@@ -63,6 +65,7 @@
 - Fix: リノートをリノートできるのを修正
 - Fix: アクセストークンを削除すると、通知が取得できなくなる場合がある問題を修正
 - Fix: 自身の宛先なしダイレクト投稿がストリーミングで流れてこない問題を修正
+- Fix: サーバーサイドからのテスト通知を正しく行えるように修正
 
 ## 2023.10.2
 

README.md

@@ -7,6 +7,20 @@
 
 ---
 
+<a href="https://joinsharkey.org">
+		<img src="https://custom-icon-badges.herokuapp.com/badge/find_an-instance-acea31?logoColor=acea31&style=for-the-badge&logo=sharkey&labelColor=363B40" alt="find an instance"/></a>
+
+<a href="./CONTRIBUTING.md">
+		<img src="https://custom-icon-badges.herokuapp.com/badge/become_a-contributor-A371F7?logoColor=A371F7&style=for-the-badge&logo=git-merge&labelColor=363B40" alt="become a contributor"/></a>
+
+<a href="https://discord.gg/6VgKmEqHNk">
+		<img src="https://custom-icon-badges.herokuapp.com/badge/join_the-community-5865F2?logoColor=5865F2&style=for-the-badge&logo=discord&labelColor=363B40" alt="join the community"/></a>
+
+<a href="https://ko-fi.com/transfem">
+		<img src="https://custom-icon-badges.herokuapp.com/badge/donate-F96854?logoColor=F96854&style=for-the-badge&logo=kofi&labelColor=363B40" alt="donate"/></a>
+
+---
+
 </div>
 
 <div>

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "sharkey",
-	"version": "2023.11.0.beta2",
+	"version": "2023.11.0.beta3",
 	"codename": "shonk",
 	"repository": {
 		"type": "git",
@@ -48,14 +48,14 @@
 		"cssnano": "6.0.1",
 		"js-yaml": "4.1.0",
 		"postcss": "8.4.31",
-		"terser": "5.22.0",
+		"terser": "5.24.0",
 		"typescript": "5.2.2"
 	},
 	"devDependencies": {
-		"@typescript-eslint/eslint-plugin": "6.9.0",
-		"@typescript-eslint/parser": "6.9.0",
+		"@typescript-eslint/eslint-plugin": "6.9.1",
+		"@typescript-eslint/parser": "6.9.1",
 		"cross-env": "7.0.3",
-		"cypress": "13.3.3",
+		"cypress": "13.4.0",
 		"eslint": "8.52.0",
 		"start-server-and-test": "2.0.1"
 	},

packages/backend/package.json

@@ -66,17 +66,17 @@
 		"@discordapp/twemoji": "14.1.2",
 		"@fastify/accepts": "4.2.0",
 		"@fastify/cookie": "9.1.0",
-		"@fastify/cors": "8.4.0",
+		"@fastify/cors": "8.4.1",
 		"@fastify/express": "2.3.0",
 		"@fastify/http-proxy": "9.2.1",
 		"@fastify/multipart": "8.0.0",
-		"@fastify/static": "6.11.2",
+		"@fastify/static": "6.12.0",
 		"@fastify/view": "8.2.0",
 		"@nestjs/common": "10.2.7",
 		"@nestjs/core": "10.2.7",
 		"@nestjs/testing": "10.2.7",
 		"@peertube/http-signature": "1.7.0",
-		"@simplewebauthn/server": "8.3.4",
+		"@simplewebauthn/server": "8.3.5",
 		"@sinonjs/fake-timers": "11.2.2",
 		"@swc/cli": "0.1.62",
 		"@swc/core": "1.3.95",
@@ -88,7 +88,7 @@
 		"bcryptjs": "2.4.3",
 		"blurhash": "2.0.5",
 		"body-parser": "1.20.2",
-		"bullmq": "4.12.6",
+		"bullmq": "4.12.7",
 		"cacheable-lookup": "7.0.0",
 		"cbor": "9.0.1",
 		"chalk": "5.3.0",
@@ -141,7 +141,7 @@
 		"probe-image-size": "7.2.3",
 		"promise-limit": "2.7.0",
 		"pug": "3.0.2",
-		"punycode": "2.3.0",
+		"punycode": "2.3.1",
 		"pureimage": "0.3.17",
 		"qrcode": "1.5.3",
 		"random-seed": "0.3.0",
@@ -179,22 +179,22 @@
 		"@simplewebauthn/typescript-types": "8.3.4",
 		"@swc/jest": "0.2.29",
 		"@types/accepts": "1.3.6",
-		"@types/archiver": "5.3.4",
+		"@types/archiver": "6.0.0",
 		"@types/bcryptjs": "2.4.5",
 		"@types/body-parser": "1.19.4",
 		"@types/cbor": "6.0.0",
 		"@types/color-convert": "2.0.2",
 		"@types/content-disposition": "0.5.7",
 		"@types/fluent-ffmpeg": "2.1.23",
 		"@types/http-link-header": "1.0.4",
-		"@types/jest": "29.5.6",
+		"@types/jest": "29.5.7",
 		"@types/js-yaml": "4.0.8",
 		"@types/jsdom": "21.1.4",
 		"@types/jsonld": "1.5.11",
 		"@types/jsrsasign": "10.5.11",
 		"@types/mime-types": "2.1.3",
 		"@types/ms": "0.7.33",
-		"@types/node": "20.8.9",
+		"@types/node": "20.8.10",
 		"@types/node-fetch": "3.0.3",
 		"@types/nodemailer": "6.4.13",
 		"@types/oauth": "0.9.3",
@@ -218,8 +218,8 @@
 		"@types/web-push": "3.6.2",
 		"@types/uuid": "^9.0.4",
 		"@types/ws": "8.5.8",
-		"@typescript-eslint/eslint-plugin": "6.9.0",
-		"@typescript-eslint/parser": "6.9.0",
+		"@typescript-eslint/eslint-plugin": "6.9.1",
+		"@typescript-eslint/parser": "6.9.1",
 		"aws-sdk-client-mock": "3.0.0",
 		"cross-env": "7.0.3",
 		"eslint": "8.52.0",

packages/backend/src/config.ts

@@ -258,7 +258,7 @@ export function loadConfig(): Config {
 		clientEntry: clientManifest['src/_boot_.ts'],
 		clientManifestExists: clientManifestExists,
 		perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000,
-		perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 300,
+		perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 500,
 		deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7),
 		pidFile: config.pidFile,
 	};

packages/backend/src/core/FeaturedService.ts

@@ -35,10 +35,16 @@ export class FeaturedService {
 			`${name}:${currentWindow}`,
 			score,
 			element);
-		redisTransaction.expire(
-			`${name}:${currentWindow}`,
-			(windowRange * 3) / 1000,
-			'NX'); // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
+
+		const TTL = await this.redisClient.ttl(`${name}:${currentWindow}`);
+
+		if (TTL === -1) {
+			this.redisClient.expire(`${name}:${currentWindow}`,
+				(windowRange * 3) / 1000, // 1時間
+				//'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
+			);
+		}
+
 		await redisTransaction.exec();
 	}
 

packages/backend/src/core/HashtagService.ts

@@ -176,20 +176,30 @@ export class HashtagService {
 
 		// チャート用
 		redisPipeline.pfadd(`hashtagUsers:${hashtag}:${window}`, userId);
-		redisPipeline.expire(`hashtagUsers:${hashtag}:${window}`,
-			60 * 60 * 24 * 3, // 3日間
-			'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
-		);
 
 		// ユニークカウント用
 		// TODO: Bloom Filter を使うようにしても良さそう
 		redisPipeline.sadd(`hashtagUsers:${hashtag}`, userId);
-		redisPipeline.expire(`hashtagUsers:${hashtag}`,
-			60 * 60, // 1時間
-			'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
-		);
 
-		redisPipeline.exec();
+		await redisPipeline.exec();
+
+		const TTL = await this.redisClient.ttl(`hashtagUsers:${hashtag}`);
+
+		if (TTL === -1) {
+			this.redisClient.expire(`hashtagUsers:${hashtag}`,
+				60 * 60, // 1時間
+				//'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
+			);
+		}
+
+		const TTLwindow = await this.redisClient.ttl(`hashtagUsers:${hashtag}:${window}`);
+
+		if (TTLwindow === -1) {
+			this.redisClient.expire(`hashtagUsers:${hashtag}:${window}`,
+				60 * 60 * 24 * 3, // 3日間
+				//'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
+			);
+		}
 	}
 
 	@bindThis

packages/backend/src/core/MfmService.ts

@@ -389,6 +389,8 @@ export class MfmService {
 		return `<p>${doc.body.innerHTML}</p>`;
 	}
 
+	// the toMastoHtml function was taken from Iceshrimp and written by zotan and modified by marie to work with the current MK version
+
 	@bindThis
 	public async toMastoHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMentionedRemoteUsers = [], inline = false, quoteUri: string | null = null) {
 		if (nodes == null) {

packages/backend/src/core/NotificationService.ts

@@ -144,7 +144,9 @@ export class NotificationService implements OnApplicationShutdown {
 		this.globalEventService.publishMainStream(notifieeId, 'notification', packed);
 
 		// 2秒経っても(今回作成した)通知が既読にならなかったら「未読の通知がありますよ」イベントを発行する
-		setTimeout(2000, 'unread notification', { signal: this.#shutdownController.signal }).then(async () => {
+		// テスト通知の場合は即時発行
+		const interval = notification.type === 'test' ? 0 : 2000;
+		setTimeout(interval, 'unread notification', { signal: this.#shutdownController.signal }).then(async () => {
 			const latestReadNotificationId = await this.redisClient.get(`latestReadNotification:${notifieeId}`);
 			if (latestReadNotificationId && (latestReadNotificationId >= (await redisIdPromise)!)) return;
 

packages/backend/src/core/SearchService.ts

@@ -78,7 +78,7 @@ export class SearchService {
 		private idService: IdService,
 	) {
 		if (meilisearch) {
-			this.meilisearchNoteIndex = meilisearch.index(`${config.meilisearch!.index}---notes`);
+			this.meilisearchNoteIndex = meilisearch.index(`${this.config.meilisearch?.index}---notes`);
 			this.meilisearchNoteIndex.updateSettings({
 				searchableAttributes: [
 					'text',
@@ -103,8 +103,8 @@ export class SearchService {
 			});
 		}
 
-		if (config.meilisearch?.scope) {
-			this.meilisearchIndexScope = config.meilisearch.scope;
+		if (this.config.meilisearch?.scope) {
+			this.meilisearchIndexScope = this.config.meilisearch.scope;
 		}
 	}
 

packages/backend/src/core/entities/ChannelEntityService.ts

@@ -5,7 +5,7 @@
 
 import { Inject, Injectable } from '@nestjs/common';
 import { DI } from '@/di-symbols.js';
-import type { ChannelFavoritesRepository, ChannelFollowingsRepository, ChannelsRepository, DriveFilesRepository, NoteUnreadsRepository, NotesRepository } from '@/models/_.js';
+import type { ChannelFavoritesRepository, ChannelFollowingsRepository, ChannelsRepository, DriveFilesRepository, NotesRepository } from '@/models/_.js';
 import type { Packed } from '@/misc/json-schema.js';
 import type { } from '@/models/Blocking.js';
 import type { MiUser } from '@/models/User.js';
@@ -31,9 +31,6 @@ export class ChannelEntityService {
 		@Inject(DI.notesRepository)
 		private notesRepository: NotesRepository,
 
-		@Inject(DI.noteUnreadsRepository)
-		private noteUnreadsRepository: NoteUnreadsRepository,
-
 		@Inject(DI.driveFilesRepository)
 		private driveFilesRepository: DriveFilesRepository,
 
@@ -54,13 +51,6 @@ export class ChannelEntityService {
 
 		const banner = channel.bannerId ? await this.driveFilesRepository.findOneBy({ id: channel.bannerId }) : null;
 
-		const hasUnreadNote = meId ? await this.noteUnreadsRepository.exist({
-			where: {
-				noteChannelId: channel.id,
-				userId: meId,
-			},
-		}) : undefined;
-
 		const isFollowing = meId ? await this.channelFollowingsRepository.exist({
 			where: {
 				followerId: meId,
@@ -99,7 +89,7 @@ export class ChannelEntityService {
 			...(me ? {
 				isFollowing,
 				isFavorited,
-				hasUnreadNote,
+				hasUnreadNote: false, // 後方互換性のため
 			} : {}),
 
 			...(detailed ? {

packages/backend/src/core/entities/UserEntityService.ts

@@ -15,6 +15,7 @@ import { awaitAll } from '@/misc/prelude/await-all.js';
 import { USER_ACTIVE_THRESHOLD, USER_ONLINE_THRESHOLD } from '@/const.js';
 import type { MiLocalUser, MiPartialLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser } from '@/models/User.js';
 import { birthdaySchema, listenbrainzSchema, descriptionSchema, localUsernameSchema, locationSchema, nameSchema, passwordSchema } from '@/models/User.js';
+import { MiNotification } from '@/models/Notification.js';
 import type { UsersRepository, UserSecurityKeysRepository, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, MutingsRepository, DriveFilesRepository, NoteUnreadsRepository, UserNotePiningsRepository, UserProfilesRepository, AnnouncementReadsRepository, AnnouncementsRepository, MiUserProfile, RenoteMutingsRepository, UserMemoRepository } from '@/models/_.js';
 import { bindThis } from '@/decorators.js';
 import { RoleService } from '@/core/RoleService.js';
@@ -236,17 +237,34 @@ export class UserEntityService implements OnModuleInit {
 	}
 
 	@bindThis
-	public async getHasUnreadNotification(userId: MiUser['id']): Promise<boolean> {
+	public async getNotificationsInfo(userId: MiUser['id']): Promise<{
+		hasUnread: boolean;
+		unreadCount: number;
+	}> {
+		const response = {
+			hasUnread: false,
+			unreadCount: 0,
+		};
+
 		const latestReadNotificationId = await this.redisClient.get(`latestReadNotification:${userId}`);
 
-		const latestNotificationIdsRes = await this.redisClient.xrevrange(
-			`notificationTimeline:${userId}`,
-			'+',
-			'-',
-			'COUNT', 1);
-		const latestNotificationId = latestNotificationIdsRes[0]?.[0];
+		if (!latestReadNotificationId) {
+			response.unreadCount = await this.redisClient.xlen(`notificationTimeline:${userId}`);
+		} else {
+			const latestNotificationIdsRes = await this.redisClient.xrevrange(
+				`notificationTimeline:${userId}`,
+				'+',
+				latestReadNotificationId,
+			);
+
+			response.unreadCount = (latestNotificationIdsRes.length - 1 >= 0) ? latestNotificationIdsRes.length - 1 : 0;
+		}
+
+		if (response.unreadCount > 0) {
+			response.hasUnread = true;
+		}
 
-		return latestNotificationId != null && (latestReadNotificationId == null || latestReadNotificationId < latestNotificationId);
+		return response;
 	}
 
 	@bindThis
@@ -361,6 +379,8 @@ export class UserEntityService implements OnModuleInit {
 			})) : null;
 
 		const checkHost = user.host == null ? this.config.host : user.host;
+
+		const notificationsInfo = isMe && opts.detail ? await this.getNotificationsInfo(user.id) : null;
 
 		const packed = {
 			id: user.id,
@@ -486,8 +506,9 @@ export class UserEntityService implements OnModuleInit {
 				unreadAnnouncements,
 				hasUnreadAntenna: this.getHasUnreadAntenna(user.id),
 				hasUnreadChannel: false, // 後方互換性のため
-				hasUnreadNotification: this.getHasUnreadNotification(user.id),
+				hasUnreadNotification: notificationsInfo?.hasUnread, // 後方互換性のため
 				hasPendingReceivedFollowRequest: this.getHasPendingReceivedFollowRequest(user.id),
+				unreadNotificationsCount: notificationsInfo?.unreadCount,
 				mutedWords: profile!.mutedWords,
 				mutedInstances: profile!.mutedInstances,
 				mutingNotificationTypes: [], // 後方互換性のため

packages/backend/src/models/json-schema/user.ts

@@ -423,6 +423,10 @@ export const packedMeDetailedOnlySchema = {
 			type: 'boolean',
 			nullable: false, optional: false,
 		},
+		unreadNotificationsCount: {
+			type: 'number',
+			nullable: false, optional: false,
+		},
 		mutedWords: {
 			type: 'array',
 			nullable: false, optional: false,

packages/backend/test/e2e/clips.ts

@@ -720,7 +720,7 @@ describe('クリップ', () => {
 		test('を追加できる。', async () => {
 			await addNote({ clipId: aliceClip.id, noteId: aliceNote.id });
 			const res = await show({ clipId: aliceClip.id });
-			assert.strictEqual(res.lastClippedAt, new Date(res.lastClippedAt ?? '').toISOString());
+			assert.strictEqual(res.lastClippedAt, res.lastClippedAt ? new Date(res.lastClippedAt).toISOString() : null);
 			assert.deepStrictEqual((await notes({ clipId: aliceClip.id })).map(x => x.id), [aliceNote.id]);
 
 			// 他人の非公開ノートも突っ込める