Skip to content

Commit

Permalink
enhance(backend): タイムライン等の有効期限を設定可能に
Browse files Browse the repository at this point in the history 
Resolve #11773
  • Loading branch information
@syuilo
syuilo committed Sep 5, 2023
1 parent e7d30c8 commit f53cffa
Show file tree
Hide file tree
Showing 4 changed files with 128 additions and 45 deletions.
162 changes: 120 additions & 42 deletions packages/backend/src/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,9 @@ type RedisOptionsSource = Partial<RedisOptions> & {
};

/**
* ユーザーが設定する必要のある情報
* 設定ファイルの型
*/
export type Source = {
repository_url?: string;
feedback_url?: string;
type Source = {
url: string;
port?: number;
socket?: string;
Expand Down Expand Up @@ -93,12 +91,63 @@ export type Source = {
videoThumbnailGenerator?: string;

signToActivityPubGet?: boolean;

perChannelMaxNoteCacheCount?: number;
perUserNotificationsMaxCount?: number;
};

/**
* Misskeyが自動的に(ユーザーが設定した情報から推論して)設定する情報
*/
export type Mixin = {
export type Config = {
url: string;
port: number;
socket: string | undefined;
chmodSocket: string | undefined;
disableHsts: boolean | undefined;
db: {
host: string;
port: number;
db: string;
user: string;
pass: string;
disableCache?: boolean;
extra?: { [x: string]: string };
};
dbReplications: boolean | undefined;
dbSlaves: {
host: string;
port: number;
db: string;
user: string;
pass: string;
}[] | undefined;
meilisearch: {
host: string;
port: string;
apiKey: string;
ssl?: boolean;
index: string;
scope?: 'local' | 'global' | string[];
} | undefined;
proxy: string | undefined;
proxySmtp: string | undefined;
proxyBypassHosts: string[] | undefined;
allowedPrivateNetworks: string[] | undefined;
maxFileSize: number | undefined;
accesslog: string | undefined;
clusterLimit: number | undefined;
id: string;
outgoingAddress: string | undefined;
outgoingAddressFamily: 'ipv4' | 'ipv6' | 'dual' | undefined;
deliverJobConcurrency: number | undefined;
inboxJobConcurrency: number | undefined;
relashionshipJobConcurrency: number | undefined;
deliverJobPerSec: number | undefined;
inboxJobPerSec: number | undefined;
relashionshipJobPerSec: number | undefined;
deliverJobMaxAttempts: number | undefined;
inboxJobMaxAttempts: number | undefined;
proxyRemoteFiles: boolean | undefined;
signToActivityPubGet: boolean | undefined;

version: string;
host: string;
hostname: string;
Expand All @@ -117,10 +166,10 @@ export type Mixin = {
redis: RedisOptions & RedisOptionsSource;
redisForPubsub: RedisOptions & RedisOptionsSource;
redisForJobQueue: RedisOptions & RedisOptionsSource;
perChannelMaxNoteCacheCount: number;
perUserNotificationsMaxCount: number;
};

export type Config = Source & Mixin;

const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);

Expand All @@ -138,51 +187,80 @@ const path = process.env.MISSKEY_CONFIG_YML
? resolve(dir, 'test.yml')
: resolve(dir, 'default.yml');

export function loadConfig() {
export function loadConfig(): Config {
const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8'));
const clientManifestExists = fs.existsSync(_dirname + '/../../../built/_vite_/manifest.json');
const clientManifest = clientManifestExists ?
JSON.parse(fs.readFileSync(`${_dirname}/../../../built/_vite_/manifest.json`, 'utf-8'))
: { 'src/_boot_.ts': { file: 'src/_boot_.ts' } };
const config = yaml.load(fs.readFileSync(path, 'utf-8')) as Source;

const mixin = {} as Mixin;

const url = tryCreateUrl(config.url);

config.url = url.origin;

config.port = config.port ?? parseInt(process.env.PORT ?? '', 10);

mixin.version = meta.version;
mixin.host = url.host;
mixin.hostname = url.hostname;
mixin.scheme = url.protocol.replace(/:$/, '');
mixin.wsScheme = mixin.scheme.replace('http', 'ws');
mixin.wsUrl = `${mixin.wsScheme}://${mixin.host}`;
mixin.apiUrl = `${mixin.scheme}://${mixin.host}/api`;
mixin.authUrl = `${mixin.scheme}://${mixin.host}/auth`;
mixin.driveUrl = `${mixin.scheme}://${mixin.host}/files`;
mixin.userAgent = `Misskey/${meta.version} (${config.url})`;
mixin.clientEntry = clientManifest['src/_boot_.ts'];
mixin.clientManifestExists = clientManifestExists;
const version = meta.version;
const host = url.host;
const hostname = url.hostname;
const scheme = url.protocol.replace(/:$/, '');
const wsScheme = scheme.replace('http', 'ws');

const externalMediaProxy = config.mediaProxy ?
config.mediaProxy.endsWith('/') ? config.mediaProxy.substring(0, config.mediaProxy.length - 1) : config.mediaProxy
: null;
const internalMediaProxy = `${mixin.scheme}://${mixin.host}/proxy`;
mixin.mediaProxy = externalMediaProxy ?? internalMediaProxy;
mixin.externalMediaProxyEnabled = externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy;
const internalMediaProxy = `${scheme}://${host}/proxy`;
const redis = convertRedisOptions(config.redis, host);

mixin.videoThumbnailGenerator = config.videoThumbnailGenerator ?
config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator
: null;

mixin.redis = convertRedisOptions(config.redis, mixin.host);
mixin.redisForPubsub = config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, mixin.host) : mixin.redis;
mixin.redisForJobQueue = config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, mixin.host) : mixin.redis;

return Object.assign(config, mixin);
return {
version,
url: url.origin,
port: config.port ?? parseInt(process.env.PORT ?? '', 10),
socket: config.socket,
chmodSocket: config.chmodSocket,
disableHsts: config.disableHsts,
host,
hostname,
scheme,
wsScheme,
wsUrl: `${wsScheme}://${host}`,
apiUrl: `${scheme}://${host}/api`,
authUrl: `${scheme}://${host}/auth`,
driveUrl: `${scheme}://${host}/files`,
db: config.db,
dbReplications: config.dbReplications,
dbSlaves: config.dbSlaves,
meilisearch: config.meilisearch,
redis,
redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis,
redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis,
id: config.id,
proxy: config.proxy,
proxySmtp: config.proxySmtp,
proxyBypassHosts: config.proxyBypassHosts,
allowedPrivateNetworks: config.allowedPrivateNetworks,
maxFileSize: config.maxFileSize,
accesslog: config.accesslog,
clusterLimit: config.clusterLimit,
outgoingAddress: config.outgoingAddress,
outgoingAddressFamily: config.outgoingAddressFamily,
deliverJobConcurrency: config.deliverJobConcurrency,
inboxJobConcurrency: config.inboxJobConcurrency,
relashionshipJobConcurrency: config.relashionshipJobConcurrency,
deliverJobPerSec: config.deliverJobPerSec,
inboxJobPerSec: config.inboxJobPerSec,
relashionshipJobPerSec: config.relashionshipJobPerSec,
deliverJobMaxAttempts: config.deliverJobMaxAttempts,
inboxJobMaxAttempts: config.inboxJobMaxAttempts,
proxyRemoteFiles: config.proxyRemoteFiles,
signToActivityPubGet: config.signToActivityPubGet,
mediaProxy: externalMediaProxy ?? internalMediaProxy,
externalMediaProxyEnabled: externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy,
videoThumbnailGenerator: config.videoThumbnailGenerator ?
config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator
: null,
userAgent: `Misskey/${version} (${config.url})`,
clientEntry: clientManifest['src/_boot_.ts'],
clientManifestExists: clientManifestExists,
perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000,
perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 300,
};
}

function tryCreateUrl(url: string) {
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/core/NoteCreateService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ export class NoteCreateService implements OnApplicationShutdown {
if (data.channel) {
this.redisClient.xadd(
`channelTimeline:${data.channel.id}`,
'MAXLEN', '~', '1000',
'MAXLEN', '~', this.config.perChannelMaxNoteCacheCount.toString(),
'*',
'note', note.id);
}
Expand Down
6 changes: 5 additions & 1 deletion packages/backend/src/core/NotificationService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,16 @@ import { PushNotificationService } from '@/core/PushNotificationService.js';
import { NotificationEntityService } from '@/core/entities/NotificationEntityService.js';
import { IdService } from '@/core/IdService.js';
import { CacheService } from '@/core/CacheService.js';
import { Config } from '@/config.js';

@Injectable()
export class NotificationService implements OnApplicationShutdown {
#shutdownController = new AbortController();

constructor(
@Inject(DI.config)
private config: Config,

@Inject(DI.redis)
private redisClient: Redis.Redis,

Expand Down Expand Up @@ -96,7 +100,7 @@ export class NotificationService implements OnApplicationShutdown {

const redisIdPromise = this.redisClient.xadd(
`notificationTimeline:${notifieeId}`,
'MAXLEN', '~', '300',
'MAXLEN', '~', this.config.perUserNotificationsMaxCount.toString(),
'*',
'data', JSON.stringify(notification));

Expand Down
3 changes: 2 additions & 1 deletion packages/backend/src/server/api/endpoints/notes/timeline.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
//#region Construct query
const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'),
ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
.andWhere('note.id > :minId', { minId: this.idService.genId(new Date(Date.now() - (1000 * 60 * 60 * 24 * 10))) }) // 10日前まで
// パフォーマンス上の利点が無さそう？
//.andWhere('note.id > :minId', { minId: this.idService.genId(new Date(Date.now() - (1000 * 60 * 60 * 24 * 10))) }) // 10日前まで
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('note.renote', 'renote')
Expand Down

0 comments on commit f53cffa

Please sign in to comment.