Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(backend): core/activitypub/models #11067

Merged
merged 100 commits into from
Jul 7, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
100 commits
Select commit Hold shift + click to select a range
d66322c
cleanup(`ApImageService.ts`)
Jul 2, 2023
1abb06d
refactor(`ApImageService.ts`)
Jul 2, 2023
16ffe72
cleanup(`check-https.ts`)
Jul 2, 2023
c9adcfa
cleanup(`ApMentionService.ts`)
Jul 2, 2023
9889ad6
refactor(`ApMentionService.ts`)
Jul 2, 2023
fb459de
cleanup(`ApNoteService.ts`): unneeded `eslint-disable-next-line`
Jul 2, 2023
9dc2e98
cleanup(`ApNoteService.ts`)
Jul 2, 2023
aeefb84
WIP(`ApImageService.ts`): `image.url`を`getApHrefNullable()`に通すかどうか悩んでいる
Jul 2, 2023
1246382
refactor(`ApNoteService.ts`): function return type
Jul 2, 2023
b203dab
cleanup(`ApNoteService.ts`): deadcode
Jul 2, 2023
acce210
refactor(`ApNoteService.ts`): `eslint-disable-next-line`
Jul 2, 2023
47a07d3
refactor(`ApNoteService.ts`): non-null assertion
Jul 2, 2023
370c1fb
cleanup(`ApNoteService.ts`): unneeded await
Jul 2, 2023
dc291c5
refactor(`ApNoteService.ts`): note.attachment
Jul 2, 2023
1af9392
cleanup(`ApNoteService.ts`)
Jul 2, 2023
c234f3a
refactor(`ApNoteService.ts`): よりよい型定義
Jul 2, 2023
ad7b4e2
refactor(`ApNoteService.ts`): 不要な条件を削除
Jul 2, 2023
688ad27
cleanup(`ApNoteService.ts`)
Jul 2, 2023
c2b184b
cleanup(`ApNoteService.ts`): 重要でない`as`を削除
Jul 2, 2023
394678e
refactor(`ApNoteService.ts`): `eslint-disable-next-line`
Jul 2, 2023
c788cbb
cleanup(`ApNoteService.ts`): deadcode
Jul 2, 2023
ad92cb6
cleanup(`ApNoteService.ts`): unneeded non-null assertion
Jul 2, 2023
a24e709
refactor(`ApNoteService.ts`): 不要な条件を削除
Jul 2, 2023
f688d7a
WIP(`ApNoteService.ts`): `as`をなくす
Jul 2, 2023
8b3641e
cleanup(`ApNoteService.ts`): 不要な`as`を削除
Jul 2, 2023
3c17a9b
cleanup(`ApPersonService.ts`): `no-unused-vars`
Jul 2, 2023
58abc86
cleanup(`ApPersonService.ts`): deadcode
Jul 2, 2023
d3e23d9
refactor(`ApPersonService.ts`): function return type
Jul 2, 2023
c7fd197
cleanup(`ApPersonService.ts`): deadcode
Jul 2, 2023
5095542
cleanup(`ApPersonService.ts`): deadcode
Jul 2, 2023
39ad3da
WIP(`ApPersonService.ts`): `as`を調整
Jul 2, 2023
44a1b6d
refactor(`ApPersonService.ts`): `eslint-disable-next-line`
Jul 2, 2023
b1ecaec
WIP(`ApPersonService.ts`): `as any`をなくした
Jul 2, 2023
71ca66e
WIP(`ApNoteService.ts`): non-null assertion
Jul 2, 2023
8ee31e8
refactor(`ApNoteService.ts`): non-null assertion -> optional chaining
Jul 2, 2023
be415a9
refactor(`ApPersonService.ts`): `eslint-disable-next-line`
Jul 2, 2023
4535728
refactor(`ApPersonService.ts`): `eslint-disable-next-line`
Jul 2, 2023
a40ea81
refactor(`ApPersonService.ts`): function return type
Jul 2, 2023
d276dab
refactor(`ApPersonService.ts`): type guardによるnon-null assertionの削除
Jul 2, 2023
785a5f6
WIP(`ApPersonService.ts`): `analyzeAttachments`
Jul 2, 2023
41fe1b6
Revert "WIP(`ApImageService.ts`): `image.url`を`getApHrefNullable()`に通…
Jul 2, 2023
1454d04
cleanup(`ApImageService.ts`): `import`
Jul 2, 2023
67f9d9d
refactor(`ApImageService.ts`): 冗長だった部分を短く
Jul 2, 2023
244f672
cleanup(`ApMentionService.ts`): `import`
Jul 2, 2023
b651fb5
refactor(`ApImageService.ts`): `JSON.stringify()`でのindentationを追加
Jul 2, 2023
d8f0d76
cleanup(`ApNoteService.ts`): `import`
Jul 2, 2023
bfc22e9
cleanup(`ApNoteService.ts`)
Jul 2, 2023
42b90cd
cleanup(`ApNoteService.ts`)
Jul 2, 2023
7a82575
cleanup(`ApNoteService.ts`)
Jul 2, 2023
3bd784e
cleanup(`ApNoteService.ts`): `any`に対するnon-null assertion
Jul 2, 2023
451a0e3
refactor(`ApNoteService.ts`): 添付ファイル
Jul 2, 2023
5190ef9
cleanup(`ApPersonService.ts`): `import`
Jul 2, 2023
f20381d
refactor(`ApPersonService.ts`): より実情に即した`as`に
Jul 2, 2023
541429f
cleanup(`ApPersonService.ts`)
Jul 2, 2023
a003fb2
refactor(`ApPersonService.ts`): 冗長だった部分を修正
Jul 2, 2023
8cd1152
cleanup(`ApPersonService.ts`): deadcode
Jul 2, 2023
f4a8e74
cleanup(`ApPersonService.ts`)
Jul 2, 2023
778585e
cleanup(`ApQuestionService.ts`): `import`
Jul 2, 2023
fcc6042
refactor(`ApQuestionService.ts`): `eslint-disable-next-line`
Jul 2, 2023
f70fd08
refactor(`ApQuestionService.ts`): `eslint-disable-next-line`
Jul 2, 2023
279321e
cleanup(`ApQuestionService.ts`)
Jul 2, 2023
0f8686d
refactor(`ApQuestionService.ts`): non-null assertionを消した
Jul 2, 2023
5e715b2
cleanup(`ApQuestionService.ts`)
Jul 2, 2023
d0e00c8
WIP(`ApQuestionService.ts`): non-null assertionを消す
Jul 2, 2023
630078f
refactor(`ApQuestionService.ts`): `any`を消す
Jul 2, 2023
afe16d7
refactor(`ApQuestionService.ts`): function return type
Jul 2, 2023
09f6720
WIP(`ApPersonService.ts`): 可読性の低い三項演算子を削除しつつnon-null assertionを回避
Jul 3, 2023
eb0607e
cleanup(`ApPersonService.ts`): 不必要な三項演算子を削除
Jul 3, 2023
9801b20
cleanup(`ApPersonService.ts`): 不要な`as`
Jul 3, 2023
8e46204
cleanup(`ApPersonService.ts`)
Jul 3, 2023
56a6608
refactor(`ApPersonService.ts`)
Jul 3, 2023
37d5c2c
refactor(`ApPersonService.ts`): 可読性の低い三項演算子を削除
Jul 3, 2023
a234675
cleanup(`ApPersonService.ts`)
Jul 3, 2023
c2c2efb
cleanup(`ApPersonService.ts`)
Jul 3, 2023
bfa0fcd
refactor(`ApPersonService.ts`): 返り値を`void`に統一
Jul 3, 2023
083cd67
fixup! refactor(`ApPersonService.ts`): 返り値を`void`に統一
Jul 3, 2023
d4ffb25
refactor(`ApNoteService.ts`)
Jul 3, 2023
12516b9
refactor(`ApPersonService.ts`)
Jul 3, 2023
5bf82ff
cleanup(`ApPersonService.ts`)
Jul 3, 2023
ff57dc0
cleanup(`ApPersonService.ts`)
Jul 3, 2023
662d2b1
refactor(`ApPersonService.ts`): 返り値の`void`統一と条件式の調整
Jul 3, 2023
264ae11
cleanup(`ApQuestionService.ts`)
Jul 3, 2023
42fed0c
refactor(`ApQuestionService.ts`)
Jul 3, 2023
58dbf61
refactor(`ApQuestionService.ts`)
Jul 3, 2023
c6897ea
refactor(`tag.ts`): function return type
Jul 3, 2023
6dfbf60
fixup! enhance: account migration (#10592)
Jul 3, 2023
f2bc374
fixup! WIP(`ApPersonService.ts`): 可読性の低い三項演算子を削除しつつnon-null assertionを回避
Jul 3, 2023
6f61e66
fixup! cleanup(`ApPersonService.ts`): 不要な`as`
Jul 3, 2023
01e6dfb
refactor: エラーメッセージを見繕った
Jul 4, 2023
890faa8
Merge branch 'develop' into refactor-backend-core-activitypub
Jul 4, 2023
f11c601
Revert "cleanup(`ApImageService.ts`): `import`"
Jul 4, 2023
abecfc4
Revert "cleanup(`ApMentionService.ts`): `import`"
Jul 4, 2023
4522ad1
Revert "cleanup(`ApNoteService.ts`): `import`"
Jul 4, 2023
0988437
Revert "cleanup(`ApPersonService.ts`): `import`"
Jul 4, 2023
bd2bc1c
Revert "cleanup(`ApQuestionService.ts`): `import`"
Jul 4, 2023
ac8f9fa
Merge branch 'develop' into pr/okayurisotto/11067
tamaina Jul 7, 2023
99c8f7c
processRemoteMoveはそのままにしてほしい
tamaina Jul 7, 2023
081df38
Revert "fixup! refactor(`ApPersonService.ts`): 返り値を`void`に統一"
Jul 7, 2023
61ba747
Revert "refactor(`ApPersonService.ts`): 返り値を`void`に統一"
Jul 7, 2023
86c0dc0
Merge branch 'develop' into refactor-backend-core-activitypub
Jul 7, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 15 additions & 20 deletions packages/backend/src/core/activitypub/models/ApImageService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@ import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/const.js';
import { DriveService } from '@/core/DriveService.js';
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { checkHttps } from '@/misc/check-https.js';
import { ApResolverService } from '../ApResolverService.js';
import { ApLoggerService } from '../ApLoggerService.js';
import { checkHttps } from '@/misc/check-https.js';
import type { IObject } from '../type.js';

@Injectable()
export class ApImageService {
Expand All @@ -37,18 +38,22 @@ export class ApImageService {
* Imageを作成します。
*/
@bindThis
public async createImage(actor: RemoteUser, value: any): Promise<DriveFile> {
public async createImage(actor: RemoteUser, value: string | IObject): Promise<DriveFile> {
// 投稿者が凍結されていたらスキップ
if (actor.isSuspended) {
throw new Error('actor has been suspended');
}

const image = await this.apResolverService.createResolver().resolve(value) as any;
const image = await this.apResolverService.createResolver().resolve(value);

if (image.url == null) {
throw new Error('invalid image: url not privided');
}

if (typeof image.url !== 'string') {
throw new Error('invalid image: unexpected type of url: ' + JSON.stringify(image.url, null, 2));
}

if (!checkHttps(image.url)) {
throw new Error('invalid image: unexpected schema of url: ' + image.url);
}
Expand All @@ -57,29 +62,19 @@ export class ApImageService {

const instance = await this.metaService.fetch();

let file = await this.driveService.uploadFromUrl({
const file = await this.driveService.uploadFromUrl({
url: image.url,
user: actor,
uri: image.url,
sensitive: image.sensitive,
isLink: !instance.cacheRemoteFiles,
comment: truncate(image.name, DB_MAX_IMAGE_COMMENT_LENGTH),
comment: truncate(image.name ?? undefined, DB_MAX_IMAGE_COMMENT_LENGTH),
});
if (!file.isLink || file.url === image.url) return file;

if (file.isLink) {
// URLが異なっている場合、同じ画像が以前に異なるURLで登録されていたということなので、
// URLを更新する
if (file.url !== image.url) {
await this.driveFilesRepository.update({ id: file.id }, {
url: image.url,
uri: image.url,
});

file = await this.driveFilesRepository.findOneByOrFail({ id: file.id });
}
}

return file;
// URLが異なっている場合、同じ画像が以前に異なるURLで登録されていたということなので、URLを更新する
await this.driveFilesRepository.update({ id: file.id }, { url: image.url, uri: image.url });
return await this.driveFilesRepository.findOneByOrFail({ id: file.id });
}

/**
Expand All @@ -89,7 +84,7 @@ export class ApImageService {
* リモートサーバーからフェッチしてMisskeyに登録しそれを返します。
*/
@bindThis
public async resolveImage(actor: RemoteUser, value: any): Promise<DriveFile> {
public async resolveImage(actor: RemoteUser, value: string | IObject): Promise<DriveFile> {
// TODO

// リモートサーバーからフェッチしてきて登録
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ export class ApMentionService {
}

@bindThis
public async extractApMentions(tags: IObject | IObject[] | null | undefined, resolver: Resolver) {
const hrefs = unique(this.extractApMentionObjects(tags).map(x => x.href as string));
public async extractApMentions(tags: IObject | IObject[] | null | undefined, resolver: Resolver): Promise<User[]> {
const hrefs = unique(this.extractApMentionObjects(tags).map(x => x.href));

const limit = promiseLimit<User | null>(2);
const mentionedUsers = (await Promise.all(
Expand Down
137 changes: 61 additions & 76 deletions packages/backend/src/core/activitypub/models/ApNoteService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { checkHttps } from '@/misc/check-https.js';
import { getOneApId, getApId, getOneApHrefNullable, validPost, isEmoji, getApType } from '../type.js';
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
import { ApLoggerService } from '../ApLoggerService.js';
import { ApMfmService } from '../ApMfmService.js';
import { ApDbResolverService } from '../ApDbResolverService.js';
Expand Down Expand Up @@ -72,13 +71,9 @@ export class ApNoteService {
}

@bindThis
public validateNote(object: IObject, uri: string) {
public validateNote(object: IObject, uri: string): Error | null {
const expectHost = this.utilityService.extractDbHost(uri);

if (object == null) {
return new Error('invalid Note: object is null');
}

if (!validPost.includes(getApType(object))) {
return new Error(`invalid Note: invalid object type ${getApType(object)}`);
}
Expand Down Expand Up @@ -110,19 +105,18 @@ export class ApNoteService {
*/
@bindThis
public async createNote(value: string | IObject, resolver?: Resolver, silent = false): Promise<Note | null> {
// eslint-disable-next-line no-param-reassign
if (resolver == null) resolver = this.apResolverService.createResolver();

const object = await resolver.resolve(value);

const entryUri = getApId(value);
const err = this.validateNote(object, entryUri);
if (err) {
this.logger.error(`${err.message}`, {
resolver: {
history: resolver.getHistory(),
},
value: value,
object: object,
this.logger.error(err.message, {
resolver: { history: resolver.getHistory() },
value,
object,
});
throw new Error('invalid note');
}
Expand All @@ -144,7 +138,11 @@ export class ApNoteService {
this.logger.info(`Creating the Note: ${note.id}`);

// 投稿者をフェッチ
const actor = await this.apPersonService.resolvePerson(getOneApId(note.attributedTo!), resolver) as RemoteUser;
if (note.attributedTo == null) {
throw new Error('invalid note.attributedTo: ' + note.attributedTo);
}

const actor = await this.apPersonService.resolvePerson(getOneApId(note.attributedTo), resolver) as RemoteUser;

// 投稿者が凍結されていたらスキップ
if (actor.isSuspended) {
Expand All @@ -164,59 +162,49 @@ export class ApNoteService {
}

const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver);
const apHashtags = await extractApHashtags(note.tag);
const apHashtags = extractApHashtags(note.tag);

// 添付ファイル
// TODO: attachmentは必ずしもImageではない
// TODO: attachmentは必ずしも配列ではない
// Noteがsensitiveなら添付もsensitiveにする
const limit = promiseLimit(2);

note.attachment = Array.isArray(note.attachment) ? note.attachment : note.attachment ? [note.attachment] : [];
const files = note.attachment
.map(attach => attach.sensitive = note.sensitive)
? (await Promise.all(note.attachment.map(x => limit(() => this.apImageService.resolveImage(actor, x)) as Promise<DriveFile>)))
.filter(image => image != null)
: [];
const limit = promiseLimit<DriveFile>(2);
const files = (await Promise.all(toArray(note.attachment).map(attach => (
limit(() => this.apImageService.resolveImage(actor, {
...attach,
sensitive: note.sensitive, // Noteがsensitiveなら添付もsensitiveにする
}))
))));

// リプライ
const reply: Note | null = note.inReplyTo
? await this.resolveNote(note.inReplyTo, resolver).then(x => {
if (x == null) {
this.logger.warn('Specified inReplyTo, but not found');
throw new Error('inReplyTo not found');
} else {
? await this.resolveNote(note.inReplyTo, resolver)
.then(x => {
if (x == null) {
this.logger.warn('Specified inReplyTo, but not found');
throw new Error('inReplyTo not found');
}

return x;
}
}).catch(async err => {
this.logger.warn(`Error in inReplyTo ${note.inReplyTo} - ${err.statusCode ?? err}`);
throw err;
})
})
.catch(async err => {
this.logger.warn(`Error in inReplyTo ${note.inReplyTo} - ${err.statusCode ?? err}`);
throw err;
})
: null;

// 引用
let quote: Note | undefined | null;
let quote: Note | undefined | null = null;

if (note._misskey_quote || note.quoteUrl) {
const tryResolveNote = async (uri: string): Promise<{
status: 'ok';
res: Note | null;
} | {
status: 'permerror' | 'temperror';
}> => {
if (typeof uri !== 'string' || !uri.match(/^https?:/)) return { status: 'permerror' };
const tryResolveNote = async (uri: string): Promise<
| { status: 'ok'; res: Note }
| { status: 'permerror' | 'temperror' }
> => {
if (!uri.match(/^https?:/)) return { status: 'permerror' };
try {
const res = await this.resolveNote(uri);
if (res) {
return {
status: 'ok',
res,
};
} else {
return {
status: 'permerror',
};
}
if (res == null) return { status: 'permerror' };
return { status: 'ok', res };
} catch (e) {
return {
status: (e instanceof StatusError && e.isClientError) ? 'permerror' : 'temperror',
Expand All @@ -225,9 +213,9 @@ export class ApNoteService {
};

const uris = unique([note._misskey_quote, note.quoteUrl].filter((x): x is string => typeof x === 'string'));
const results = await Promise.all(uris.map(uri => tryResolveNote(uri)));
const results = await Promise.all(uris.map(tryResolveNote));

quote = results.filter((x): x is { status: 'ok', res: Note | null } => x.status === 'ok').map(x => x.res).find(x => x);
quote = results.filter((x): x is { status: 'ok', res: Note } => x.status === 'ok').map(x => x.res).at(0);
if (!quote) {
if (results.some(x => x.status === 'temperror')) {
throw new Error('quote resolve failed');
Expand Down Expand Up @@ -271,7 +259,7 @@ export class ApNoteService {

const emojis = await this.extractEmojis(note.tag ?? [], actor.host).catch(e => {
this.logger.info(`extractEmojis: ${e}`);
return [] as Emoji[];
return [];
});

const apEmojis = emojis.map(emoji => emoji.name);
Expand Down Expand Up @@ -309,19 +297,18 @@ export class ApNoteService {
const uri = typeof value === 'string' ? value : value.id;
if (uri == null) throw new Error('missing uri');

// ブロックしてたら中断
// ブロックしていたら中断
const meta = await this.metaService.fetch();
if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.extractDbHost(uri))) throw new StatusError('blocked host', 451);
if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.extractDbHost(uri))) {
throw new StatusError('blocked host', 451);
}

const unlock = await this.appLockService.getApLock(uri);

try {
//#region このサーバーに既に登録されていたらそれを返す
const exist = await this.fetchNote(uri);

if (exist) {
return exist;
}
if (exist) return exist;
//#endregion

if (uri.startsWith(this.config.url)) {
Expand All @@ -339,43 +326,41 @@ export class ApNoteService {

@bindThis
public async extractEmojis(tags: IObject | IObject[], host: string): Promise<Emoji[]> {
// eslint-disable-next-line no-param-reassign
host = this.utilityService.toPuny(host);

if (!tags) return [];

const eomjiTags = toArray(tags).filter(isEmoji);

const existingEmojis = await this.emojisRepository.findBy({
host,
name: In(eomjiTags.map(tag => tag.name!.replaceAll(':', ''))),
name: In(eomjiTags.map(tag => tag.name.replaceAll(':', ''))),
});

return await Promise.all(eomjiTags.map(async tag => {
const name = tag.name!.replaceAll(':', '');
const name = tag.name.replaceAll(':', '');
tag.icon = toSingle(tag.icon);

const exists = existingEmojis.find(x => x.name === name);

if (exists) {
if ((tag.updated != null && exists.updatedAt == null)
if ((exists.updatedAt == null)
|| (tag.id != null && exists.uri == null)
|| (tag.updated != null && exists.updatedAt != null && new Date(tag.updated) > exists.updatedAt)
|| (tag.icon!.url !== exists.originalUrl)
|| (new Date(tag.updated) > exists.updatedAt)
|| (tag.icon.url !== exists.originalUrl)
) {
await this.emojisRepository.update({
host,
name,
}, {
uri: tag.id,
originalUrl: tag.icon!.url,
publicUrl: tag.icon!.url,
originalUrl: tag.icon.url,
publicUrl: tag.icon.url,
updatedAt: new Date(),
});

return await this.emojisRepository.findOneBy({
host,
name,
}) as Emoji;
const emoji = await this.emojisRepository.findOneBy({ host, name });
if (emoji == null) throw new Error('emoji update failed');
return emoji;
}

return exists;
Expand All @@ -388,11 +373,11 @@ export class ApNoteService {
host,
name,
uri: tag.id,
originalUrl: tag.icon!.url,
publicUrl: tag.icon!.url,
originalUrl: tag.icon.url,
publicUrl: tag.icon.url,
updatedAt: new Date(),
aliases: [],
} as Partial<Emoji>).then(x => this.emojisRepository.findOneByOrFail(x.identifiers[0]));
}).then(x => this.emojisRepository.findOneByOrFail(x.identifiers[0]));
}));
}
}
Loading