From b4894aae3d0ad9dcac9b18745f11b18ba2b3c3c5 Mon Sep 17 00:00:00 2001
From: shiosyakeyakini-info <sorairo@shiosyakeyakini.info>
Date: Sat, 31 Aug 2024 20:35:24 +0900
Subject: [PATCH] =?UTF-8?q?Twitter=E3=82=A4=E3=83=B3=E3=83=9D=E3=83=BC?=
 =?UTF-8?q?=E3=83=88=E6=A9=9F=E8=83=BD=E3=81=AE=E5=BC=B7=E5=8C=96=20(#55)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../processors/ImportNotesProcessorService.ts | 32 +++++++++++++++----
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts
index f6b745768ba7..e0da4f2c9eb3 100644
--- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts
+++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts
@@ -141,7 +141,7 @@ export class ImportNotesProcessorService {
 
 	@bindThis
 	private parseTwitterFile(str : string) : { tweet: object }[] {
-		const jsonStr = str.replace(/^\s*window\.YTD\.tweets\.part0\s*=\s*/, '');
+		const jsonStr = str.replace(/^\s*window\.YTD\.tweets\.part\d+\s*=\s*/, '');
 
 		try {
 			return JSON.parse(jsonStr);
@@ -197,9 +197,18 @@ export class ImportNotesProcessorService {
 			try {
 				this.logger.succ(`Unzipping to ${outputPath}`);
 				ZipReader.withDestinationPath(outputPath).viaBuffer(await fsp.readFile(destPath));
-
+				this.logger.succ('processing ' + outputPath + '/data/tweets.js');
 				const unprocessedTweets = this.parseTwitterFile(await fsp.readFile(outputPath + '/data/tweets.js', 'utf-8'));
 
+				const files = await fsp.readdir(outputPath + '/data/');
+				const additionalTweetFiles = files.filter((file) => /tweets-part\d+?\.js/.test(file));
+				for await (const file of additionalTweetFiles) {
+					this.logger.succ('processing ' + outputPath + '/data/' + file);
+					const additionalUnprocessedTweet = this.parseTwitterFile(
+						await fsp.readFile(outputPath + '/data/' + file, 'utf-8'));
+					unprocessedTweets.push(...additionalUnprocessedTweet);
+				}
+
 				const tweets = unprocessedTweets.map(e => e.tweet);
 				const processedTweets = await this.recreateChain(['id_str'], ['in_reply_to_status_id_str'], tweets, false);
 				this.queueService.createImportTweetsToDbJob(job.data.user, processedTweets, null);
@@ -587,22 +596,33 @@ export class ImportNotesProcessorService {
 			return full_textedit;
 		}
 
-		async function replaceTwitterMentions(full_text: string, mentions: any) {
+		async function replaceTwitterMentions(full_text: string, mentions: any, isRetweet: boolean) {
 			let full_textedit = full_text;
 			mentions.forEach((mention: any) => {
-				full_textedit = full_textedit.replaceAll(`@${mention.screen_name}`, `[@${mention.screen_name}](https://twitter.com/${mention.screen_name})`);
+				full_textedit = full_textedit.replaceAll(`@${mention.screen_name}`, `${isRetweet ? '?' : ''}[@${mention.screen_name}](https://twitter.com/${mention.screen_name})`);
 			});
 			return full_textedit;
 		}
 
+		async function replaceAllMentions(full_text: string) {
+			return full_text.replaceAll(/(?<!https?:\/\/\S*)@([A-Za-z0-9_\.]+)/g, '<plain>@$1</plain>');
+		}
+
+		async function replaceRetweet(full_text: string, isRetweet: boolean) {
+			if (!isRetweet) return full_text;
+			return '[RT](https://x.com/i/status/' + tweet.id_str + ')' + full_text.substring(2);
+		}
+
 		try {
 			const date = new Date(tweet.created_at);
+			const isRetweet = tweet.full_text.startsWith('RT @');
 			const decodedText = tweet.full_text.replaceAll('&gt;', '>').replaceAll('&lt;', '<').replaceAll('&amp;', '&');
 			const textReplaceURLs = tweet.entities.urls && tweet.entities.urls.length > 0 ? await replaceTwitterUrls(decodedText, tweet.entities.urls) : decodedText;
-			const text = tweet.entities.user_mentions && tweet.entities.user_mentions.length > 0 ? await replaceTwitterMentions(textReplaceURLs, tweet.entities.user_mentions) : textReplaceURLs;
+			const mentionReplaceURLs = tweet.entities.user_mentions && tweet.entities.user_mentions.length > 0 ? await replaceTwitterMentions(textReplaceURLs, tweet.entities.user_mentions, isRetweet) : textReplaceURLs;
+			const text = await replaceRetweet(await replaceAllMentions(mentionReplaceURLs), isRetweet);
 			const files: MiDriveFile[] = [];
 
-			if (tweet.extended_entities && this.isIterable(tweet.extended_entities.media)) {
+			if (!isRetweet && tweet.extended_entities && this.isIterable(tweet.extended_entities.media)) {
 				let twitFolder = await this.driveFoldersRepository.findOneBy({ name: 'Twitter', userId: job.data.user.id, parentId: folder.id });
 				if (twitFolder == null) {
 					await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Twitter', userId: job.data.user.id, parentId: folder.id });