From 0b7c45e75b58b1e121348ed96dcc9d88132bf2e6 Mon Sep 17 00:00:00 2001 From: Andy Neillans Date: Tue, 17 Oct 2023 15:22:40 +0100 Subject: [PATCH 1/8] WIP: Refactoring underway --- commons/src/index.ts | 4 +- .../megalodon.account.interface.ts} | 2 +- .../pixelfed/pixelfed.account.interface.ts | 5 - .../pleroma/pleroma.account.interface.ts | 6 - electron-app/package-lock.json | 4 +- .../websites/mastodon/mastodon.service.ts | 12 +- .../websites/megalodon/megalodon.service.ts | 352 ++++++++++++++++++ .../websites/pixelfed/pixelfed.service.ts | 14 +- .../websites/pleroma/pleroma.service.ts | 28 +- package-lock.json | 4 +- ui/package-lock.json | 4 +- ui/src/websites/mastodon/MastodonLogin.tsx | 4 +- ui/src/websites/pixelfed/PixelfedLogin.tsx | 4 +- ui/src/websites/pleroma/PleromaLogin.tsx | 6 +- 14 files changed, 394 insertions(+), 55 deletions(-) rename commons/src/interfaces/websites/{mastodon/mastodon.account.interface.ts => megalodon/megalodon.account.interface.ts} (59%) delete mode 100644 commons/src/interfaces/websites/pixelfed/pixelfed.account.interface.ts delete mode 100644 commons/src/interfaces/websites/pleroma/pleroma.account.interface.ts create mode 100644 electron-app/src/server/websites/megalodon/megalodon.service.ts diff --git a/commons/src/index.ts b/commons/src/index.ts index 2c65bef8..f977f505 100644 --- a/commons/src/index.ts +++ b/commons/src/index.ts @@ -60,13 +60,12 @@ export * from './interfaces/websites/hentai-foundry/hentai-foundry.file.options. export * from './interfaces/websites/inkbunny/inkbunny.file.options.interface'; export * from './interfaces/websites/ko-fi/ko-fi.file.options.interface'; export * from './interfaces/websites/manebooru/manebooru.file.options.interface'; -export * from './interfaces/websites/mastodon/mastodon.account.interface'; +export * from './interfaces/websites/megalodon/megalodon.account.interface'; export * from './interfaces/websites/mastodon/mastodon.file.options.interface'; export * from './interfaces/websites/mastodon/mastodon.notification.options.interface'; export * from './interfaces/websites/misskey/misskey.account.interface'; export * from './interfaces/websites/misskey/misskey.file.options.interface'; export * from './interfaces/websites/misskey/misskey.notification.options.interface'; -export * from './interfaces/websites/pleroma/pleroma.account.interface'; export * from './interfaces/websites/pleroma/pleroma.file.options.interface'; export * from './interfaces/websites/pleroma/pleroma.notification.options.interface'; export * from './interfaces/websites/newgrounds/newgrounds.file.options.interface'; @@ -91,7 +90,6 @@ export * from './interfaces/websites/itaku/itaku.notification.options.interface' export * from './interfaces/websites/telegram/telegram.account.interface'; export * from './interfaces/websites/telegram/telegram.file.options.interface'; export * from './interfaces/websites/telegram/telegram.notification.options.interface'; -export * from './interfaces/websites/pixelfed/pixelfed.account.interface'; export * from './interfaces/websites/pixelfed/pixelfed.file.options.interface'; export * from './interfaces/websites/bluesky/bluesky.account.interface'; export * from './interfaces/websites/bluesky/bluesky.file.options.interface'; diff --git a/commons/src/interfaces/websites/mastodon/mastodon.account.interface.ts b/commons/src/interfaces/websites/megalodon/megalodon.account.interface.ts similarity index 59% rename from commons/src/interfaces/websites/mastodon/mastodon.account.interface.ts rename to commons/src/interfaces/websites/megalodon/megalodon.account.interface.ts index 6a6c07fb..56a4dff1 100644 --- a/commons/src/interfaces/websites/mastodon/mastodon.account.interface.ts +++ b/commons/src/interfaces/websites/megalodon/megalodon.account.interface.ts @@ -1,4 +1,4 @@ -export interface MastodonAccountData { +export interface MegalodonAccountData { token: string; website: string; username: string; diff --git a/commons/src/interfaces/websites/pixelfed/pixelfed.account.interface.ts b/commons/src/interfaces/websites/pixelfed/pixelfed.account.interface.ts deleted file mode 100644 index 8a552bd3..00000000 --- a/commons/src/interfaces/websites/pixelfed/pixelfed.account.interface.ts +++ /dev/null @@ -1,5 +0,0 @@ -export interface PixelfedAccountData { - token: string; - website: string; - username: string; -} diff --git a/commons/src/interfaces/websites/pleroma/pleroma.account.interface.ts b/commons/src/interfaces/websites/pleroma/pleroma.account.interface.ts deleted file mode 100644 index 88377918..00000000 --- a/commons/src/interfaces/websites/pleroma/pleroma.account.interface.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { OAuth } from 'megalodon' -export interface PleromaAccountData { - tokenData: OAuth.TokenData | null; - website: string; - username: string; -} diff --git a/electron-app/package-lock.json b/electron-app/package-lock.json index 607a60de..ee212ae4 100644 --- a/electron-app/package-lock.json +++ b/electron-app/package-lock.json @@ -1,12 +1,12 @@ { "name": "postybirb-plus", - "version": "3.1.30", + "version": "3.1.31", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "postybirb-plus", - "version": "3.1.30", + "version": "3.1.31", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { diff --git a/electron-app/src/server/websites/mastodon/mastodon.service.ts b/electron-app/src/server/websites/mastodon/mastodon.service.ts index 9327dd5b..8cb9f5c7 100644 --- a/electron-app/src/server/websites/mastodon/mastodon.service.ts +++ b/electron-app/src/server/websites/mastodon/mastodon.service.ts @@ -5,7 +5,7 @@ import { FileRecord, FileSubmission, FileSubmissionType, - MastodonAccountData, + MegalodonAccountData, MastodonFileOptions, MastodonNotificationOptions, PostResponse, @@ -93,7 +93,7 @@ export class Mastodon extends Website { async checkLoginStatus(data: UserAccountEntity): Promise { const status: LoginResponse = { loggedIn: false, username: null }; - const accountData: MastodonAccountData = data.data; + const accountData: MegalodonAccountData = data.data; if (accountData && accountData.token) { await this.getAndStoreInstanceInfo(data._id, accountData); @@ -103,7 +103,7 @@ export class Mastodon extends Website { return status; } - private async getAndStoreInstanceInfo(profileId: string, data: MastodonAccountData) { + private async getAndStoreInstanceInfo(profileId: string, data: MegalodonAccountData) { const client = generator('mastodon', data.website, data.token); const instance = await client.getInstance(); @@ -137,7 +137,7 @@ export class Mastodon extends Website { // Megaladon api has uploadMedia method, hovewer, it does not work with mastodon private async uploadMedia( - data: MastodonAccountData, + data: MegalodonAccountData, file: PostFile, altText: string, ): Promise { @@ -196,7 +196,7 @@ export class Mastodon extends Website { async postFileSubmission( cancellationToken: CancellationToken, data: FilePostData, - accountData: MastodonAccountData, + accountData: MegalodonAccountData, ): Promise { const M = generator('mastodon', accountData.website, accountData.token); @@ -267,7 +267,7 @@ export class Mastodon extends Website { async postNotificationSubmission( cancellationToken: CancellationToken, data: PostData, - accountData: MastodonAccountData, + accountData: MegalodonAccountData, ): Promise { const M = generator('mastodon', accountData.website, accountData.token); const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(data.part.accountId, INFO_KEY); diff --git a/electron-app/src/server/websites/megalodon/megalodon.service.ts b/electron-app/src/server/websites/megalodon/megalodon.service.ts new file mode 100644 index 00000000..8a5a3b72 --- /dev/null +++ b/electron-app/src/server/websites/megalodon/megalodon.service.ts @@ -0,0 +1,352 @@ +import { Injectable } from '@nestjs/common'; +import generator, { Entity, Response } from 'megalodon'; +import { + DefaultOptions, + FileRecord, + FileSubmission, + FileSubmissionType, + MegalodonAccountData, + MastodonFileOptions, + MastodonNotificationOptions, + PostResponse, + Submission, + SubmissionPart, + SubmissionRating, +} from 'postybirb-commons'; +import { ScalingOptions } from '../interfaces/scaling-options.interface'; +import UserAccountEntity from 'src/server//account/models/user-account.entity'; +import { PlaintextParser } from 'src/server/description-parsing/plaintext/plaintext.parser'; +import ImageManipulator from 'src/server/file-manipulation/manipulators/image.manipulator'; +import Http from 'src/server/http/http.util'; +import { CancellationToken } from 'src/server/submission/post/cancellation/cancellation-token'; +import { + FilePostData, + PostFile, +} from 'src/server/submission/post/interfaces/file-post-data.interface'; +import { PostData } from 'src/server/submission/post/interfaces/post-data.interface'; +import { ValidationParts } from 'src/server/submission/validator/interfaces/validation-parts.interface'; +import FileSize from 'src/server/utils/filesize.util'; +import FormContent from 'src/server/utils/form-content.util'; +import WebsiteValidator from 'src/server/utils/website-validator.util'; +import { LoginResponse } from '../interfaces/login-response.interface'; +import { Website } from '../website.base'; +import _ from 'lodash'; +import WaitUtil from 'src/server/utils/wait.util'; +import { FileManagerService } from 'src/server/file-manager/file-manager.service'; + +const INFO_KEY = 'INSTANCE INFO'; + +export abstract class Megalodon extends Website { + constructor(private readonly fileRepository: FileManagerService) { + super(); + } + + readonly megalodonService = 'mastodon'; // Set this as appropriate in your constructor + readonly maxCharLength = 500; // Set this off the instance information! + + readonly BASE_URL: string; + readonly enableAdvertisement = false; + readonly acceptsAdditionalFiles = true; + readonly defaultDescriptionParser = PlaintextParser.parse; + readonly acceptsFiles = [ // Override, or extend this list in your inherited classes! + 'png', + 'jpeg', + 'jpg', + 'gif', + 'webp', + 'm4v', + 'mov' + ]; + + // Boiler plate login check code across all versions of services using the megalodon library + async checkLoginStatus(data: UserAccountEntity): Promise { + const status: LoginResponse = { loggedIn: false, username: null }; + const accountData: MegalodonAccountData = data.data; + if (accountData && accountData.token) { + await this.getAndStoreInstanceInfo(data._id, accountData); + + status.loggedIn = true; + status.username = accountData.username; + } + return status; + } + + private async getAndStoreInstanceInfo(profileId: string, data: MegalodonAccountData) { + const client = generator(this.megalodonService, data.website, data.token); + const instance = await client.getInstance(); + + this.storeAccountInformation(profileId, INFO_KEY, instance.data); + } + + // TODO: Refactor + + getScalingOptions(file: FileRecord, accountId: string): ScalingOptions { + const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(accountId, INFO_KEY); + if (instanceInfo?.configuration?.media_attachments) { + const maxPixels = + file.type === FileSubmissionType.IMAGE + ? instanceInfo.configuration.media_attachments.image_matrix_limit + : instanceInfo.configuration.media_attachments.video_matrix_limit; + + return { + maxHeight: Math.round(Math.sqrt(maxPixels * (file.width / file.height))), + maxWidth: Math.round(Math.sqrt(maxPixels * (file.height / file.width))), + maxSize: + file.type === FileSubmissionType.IMAGE + ? instanceInfo.configuration.media_attachments.image_size_limit + : instanceInfo.configuration.media_attachments.video_size_limit, + }; + } else if (instanceInfo?.upload_limit) { + return { + maxSize: instanceInfo?.upload_limit, + }; + } else { + return undefined; + } + } + + // TODO: Add common uploadMedia code from Pleroma codebase + + // TODO: Refactor + + async postFileSubmission( + cancellationToken: CancellationToken, + data: FilePostData, + accountData: MegalodonAccountData, + ): Promise { + const M = generator('mastodon', accountData.website, accountData.token); + + const files = [data.primary, ...data.additional]; + const uploadedMedias: string[] = []; + for (const file of files) { + this.checkCancelled(cancellationToken); + uploadedMedias.push(await this.uploadMedia(accountData, file.file, data.options.altText)); + } + + const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(data.part.accountId, INFO_KEY); + const chunkCount = + instanceInfo?.configuration?.statuses?.max_media_attachments ?? + instanceInfo?.max_media_attachments ?? + (instanceInfo?.upload_limit ? 1000 : 4); + const maxChars = + instanceInfo?.configuration?.statuses?.max_characters ?? instanceInfo?.max_toot_chars ?? 500; + + const isSensitive = data.rating !== SubmissionRating.GENERAL; + const chunks = _.chunk(uploadedMedias, chunkCount); + let status = `${data.options.useTitle && data.title ? `${data.title}\n` : ''}${ + data.description + }`.substring(0, maxChars); + let lastId = ''; + let source = ''; + const replyToId = this.getPostIdFromUrl(data.options.replyToUrl); + + for (let i = 0; i < chunks.length; i++) { + this.checkCancelled(cancellationToken); + const statusOptions: any = { + sensitive: isSensitive, + visibility: data.options.visibility || 'public', + media_ids: chunks[i], + }; + + if (i !== 0) { + statusOptions.in_reply_to_id = lastId; + } else if (replyToId) { + statusOptions.in_reply_to_id = replyToId; + } + + if (data.options.spoilerText) { + statusOptions.spoiler_text = data.options.spoilerText; + } + + status = this.appendTags(this.formatTags(data.tags), status, maxChars); + + try { + const result = (await M.postStatus(status, statusOptions)).data as Entity.Status; + if (!source) source = result.url; + lastId = result.id; + } catch (err) { + return Promise.reject( + this.createPostResponse({ + message: err.message, + stack: err.stack, + additionalInfo: { chunkNumber: i }, + }), + ); + } + } + + this.checkCancelled(cancellationToken); + + return this.createPostResponse({ source }); + } + + // TODO: Refactor + + async postNotificationSubmission( + cancellationToken: CancellationToken, + data: PostData, + accountData: MegalodonAccountData, + ): Promise { + const M = generator('mastodon', accountData.website, accountData.token); + const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(data.part.accountId, INFO_KEY); + const maxChars = instanceInfo?.configuration?.statuses?.max_characters ?? 500; + + const isSensitive = data.rating !== SubmissionRating.GENERAL; + const statusOptions: any = { + sensitive: isSensitive, + visibility: data.options.visibility || 'public', + }; + let status = `${data.options.useTitle && data.title ? `${data.title}\n` : ''}${ + data.description + }`; + if (data.options.spoilerText) { + statusOptions.spoiler_text = data.options.spoilerText; + } + status = this.appendTags(this.formatTags(data.tags), status, maxChars); + + const replyToId = this.getPostIdFromUrl(data.options.replyToUrl); + if (replyToId) { + statusOptions.in_reply_to_id = replyToId; + } + + this.checkCancelled(cancellationToken); + try { + const result = (await M.postStatus(status, statusOptions)).data as Entity.Status; + return this.createPostResponse({ source: result.url }); + } catch (error) { + return Promise.reject(this.createPostResponse(error)); + } + } + + // TODO: Make sure this has the strip preceeding space ? + formatTags(tags: string[]) { + return this.parseTags( + tags + .map(tag => tag.replace(/[^a-z0-9]/gi, ' ')) + .map(tag => + tag + .split(' ') + .join(''), + ), + { spaceReplacer: '_' }, + ).map(tag => `#${tag}`); + } + + // TODO REFACTOR + + validateFileSubmission( + submission: FileSubmission, + submissionPart: SubmissionPart, + defaultPart: SubmissionPart, + ): ValidationParts { + const problems: string[] = []; + const warnings: string[] = []; + const isAutoscaling: boolean = submissionPart.data.autoScale; + + const description = this.defaultDescriptionParser( + FormContent.getDescription(defaultPart.data.description, submissionPart.data.description), + ); + + const instanceInfo: MastodonInstanceInfo = this.getAccountInfo( + submissionPart.accountId, + INFO_KEY, + ); + const maxChars = instanceInfo?.configuration?.statuses?.max_characters ?? 500; + + if (description.length > maxChars) { + warnings.push( + `Max description length allowed is ${maxChars} characters (for this instance).`, + ); + } + + const files = [ + submission.primary, + ...(submission.additional || []).filter( + f => !f.ignoredAccounts!.includes(submissionPart.accountId), + ), + ]; + + files.forEach(file => { + const { type, size, name, mimetype } = file; + if (!WebsiteValidator.supportsFileType(file, this.acceptsFiles)) { + problems.push(`Does not support file format: (${name}) ${mimetype}.`); + } + + const scalingOptions = this.getScalingOptions(file, submissionPart.accountId); + + if (scalingOptions && scalingOptions.maxSize < size) { + if ( + isAutoscaling && + type === FileSubmissionType.IMAGE && + ImageManipulator.isMimeType(mimetype) + ) { + warnings.push( + `${name} will be scaled down to ${FileSize.BytesToMB(scalingOptions.maxSize)}MB`, + ); + } else { + problems.push( + `This instance limits ${mimetype} to ${FileSize.BytesToMB(scalingOptions.maxSize)}MB`, + ); + } + } + + if ( + scalingOptions && + isAutoscaling && + type === FileSubmissionType.IMAGE && + scalingOptions.maxWidth && + scalingOptions.maxHeight && + (file.height > scalingOptions.maxHeight || file.width > scalingOptions.maxWidth) + ) { + warnings.push( + `${name} will be scaled down to a maximum size of ${scalingOptions.maxWidth}x${scalingOptions.maxHeight}, while maintaining aspect ratio`, + ); + } + }); + + if ( + (submissionPart.data.tags.value.length > 1 || defaultPart.data.tags.value.length > 1) && + submissionPart.data.visibility != 'public' + ) { + warnings.push( + `This post won't be listed under any hashtag as it is not public. Only public posts can be searched by hashtag.`, + ); + } + + this.validateReplyToUrl(problems, submissionPart.data.replyToUrl); + + return { problems, warnings }; + } + + validateNotificationSubmission( + submission: Submission, + submissionPart: SubmissionPart, + defaultPart: SubmissionPart, + ): ValidationParts { + const problems = []; + const warnings = []; + + const description = this.defaultDescriptionParser( + FormContent.getDescription(defaultPart.data.description, submissionPart.data.description), + ); + + if (description.length > this.maxCharLength) { + warnings.push( + `Max description length allowed is ${this.maxCharLength} characters.`, + ); + } + + this.validateReplyToUrl(problems, submissionPart.data.replyToUrl); + + return { problems, warnings }; + } + + private validateReplyToUrl(problems: string[], url?: string): void { + if(url?.trim() && !this.getPostIdFromUrl(url)) { + problems.push("Invalid post URL to reply to."); + } + } + + abstract getPostIdFromUrl(url: string): string | null; + +} diff --git a/electron-app/src/server/websites/pixelfed/pixelfed.service.ts b/electron-app/src/server/websites/pixelfed/pixelfed.service.ts index 5493ec9d..bbc01e49 100644 --- a/electron-app/src/server/websites/pixelfed/pixelfed.service.ts +++ b/electron-app/src/server/websites/pixelfed/pixelfed.service.ts @@ -5,7 +5,7 @@ import { FileRecord, FileSubmission, FileSubmissionType, - PixelfedAccountData, + MegalodonAccountData, PixelfedFileOptions, PostResponse, Submission, @@ -62,7 +62,7 @@ export class Pixelfed extends Website { async checkLoginStatus(data: UserAccountEntity): Promise { const status: LoginResponse = { loggedIn: false, username: null }; - const accountData: PixelfedAccountData = data.data; + const accountData: MegalodonAccountData = data.data; if (accountData && accountData.token) { const refresh = await this.refreshToken(accountData); if (refresh) { @@ -74,7 +74,7 @@ export class Pixelfed extends Website { return status; } - private async getInstanceInfo(profileId: string, data: PixelfedAccountData) { + private async getInstanceInfo(profileId: string, data: MegalodonAccountData) { const client = generator('mastodon', data.website, data.token); const instance = await client.getInstance(); @@ -82,12 +82,12 @@ export class Pixelfed extends Website { } // TBH not entirely sure why this is a thing, but its in the old code so... :shrug: - private async refreshToken(data: PixelfedAccountData): Promise { + private async refreshToken(data: MegalodonAccountData): Promise { const M = this.getPixelfedInstance(data); return true; } - private getPixelfedInstance(data: PixelfedAccountData): Entity.Instance { + private getPixelfedInstance(data: MegalodonAccountData): Entity.Instance { const client = generator('mastodon', data.website, data.token); client.getInstance().then(res => { return res.data; @@ -114,7 +114,7 @@ export class Pixelfed extends Website { } private async uploadMedia( - data: PixelfedAccountData, + data: MegalodonAccountData, file: PostFile, altText: string, ): Promise<{ id: string }> { @@ -173,7 +173,7 @@ export class Pixelfed extends Website { async postFileSubmission( cancellationToken: CancellationToken, data: FilePostData, - accountData: PixelfedAccountData, + accountData: MegalodonAccountData, ): Promise { const M = generator('mastodon', accountData.website, accountData.token); diff --git a/electron-app/src/server/websites/pleroma/pleroma.service.ts b/electron-app/src/server/websites/pleroma/pleroma.service.ts index dae9da34..850868d2 100644 --- a/electron-app/src/server/websites/pleroma/pleroma.service.ts +++ b/electron-app/src/server/websites/pleroma/pleroma.service.ts @@ -5,7 +5,7 @@ import { FileRecord, FileSubmission, FileSubmissionType, - PleromaAccountData, + MegalodonAccountData, PleromaFileOptions, PleromaNotificationOptions, PostResponse, @@ -77,8 +77,8 @@ export class Pleroma extends Website { async checkLoginStatus(data: UserAccountEntity): Promise { const status: LoginResponse = { loggedIn: false, username: null }; - const accountData: PleromaAccountData = data.data; - if (accountData && accountData.tokenData) { + const accountData: MegalodonAccountData = data.data; + if (accountData && accountData.token) { const refresh = await this.refreshToken(accountData); if (refresh) { status.loggedIn = true; @@ -89,20 +89,20 @@ export class Pleroma extends Website { return status; } - private async getInstanceInfo(profileId: string, data: PleromaAccountData) { - const client = generator('pleroma', `https://${data.website}`, data.tokenData.access_token); + private async getInstanceInfo(profileId: string, data: MegalodonAccountData) { + const client = generator('pleroma', `https://${data.website}`, data.token); // token => tokenData.access_token const instance = await client.getInstance(); this.storeAccountInformation(profileId, INFO_KEY, instance.data); } // TBH not entirely sure why this is a thing, but its in the old code so... :shrug: - private async refreshToken(data: PleromaAccountData): Promise { + private async refreshToken(data: MegalodonAccountData): Promise { const M = this.getPleromaInstance(data); return true; } - private getPleromaInstance(data: PleromaAccountData) : Entity.Instance { - const client = generator('pleroma', `https://${data.website}`, data.tokenData.access_token); + private getPleromaInstance(data: MegalodonAccountData) : Entity.Instance { + const client = generator('pleroma', `https://${data.website}`, data.token); client.getInstance().then((res) => { return res.data; }); @@ -128,12 +128,12 @@ export class Pleroma extends Website { } private async uploadMedia( - data: PleromaAccountData, + data: MegalodonAccountData, file: PostFile, altText: string, ): Promise<{ id: string }> { this.logger.log("Uploading media") - const M = generator('pleroma', `https://${data.website}`, data.tokenData.access_token); + const M = generator('pleroma', `https://${data.website}`, data.token); // megalodon is odd, and doesnt seem to like the buffer being passed to the upload media call // So lets write the file out to a temp file, then pass that to createReadStream, then that to uploadMedia. @@ -168,9 +168,9 @@ export class Pleroma extends Website { async postFileSubmission( cancellationToken: CancellationToken, data: FilePostData, - accountData: PleromaAccountData, + accountData: MegalodonAccountData, ): Promise { - const M = generator('pleroma', `https://${accountData.website}`, accountData.tokenData.access_token); + const M = generator('pleroma', `https://${accountData.website}`, accountData.token); const files = [data.primary, ...data.additional]; @@ -270,10 +270,10 @@ export class Pleroma extends Website { async postNotificationSubmission( cancellationToken: CancellationToken, data: PostData, - accountData: PleromaAccountData, + accountData: MegalodonAccountData, ): Promise { const mInstance = this.getPleromaInstance(accountData); - const M = generator('pleroma', `https://${accountData.website}`, accountData.tokenData.access_token); + const M = generator('pleroma', `https://${accountData.website}`, accountData.token); const maxChars = mInstance ? mInstance?.configuration?.statuses?.max_characters : 500; diff --git a/package-lock.json b/package-lock.json index 9bd3bc63..a5100ba6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "postybirb-plus", - "version": "3.1.30", + "version": "3.1.31", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "postybirb-plus", - "version": "3.1.30", + "version": "3.1.31", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { diff --git a/ui/package-lock.json b/ui/package-lock.json index cc91c8ce..5bedab72 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "postybirb-plus-ui", - "version": "3.1.30", + "version": "3.1.31", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "postybirb-plus-ui", - "version": "3.1.30", + "version": "3.1.31", "license": "BSD-3-Clause", "dependencies": { "@atproto/api": "^0.6.4", diff --git a/ui/src/websites/mastodon/MastodonLogin.tsx b/ui/src/websites/mastodon/MastodonLogin.tsx index 7ee76e8a..5b60fc66 100644 --- a/ui/src/websites/mastodon/MastodonLogin.tsx +++ b/ui/src/websites/mastodon/MastodonLogin.tsx @@ -2,11 +2,11 @@ import { Button, Form, Input, message, Spin } from 'antd'; import Axios from 'axios'; import React from 'react'; import ReactDOM from 'react-dom'; -import { MastodonAccountData } from 'postybirb-commons'; +import { MegalodonAccountData } from 'postybirb-commons'; import LoginService from '../../services/login.service'; import { LoginDialogProps } from '../interfaces/website.interface'; -interface State extends MastodonAccountData { +interface State extends MegalodonAccountData { code: string; loading: boolean; } diff --git a/ui/src/websites/pixelfed/PixelfedLogin.tsx b/ui/src/websites/pixelfed/PixelfedLogin.tsx index 6b8215e9..396e7cb1 100644 --- a/ui/src/websites/pixelfed/PixelfedLogin.tsx +++ b/ui/src/websites/pixelfed/PixelfedLogin.tsx @@ -2,12 +2,12 @@ import { Button, Form, Input, message, Spin } from 'antd'; import Axios from 'axios'; import React from 'react'; import ReactDOM from 'react-dom'; -import { PixelfedAccountData } from 'postybirb-commons'; +import { MegalodonAccountData } from 'postybirb-commons'; import LoginService from '../../services/login.service'; import { LoginDialogProps } from '../interfaces/website.interface'; import { stringify } from 'querystring'; -interface State extends PixelfedAccountData { +interface State extends MegalodonAccountData { code: string; loading: boolean; } diff --git a/ui/src/websites/pleroma/PleromaLogin.tsx b/ui/src/websites/pleroma/PleromaLogin.tsx index b987fef0..a28da9ba 100644 --- a/ui/src/websites/pleroma/PleromaLogin.tsx +++ b/ui/src/websites/pleroma/PleromaLogin.tsx @@ -2,13 +2,13 @@ import { Button, Form, Input, message, Spin } from 'antd'; import Axios from 'axios'; import React from 'react'; import ReactDOM from 'react-dom'; -import { PleromaAccountData } from 'postybirb-commons'; +import { MegalodonAccountData } from 'postybirb-commons'; import LoginService from '../../services/login.service'; import { LoginDialogProps } from '../interfaces/website.interface'; import generator, { OAuth } from 'megalodon' -interface State extends PleromaAccountData { +interface State extends MegalodonAccountData { code: string; client_id: string; client_secret: string; @@ -76,7 +76,7 @@ export default class PleromaLogin extends React.Component{ let website = `https://${this.state.website}`; this.state.username = res.data.username; - this.state.tokenData = value; + this.state.token = value.access_token; LoginService.setAccountData(this.props.account._id, this.state ).then( () => { message.success(`${this.state.website} authenticated.`); From f0b1234ee472917cee29f6dddfe08940ba344c4a Mon Sep 17 00:00:00 2001 From: Andy Neillans Date: Tue, 17 Oct 2023 20:38:11 +0100 Subject: [PATCH 2/8] WIP: Refactoring --- .../websites/mastodon/mastodon.service.ts | 130 ++---------------- .../websites/megalodon/megalodon.service.ts | 42 +++++- .../websites/pleroma/pleroma.service.ts | 2 +- 3 files changed, 51 insertions(+), 123 deletions(-) diff --git a/electron-app/src/server/websites/mastodon/mastodon.service.ts b/electron-app/src/server/websites/mastodon/mastodon.service.ts index 8cb9f5c7..231f56e2 100644 --- a/electron-app/src/server/websites/mastodon/mastodon.service.ts +++ b/electron-app/src/server/websites/mastodon/mastodon.service.ts @@ -33,6 +33,7 @@ import { Website } from '../website.base'; import _ from 'lodash'; import WaitUtil from 'src/server/utils/wait.util'; import { FileManagerService } from 'src/server/file-manager/file-manager.service'; +import { Megalodon } from '../megalodon/megalodon.service'; const INFO_KEY = 'INSTANCE INFO'; @@ -56,14 +57,10 @@ type MastodonInstanceInfo = { }; @Injectable() -export class Mastodon extends Website { - constructor(private readonly fileRepository: FileManagerService) { - super(); - } - readonly BASE_URL: string; - readonly enableAdvertisement = false; - readonly acceptsAdditionalFiles = true; - readonly defaultDescriptionParser = PlaintextParser.parse; +export class Mastodon extends Megalodon { + + readonly megalodonService = 'mastodon'; + readonly acceptsFiles = [ 'png', 'jpeg', @@ -91,25 +88,6 @@ export class Mastodon extends Website { 'wma', ]; - async checkLoginStatus(data: UserAccountEntity): Promise { - const status: LoginResponse = { loggedIn: false, username: null }; - const accountData: MegalodonAccountData = data.data; - if (accountData && accountData.token) { - await this.getAndStoreInstanceInfo(data._id, accountData); - - status.loggedIn = true; - status.username = accountData.username; - } - return status; - } - - private async getAndStoreInstanceInfo(profileId: string, data: MegalodonAccountData) { - const client = generator('mastodon', data.website, data.token); - const instance = await client.getInstance(); - - this.storeAccountInformation(profileId, INFO_KEY, instance.data); - } - getScalingOptions(file: FileRecord, accountId: string): ScalingOptions { const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(accountId, INFO_KEY); if (instanceInfo?.configuration?.media_attachments) { @@ -135,64 +113,6 @@ export class Mastodon extends Website { } } - // Megaladon api has uploadMedia method, hovewer, it does not work with mastodon - private async uploadMedia( - data: MegalodonAccountData, - file: PostFile, - altText: string, - ): Promise { - const upload = await Http.post<{ id: string; errors: any; url: string }>( - `${data.website}/api/v2/media`, - undefined, - { - type: 'multipart', - data: { - file, - description: altText, - }, - requestOptions: { json: true }, - headers: { - Accept: '*/*', - 'User-Agent': 'node-mastodon-client/PostyBirb', - Authorization: `Bearer ${data.token}`, - }, - }, - ); - - this.verifyResponse(upload, 'Verify upload'); - - // Processing - if (upload.response.statusCode === 202 || !upload.body.url) { - for (let i = 0; i < 10; i++) { - await WaitUtil.wait(4000); - const checkUpload = await Http.get<{ id: string; errors: any; url: string }>( - `${data.website}/api/v1/media/${upload.body.id}`, - undefined, - { - requestOptions: { json: true }, - headers: { - Accept: '*/*', - 'User-Agent': 'node-mastodon-client/PostyBirb', - Authorization: `Bearer ${data.token}`, - }, - }, - ); - - if (checkUpload.body.url) { - break; - } - } - } - - if (upload.body.errors) { - return Promise.reject( - this.createPostResponse({ additionalInfo: upload.body, message: upload.body.errors }), - ); - } - - return upload.body.id; - } - async postFileSubmission( cancellationToken: CancellationToken, data: FilePostData, @@ -201,7 +121,9 @@ export class Mastodon extends Website { const M = generator('mastodon', accountData.website, accountData.token); const files = [data.primary, ...data.additional]; - const uploadedMedias: string[] = []; + const uploadedMedias: { + id: string; + }[] = []; for (const file of files) { this.checkCancelled(cancellationToken); uploadedMedias.push(await this.uploadMedia(accountData, file.file, data.options.altText)); @@ -398,41 +320,7 @@ export class Mastodon extends Website { return { problems, warnings }; } - validateNotificationSubmission( - submission: Submission, - submissionPart: SubmissionPart, - defaultPart: SubmissionPart, - ): ValidationParts { - const problems = []; - const warnings = []; - const description = this.defaultDescriptionParser( - FormContent.getDescription(defaultPart.data.description, submissionPart.data.description), - ); - - const instanceInfo: MastodonInstanceInfo = this.getAccountInfo( - submissionPart.accountId, - INFO_KEY, - ); - const maxChars = - instanceInfo?.configuration?.statuses?.max_characters ?? instanceInfo?.max_toot_chars ?? 500; - if (description.length > maxChars) { - warnings.push( - `Max description length allowed is ${maxChars} characters (for this instance).`, - ); - } - - this.validateReplyToUrl(problems, submissionPart.data.replyToUrl); - - return { problems, warnings }; - } - - private validateReplyToUrl(problems: string[], url?: string): void { - if(url?.trim() && !this.getPostIdFromUrl(url)) { - problems.push("Invalid post URL to reply to."); - } - } - - private getPostIdFromUrl(url: string): string | null { + getPostIdFromUrl(url: string): string | null { // We expect this to a post URL like https://{instance}/@{user}/{id} or // https://:instance/deck/@{user}/{id}. We grab the id after the @ part. const match = /\/@[^\/]+\/([0-9]+)/.exec(url); diff --git a/electron-app/src/server/websites/megalodon/megalodon.service.ts b/electron-app/src/server/websites/megalodon/megalodon.service.ts index 8a5a3b72..3bda7c9a 100644 --- a/electron-app/src/server/websites/megalodon/megalodon.service.ts +++ b/electron-app/src/server/websites/megalodon/megalodon.service.ts @@ -33,6 +33,9 @@ import { Website } from '../website.base'; import _ from 'lodash'; import WaitUtil from 'src/server/utils/wait.util'; import { FileManagerService } from 'src/server/file-manager/file-manager.service'; +import * as fs from 'fs'; +import { tmpdir } from 'os'; +import * as path from 'path'; const INFO_KEY = 'INSTANCE INFO'; @@ -341,7 +344,7 @@ export abstract class Megalodon extends Website { return { problems, warnings }; } - private validateReplyToUrl(problems: string[], url?: string): void { + validateReplyToUrl(problems: string[], url?: string): void { if(url?.trim() && !this.getPostIdFromUrl(url)) { problems.push("Invalid post URL to reply to."); } @@ -349,4 +352,41 @@ export abstract class Megalodon extends Website { abstract getPostIdFromUrl(url: string): string | null; + async uploadMedia( + data: MegalodonAccountData, + file: PostFile, + altText: string, + ): Promise<{ id: string }> { + this.logger.log("Uploading media") + const M = generator(this.megalodonService, `https://${data.website}`, data.token); + + // megalodon is odd, and doesnt seem to like the buffer being passed to the upload media call + // So lets write the file out to a temp file, then pass that to createReadStream, then that to uploadMedia. + // That works .... \o/ + const tempDir = tmpdir(); + + fs.writeFileSync(path.join(tempDir, file.options.filename), file.value); + + const upload = await M.uploadMedia(fs.createReadStream(path.join(tempDir, file.options.filename)), { description: altText }); + + this.logger.log(upload) + + fs.unlink(path.join(tempDir, file.options.filename), (err) => { + if (err) { + this.logger.error("Unable to remove the temp file", err.stack, err.message); + } + }); + + if (upload.status > 300) { + this.logger.log(upload); + return Promise.reject( + this.createPostResponse({ additionalInfo: upload.status, message: upload.statusText }), + ); + } + + this.logger.log("Image uploaded"); + + return { id: upload.data.id }; + } + } diff --git a/electron-app/src/server/websites/pleroma/pleroma.service.ts b/electron-app/src/server/websites/pleroma/pleroma.service.ts index 850868d2..6e076fcf 100644 --- a/electron-app/src/server/websites/pleroma/pleroma.service.ts +++ b/electron-app/src/server/websites/pleroma/pleroma.service.ts @@ -127,7 +127,7 @@ export class Pleroma extends Website { }; } - private async uploadMedia( + public async uploadMedia( data: MegalodonAccountData, file: PostFile, altText: string, From fd32381848444a9d2ebb613811383734e1f42b53 Mon Sep 17 00:00:00 2001 From: Andy Neillans Date: Tue, 17 Oct 2023 20:54:23 +0100 Subject: [PATCH 3/8] WIP: Refactoring --- .../websites/mastodon/mastodon.service.ts | 20 +++-------- .../websites/megalodon/megalodon.service.ts | 33 +++---------------- 2 files changed, 8 insertions(+), 45 deletions(-) diff --git a/electron-app/src/server/websites/mastodon/mastodon.service.ts b/electron-app/src/server/websites/mastodon/mastodon.service.ts index 231f56e2..6e3369ad 100644 --- a/electron-app/src/server/websites/mastodon/mastodon.service.ts +++ b/electron-app/src/server/websites/mastodon/mastodon.service.ts @@ -51,10 +51,7 @@ type MastodonInstanceInfo = { video_matrix_limit: number; }; }; - upload_limit?: number; // Pleroma, Akkoma - max_toot_chars?: number; // Pleroma, Akkoma - max_media_attachments?: number; //Pleroma -}; +} @Injectable() export class Mastodon extends Megalodon { @@ -104,10 +101,6 @@ export class Mastodon extends Megalodon { ? instanceInfo.configuration.media_attachments.image_size_limit : instanceInfo.configuration.media_attachments.video_size_limit, }; - } else if (instanceInfo?.upload_limit) { - return { - maxSize: instanceInfo?.upload_limit, - }; } else { return undefined; } @@ -130,18 +123,13 @@ export class Mastodon extends Megalodon { } const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(data.part.accountId, INFO_KEY); - const chunkCount = - instanceInfo?.configuration?.statuses?.max_media_attachments ?? - instanceInfo?.max_media_attachments ?? - (instanceInfo?.upload_limit ? 1000 : 4); - const maxChars = - instanceInfo?.configuration?.statuses?.max_characters ?? instanceInfo?.max_toot_chars ?? 500; + const chunkCount = instanceInfo ? instanceInfo?.configuration?.statuses?.max_media_attachments : 4; const isSensitive = data.rating !== SubmissionRating.GENERAL; const chunks = _.chunk(uploadedMedias, chunkCount); let status = `${data.options.useTitle && data.title ? `${data.title}\n` : ''}${ data.description - }`.substring(0, maxChars); + }`.substring(0, this.maxCharLength); let lastId = ''; let source = ''; const replyToId = this.getPostIdFromUrl(data.options.replyToUrl); @@ -164,7 +152,7 @@ export class Mastodon extends Megalodon { statusOptions.spoiler_text = data.options.spoilerText; } - status = this.appendTags(this.formatTags(data.tags), status, maxChars); + status = this.appendTags(this.formatTags(data.tags), status, this.maxCharLength); try { const result = (await M.postStatus(status, statusOptions)).data as Entity.Status; diff --git a/electron-app/src/server/websites/megalodon/megalodon.service.ts b/electron-app/src/server/websites/megalodon/megalodon.service.ts index 3bda7c9a..ab9fe3e5 100644 --- a/electron-app/src/server/websites/megalodon/megalodon.service.ts +++ b/electron-app/src/server/websites/megalodon/megalodon.service.ts @@ -81,34 +81,7 @@ export abstract class Megalodon extends Website { this.storeAccountInformation(profileId, INFO_KEY, instance.data); } - // TODO: Refactor - - getScalingOptions(file: FileRecord, accountId: string): ScalingOptions { - const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(accountId, INFO_KEY); - if (instanceInfo?.configuration?.media_attachments) { - const maxPixels = - file.type === FileSubmissionType.IMAGE - ? instanceInfo.configuration.media_attachments.image_matrix_limit - : instanceInfo.configuration.media_attachments.video_matrix_limit; - - return { - maxHeight: Math.round(Math.sqrt(maxPixels * (file.width / file.height))), - maxWidth: Math.round(Math.sqrt(maxPixels * (file.height / file.width))), - maxSize: - file.type === FileSubmissionType.IMAGE - ? instanceInfo.configuration.media_attachments.image_size_limit - : instanceInfo.configuration.media_attachments.video_size_limit, - }; - } else if (instanceInfo?.upload_limit) { - return { - maxSize: instanceInfo?.upload_limit, - }; - } else { - return undefined; - } - } - - // TODO: Add common uploadMedia code from Pleroma codebase + abstract getScalingOptions(file: FileRecord, accountId: string): ScalingOptions; // TODO: Refactor @@ -120,7 +93,9 @@ export abstract class Megalodon extends Website { const M = generator('mastodon', accountData.website, accountData.token); const files = [data.primary, ...data.additional]; - const uploadedMedias: string[] = []; + const uploadedMedias: { + id: string; + }[] = []; for (const file of files) { this.checkCancelled(cancellationToken); uploadedMedias.push(await this.uploadMedia(accountData, file.file, data.options.altText)); From 747cab62b1df14ec8c85e29197f4991dfdada091 Mon Sep 17 00:00:00 2001 From: Andy Neillans Date: Wed, 18 Oct 2023 20:30:28 +0100 Subject: [PATCH 4/8] Refactor tested with Mastodon - Passing --- .../websites/mastodon/mastodon.service.ts | 237 +----------------- .../websites/megalodon/megalodon.service.ts | 75 +++--- .../websites/pixelfed/pixelfed.service.ts | 3 +- ui/src/websites/pleroma/PleromaLogin.tsx | 6 +- 4 files changed, 38 insertions(+), 283 deletions(-) diff --git a/electron-app/src/server/websites/mastodon/mastodon.service.ts b/electron-app/src/server/websites/mastodon/mastodon.service.ts index 6e3369ad..38f81018 100644 --- a/electron-app/src/server/websites/mastodon/mastodon.service.ts +++ b/electron-app/src/server/websites/mastodon/mastodon.service.ts @@ -1,38 +1,10 @@ import { Injectable } from '@nestjs/common'; -import generator, { Entity, Response } from 'megalodon'; import { - DefaultOptions, FileRecord, - FileSubmission, FileSubmissionType, - MegalodonAccountData, - MastodonFileOptions, - MastodonNotificationOptions, - PostResponse, - Submission, - SubmissionPart, - SubmissionRating, } from 'postybirb-commons'; import { ScalingOptions } from '../interfaces/scaling-options.interface'; -import UserAccountEntity from 'src/server//account/models/user-account.entity'; -import { PlaintextParser } from 'src/server/description-parsing/plaintext/plaintext.parser'; -import ImageManipulator from 'src/server/file-manipulation/manipulators/image.manipulator'; -import Http from 'src/server/http/http.util'; -import { CancellationToken } from 'src/server/submission/post/cancellation/cancellation-token'; -import { - FilePostData, - PostFile, -} from 'src/server/submission/post/interfaces/file-post-data.interface'; -import { PostData } from 'src/server/submission/post/interfaces/post-data.interface'; -import { ValidationParts } from 'src/server/submission/validator/interfaces/validation-parts.interface'; -import FileSize from 'src/server/utils/filesize.util'; -import FormContent from 'src/server/utils/form-content.util'; -import WebsiteValidator from 'src/server/utils/website-validator.util'; -import { LoginResponse } from '../interfaces/login-response.interface'; -import { Website } from '../website.base'; import _ from 'lodash'; -import WaitUtil from 'src/server/utils/wait.util'; -import { FileManagerService } from 'src/server/file-manager/file-manager.service'; import { Megalodon } from '../megalodon/megalodon.service'; const INFO_KEY = 'INSTANCE INFO'; @@ -85,6 +57,13 @@ export class Mastodon extends Megalodon { 'wma', ]; + getInstanceSettings(accountId: string) { + const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(accountId, INFO_KEY); + + this.maxCharLength = instanceInfo?.configuration?.statuses?.max_characters ?? 500; + this.maxMediaCount = instanceInfo ? instanceInfo?.configuration?.statuses?.max_media_attachments : 4; + } + getScalingOptions(file: FileRecord, accountId: string): ScalingOptions { const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(accountId, INFO_KEY); if (instanceInfo?.configuration?.media_attachments) { @@ -106,208 +85,6 @@ export class Mastodon extends Megalodon { } } - async postFileSubmission( - cancellationToken: CancellationToken, - data: FilePostData, - accountData: MegalodonAccountData, - ): Promise { - const M = generator('mastodon', accountData.website, accountData.token); - - const files = [data.primary, ...data.additional]; - const uploadedMedias: { - id: string; - }[] = []; - for (const file of files) { - this.checkCancelled(cancellationToken); - uploadedMedias.push(await this.uploadMedia(accountData, file.file, data.options.altText)); - } - - const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(data.part.accountId, INFO_KEY); - const chunkCount = instanceInfo ? instanceInfo?.configuration?.statuses?.max_media_attachments : 4; - - const isSensitive = data.rating !== SubmissionRating.GENERAL; - const chunks = _.chunk(uploadedMedias, chunkCount); - let status = `${data.options.useTitle && data.title ? `${data.title}\n` : ''}${ - data.description - }`.substring(0, this.maxCharLength); - let lastId = ''; - let source = ''; - const replyToId = this.getPostIdFromUrl(data.options.replyToUrl); - - for (let i = 0; i < chunks.length; i++) { - this.checkCancelled(cancellationToken); - const statusOptions: any = { - sensitive: isSensitive, - visibility: data.options.visibility || 'public', - media_ids: chunks[i], - }; - - if (i !== 0) { - statusOptions.in_reply_to_id = lastId; - } else if (replyToId) { - statusOptions.in_reply_to_id = replyToId; - } - - if (data.options.spoilerText) { - statusOptions.spoiler_text = data.options.spoilerText; - } - - status = this.appendTags(this.formatTags(data.tags), status, this.maxCharLength); - - try { - const result = (await M.postStatus(status, statusOptions)).data as Entity.Status; - if (!source) source = result.url; - lastId = result.id; - } catch (err) { - return Promise.reject( - this.createPostResponse({ - message: err.message, - stack: err.stack, - additionalInfo: { chunkNumber: i }, - }), - ); - } - } - - this.checkCancelled(cancellationToken); - - return this.createPostResponse({ source }); - } - - async postNotificationSubmission( - cancellationToken: CancellationToken, - data: PostData, - accountData: MegalodonAccountData, - ): Promise { - const M = generator('mastodon', accountData.website, accountData.token); - const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(data.part.accountId, INFO_KEY); - const maxChars = instanceInfo?.configuration?.statuses?.max_characters ?? 500; - - const isSensitive = data.rating !== SubmissionRating.GENERAL; - const statusOptions: any = { - sensitive: isSensitive, - visibility: data.options.visibility || 'public', - }; - let status = `${data.options.useTitle && data.title ? `${data.title}\n` : ''}${ - data.description - }`; - if (data.options.spoilerText) { - statusOptions.spoiler_text = data.options.spoilerText; - } - status = this.appendTags(this.formatTags(data.tags), status, maxChars); - - const replyToId = this.getPostIdFromUrl(data.options.replyToUrl); - if (replyToId) { - statusOptions.in_reply_to_id = replyToId; - } - - this.checkCancelled(cancellationToken); - try { - const result = (await M.postStatus(status, statusOptions)).data as Entity.Status; - return this.createPostResponse({ source: result.url }); - } catch (error) { - return Promise.reject(this.createPostResponse(error)); - } - } - - formatTags(tags: string[]) { - return this.parseTags( - tags - .map(tag => tag.replace(/[^a-z0-9]/gi, ' ')) - .map(tag => - tag - .split(' ') - // .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(''), - ), - { spaceReplacer: '_' }, - ).map(tag => `#${tag}`); - } - - validateFileSubmission( - submission: FileSubmission, - submissionPart: SubmissionPart, - defaultPart: SubmissionPart, - ): ValidationParts { - const problems: string[] = []; - const warnings: string[] = []; - const isAutoscaling: boolean = submissionPart.data.autoScale; - - const description = this.defaultDescriptionParser( - FormContent.getDescription(defaultPart.data.description, submissionPart.data.description), - ); - - const instanceInfo: MastodonInstanceInfo = this.getAccountInfo( - submissionPart.accountId, - INFO_KEY, - ); - const maxChars = instanceInfo?.configuration?.statuses?.max_characters ?? 500; - - if (description.length > maxChars) { - warnings.push( - `Max description length allowed is ${maxChars} characters (for this instance).`, - ); - } - - const files = [ - submission.primary, - ...(submission.additional || []).filter( - f => !f.ignoredAccounts!.includes(submissionPart.accountId), - ), - ]; - - files.forEach(file => { - const { type, size, name, mimetype } = file; - if (!WebsiteValidator.supportsFileType(file, this.acceptsFiles)) { - problems.push(`Does not support file format: (${name}) ${mimetype}.`); - } - - const scalingOptions = this.getScalingOptions(file, submissionPart.accountId); - - if (scalingOptions && scalingOptions.maxSize < size) { - if ( - isAutoscaling && - type === FileSubmissionType.IMAGE && - ImageManipulator.isMimeType(mimetype) - ) { - warnings.push( - `${name} will be scaled down to ${FileSize.BytesToMB(scalingOptions.maxSize)}MB`, - ); - } else { - problems.push( - `This instance limits ${mimetype} to ${FileSize.BytesToMB(scalingOptions.maxSize)}MB`, - ); - } - } - - if ( - scalingOptions && - isAutoscaling && - type === FileSubmissionType.IMAGE && - scalingOptions.maxWidth && - scalingOptions.maxHeight && - (file.height > scalingOptions.maxHeight || file.width > scalingOptions.maxWidth) - ) { - warnings.push( - `${name} will be scaled down to a maximum size of ${scalingOptions.maxWidth}x${scalingOptions.maxHeight}, while maintaining aspect ratio`, - ); - } - }); - - if ( - (submissionPart.data.tags.value.length > 1 || defaultPart.data.tags.value.length > 1) && - submissionPart.data.visibility != 'public' - ) { - warnings.push( - `This post won't be listed under any hashtag as it is not public. Only public posts can be searched by hashtag.`, - ); - } - - this.validateReplyToUrl(problems, submissionPart.data.replyToUrl); - - return { problems, warnings }; - } - getPostIdFromUrl(url: string): string | null { // We expect this to a post URL like https://{instance}/@{user}/{id} or // https://:instance/deck/@{user}/{id}. We grab the id after the @ part. diff --git a/electron-app/src/server/websites/megalodon/megalodon.service.ts b/electron-app/src/server/websites/megalodon/megalodon.service.ts index ab9fe3e5..61e649ff 100644 --- a/electron-app/src/server/websites/megalodon/megalodon.service.ts +++ b/electron-app/src/server/websites/megalodon/megalodon.service.ts @@ -1,4 +1,3 @@ -import { Injectable } from '@nestjs/common'; import generator, { Entity, Response } from 'megalodon'; import { DefaultOptions, @@ -17,7 +16,6 @@ import { ScalingOptions } from '../interfaces/scaling-options.interface'; import UserAccountEntity from 'src/server//account/models/user-account.entity'; import { PlaintextParser } from 'src/server/description-parsing/plaintext/plaintext.parser'; import ImageManipulator from 'src/server/file-manipulation/manipulators/image.manipulator'; -import Http from 'src/server/http/http.util'; import { CancellationToken } from 'src/server/submission/post/cancellation/cancellation-token'; import { FilePostData, @@ -31,7 +29,6 @@ import WebsiteValidator from 'src/server/utils/website-validator.util'; import { LoginResponse } from '../interfaces/login-response.interface'; import { Website } from '../website.base'; import _ from 'lodash'; -import WaitUtil from 'src/server/utils/wait.util'; import { FileManagerService } from 'src/server/file-manager/file-manager.service'; import * as fs from 'fs'; import { tmpdir } from 'os'; @@ -45,7 +42,8 @@ export abstract class Megalodon extends Website { } readonly megalodonService = 'mastodon'; // Set this as appropriate in your constructor - readonly maxCharLength = 500; // Set this off the instance information! + maxCharLength = 500; // Set this off the instance information! + maxMediaCount = 4; readonly BASE_URL: string; readonly enableAdvertisement = false; @@ -83,37 +81,30 @@ export abstract class Megalodon extends Website { abstract getScalingOptions(file: FileRecord, accountId: string): ScalingOptions; - // TODO: Refactor + abstract getInstanceSettings(accountId: string); async postFileSubmission( cancellationToken: CancellationToken, data: FilePostData, accountData: MegalodonAccountData, ): Promise { - const M = generator('mastodon', accountData.website, accountData.token); + this.logger.log("Posting a file") + this.getInstanceSettings(data.part.accountId); + + const M = generator(this.megalodonService, accountData.website, accountData.token); const files = [data.primary, ...data.additional]; - const uploadedMedias: { - id: string; - }[] = []; + const uploadedMedias: string[] = []; for (const file of files) { this.checkCancelled(cancellationToken); uploadedMedias.push(await this.uploadMedia(accountData, file.file, data.options.altText)); } - const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(data.part.accountId, INFO_KEY); - const chunkCount = - instanceInfo?.configuration?.statuses?.max_media_attachments ?? - instanceInfo?.max_media_attachments ?? - (instanceInfo?.upload_limit ? 1000 : 4); - const maxChars = - instanceInfo?.configuration?.statuses?.max_characters ?? instanceInfo?.max_toot_chars ?? 500; - const isSensitive = data.rating !== SubmissionRating.GENERAL; - const chunks = _.chunk(uploadedMedias, chunkCount); + const chunks = _.chunk(uploadedMedias, this.maxMediaCount); let status = `${data.options.useTitle && data.title ? `${data.title}\n` : ''}${ data.description - }`.substring(0, maxChars); + }`.substring(0, this.maxCharLength); let lastId = ''; let source = ''; const replyToId = this.getPostIdFromUrl(data.options.replyToUrl); @@ -136,7 +127,7 @@ export abstract class Megalodon extends Website { statusOptions.spoiler_text = data.options.spoilerText; } - status = this.appendTags(this.formatTags(data.tags), status, maxChars); + status = this.appendTags(this.formatTags(data.tags), status, this.maxCharLength); try { const result = (await M.postStatus(status, statusOptions)).data as Entity.Status; @@ -158,16 +149,15 @@ export abstract class Megalodon extends Website { return this.createPostResponse({ source }); } - // TODO: Refactor - async postNotificationSubmission( cancellationToken: CancellationToken, data: PostData, accountData: MegalodonAccountData, ): Promise { - const M = generator('mastodon', accountData.website, accountData.token); - const instanceInfo: MastodonInstanceInfo = this.getAccountInfo(data.part.accountId, INFO_KEY); - const maxChars = instanceInfo?.configuration?.statuses?.max_characters ?? 500; + this.logger.log("Posting a notification") + this.getInstanceSettings(data.part.accountId); + + const M = generator(this.megalodonService, accountData.website, accountData.token); const isSensitive = data.rating !== SubmissionRating.GENERAL; const statusOptions: any = { @@ -180,7 +170,7 @@ export abstract class Megalodon extends Website { if (data.options.spoilerText) { statusOptions.spoiler_text = data.options.spoilerText; } - status = this.appendTags(this.formatTags(data.tags), status, maxChars); + status = this.appendTags(this.formatTags(data.tags), status, this.maxCharLength); const replyToId = this.getPostIdFromUrl(data.options.replyToUrl); if (replyToId) { @@ -196,27 +186,20 @@ export abstract class Megalodon extends Website { } } - // TODO: Make sure this has the strip preceeding space ? formatTags(tags: string[]) { return this.parseTags( - tags - .map(tag => tag.replace(/[^a-z0-9]/gi, ' ')) - .map(tag => - tag - .split(' ') - .join(''), - ), + tags.map(tag => tag.replace(/[^a-z0-9]/gi, ' ')).map(tag => tag.split(' ').join('')), { spaceReplacer: '_' }, ).map(tag => `#${tag}`); } - // TODO REFACTOR - validateFileSubmission( submission: FileSubmission, submissionPart: SubmissionPart, defaultPart: SubmissionPart, ): ValidationParts { + this.getInstanceSettings(defaultPart.accountId); + const problems: string[] = []; const warnings: string[] = []; const isAutoscaling: boolean = submissionPart.data.autoScale; @@ -225,15 +208,9 @@ export abstract class Megalodon extends Website { FormContent.getDescription(defaultPart.data.description, submissionPart.data.description), ); - const instanceInfo: MastodonInstanceInfo = this.getAccountInfo( - submissionPart.accountId, - INFO_KEY, - ); - const maxChars = instanceInfo?.configuration?.statuses?.max_characters ?? 500; - - if (description.length > maxChars) { + if (description.length > this.maxCharLength) { warnings.push( - `Max description length allowed is ${maxChars} characters (for this instance).`, + `Max description length allowed is ${this.maxCharLength} characters.`, ); } @@ -301,6 +278,8 @@ export abstract class Megalodon extends Website { submissionPart: SubmissionPart, defaultPart: SubmissionPart, ): ValidationParts { + this.getInstanceSettings(defaultPart.accountId); + const problems = []; const warnings = []; @@ -331,9 +310,10 @@ export abstract class Megalodon extends Website { data: MegalodonAccountData, file: PostFile, altText: string, - ): Promise<{ id: string }> { + ): Promise { + //): Promise<{ id: string }> {\ this.logger.log("Uploading media") - const M = generator(this.megalodonService, `https://${data.website}`, data.token); + const M = generator(this.megalodonService, data.website, data.token); // megalodon is odd, and doesnt seem to like the buffer being passed to the upload media call // So lets write the file out to a temp file, then pass that to createReadStream, then that to uploadMedia. @@ -361,7 +341,8 @@ export abstract class Megalodon extends Website { this.logger.log("Image uploaded"); - return { id: upload.data.id }; +// return { id: upload.data.id }; + return upload.data.id; } } diff --git a/electron-app/src/server/websites/pixelfed/pixelfed.service.ts b/electron-app/src/server/websites/pixelfed/pixelfed.service.ts index bbc01e49..096473dc 100644 --- a/electron-app/src/server/websites/pixelfed/pixelfed.service.ts +++ b/electron-app/src/server/websites/pixelfed/pixelfed.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@nestjs/common'; -import generator, { Entity, Response } from 'megalodon'; +import generator, { Entity } from 'megalodon'; import { DefaultOptions, FileRecord, @@ -8,7 +8,6 @@ import { MegalodonAccountData, PixelfedFileOptions, PostResponse, - Submission, SubmissionPart, SubmissionRating, } from 'postybirb-commons'; diff --git a/ui/src/websites/pleroma/PleromaLogin.tsx b/ui/src/websites/pleroma/PleromaLogin.tsx index a28da9ba..164a57c6 100644 --- a/ui/src/websites/pleroma/PleromaLogin.tsx +++ b/ui/src/websites/pleroma/PleromaLogin.tsx @@ -1,11 +1,9 @@ import { Button, Form, Input, message, Spin } from 'antd'; -import Axios from 'axios'; import React from 'react'; import ReactDOM from 'react-dom'; import { MegalodonAccountData } from 'postybirb-commons'; import LoginService from '../../services/login.service'; import { LoginDialogProps } from '../interfaces/website.interface'; - import generator, { OAuth } from 'megalodon' interface State extends MegalodonAccountData { @@ -22,8 +20,8 @@ export default class PleromaLogin extends React.Component Date: Wed, 18 Oct 2023 20:40:19 +0100 Subject: [PATCH 5/8] Pixelfed converted to the refactor - passing test --- .../websites/pixelfed/pixelfed.service.ts | 302 +----------------- 1 file changed, 11 insertions(+), 291 deletions(-) diff --git a/electron-app/src/server/websites/pixelfed/pixelfed.service.ts b/electron-app/src/server/websites/pixelfed/pixelfed.service.ts index 096473dc..08e93cb9 100644 --- a/electron-app/src/server/websites/pixelfed/pixelfed.service.ts +++ b/electron-app/src/server/websites/pixelfed/pixelfed.service.ts @@ -1,36 +1,12 @@ import { Injectable } from '@nestjs/common'; -import generator, { Entity } from 'megalodon'; import { - DefaultOptions, FileRecord, - FileSubmission, FileSubmissionType, - MegalodonAccountData, - PixelfedFileOptions, - PostResponse, - SubmissionPart, - SubmissionRating, } from 'postybirb-commons'; import { ScalingOptions } from '../interfaces/scaling-options.interface'; -import UserAccountEntity from 'src/server//account/models/user-account.entity'; -import { PlaintextParser } from 'src/server/description-parsing/plaintext/plaintext.parser'; -import ImageManipulator from 'src/server/file-manipulation/manipulators/image.manipulator'; -import Http from 'src/server/http/http.util'; -import { CancellationToken } from 'src/server/submission/post/cancellation/cancellation-token'; -import { - FilePostData, - PostFile, -} from 'src/server/submission/post/interfaces/file-post-data.interface'; -import { PostData } from 'src/server/submission/post/interfaces/post-data.interface'; -import { ValidationParts } from 'src/server/submission/validator/interfaces/validation-parts.interface'; import FileSize from 'src/server/utils/filesize.util'; -import FormContent from 'src/server/utils/form-content.util'; -import WebsiteValidator from 'src/server/utils/website-validator.util'; -import { LoginResponse } from '../interfaces/login-response.interface'; -import { Website } from '../website.base'; import _ from 'lodash'; -import WaitUtil from 'src/server/utils/wait.util'; -import { FileManagerService } from 'src/server/file-manager/file-manager.service'; +import { Megalodon } from '../megalodon/megalodon.service'; const INFO_KEY = 'INSTANCE INFO'; @@ -49,49 +25,18 @@ type PixelfedInstanceInfo = { }; @Injectable() -export class Pixelfed extends Website { - constructor(private readonly fileRepository: FileManagerService) { - super(); - } - readonly BASE_URL: string; +export class Pixelfed extends Megalodon { + readonly enableAdvertisement = false; - readonly acceptsAdditionalFiles = true; - readonly defaultDescriptionParser = PlaintextParser.parse; readonly acceptsFiles = ['png', 'jpeg', 'jpg', 'gif', 'swf', 'flv', 'mp4']; - async checkLoginStatus(data: UserAccountEntity): Promise { - const status: LoginResponse = { loggedIn: false, username: null }; - const accountData: MegalodonAccountData = data.data; - if (accountData && accountData.token) { - const refresh = await this.refreshToken(accountData); - if (refresh) { - status.loggedIn = true; - status.username = accountData.username; - this.getInstanceInfo(data._id, accountData); - } - } - return status; - } - - private async getInstanceInfo(profileId: string, data: MegalodonAccountData) { - const client = generator('mastodon', data.website, data.token); - const instance = await client.getInstance(); - - this.storeAccountInformation(profileId, INFO_KEY, instance.data); - } + readonly megalodonService = 'mastodon'; // At some point will change this when they get Pixelfed support natively - // TBH not entirely sure why this is a thing, but its in the old code so... :shrug: - private async refreshToken(data: MegalodonAccountData): Promise { - const M = this.getPixelfedInstance(data); - return true; - } + getInstanceSettings(accountId: string) { + const instanceInfo: PixelfedInstanceInfo = this.getAccountInfo(accountId, INFO_KEY); - private getPixelfedInstance(data: MegalodonAccountData): Entity.Instance { - const client = generator('mastodon', data.website, data.token); - client.getInstance().then(res => { - return res.data; - }); - return null; + this.maxCharLength = instanceInfo?.configuration?.statuses?.max_characters ?? 500; + this.maxMediaCount = instanceInfo ? instanceInfo?.configuration?.statuses?.max_media_attachments : 4; } getScalingOptions(file: FileRecord, accountId: string): ScalingOptions { @@ -112,233 +57,8 @@ export class Pixelfed extends Website { }; } - private async uploadMedia( - data: MegalodonAccountData, - file: PostFile, - altText: string, - ): Promise<{ id: string }> { - const upload = await Http.post<{ id: string; errors: any; url: string }>( - `${data.website}/api/v2/media`, - undefined, - { - type: 'multipart', - data: { - file, - description: altText, - }, - requestOptions: { json: true }, - headers: { - Accept: '*/*', - 'User-Agent': 'node-mastodon-client/PostyBirb', - Authorization: `Bearer ${data.token}`, - }, - }, - ); - - this.verifyResponse(upload, 'Verify upload'); - - // Processing - if (upload.response.statusCode === 202 || !upload.body.url) { - for (let i = 0; i < 10; i++) { - await WaitUtil.wait(4000); - const checkUpload = await Http.get<{ id: string; errors: any; url: string }>( - `${data.website}/api/v1/media/${upload.body.id}`, - undefined, - { - requestOptions: { json: true }, - headers: { - Accept: '*/*', - 'User-Agent': 'node-mastodon-client/PostyBirb', - Authorization: `Bearer ${data.token}`, - }, - }, - ); - - if (checkUpload.body.url) { - break; - } - } - } - - if (upload.body.errors) { - return Promise.reject( - this.createPostResponse({ additionalInfo: upload.body, message: upload.body.errors }), - ); - } - - return { id: upload.body.id }; - } - - async postFileSubmission( - cancellationToken: CancellationToken, - data: FilePostData, - accountData: MegalodonAccountData, - ): Promise { - const M = generator('mastodon', accountData.website, accountData.token); - - const files = [data.primary, ...data.additional]; - this.checkCancelled(cancellationToken); - const uploadedMedias: { - id: string; - }[] = []; - for (const file of files) { - uploadedMedias.push(await this.uploadMedia(accountData, file.file, data.options.altText)); - } - - const instanceInfo: PixelfedInstanceInfo = this.getAccountInfo(data.part.accountId, INFO_KEY); - const chunkCount = instanceInfo - ? instanceInfo?.configuration?.statuses?.max_media_attachments - : 4; - const maxChars = instanceInfo ? instanceInfo?.configuration?.statuses?.max_characters : 500; - - const isSensitive = data.rating !== SubmissionRating.GENERAL; - const { options } = data; - const chunks = _.chunk(uploadedMedias, chunkCount); - let lastId = undefined; - let statusOptions: any = { - sensitive: isSensitive, - visibility: options.visibility || 'public', - in_reply_to_id: lastId, - spoiler_text: '', - }; - let status = ''; - - for (let i = 0; i < chunks.length; i++) { - if (i === 0) { - status = `${options.useTitle && data.title ? `${data.title}\n` : ''}${ - data.description - }`.substring(0, maxChars); - statusOptions.media_ids = chunks[i].map(media => media.id); - } - - const tags = this.formatTags(data.tags); - - this.logger.debug(`Number of tags set ${tags.length}`); - - // Update the post content with the Tags if any are specified - for Pixelfed, we need to append - // these onto the post, *IF* there is character count available. - if (tags.length > 0) { - status += '\n\n'; - } - - tags.forEach(tag => { - let remain = maxChars - status.length; - let tagToInsert = tag; - if (remain > tagToInsert.length) { - status += ` ${tagToInsert}`; - } - // We don't exit the loop, so we can cram in every possible tag, even if there are short ones! - }); - - if (options.spoilerText) { - statusOptions.spoiler_text = options.spoilerText; - } - - this.checkCancelled(cancellationToken); - - await M.postStatus(status, statusOptions) - .then(result => { - lastId = result.data.id; - let res = result.data as Entity.Status; - return this.createPostResponse({ source: res.url }); - }) - .catch((err: Error) => { - return Promise.reject(this.createPostResponse({ message: err.message })); - }); - } - - return this.createPostResponse({}); - } - - formatTags(tags: string[]) { - return this.parseTags( - tags - .map(tag => tag.replace(/[^a-z0-9]/gi, ' ')) - .map(tag => - tag - .split(' ') - // .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(''), - ), - { spaceReplacer: '_' }, - ).map(tag => `#${tag}`); - } - - validateFileSubmission( - submission: FileSubmission, - submissionPart: SubmissionPart, - defaultPart: SubmissionPart, - ): ValidationParts { - const problems: string[] = []; - const warnings: string[] = []; - const isAutoscaling: boolean = submissionPart.data.autoScale; - - const description = this.defaultDescriptionParser( - FormContent.getDescription(defaultPart.data.description, submissionPart.data.description), - ); - - const instanceInfo: PixelfedInstanceInfo = this.getAccountInfo( - submissionPart.accountId, - INFO_KEY, - ); - const maxChars = instanceInfo ? instanceInfo?.configuration?.statuses?.max_characters : 500; - - if (description.length > maxChars) { - warnings.push( - `Max description length allowed is ${maxChars} characters (for this Pixelfed client).`, - ); - } - - const files = [ - submission.primary, - ...(submission.additional || []).filter( - f => !f.ignoredAccounts!.includes(submissionPart.accountId), - ), - ]; - - const maxImageSize = instanceInfo - ? instanceInfo?.configuration?.media_attachments?.image_size_limit - : FileSize.MBtoBytes(50); - - files.forEach(file => { - const { type, size, name, mimetype } = file; - if (!WebsiteValidator.supportsFileType(file, this.acceptsFiles)) { - problems.push(`Does not support file format: (${name}) ${mimetype}.`); - } - - if (maxImageSize < size) { - if ( - isAutoscaling && - type === FileSubmissionType.IMAGE && - ImageManipulator.isMimeType(mimetype) - ) { - warnings.push(`${name} will be scaled down to ${FileSize.BytesToMB(maxImageSize)}MB`); - } else { - problems.push(`Pixelfed limits ${mimetype} to ${FileSize.BytesToMB(maxImageSize)}MB`); - } - } - - // Check the image dimensions are not over 4000 x 4000 - this is the Pixelfed server max - if ( - isAutoscaling && - type === FileSubmissionType.IMAGE && - (file.height > 4000 || file.width > 4000) - ) { - warnings.push(`${name} will be scaled down to a maximum size of 4000x4000, while maintaining - aspect ratio`); - } - }); - - if ( - (submissionPart.data.tags.value.length > 1 || defaultPart.data.tags.value.length > 1) && - submissionPart.data.visibility != 'public' - ) { - warnings.push( - `This post won't be listed under any hashtag as it is not public. Only public posts - can be searched by hashtag.`, - ); - } - - return { problems, warnings }; + // https://{instance}/i/web/post/{id} + getPostIdFromUrl(url: string): string | null { + return url.slice(url.lastIndexOf('/') + 1); } } From 4469b16dbf7cf2592e89bf1163f57f927eeac355 Mon Sep 17 00:00:00 2001 From: Andy Neillans Date: Wed, 18 Oct 2023 21:04:04 +0100 Subject: [PATCH 6/8] WIP: Pleroma refactored; failed test --- .../websites/megalodon/megalodon.service.ts | 2 +- .../websites/pleroma/pleroma.service.ts | 385 +----------------- 2 files changed, 15 insertions(+), 372 deletions(-) diff --git a/electron-app/src/server/websites/megalodon/megalodon.service.ts b/electron-app/src/server/websites/megalodon/megalodon.service.ts index 61e649ff..406ff704 100644 --- a/electron-app/src/server/websites/megalodon/megalodon.service.ts +++ b/electron-app/src/server/websites/megalodon/megalodon.service.ts @@ -41,7 +41,7 @@ export abstract class Megalodon extends Website { super(); } - readonly megalodonService = 'mastodon'; // Set this as appropriate in your constructor + megalodonService: 'mastodon' | 'pleroma' | 'misskey' | 'friendica' = 'mastodon'; // Set this as appropriate in your constructor maxCharLength = 500; // Set this off the instance information! maxMediaCount = 4; diff --git a/electron-app/src/server/websites/pleroma/pleroma.service.ts b/electron-app/src/server/websites/pleroma/pleroma.service.ts index 6e076fcf..139e3742 100644 --- a/electron-app/src/server/websites/pleroma/pleroma.service.ts +++ b/electron-app/src/server/websites/pleroma/pleroma.service.ts @@ -35,32 +35,27 @@ import { Readable } from 'stream'; import * as fs from 'fs'; import { tmpdir } from 'os'; import * as path from 'path'; +import { Megalodon } from '../megalodon/megalodon.service'; const INFO_KEY = 'INSTANCE INFO'; type PleromaInstanceInfo = { + upload_limit?: number; // Pleroma, Akkoma + max_toot_chars?: number; // Pleroma, Akkoma + max_media_attachments?: number; //Pleroma configuration: { - statuses: { - max_characters: number; - max_media_attachments: number; - }; media_attachments: { - supported_mime_types: string[]; image_size_limit: number; video_size_limit: number; - }; - }; + }; + } }; @Injectable() -export class Pleroma extends Website { - constructor(private readonly fileRepository: FileManagerService) { - super(); - } - readonly BASE_URL: string; - readonly enableAdvertisement = false; +export class Pleroma extends Megalodon { + readonly acceptsAdditionalFiles = true; - readonly defaultDescriptionParser = PlaintextParser.parse; + megalodonService: 'mastodon' | 'pleroma' | 'misskey' | 'friendica' = 'pleroma'; readonly acceptsFiles = [ 'png', 'jpeg', @@ -75,38 +70,11 @@ export class Pleroma extends Website { 'mp3', ]; - async checkLoginStatus(data: UserAccountEntity): Promise { - const status: LoginResponse = { loggedIn: false, username: null }; - const accountData: MegalodonAccountData = data.data; - if (accountData && accountData.token) { - const refresh = await this.refreshToken(accountData); - if (refresh) { - status.loggedIn = true; - status.username = accountData.username; - this.getInstanceInfo(data._id, accountData); - } - } - return status; - } - - private async getInstanceInfo(profileId: string, data: MegalodonAccountData) { - const client = generator('pleroma', `https://${data.website}`, data.token); // token => tokenData.access_token - const instance = await client.getInstance(); - this.storeAccountInformation(profileId, INFO_KEY, instance.data); - } - - // TBH not entirely sure why this is a thing, but its in the old code so... :shrug: - private async refreshToken(data: MegalodonAccountData): Promise { - const M = this.getPleromaInstance(data); - return true; - } + getInstanceSettings(accountId: string) { + const instanceInfo: PleromaInstanceInfo = this.getAccountInfo(accountId, INFO_KEY); - private getPleromaInstance(data: MegalodonAccountData) : Entity.Instance { - const client = generator('pleroma', `https://${data.website}`, data.token); - client.getInstance().then((res) => { - return res.data; - }); - return null; + this.maxCharLength = instanceInfo?.max_toot_chars ?? 500; + this.maxMediaCount = instanceInfo?.max_media_attachments ?? 4; } getScalingOptions(file: FileRecord, accountId: string): ScalingOptions { @@ -127,332 +95,7 @@ export class Pleroma extends Website { }; } - public async uploadMedia( - data: MegalodonAccountData, - file: PostFile, - altText: string, - ): Promise<{ id: string }> { - this.logger.log("Uploading media") - const M = generator('pleroma', `https://${data.website}`, data.token); - - // megalodon is odd, and doesnt seem to like the buffer being passed to the upload media call - // So lets write the file out to a temp file, then pass that to createReadStream, then that to uploadMedia. - // That works .... \o/ - - const tempDir = tmpdir(); - - fs.writeFileSync(path.join(tempDir, file.options.filename), file.value); - - const upload = await M.uploadMedia(fs.createReadStream(path.join(tempDir, file.options.filename)), { description: altText }); - - this.logger.log(upload) - - fs.unlink(path.join(tempDir, file.options.filename), (err) => { - if (err) { - this.logger.error("Unable to remove the temp file", err.stack, err.message); - } - }); - - if (upload.status > 300) { - this.logger.log(upload); - return Promise.reject( - this.createPostResponse({ additionalInfo: upload.status, message: upload.statusText }), - ); - } - - this.logger.log("Pleroma image uploaded"); - - return { id: upload.data.id }; - } - - async postFileSubmission( - cancellationToken: CancellationToken, - data: FilePostData, - accountData: MegalodonAccountData, - ): Promise { - const M = generator('pleroma', `https://${accountData.website}`, accountData.token); - - const files = [data.primary, ...data.additional]; - - this.checkCancelled(cancellationToken); - - const uploadedMedias: { - id: string; - }[] = []; - for (const file of files) { - uploadedMedias.push(await this.uploadMedia(accountData, file.file, data.options.altText)); - } - - this.logger.log("All Media uploaded!") - - const instanceInfo: PleromaInstanceInfo = this.getAccountInfo(data.part.accountId, INFO_KEY); - const chunkCount = instanceInfo ? instanceInfo?.configuration?.statuses?.max_media_attachments : 4; - const maxChars = instanceInfo ? instanceInfo?.configuration?.statuses?.max_characters : 500; - - const isSensitive = data.rating !== SubmissionRating.GENERAL; - const { options } = data; - const chunks = _.chunk(uploadedMedias, chunkCount); - let lastId = undefined; - this.logger.log("Prepping post content") - for (let i = 0; i < chunks.length; i++) { - let statusOptions: any = { - status: '', - sensitive: isSensitive, - visibility: options.visibility || 'public', - spoiler_text: '' - }; - let status = undefined; - - if (i === 0) { - status = `${options.useTitle && data.title ? `${data.title}\n` : ''}${ - data.description - }`.substring(0, maxChars) - this.logger.log(`Initial Status: ${status}`) - - statusOptions = { - sensitive: isSensitive, - visibility: options.visibility || 'public', - media_ids: chunks[i].map((media) => media.id), - spoiler_text: "", - } - } else { - statusOptions = { - sensitive: isSensitive, - visibility: options.visibility || 'public', - media_ids: chunks[i].map((media) => media.id), - in_reply_to_id: lastId, - spoiler_text: "", - } - } - - const replyToId = this.getPostIdFromUrl(data.options.replyToUrl); - if (replyToId) { - statusOptions.in_reply_to_id = replyToId; - } - - const tags = this.formatTags(data.tags); - - // Update the post content with the Tags if any are specified - for Pleroma, we need to append - // these onto the post, *IF* there is character count available. - if (tags.length > 0) { - status += "\n\n"; - } - - tags.forEach(tag => { - let remain = maxChars - status.length; - let tagToInsert = tag; - if (remain > (tagToInsert.length)) { - status += ` ${tagToInsert}` - } - // We don't exit the loop, so we can cram in every possible tag, even if there are short ones! - }) - - if (options.spoilerText) { - statusOptions.spoiler_text = options.spoilerText; - } - - this.checkCancelled(cancellationToken); - - await M.postStatus(status, statusOptions).then((result) => { - lastId = result.data.id; - let res = result.data as Entity.Status; - return this.createPostResponse({ source: res.url }); - }).catch((err: Error) => { - return Promise.reject( - this.createPostResponse({ message: err.message }), - ); - }) - } - - return this.createPostResponse({}); - } - - async postNotificationSubmission( - cancellationToken: CancellationToken, - data: PostData, - accountData: MegalodonAccountData, - ): Promise { - const mInstance = this.getPleromaInstance(accountData); - const M = generator('pleroma', `https://${accountData.website}`, accountData.token); - - const maxChars = mInstance ? mInstance?.configuration?.statuses?.max_characters : 500; - - const isSensitive = data.rating !== SubmissionRating.GENERAL; - - const { options } = data; - let status = `${options.useTitle && data.title ? `${data.title}\n` : ''}${data.description}`; - const statusOptions: any = { - sensitive: isSensitive, - visibility: options.visibility || 'public', - spoiler_text: "" - }; - - const replyToId = this.getPostIdFromUrl(data.options.replyToUrl); - if (replyToId) { - statusOptions.in_reply_to_id = replyToId; - } - - const tags = this.formatTags(data.tags); - - // Update the post content with the Tags if any are specified - for Pleroma, we need to append - // these onto the post, *IF* there is character count available. - if (tags.length > 0) { - status += "\n\n"; - } - - tags.forEach(tag => { - let remain = maxChars - status.length; - let tagToInsert = tag; - if (remain > (tagToInsert.length)) { - status += ` ${tagToInsert}` - } - // We don't exit the loop, so we can cram in every possible tag, even if there are short ones! - }) - - if (options.spoilerText) { - statusOptions.spoiler_text = options.spoilerText; - } - - this.checkCancelled(cancellationToken); - - await M.postStatus(status, statusOptions).then((result) => { - let res = result.data as Entity.Status; - return this.createPostResponse({ source: res.url }); - }).catch((err: Error) => { - return Promise.reject( - this.createPostResponse({ message: err.message }), - ); - }) - return this.createPostResponse({}); - } - - formatTags(tags: string[]) { - return this.parseTags( - tags - .map((tag) => tag.replace(/[^a-z0-9]/gi, ' ')) - .map((tag) => - tag - .split(' ') - // .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(''), - ), - { spaceReplacer: '_' }, - ).map((tag) => `#${tag}`); - } - - validateFileSubmission( - submission: FileSubmission, - submissionPart: SubmissionPart, - defaultPart: SubmissionPart, - ): ValidationParts { - this.logger.log(submission.primary.location) - - const problems: string[] = []; - const warnings: string[] = []; - const isAutoscaling: boolean = submissionPart.data.autoScale; - - const description = this.defaultDescriptionParser( - FormContent.getDescription(defaultPart.data.description, submissionPart.data.description), - ); - - const instanceInfo: PleromaInstanceInfo = this.getAccountInfo( - submissionPart.accountId, - INFO_KEY, - ); - const maxChars = instanceInfo ? instanceInfo?.configuration?.statuses?.max_characters : 500; - - if (description.length > maxChars) { - warnings.push( - `Max description length allowed is ${maxChars} characters (for this Pleroma client).`, - ); - } - - const files = [ - submission.primary, - ...(submission.additional || []).filter( - (f) => !f.ignoredAccounts!.includes(submissionPart.accountId), - ), - ]; - - const maxImageSize = instanceInfo - ? instanceInfo?.configuration?.media_attachments?.image_size_limit - : FileSize.MBtoBytes(50); - - files.forEach((file) => { - const { type, size, name, mimetype } = file; - if (!WebsiteValidator.supportsFileType(file, this.acceptsFiles)) { - problems.push(`Does not support file format: (${name}) ${mimetype}.`); - } - - if (maxImageSize < size) { - if ( - isAutoscaling && - type === FileSubmissionType.IMAGE && - ImageManipulator.isMimeType(mimetype) - ) { - warnings.push(`${name} will be scaled down to ${FileSize.BytesToMB(maxImageSize)}MB`); - } else { - problems.push(`Pleroma limits ${mimetype} to ${FileSize.BytesToMB(maxImageSize)}MB`); - } - } - - // Check the image dimensions are not over 4000 x 4000 - this is the Pleroma server max - if ( - isAutoscaling && - type === FileSubmissionType.IMAGE && - (file.height > 4000 || file.width > 4000)) { - warnings.push(`${name} will be scaled down to a maximum size of 4000x4000, while maintaining - aspect ratio`); - } - }); - - if ((submissionPart.data.tags.value.length > 1 || defaultPart.data.tags.value.length > 1) && - submissionPart.data.visibility != "public") { - warnings.push( - `This post won't be listed under any hashtag as it is not public. Only public posts - can be searched by hashtag.`, - ); - } - - this.validateReplyToUrl(problems, submissionPart.data.replyToUrl); - - return { problems, warnings }; - } - - validateNotificationSubmission( - submission: Submission, - submissionPart: SubmissionPart, - defaultPart: SubmissionPart, - ): ValidationParts { - const warnings = []; - const problems = []; - const description = this.defaultDescriptionParser( - FormContent.getDescription(defaultPart.data.description, submissionPart.data.description), - ); - - const instanceInfo: PleromaInstanceInfo = this.getAccountInfo( - submissionPart.accountId, - INFO_KEY, - ); - const maxChars = instanceInfo ? instanceInfo?.configuration?.statuses?.max_characters : 500; - if (description.length > maxChars) { - warnings.push( - `Max description length allowed is ${maxChars} characters (for this Pleroma client).`, - ); - } - - this.validateReplyToUrl(problems, submissionPart.data.replyToUrl); - - return { problems, warnings }; - } - - private validateReplyToUrl(problems: string[], url?: string): void { - if(url?.trim() && !this.getPostIdFromUrl(url)) { - problems.push("Invalid post URL to reply to."); - } - } - - private getPostIdFromUrl(url: string): string | null { + getPostIdFromUrl(url: string): string | null { if (url) { const match = url.slice(url.lastIndexOf('/')+1) return match ? match[1] : null; From 89c904b8f31b063459f130512ecab5c47cbf35ad Mon Sep 17 00:00:00 2001 From: Andy Neillans Date: Thu, 19 Oct 2023 15:26:25 +0100 Subject: [PATCH 7/8] Pleroma moved over - tests passing --- .../websites/megalodon/megalodon.service.ts | 3 +- .../websites/pleroma/pleroma.service.ts | 30 +------------------ ui/src/websites/pleroma/PleromaLogin.tsx | 23 ++++++++------ 3 files changed, 17 insertions(+), 39 deletions(-) diff --git a/electron-app/src/server/websites/megalodon/megalodon.service.ts b/electron-app/src/server/websites/megalodon/megalodon.service.ts index 406ff704..335483b7 100644 --- a/electron-app/src/server/websites/megalodon/megalodon.service.ts +++ b/electron-app/src/server/websites/megalodon/megalodon.service.ts @@ -1,4 +1,4 @@ -import generator, { Entity, Response } from 'megalodon'; +import generator, { Entity } from 'megalodon'; import { DefaultOptions, FileRecord, @@ -63,6 +63,7 @@ export abstract class Megalodon extends Website { async checkLoginStatus(data: UserAccountEntity): Promise { const status: LoginResponse = { loggedIn: false, username: null }; const accountData: MegalodonAccountData = data.data; + this.logger.debug(`Login check: ${data._id} = ${accountData.website}`); if (accountData && accountData.token) { await this.getAndStoreInstanceInfo(data._id, accountData); diff --git a/electron-app/src/server/websites/pleroma/pleroma.service.ts b/electron-app/src/server/websites/pleroma/pleroma.service.ts index 139e3742..7e1106f0 100644 --- a/electron-app/src/server/websites/pleroma/pleroma.service.ts +++ b/electron-app/src/server/websites/pleroma/pleroma.service.ts @@ -1,40 +1,11 @@ import { Injectable } from '@nestjs/common'; -import generator, { Entity, Response } from 'megalodon' import { - DefaultOptions, FileRecord, - FileSubmission, FileSubmissionType, - MegalodonAccountData, - PleromaFileOptions, - PleromaNotificationOptions, - PostResponse, - Submission, - SubmissionPart, - SubmissionRating, } from 'postybirb-commons'; import { ScalingOptions } from '../interfaces/scaling-options.interface'; -import UserAccountEntity from 'src/server//account/models/user-account.entity'; -import { PlaintextParser } from 'src/server/description-parsing/plaintext/plaintext.parser'; -import ImageManipulator from 'src/server/file-manipulation/manipulators/image.manipulator'; -import { CancellationToken } from 'src/server/submission/post/cancellation/cancellation-token'; -import { - FilePostData, - PostFile, -} from 'src/server/submission/post/interfaces/file-post-data.interface'; -import { PostData } from 'src/server/submission/post/interfaces/post-data.interface'; -import { ValidationParts } from 'src/server/submission/validator/interfaces/validation-parts.interface'; import FileSize from 'src/server/utils/filesize.util'; -import FormContent from 'src/server/utils/form-content.util'; -import WebsiteValidator from 'src/server/utils/website-validator.util'; -import { LoginResponse } from '../interfaces/login-response.interface'; -import { Website } from '../website.base'; import _ from 'lodash'; -import { FileManagerService } from 'src/server/file-manager/file-manager.service'; -import { Readable } from 'stream'; -import * as fs from 'fs'; -import { tmpdir } from 'os'; -import * as path from 'path'; import { Megalodon } from '../megalodon/megalodon.service'; const INFO_KEY = 'INSTANCE INFO'; @@ -71,6 +42,7 @@ export class Pleroma extends Megalodon { ]; getInstanceSettings(accountId: string) { + console.log(this.getAccountInfo(accountId, INFO_KEY)); const instanceInfo: PleromaInstanceInfo = this.getAccountInfo(accountId, INFO_KEY); this.maxCharLength = instanceInfo?.max_toot_chars ?? 500; diff --git a/ui/src/websites/pleroma/PleromaLogin.tsx b/ui/src/websites/pleroma/PleromaLogin.tsx index 164a57c6..ee923c3a 100644 --- a/ui/src/websites/pleroma/PleromaLogin.tsx +++ b/ui/src/websites/pleroma/PleromaLogin.tsx @@ -48,11 +48,15 @@ export default class PleromaLogin extends React.Component { // Get the username so we have complete data. - const usernameClient = generator('pleroma', `https://${this.state.website}`, value.accessToken); + const usernameClient = generator('pleroma', website, value.accessToken); usernameClient.verifyAccountCredentials().then((res)=>{ - let website = `https://${this.state.website}`; this.state.username = res.data.username; this.state.token = value.access_token; - LoginService.setAccountData(this.props.account._id, this.state ).then( + + LoginService.setAccountData(this.props.account._id, { ...this.state, website } ).then( () => { - message.success(`${this.state.website} authenticated.`); + message.success(`${website} authenticated.`); }); }); }) .catch((err: Error) => { - message.error(`Failed to authenticate ${this.state.website}.`); + message.error(`Failed to authenticate ${website}.`); }) } @@ -103,7 +108,7 @@ export default class PleromaLogin extends React.Component { const website = target.value.replace(/(https:\/\/|http:\/\/)/, ''); this.view.loadURL(this.getAuthURL(website)); - this.setState({ website }); + this.setState({ website: website }); }} /> From 39a674602510a51f337b45eef43253506a6da53e Mon Sep 17 00:00:00 2001 From: Andy Neillans Date: Fri, 20 Oct 2023 06:42:53 +0100 Subject: [PATCH 8/8] Pixelfed was missing a null check --- .../src/server/websites/pixelfed/pixelfed.service.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/electron-app/src/server/websites/pixelfed/pixelfed.service.ts b/electron-app/src/server/websites/pixelfed/pixelfed.service.ts index 08e93cb9..e0968637 100644 --- a/electron-app/src/server/websites/pixelfed/pixelfed.service.ts +++ b/electron-app/src/server/websites/pixelfed/pixelfed.service.ts @@ -59,6 +59,10 @@ export class Pixelfed extends Megalodon { // https://{instance}/i/web/post/{id} getPostIdFromUrl(url: string): string | null { - return url.slice(url.lastIndexOf('/') + 1); + if (url && url.lastIndexOf('/') > -1) { + return url.slice(url.lastIndexOf('/') + 1); + } else { + return null; + } } }