refactor: Telegram/tglib for Hono

lib/config.ts

@@ -243,6 +243,10 @@ export type Config = {
     };
     telegram: {
         token?: string;
+        session?: string;
+        apiId?: string;
+        apiHash?: string;
+        maxConcurrentDownloads?: string;
     };
     tophub: {
         cookie?: string;

lib/middleware/cache.ts

@@ -38,6 +38,10 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
     // Doesn't hit the cache? We need to let others know!
     await cacheModule.globalCache.set(controlKey, '1', config.cache.requestTimeout);
 
+    // let routers control cache
+    ctx.set('cacheKey', key);
+    ctx.set('cacheControlKey', controlKey);
+
     try {
         await next();
     } catch (error) {

lib/middleware/template.ts

@@ -10,6 +10,7 @@ const __dirname = getCurrentPath(import.meta.url);
 
 const middleware: MiddlewareHandler = async (ctx, next) => {
     await next();
+    if (ctx.get('no-template')) { return; }
 
     const data: Data = ctx.get('data');
     const outputType = ctx.req.query('format') || 'rss';

lib/routes/telegram/tglib/channel-media.ts

@@ -0,0 +1,224 @@
+import { Context } from 'hono';
+import { stream } from 'hono/streaming';
+import { Api, TelegramClient } from 'telegram';
+import { IterDownloadFunction } from 'telegram/client/downloads';
+import { getAppropriatedPartSize } from 'telegram/Utils';
+import { config } from '@/config';
+import cacheModule from '@/utils/cache/index';
+import { getClient, getDocument, getFilename } from './client';
+import { returnBigInt as bigInt } from 'telegram/Helpers';
+
+/**
+ * https://core.telegram.org/api/files#stripped-thumbnails
+ * @param bytes Buffer
+ * @returns Buffer jpeg
+ */
+function ExpandInlineBytes(bytes: Buffer) {
+    if (bytes.length < 3 || bytes[0] !== 0x1) {
+        throw new Error('cannot inflate a stripped jpeg');
+    }
+    const header = Buffer.from([
+        0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0xff, 0xdb, 0x00, 0x43, 0x00, 0x28, 0x1c, 0x1e, 0x23, 0x1e, 0x19, 0x28, 0x23, 0x21, 0x23, 0x2d, 0x2b,
+        0x28, 0x30, 0x3c, 0x64, 0x41, 0x3c, 0x37, 0x37, 0x3c, 0x7b, 0x58, 0x5d, 0x49, 0x64, 0x91, 0x80, 0x99, 0x96, 0x8f, 0x80, 0x8c, 0x8a, 0xa0, 0xb4, 0xe6, 0xc3, 0xa0, 0xaa, 0xda, 0xad, 0x8a, 0x8c, 0xc8, 0xff, 0xcb, 0xda, 0xee,
+        0xf5, 0xff, 0xff, 0xff, 0x9b, 0xc1, 0xff, 0xff, 0xff, 0xfa, 0xff, 0xe6, 0xfd, 0xff, 0xf8, 0xff, 0xdb, 0x00, 0x43, 0x01, 0x2b, 0x2d, 0x2d, 0x3c, 0x35, 0x3c, 0x76, 0x41, 0x41, 0x76, 0xf8, 0xa5, 0x8c, 0xa5, 0xf8, 0xf8, 0xf8,
+        0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8,
+        0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xff, 0xc0, 0x00, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, 0x22, 0x00, 0x02, 0x11, 0x01, 0x03, 0x11, 0x01, 0xff, 0xc4, 0x00, 0x1f, 0x00, 0x00, 0x01, 0x05,
+        0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xc4, 0x00, 0xb5, 0x10, 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04,
+        0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7d, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1,
+        0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a,
+        0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96,
+        0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+        0xd8, 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xff, 0xc4, 0x00, 0x1f, 0x01, 0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+        0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xc4, 0x00, 0xb5, 0x11, 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04, 0x07, 0x05, 0x04, 0x04, 0x00,
+        0x01, 0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, 0x15, 0x62,
+        0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57,
+        0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a,
+        0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe2,
+        0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xff, 0xda, 0x00, 0x0c, 0x03, 0x01, 0x00, 0x02, 0x11, 0x03, 0x11, 0x00, 0x3f, 0x00,
+    ]);
+    const footer = Buffer.from([0xff, 0xd9]);
+    const real = Buffer.alloc(header.length + bytes.length + footer.length);
+    header.copy(real);
+    bytes.copy(real, header.length, 3);
+    bytes.copy(real, 164, 1, 2);
+    bytes.copy(real, 166, 2, 3);
+    footer.copy(real, header.length + bytes.length, 0);
+    return real;
+}
+
+function sortThumb(thumb: Api.TypePhotoSize) {
+    if (thumb instanceof Api.PhotoStrippedSize) {
+        return thumb.bytes.length;
+    }
+    if (thumb instanceof Api.PhotoCachedSize) {
+        return thumb.bytes.length;
+    }
+    if (thumb instanceof Api.PhotoSize) {
+        return thumb.size;
+    }
+    if (thumb instanceof Api.PhotoSizeProgressive) {
+        return Math.max(...thumb.sizes);
+    }
+    return 0;
+}
+
+function chooseLargestThumb(thumbs:  Api.TypePhotoSize[]) {
+    thumbs = [...thumbs].sort((a, b) => sortThumb(a) - sortThumb(b));
+    return thumbs.pop();
+}
+
+export async function* streamThumbnail(client: TelegramClient, doc: Api.Document) {
+    if (doc.thumbs?.length ?? 0 > 0) {
+        const size = chooseLargestThumb(doc.thumbs!);
+        if (size instanceof Api.PhotoCachedSize || size instanceof Api.PhotoStrippedSize) {
+            yield ExpandInlineBytes(size.bytes);
+        } else {
+            yield* streamDocument(client, doc, size && 'type' in size ? size.type : '');
+        }
+        return;
+    }
+    throw new Error('no thumbnails available');
+}
+
+export async function decodeMedia(client: TelegramClient, channelName: string, x: string, retry = false) {
+    const [channel, msg] = x.split('_');
+
+    try {
+        const msgs = await client.getMessages(channel, {
+            ids: [Number(msg)],
+        });
+        return msgs[0]?.media;
+    } catch (error) {
+        if (!retry) {
+            // channel likely not seen before, we need to resolve ID and retry
+            await client.getInputEntity(channelName);
+            return decodeMedia(client, channelName, x, true);
+        }
+        throw error;
+    }
+}
+
+export async function* streamDocument(client: TelegramClient, obj: Api.Document, thumbSize = '', offset?: bigInt.BigInteger, limit?: bigInt.BigInteger) {
+    const chunkSize = (obj.size ? getAppropriatedPartSize(obj.size) : 64) * 1024;
+    const iterFileParams: IterDownloadFunction = {
+        file: new Api.InputDocumentFileLocation({
+            id: obj.id,
+            accessHash: obj.accessHash,
+            fileReference: obj.fileReference,
+            thumbSize,
+        }),
+        chunkSize,
+        requestSize: 512 * 1024, // MAX_CHUNK_SIZE
+        dcId: obj.dcId,
+        offset: undefined,
+        limit: undefined
+    };
+    if (offset) {
+        iterFileParams.offset = offset;
+    }
+    if (limit) {
+        iterFileParams.limit = limit.valueOf();
+    }
+    const stream = client.iterDownload(iterFileParams);
+    yield* stream;
+    await stream.close();
+}
+
+function parseRange(range: string, length: bigInt.BigInteger) {
+    if (!range) {
+        return [];
+    }
+    const [typ, segstr] = range.split('=');
+    if (typ !== 'bytes') {
+        throw new Error(`unsupported range: ${typ}`);
+    }
+    const segs = segstr.split(',').map((s) => s.trim());
+    const parsedSegs: bigInt.BigInteger[][] = [];
+    for (const seg of segs) {
+        const range = seg
+            .split('-', 2)
+            .filter((v) => !!v)
+            .map(bigInt);
+        if (range.length < 2) {
+            if (seg.startsWith('-')) {
+                range.unshift(bigInt(0));
+            } else {
+                range.push(length.subtract(bigInt(1)));
+            }
+        }
+        parsedSegs.push(range);
+    }
+    return parsedSegs;
+}
+
+async function configureMiddlewares(ctx: Context) {
+    // media is too heavy to cache in memory or redis, and lock-up is not needed
+    await cacheModule.set(ctx.get('cacheControlKey'), '0', config.cache.requestTimeout);
+    ctx.req.raw.headers.delete('Accept-Encoding'); // avoid hono compress() middleware detecting Accept-Encoding on req
+    ctx.set('no-template', true); // skip RSSHub template middleware
+}
+
+function streamResponse(c: Context, bodyIter: AsyncGenerator<Buffer>) {
+    return stream(c, async (stream) => {
+        let aborted = false;
+        stream.onAbort(() => {
+            // console.log(`stream aborted`);
+            aborted = true;
+        });
+        for await (const chunk of bodyIter) {
+            if (aborted) { break; }
+            // console.log(`writing ${chunk.length / 1024}kB`);
+            await stream.write(chunk);
+        }
+    });
+}
+
+export default async function handler(ctx: Context) {
+    await configureMiddlewares(ctx);
+    const client = await getClient();
+    const media = await decodeMedia(client, ctx.req.param('username'), ctx.req.param('media'));
+    if (!media) {
+        return ctx.text('Unknown media', 404);
+    }
+
+    if (media instanceof Api.MessageMediaPhoto) {
+        const buf = await client.downloadMedia(media);
+        return new Response(buf, {headers: {'Content-Type': 'image/jpeg'}});
+    }
+
+    const doc = getDocument(media);
+    if (doc) {
+        if ('thumb' in ctx.req.query()) {
+            ctx.header('Content-Type', 'image/jpeg');
+            return streamResponse(ctx, streamThumbnail(client, doc));
+        }
+        ctx.header('Content-Type', doc.mimeType);
+        ctx.header('Accept-Ranges', 'bytes');
+        ctx.header('Content-Security-Policy', "default-src 'self'; script-src 'none'");
+
+        const rangeHeader = ctx.req.header('Range') ?? '';
+        const range = parseRange(rangeHeader, doc.size);
+        if (range.length > 1) {
+            return ctx.text('Not Satisfiable', 416);
+        }
+
+        if (range.length === 0) {
+            ctx.header('Content-Length', doc.size.toString());
+            if (!doc.mimeType.startsWith('video/') &&
+                !doc.mimeType.startsWith('audio/') &&
+                !doc.mimeType.startsWith('image/')) {
+                ctx.header('Content-Disposition', `attachment; filename="${encodeURIComponent(getFilename(media))}"`);
+            }
+            return streamResponse(ctx, streamDocument(client, doc));
+        } else {
+            const [offset, limit] = range[0];
+            // console.log(`Range: ${rangeHeader}`);
+            ctx.status(206); // partial content
+            ctx.header('Content-Length', (limit.subtract(offset).add(1)).toString());
+            ctx.header('Content-Range', `bytes ${offset}-${limit}/${doc.size}`);
+            return streamResponse(ctx, streamDocument(client, doc, '', offset, limit));
+        }
+    }
+
+    return ctx.text(media.className, 415);
+}