Skip to content

Commit

Permalink
Merge branch 'master' into mojo
Browse files Browse the repository at this point in the history
  • Loading branch information
MikeZeDev committed Nov 26, 2024
2 parents 2da4fb6 + 208da85 commit 25c9f8c
Show file tree
Hide file tree
Showing 81 changed files with 710 additions and 418 deletions.
8 changes: 7 additions & 1 deletion .github/workflows/pull-request-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,10 @@ jobs:
if: steps.packages.outcome == 'success' && (success() || failure())
env:
DISPLAY: ':99'
run: npm run test:websites -- MangaDex
run: npm run test:websites -- MangaDex
- name: Upload Test Results
if: failure()
uses: actions/upload-artifact@v4
with:
name: Test Results (${{ matrix.os }})
path: ./**/screenshot_*.png
3 changes: 2 additions & 1 deletion app/electron/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
"module": "ESNext",
"moduleResolution": "bundler",
"skipLibCheck": true,
"noEmit": true
"noEmit": true,
"types": [ "chrome" ]
},
"include": [
"src/**/*.ts",
Expand Down
3 changes: 2 additions & 1 deletion app/nw/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
"module": "ESNext",
"moduleResolution": "bundler",
"skipLibCheck": true,
"noEmit": true
"noEmit": true,
"types": [ "chrome", "nw.js" ]
},
"include": [
"src/**/*.ts",
Expand Down
9 changes: 9 additions & 0 deletions test/PuppeteerFixture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ export class PuppeteerFixture {
return PuppeteerFixture.#page;
}

public async Screenshot(page: puppeteer.Page) {
await page.screenshot({
type: 'png',
fullPage: true,
captureBeyondViewport: true,
path: `./screenshot_${Date.now().toString(36)}.png`,
});
}

protected async OpenPage(url: string, ...evasions: Evasion[]): Promise<puppeteer.Page> {
const page = await (await this.GetBrowser()).newPage();
await Promise.all(evasions.map(setupEvasion => setupEvasion(page)));
Expand Down
10 changes: 10 additions & 0 deletions web/eslint.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,16 @@ export default tseslint.config({
'no-multi-spaces': 'error',
'no-throw-literal': 'error',
'tsdoc/syntax': 'warn',
/**
* Use restricted properties as workaround to address a flaw in the typescript system of global declarations.
* Globally declared modules (such as @types/node::Buffer) should be declared for e.g., tests but must not be declared for the web-application (browsers do not have `Buffer`).
* Unfortunately, once included these are available everywhere in the workspace without exception.
* See also: https://github.com/microsoft/TypeScript/issues/50424
*/
'no-restricted-properties': [ 'error', {
"object": "Buffer",
"property": "from"
} ],
'@typescript-eslint/naming-convention': [ 'error', // See: https://typescript-eslint.io/rules/naming-convention/#options
/*
{
Expand Down
22 changes: 8 additions & 14 deletions web/src/AntiBotDetectionBypass_e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ class TestFixture extends PuppeteerFixture {
public async SetupPage(url: string, ...evasions: Evasion[]): Promise<TestFixture> {
await this.ClosePage();
this.page = await this.OpenPage(url, ...evasions);
console.log('WebDriver:', await this.page.evaluate(() => window.navigator.webdriver));
console.log('User-Agent:', await this.page.evaluate(() => window.navigator.userAgent));
console.log('Console:', await this.page.evaluate(() => console.debug.toString()));
//console.log('WebDriver:', await this.page.evaluate(() => window.navigator.webdriver));
//console.log('User-Agent:', await this.page.evaluate(() => window.navigator.userAgent));
//console.log('Console:', await this.page.evaluate(() => console.debug.toString()));
return this;
}

Expand All @@ -29,17 +29,11 @@ class TestFixture extends PuppeteerFixture {

public async AssertElementText(selector: string, expected: string): Promise<void> {
try {
await this.page.waitForSelector(selector, { timeout: 5000 });
await this.page.waitForSelector(selector, { timeout: 7500 });
const actual = await this.page.$eval(selector, (element: HTMLElement) => element.innerText);
expect(actual).toBe(expected);
} catch(error) {
await this.page.screenshot({
type: 'png',
fullPage: true,
captureBeyondViewport: true,
path: `./screenshot_${Date.now().toString(16)}.png`
});
await new Promise(resolve => setTimeout(resolve, 5_000));
await this.Screenshot(this.page);
throw error;
}
}
Expand Down Expand Up @@ -68,7 +62,7 @@ describe('CloudFlare', { timeout: 15_000 }, () => {
}
});

it.skip('Should bypass Turnstile Non-Interactive Challenge', async () => {
it('Should bypass Turnstile Non-Interactive Challenge', { todo: true }, async () => {
const fixture = await new TestFixture().SetupPage('https://cloudflare.bot-check.ovh/turnstile-automatic');
try {
await fixture.AssertElementText('#hash', 'A9B6FA3DD8842CD8E2D6070784D92434');
Expand All @@ -77,7 +71,7 @@ describe('CloudFlare', { timeout: 15_000 }, () => {
}
});

it.skip('Should bypass Turnstile Invisible Challenge', async () => {
it('Should bypass Turnstile Invisible Challenge', { todo: true }, async () => {
const fixture = await new TestFixture().SetupPage('https://cloudflare.bot-check.ovh/turnstile-invisible');
try {
await fixture.AssertElementText('#hash', 'A9B6FA3DD8842CD8E2D6070784D92434');
Expand All @@ -89,7 +83,7 @@ describe('CloudFlare', { timeout: 15_000 }, () => {

describe('Vercel', { timeout: 15_000 }, () => {

it.skip('Should bypass Attack Challenge Mode', async () => {
it('Should bypass Attack Challenge Mode', { todo: true }, async () => {
const fixture = await new TestFixture().SetupPage('https://vercel.bot-check.ovh', EvadeChromeDevToolProtocolDetection);
try {
await fixture.AssertElementText('#hash', 'FDF049D4B2312945BB7AA2158F041278');
Expand Down
19 changes: 19 additions & 0 deletions web/src/engine/BufferEncoder.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
export function FromHexString(hexString: string): Uint8Array {
return Uint8Array.from(hexString.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)));
}

export function GetBytesUTF8(text: string): Uint8Array {
return new TextEncoder().encode(text);
};

export function Uint8ToHexString(array: Uint8Array): string {
return array.reduce((result, x) => result + x.toString(16).padStart(2, '0'), '');
}

export function BufferToHexString(buffer: ArrayBuffer): string {
return Uint8ToHexString(new Uint8Array(buffer));
}

export function GetBytesB64(b64string: string): Uint8Array {
return Uint8Array.from(window.atob(b64string), c => c.charCodeAt(0));
}
2 changes: 2 additions & 0 deletions web/src/engine/SettingsManager_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@ import { type StorageController, Store } from './StorageController';
import { Key } from './SettingsGlobal';

window.atob = function(encoded: string): string {
/* eslint-disable-next-line no-restricted-properties */ //=> This is supposed to run in NodeJS runtime where Buffer is available
return Buffer.from(encoded, 'base64').toString('utf-8');
};

window.btoa = function(decoded: string): string {
/* eslint-disable-next-line no-restricted-properties */ //=> This is supposed to run in NodeJS runtime where Buffer is available
return Buffer.from(decoded, 'utf-8').toString('base64');
};

Expand Down
5 changes: 3 additions & 2 deletions web/src/engine/taskpool/TaskPool_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,9 @@ describe('TaskPool', () => {
expect(actual.map(r => r.Value)).toEqual([ '③' ]);
});

(process.platform === 'win32' ? it.skip : it)('Should utilize workers for concurrent processing', {
retry: process.platform === 'win32' ? 50 : 5
it('Should utilize workers for concurrent processing', {
todo: process.platform === 'win32',
retry: process.platform === 'win32' ? 50 : 5,
}, async () => {
const testee = new TaskPool(3, Unlimited);
const start = performance.now();
Expand Down
26 changes: 26 additions & 0 deletions web/src/engine/websites/AllHentai.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { Tags } from '../Tags';
import icon from './AllHentai.webp';
import { DecoratableMangaScraper } from '../providers/MangaPlugin';
import * as Common from './decorators/Common';
import * as Grouple from './decorators/Grouple';
import { FetchWindowScript } from '../platform/FetchProvider';

@Common.MangaCSS(/^{origin}\/[^/]+$/, Grouple.queryMangaTitle)
@Common.MangasMultiPageCSS(Grouple.pathMangas, Grouple.queryMangas, 0, Grouple.pageMangaOffset, 1000, Common.AnchorInfoExtractor(true))
@Common.ChaptersSinglePageJS(Grouple.chapterScript, 500)
@Grouple.PagesSinglePageJS()
@Grouple.ImageAjaxWithMirrors()
export default class extends DecoratableMangaScraper {
public constructor() {
super('allhentai', `AllHentai`, 'https://20.allhen.online', Tags.Language.Russian, Tags.Media.Manga, Tags.Rating.Pornographic, Tags.Source.Aggregator, Tags.Accessibility.DomainRotation);
}

public override get Icon() {
return icon;
}

public override async Initialize(): Promise<void> {
this.URI.href = await FetchWindowScript(new Request(this.URI), 'window.location.origin');
console.log(`Assigned URL '${this.URI}' to ${this.Title}`);
}
}
File renamed without changes.
26 changes: 26 additions & 0 deletions web/src/engine/websites/AllHentai_e2e.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { TestFixture } from '../../../test/WebsitesFixture';

const config = {
plugin: {
id: 'allhentai',
title: 'AllHentai'
},
container: {
url: 'https://20.allhen.online/nochiu_ona_nimfomanka',
id: '/nochiu_ona_nimfomanka',
title: 'Ночью она нимфоманка'
},
/* Need to be logged, and chapter link change with user id anyway
child: {
id: '/nochiu_ona_nimfomanka/vol1/4?mtr=1',
title: '1 - 1',
},
entry: {
index: 0,
size: 155_934,
type: 'image/png'
}*/
};

new TestFixture(config).AssertWebsite();
12 changes: 6 additions & 6 deletions web/src/engine/websites/AsuraScans_e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,17 @@ const config: Config = {
title: 'Asura Scans',
},
container: {
url: 'https://asuracomic.net/series/dragon-devouring-mage-f4dac81c',
id: '/series/dragon-devouring-mage-',
title: 'Dragon-Devouring Mage',
url: 'https://asuracomic.net/series/nano-machine-b755c1b9',
id: '/series/nano-machine-',
title: 'Nano Machine',
},
child: {
id: '/series/dragon-devouring-mage-/chapter/1',
title: 'Chapter 1',
id: '/series/nano-machine-/chapter/222',
title: 'Chapter 222 76. Level (1)',
},
entry: {
index: 1,
size: 697_616,
size: 781_272,
type: 'image/webp'
}
};
Expand Down
55 changes: 48 additions & 7 deletions web/src/engine/websites/BacaManga.ts
Original file line number Diff line number Diff line change
@@ -1,21 +1,62 @@
import { Tags } from '../Tags';
import icon from './BacaManga.webp';
import { DecoratableMangaScraper } from '../providers/MangaPlugin';
import * as MangaStream from './decorators/WordPressMangaStream';
import { Chapter, DecoratableMangaScraper, type Manga } from '../providers/MangaPlugin';
import * as Common from './decorators/Common';
import { FetchHTML, FetchJSON } from '../platform/FetchProvider';

@MangaStream.MangaCSS(/^{origin}\/manga\/[^/]+\/$/)
@MangaStream.MangasSinglePageCSS()
@MangaStream.ChaptersSinglePageCSS()
@MangaStream.PagesSinglePageJS()
type APIChapters = {
list_chapter_html: string
}

function MangaInfoExtractor(anchor: HTMLAnchorElement) {
return {
id: anchor.pathname,
title: anchor.querySelector<HTMLDivElement>('h3.text-white').textContent.trim()
};
}

@Common.MangaCSS(/^{origin}\/komik\/[^/]+\/$/, 'article header h1')
@Common.MangasMultiPageCSS('/daftar-komik/page/{page}/', 'div.grid div[id*="post-"] > a', 1, 1, 0, MangaInfoExtractor)
@Common.PagesSinglePageCSS('div.img-landmine div#chimg-auh img')
@Common.ImageAjax()
export default class extends DecoratableMangaScraper {

public constructor() {
super('bacamanga', 'BacaManga', 'https://bacamanga.id', Tags.Media.Manga, Tags.Media.Manhua, Tags.Media.Manhwa, Tags.Language.Indonesian, Tags.Source.Aggregator);
super('bacamanga', 'BacaManga', 'https://bacamanga.cc', Tags.Media.Manga, Tags.Media.Manhua, Tags.Media.Manhwa, Tags.Language.Indonesian, Tags.Source.Aggregator);
}

public override get Icon() {
return icon;
}

public override async FetchChapters(manga: Manga): Promise<Chapter[]> {
const mangaUrl = new URL(manga.Identifier, this.URI);
let dom = await FetchHTML(new Request(mangaUrl));
const series_id = new URL(dom.querySelector<HTMLLinkElement>('link[rel="shortlink"]').href).searchParams.get('p');
const chapters = this.ExtractChapters(dom, manga, 'div#chapter-list > a');

const { list_chapter_html } = await FetchJSON<APIChapters>(new Request(new URL('/wp-admin/admin-ajax.php', this.URI), {
method: 'POST',
body: new URLSearchParams({
action: 'get_list_chapter',
series_id
}).toString(),
headers: {
'Content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'X-Requested-With': 'XMLHttpRequest',
Referer: mangaUrl.href,
Origin: this.URI.origin
}
}));
dom = new DOMParser().parseFromString(list_chapter_html, 'text/html');
return chapters.concat( this.ExtractChapters(dom, manga, 'a'));
}

private ExtractChapters(document: Document, manga: Manga, selector: string): Chapter[] {
return [...document.querySelectorAll<HTMLAnchorElement>(selector)].map(chapter => {
const title = chapter.querySelector<HTMLDivElement>('div.text-white').innerText.trim();
return new Chapter(this, manga, chapter.pathname, title);
});
}

}
14 changes: 7 additions & 7 deletions web/src/engine/websites/BacaManga_e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,17 @@ const config: Config = {
title: 'BacaManga'
},
container: {
url: 'https://bacamanga.id/manga/50kg-cinderella/',
id: '/manga/50kg-cinderella/',
title: '-50kg Cinderella'
url: 'https://bacamanga.cc/komik/martial-peak/',
id: '/komik/martial-peak/',
title: 'Martial Peak'
},
child: {
id: '/50kg-cinderella-chapter-1-1/',
title: 'Chapter 1.1'
id: '/martial-peak-chapter-3772/',
title: 'Chapter 3772'
},
entry: {
index: 2,
size: 57_750,
index: 1,
size: 224_158,
type: 'image/jpeg'
}
};
Expand Down
7 changes: 4 additions & 3 deletions web/src/engine/websites/Bomtoon.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { WebsiteResourceKey as R } from '../../i18n/ILocale';
import type { Priority } from '../taskpool/DeferredTask';
import DeScramble from '../transformers/ImageDescrambler';
import { DelitoonBase, type APIManga, type APIResult } from './templates/DelitoonBase';
import { GetBytesB64, GetBytesUTF8 } from '../BufferEncoder';

type APIMangas = {
content: APIManga[]
Expand Down Expand Up @@ -86,9 +87,9 @@ export default class extends DelitoonBase {
}

private async Decrypt(encrypted: string, scramblekey: string): Promise<number[]> {
const cipher = { name: 'AES-CBC', iv: Buffer.from(scramblekey.slice(0, 16)) };
const key = await crypto.subtle.importKey('raw', Buffer.from(scramblekey), { name: 'AES-CBC', length: 256 }, false, ['decrypt']);
const decrypted = await crypto.subtle.decrypt(cipher, key, Buffer.from(encrypted, 'base64'));
const cipher = { name: 'AES-CBC', iv: GetBytesUTF8(scramblekey.slice(0, 16)) };
const key = await crypto.subtle.importKey('raw', GetBytesUTF8(scramblekey), { name: 'AES-CBC', length: 256 }, false, ['decrypt']);
const decrypted = await crypto.subtle.decrypt(cipher, key, GetBytesB64(encrypted));
return JSON.parse(new TextDecoder('utf-8').decode(decrypted)) as number[];
}

Expand Down
3 changes: 2 additions & 1 deletion web/src/engine/websites/CiaoPlus.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import * as Common from './decorators/Common';
import { FetchJSON } from '../platform/FetchProvider';
import type { Priority } from '../taskpool/DeferredTask';
import DeScramble from '../transformers/ImageDescrambler';
import { BufferToHexString, GetBytesUTF8 } from '../BufferEncoder';

type APIMangas = {
title_list: {
Expand Down Expand Up @@ -203,6 +204,6 @@ export default class extends DecoratableMangaScraper {
}

private async SHA(text: string, cipher: string): Promise<string> {
return Buffer.from(await crypto.subtle.digest(cipher, Buffer.from(text))).toString('hex');
return BufferToHexString(await crypto.subtle.digest(cipher, GetBytesUTF8(text)));
}
}
Loading

0 comments on commit 25c9f8c

Please sign in to comment.