Skip to content

Commit

Permalink
Merge branch 'master' into MHKX
Browse files Browse the repository at this point in the history
  • Loading branch information
MikeZeDev committed Nov 26, 2024
2 parents 84aee49 + 208da85 commit 074b30c
Show file tree
Hide file tree
Showing 125 changed files with 1,112 additions and 1,053 deletions.
8 changes: 7 additions & 1 deletion .github/workflows/pull-request-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,10 @@ jobs:
if: steps.packages.outcome == 'success' && (success() || failure())
env:
DISPLAY: ':99'
run: npm run test:websites -- MangaDex
run: npm run test:websites -- MangaDex
- name: Upload Test Results
if: failure()
uses: actions/upload-artifact@v4
with:
name: Test Results (${{ matrix.os }})
path: ./**/screenshot_*.png
3 changes: 2 additions & 1 deletion app/electron/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
"module": "ESNext",
"moduleResolution": "bundler",
"skipLibCheck": true,
"noEmit": true
"noEmit": true,
"types": [ "chrome" ]
},
"include": [
"src/**/*.ts",
Expand Down
3 changes: 2 additions & 1 deletion app/nw/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
"module": "ESNext",
"moduleResolution": "bundler",
"skipLibCheck": true,
"noEmit": true
"noEmit": true,
"types": [ "chrome", "nw.js" ]
},
"include": [
"src/**/*.ts",
Expand Down
8 changes: 4 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,17 @@
"docs"
],
"devDependencies": {
"@stylistic/eslint-plugin": "^2.10.1",
"@stylistic/eslint-plugin": "^2.11.0",
"@types/chrome": "^0.0.283",
"@types/jsdom": "^21.1.7",
"@types/nw.js": "^0.92.0",
"eslint": "^9.15.0",
"eslint-plugin-tsdoc": "^0.3.0",
"jsdom": "^25.0.1",
"puppeteer-core": "^23.8.0",
"puppeteer-core": "^23.9.0",
"tslib": "^2.8.1",
"typescript": "^5.6.3",
"typescript-eslint": "^8.14.0",
"typescript": "^5.7.2",
"typescript-eslint": "^8.15.0",
"vite": "^5.4.11",
"vitest": "^2.1.5",
"vitest-mock-extended": "^2.0.2"
Expand Down
9 changes: 9 additions & 0 deletions test/PuppeteerFixture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ export class PuppeteerFixture {
return PuppeteerFixture.#page;
}

public async Screenshot(page: puppeteer.Page) {
await page.screenshot({
type: 'png',
fullPage: true,
captureBeyondViewport: true,
path: `./screenshot_${Date.now().toString(36)}.png`,
});
}

protected async OpenPage(url: string, ...evasions: Evasion[]): Promise<puppeteer.Page> {
const page = await (await this.GetBrowser()).newPage();
await Promise.all(evasions.map(setupEvasion => setupEvasion(page)));
Expand Down
10 changes: 10 additions & 0 deletions web/eslint.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,16 @@ export default tseslint.config({
'no-multi-spaces': 'error',
'no-throw-literal': 'error',
'tsdoc/syntax': 'warn',
/**
* Use restricted properties as workaround to address a flaw in the typescript system of global declarations.
* Globally declared modules (such as @types/node::Buffer) should be declared for e.g., tests but must not be declared for the web-application (browsers do not have `Buffer`).
* Unfortunately, once included these are available everywhere in the workspace without exception.
* See also: https://github.com/microsoft/TypeScript/issues/50424
*/
'no-restricted-properties': [ 'error', {
"object": "Buffer",
"property": "from"
} ],
'@typescript-eslint/naming-convention': [ 'error', // See: https://typescript-eslint.io/rules/naming-convention/#options
/*
{
Expand Down
10 changes: 5 additions & 5 deletions web/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
"protobufjs": "^7.4.0"
},
"devDependencies": {
"@fluentui/svg-icons": "^1.1.265",
"@fluentui/svg-icons": "^1.1.266",
"@fluentui/web-components": "^2.6.1",
"@microsoft/fast-element": "^1.14.0",
"@svelte-put/dragscroll": "^4.0.0",
Expand All @@ -23,14 +23,14 @@
"@vitejs/plugin-react": "^4.3.3",
"@vitejs/plugin-vue": "^5.2.0",
"@vscode/codicons": "^0.0.36",
"carbon-components-svelte": "^0.85.4",
"carbon-components-svelte": "^0.86.1",
"carbon-icons-svelte": "^12.13.0",
"fuse.js": "^7.0.0",
"lit": "^3.2.1",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"svelte": "^5.2.1",
"svelte-check": "^4.0.8",
"svelte": "^5.2.7",
"svelte-check": "^4.1.0",
"svelte-virtuallists": "^1.4.0",
"vue": "^3.5.13",
"vue-tsc": "^2.1.10"
Expand All @@ -42,7 +42,7 @@
"check:rules": "node ./scripts/coding-rules.mjs",
"check:svelte": "svelte-check --tsconfig=tsconfig.json --compiler-warnings a11y-click-events-have-key-events:ignore",
"check:vue": "vue-tsc --skipLibCheck --noEmit",
"check": "npm run check:ts && npm run check:lint && npm run check:svelte && npm run check:vue && npm run check:rules",
"check": "npm run check:ts && npm run check:lint && npm run check:svelte && npm run check:rules",
"build": "vite build",
"test": "vitest run",
"serve:dev": "vite --port=3000 --strictPort",
Expand Down
28 changes: 11 additions & 17 deletions web/src/AntiBotDetectionBypass_e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ class TestFixture extends PuppeteerFixture {
public async SetupPage(url: string, ...evasions: Evasion[]): Promise<TestFixture> {
await this.ClosePage();
this.page = await this.OpenPage(url, ...evasions);
console.log('WebDriver:', await this.page.evaluate(() => window.navigator.webdriver));
console.log('User-Agent:', await this.page.evaluate(() => window.navigator.userAgent));
console.log('Console:', await this.page.evaluate(() => console.debug.toString()));
//console.log('WebDriver:', await this.page.evaluate(() => window.navigator.webdriver));
//console.log('User-Agent:', await this.page.evaluate(() => window.navigator.userAgent));
//console.log('Console:', await this.page.evaluate(() => console.debug.toString()));
return this;
}

Expand All @@ -29,23 +29,17 @@ class TestFixture extends PuppeteerFixture {

public async AssertElementText(selector: string, expected: string): Promise<void> {
try {
await this.page.waitForSelector(selector, { timeout: 5000 });
await this.page.waitForSelector(selector, { timeout: 7500 });
const actual = await this.page.$eval(selector, (element: HTMLElement) => element.innerText);
expect(actual).toBe(expected);
} catch(error) {
await this.page.screenshot({
type: 'png',
fullPage: true,
captureBeyondViewport: true,
path: `./screenshot_${Date.now().toString(16)}.png`
});
await new Promise(resolve => setTimeout(resolve, 5_000));
await this.Screenshot(this.page);
throw error;
}
}
}

describe('BrowserScan', { timeout: 10_000 }, () => {
describe('BrowserScan', { timeout: 15_000 }, () => {

it('Should pass Bot Detection Test', async () => {
const fixture = await new TestFixture().SetupPage('https://www.browserscan.net/bot-detection', EvadeChromeDevToolProtocolDetection);
Expand All @@ -57,7 +51,7 @@ describe('BrowserScan', { timeout: 10_000 }, () => {
});
});

describe('CloudFlare', { timeout: 10_000 }, () => {
describe('CloudFlare', { timeout: 15_000 }, () => {

it('Should bypass JavaScript Challenge', async () => {
const fixture = await new TestFixture().SetupPage('https://cloudflare.bot-check.ovh/automatic');
Expand All @@ -68,7 +62,7 @@ describe('CloudFlare', { timeout: 10_000 }, () => {
}
});

it.skip('Should bypass Turnstile Non-Interactive Challenge', async () => {
it('Should bypass Turnstile Non-Interactive Challenge', { todo: true }, async () => {
const fixture = await new TestFixture().SetupPage('https://cloudflare.bot-check.ovh/turnstile-automatic');
try {
await fixture.AssertElementText('#hash', 'A9B6FA3DD8842CD8E2D6070784D92434');
Expand All @@ -77,7 +71,7 @@ describe('CloudFlare', { timeout: 10_000 }, () => {
}
});

it.skip('Should bypass Turnstile Invisible Challenge', async () => {
it('Should bypass Turnstile Invisible Challenge', { todo: true }, async () => {
const fixture = await new TestFixture().SetupPage('https://cloudflare.bot-check.ovh/turnstile-invisible');
try {
await fixture.AssertElementText('#hash', 'A9B6FA3DD8842CD8E2D6070784D92434');
Expand All @@ -87,9 +81,9 @@ describe('CloudFlare', { timeout: 10_000 }, () => {
});
});

describe('Vercel', { timeout: 10_000 }, () => {
describe('Vercel', { timeout: 15_000 }, () => {

it.skip('Should bypass Attack Challenge Mode', async () => {
it('Should bypass Attack Challenge Mode', { todo: true }, async () => {
const fixture = await new TestFixture().SetupPage('https://vercel.bot-check.ovh', EvadeChromeDevToolProtocolDetection);
try {
await fixture.AssertElementText('#hash', 'FDF049D4B2312945BB7AA2158F041278');
Expand Down
19 changes: 19 additions & 0 deletions web/src/engine/BufferEncoder.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
export function FromHexString(hexString: string): Uint8Array {
return Uint8Array.from(hexString.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)));
}

export function GetBytesUTF8(text: string): Uint8Array {
return new TextEncoder().encode(text);
};

export function Uint8ToHexString(array: Uint8Array): string {
return array.reduce((result, x) => result + x.toString(16).padStart(2, '0'), '');
}

export function BufferToHexString(buffer: ArrayBuffer): string {
return Uint8ToHexString(new Uint8Array(buffer));
}

export function GetBytesB64(b64string: string): Uint8Array {
return Uint8Array.from(window.atob(b64string), c => c.charCodeAt(0));
}
2 changes: 2 additions & 0 deletions web/src/engine/SettingsManager_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@ import { type StorageController, Store } from './StorageController';
import { Key } from './SettingsGlobal';

window.atob = function(encoded: string): string {
/* eslint-disable-next-line no-restricted-properties */ //=> This is supposed to run in NodeJS runtime where Buffer is available
return Buffer.from(encoded, 'base64').toString('utf-8');
};

window.btoa = function(decoded: string): string {
/* eslint-disable-next-line no-restricted-properties */ //=> This is supposed to run in NodeJS runtime where Buffer is available
return Buffer.from(decoded, 'utf-8').toString('base64');
};

Expand Down
5 changes: 3 additions & 2 deletions web/src/engine/taskpool/TaskPool_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,9 @@ describe('TaskPool', () => {
expect(actual.map(r => r.Value)).toEqual([ '③' ]);
});

(process.platform === 'win32' ? it.skip : it)('Should utilize workers for concurrent processing', {
retry: process.platform === 'win32' ? 50 : 5
it('Should utilize workers for concurrent processing', {
todo: process.platform === 'win32',
retry: process.platform === 'win32' ? 50 : 5,
}, async () => {
const testee = new TaskPool(3, Unlimited);
const start = performance.now();
Expand Down
26 changes: 26 additions & 0 deletions web/src/engine/websites/AllHentai.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { Tags } from '../Tags';
import icon from './AllHentai.webp';
import { DecoratableMangaScraper } from '../providers/MangaPlugin';
import * as Common from './decorators/Common';
import * as Grouple from './decorators/Grouple';
import { FetchWindowScript } from '../platform/FetchProvider';

@Common.MangaCSS(/^{origin}\/[^/]+$/, Grouple.queryMangaTitle)
@Common.MangasMultiPageCSS(Grouple.pathMangas, Grouple.queryMangas, 0, Grouple.pageMangaOffset, 1000, Common.AnchorInfoExtractor(true))
@Common.ChaptersSinglePageJS(Grouple.chapterScript, 500)
@Grouple.PagesSinglePageJS()
@Grouple.ImageAjaxWithMirrors()
export default class extends DecoratableMangaScraper {
public constructor() {
super('allhentai', `AllHentai`, 'https://20.allhen.online', Tags.Language.Russian, Tags.Media.Manga, Tags.Rating.Pornographic, Tags.Source.Aggregator, Tags.Accessibility.DomainRotation);
}

public override get Icon() {
return icon;
}

public override async Initialize(): Promise<void> {
this.URI.href = await FetchWindowScript(new Request(this.URI), 'window.location.origin');
console.log(`Assigned URL '${this.URI}' to ${this.Title}`);
}
}
File renamed without changes.
26 changes: 26 additions & 0 deletions web/src/engine/websites/AllHentai_e2e.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { TestFixture } from '../../../test/WebsitesFixture';

const config = {
plugin: {
id: 'allhentai',
title: 'AllHentai'
},
container: {
url: 'https://20.allhen.online/nochiu_ona_nimfomanka',
id: '/nochiu_ona_nimfomanka',
title: 'Ночью она нимфоманка'
},
/* Need to be logged, and chapter link change with user id anyway
child: {
id: '/nochiu_ona_nimfomanka/vol1/4?mtr=1',
title: '1 - 1',
},
entry: {
index: 0,
size: 155_934,
type: 'image/png'
}*/
};

new TestFixture(config).AssertWebsite();
27 changes: 18 additions & 9 deletions web/src/engine/websites/AsuraScans.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import { Tags } from '../Tags';
import icon from './AsuraScans.webp';
import { Chapter, DecoratableMangaScraper, Manga, type MangaPlugin } from '../providers/MangaPlugin';
import { DecoratableMangaScraper, Manga, type MangaPlugin } from '../providers/MangaPlugin';
import * as MangaStream from './decorators/WordPressMangaStream';
import * as Common from './decorators/Common';
import { FetchCSS, FetchWindowScript } from '../platform/FetchProvider';
import { FetchWindowScript } from '../platform/FetchProvider';

const excludes = [
/panda_gif_large/i,
Expand All @@ -13,6 +13,19 @@ const excludes = [
/EndDesignPSD/i
];

const chapterScript = `
new Promise( resolve => {
resolve( [...document.querySelectorAll('div h3 a[href*="/chapter/"]')].map(chapter => {
return {
id: chapter.pathname.replace(/(-[^-]+\\/chapter)/, '-/chapter'),
title : chapter.innerText.replace('\\n', ' ').trim()
};
}));
});
`;

const pagesScript = `[... document.querySelectorAll('div.items-center div div.center img')].map(image=> image.src);`;

function MangaInfoExtractor(anchor: HTMLAnchorElement) {
return {
id: anchor.pathname.replace(/-[^-]+$/, '-'),
Expand All @@ -21,8 +34,9 @@ function MangaInfoExtractor(anchor: HTMLAnchorElement) {
}

@Common.MangasMultiPageCSS('/series?page={page}', 'div.grid a', 1, 1, 0, MangaInfoExtractor)
@MangaStream.PagesSinglePageCSS(excludes, 'img[alt*="chapter"]')
@Common.ImageAjax()
@Common.ChaptersSinglePageJS(chapterScript, 500)
@MangaStream.PagesSinglePageJS(excludes, pagesScript, 1000)
@Common.ImageAjax(true)
export default class extends DecoratableMangaScraper {

public constructor() {
Expand All @@ -46,9 +60,4 @@ export default class extends DecoratableMangaScraper {
return new Manga(this, provider, manga.Identifier, manga.Title.replace(' - Asura Scans', '').trim());
}

public override async FetchChapters(manga: Manga): Promise<Chapter[]> {
const chapters = await FetchCSS<HTMLAnchorElement>(new Request(new URL(manga.Identifier, this.URI)), 'div.scrollbar-thumb-themecolor a.block');
return chapters.map(chapter => new Chapter(this, manga, [manga.Identifier, 'chapter', chapter.pathname.match(/(\d+)+$/)[1]].join('/'), chapter.textContent.trim()));
}

}
12 changes: 6 additions & 6 deletions web/src/engine/websites/AsuraScans_e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,17 @@ const config: Config = {
title: 'Asura Scans',
},
container: {
url: 'https://asuracomic.net/series/dragon-devouring-mage-f4dac81c',
id: '/series/dragon-devouring-mage-',
title: 'Dragon-Devouring Mage',
url: 'https://asuracomic.net/series/nano-machine-b755c1b9',
id: '/series/nano-machine-',
title: 'Nano Machine',
},
child: {
id: '/series/dragon-devouring-mage-/chapter/1',
title: 'Chapter 1',
id: '/series/nano-machine-/chapter/222',
title: 'Chapter 222 76. Level (1)',
},
entry: {
index: 1,
size: 697_616,
size: 781_272,
type: 'image/webp'
}
};
Expand Down
Loading

0 comments on commit 074b30c

Please sign in to comment.