-
Notifications
You must be signed in to change notification settings - Fork 31
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into clipstudioreader
- Loading branch information
Showing
134 changed files
with
1,282 additions
and
1,101 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| /** | ||
| * Further Reading: | ||
| * - https://github.com/paulirish/headless-cat-n-mouse/blob/master/apply-evasions.js | ||
| * - https://deviceandbrowserinfo.com/learning_zone/articles/detecting-headless-chrome-puppeteer-2024 | ||
| * - https://www.reddit.com/r/webscraping/comments/1evht3i/help_in_bypassing_cdp_detection/ | ||
| * - https://github.com/kaliiiiiiiiii/brotector | ||
| * - https://github.com/rebrowser/rebrowser-patches?tab=readme-ov-file | ||
| */ | ||
|
|
||
| import { type Browser, type Target, TargetType, type Page } from 'puppeteer-core'; | ||
|
|
||
| export type Evasion = (page: Page) => Promise<void>; | ||
|
|
||
| const conceal = ` | ||
| function conceal(obj, key, intercept) { | ||
| function toString() { | ||
| return 'function ' + this.name + '() { [native code] }'; | ||
| } | ||
| Object.defineProperty(toString, 'toString', { | ||
| value: toString, | ||
| writable: false, | ||
| enumerable: false, | ||
| }); | ||
| Object.defineProperty(toString, 'prototype', { | ||
| value: undefined, | ||
| writable: false, | ||
| enumerable: false, | ||
| }); | ||
| obj[key] = new Proxy(obj[key], { | ||
| get(target, key, receiver) { | ||
| return key === 'toString' ? toString : Reflect.get(target, key); | ||
| }, | ||
| apply: intercept, | ||
| }); | ||
| } | ||
| `; | ||
|
|
||
| /* | ||
| function EvadeWebDriverDetection(page: Page) { | ||
| //await page.evaluateOnNewDocument(`delete navigator.__proto__.webdriver`); | ||
| await page.evaluateOnNewDocument(`navigator.__proto__.webdriver = false`); | ||
| } | ||
| */ | ||
|
|
||
| /** | ||
| * When dumping an object with the console functions, the properties of the object will be evaluated. | ||
| * This evaluation only happens when the Developer Tools are opened, or a client is connected via CDP. | ||
| * A proxy object could be used to detect access to getters of the object when it is dumped. | ||
| */ | ||
| export async function EvadeChromeDevToolProtocolDetection(page: Page) { | ||
| await page.evaluateOnNewDocument(` | ||
| ${conceal} | ||
| conceal(console, 'log', () => {}); | ||
| conceal(console, 'warn', () => {}); | ||
| conceal(console, 'error', () => {}); | ||
| conceal(console, 'debug', () => {}); | ||
| `); | ||
| } | ||
|
|
||
| /** | ||
| * Avoid automation detection for Blink-based browsers when used with Puppeteer. | ||
| * This will only hide changes applied by Puppeteer to access the capabilities of the underlying Blink browser (e.g., Electron or NW.js). | ||
| */ | ||
| export function SetupBlinkEvasions(browser: Browser, ...evasions: Evasion[]) { | ||
| // TODO: This doesn't work yet due to injection being too slow | ||
| // => https://github.com/puppeteer/puppeteer/issues/3667 | ||
| browser.on('targetcreated', async (target: Target) => { | ||
| if(target.type() === TargetType.PAGE) { | ||
| const page = await target.page(); | ||
| await Promise.all(evasions.map(setupEvasion => setupEvasion(page))); | ||
| } | ||
| }); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| /** | ||
| * Check the bot-detection bypass capabilities of HakuNeko's underlying Blink-based browser engine (NW.js, Electron) | ||
| */ | ||
|
|
||
| import { describe, it, expect } from 'vitest'; | ||
| import type { Page } from 'puppeteer-core'; | ||
| import { PuppeteerFixture } from '../../test/PuppeteerFixture'; | ||
| import { | ||
| type Evasion, | ||
| EvadeChromeDevToolProtocolDetection, | ||
| } from '../../test/AutomationEvasions'; | ||
|
|
||
| class TestFixture extends PuppeteerFixture { | ||
|
|
||
| private page: Page; | ||
|
|
||
| public async SetupPage(url: string, ...evasions: Evasion[]): Promise<TestFixture> { | ||
| await this.ClosePage(); | ||
| this.page = await this.OpenPage(url, ...evasions); | ||
| //console.log('WebDriver:', await this.page.evaluate(() => window.navigator.webdriver)); | ||
| //console.log('User-Agent:', await this.page.evaluate(() => window.navigator.userAgent)); | ||
| //console.log('Console:', await this.page.evaluate(() => console.debug.toString())); | ||
| return this; | ||
| } | ||
|
|
||
| public async ClosePage(): Promise<void> { | ||
| return this.page?.close(); | ||
| } | ||
|
|
||
| public async AssertElementText(selector: string, expected: string): Promise<void> { | ||
| try { | ||
| await this.page.waitForSelector(selector, { timeout: 7500 }); | ||
| const actual = await this.page.$eval(selector, (element: HTMLElement) => element.innerText); | ||
| expect(actual).toBe(expected); | ||
| } catch(error) { | ||
| await this.Screenshot(this.page); | ||
| throw error; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| describe('BrowserScan', { timeout: 15_000 }, () => { | ||
|
|
||
| it('Should pass Bot Detection Test', async () => { | ||
| const fixture = await new TestFixture().SetupPage('https://www.browserscan.net/bot-detection', EvadeChromeDevToolProtocolDetection); | ||
| try { | ||
| await fixture.AssertElementText(`div._5h5tql svg use[*|href="#status_success"]`, undefined); | ||
| } finally { | ||
| fixture.ClosePage(); | ||
| } | ||
| }); | ||
| }); | ||
|
|
||
| describe('CloudFlare', { timeout: 15_000 }, () => { | ||
|
|
||
| it('Should bypass JavaScript Challenge', async () => { | ||
| const fixture = await new TestFixture().SetupPage('https://cloudflare.bot-check.ovh/automatic'); | ||
| try { | ||
| await fixture.AssertElementText('#hash', '801BE7B2C3621A7DE738CF77BD4EF264'); | ||
| } finally { | ||
| fixture.ClosePage(); | ||
| } | ||
| }); | ||
|
|
||
| it('Should bypass Turnstile Non-Interactive Challenge', { todo: true }, async () => { | ||
| const fixture = await new TestFixture().SetupPage('https://cloudflare.bot-check.ovh/turnstile-automatic'); | ||
| try { | ||
| await fixture.AssertElementText('#hash', 'A9B6FA3DD8842CD8E2D6070784D92434'); | ||
| } finally { | ||
| fixture.ClosePage(); | ||
| } | ||
| }); | ||
|
|
||
| it('Should bypass Turnstile Invisible Challenge', { todo: true }, async () => { | ||
| const fixture = await new TestFixture().SetupPage('https://cloudflare.bot-check.ovh/turnstile-invisible'); | ||
| try { | ||
| await fixture.AssertElementText('#hash', 'A9B6FA3DD8842CD8E2D6070784D92434'); | ||
| } finally { | ||
| fixture.ClosePage(); | ||
| } | ||
| }); | ||
| }); | ||
|
|
||
| describe('Vercel', { timeout: 15_000 }, () => { | ||
|
|
||
| it('Should bypass Attack Challenge Mode', { todo: true }, async () => { | ||
| const fixture = await new TestFixture().SetupPage('https://vercel.bot-check.ovh', EvadeChromeDevToolProtocolDetection); | ||
| try { | ||
| await fixture.AssertElementText('#hash', 'FDF049D4B2312945BB7AA2158F041278'); | ||
| } finally { | ||
| fixture.ClosePage(); | ||
| } | ||
| }); | ||
| }); |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| export function FromHexString(hexString: string): Uint8Array { | ||
| return Uint8Array.from(hexString.match(/.{1,2}/g).map((byte) => parseInt(byte, 16))); | ||
| } | ||
|
|
||
| export function GetBytesUTF8(text: string): Uint8Array { | ||
| return new TextEncoder().encode(text); | ||
| }; | ||
|
|
||
| export function Uint8ToHexString(array: Uint8Array): string { | ||
| return array.reduce((result, x) => result + x.toString(16).padStart(2, '0'), ''); | ||
| } | ||
|
|
||
| export function BufferToHexString(buffer: ArrayBuffer): string { | ||
| return Uint8ToHexString(new Uint8Array(buffer)); | ||
| } | ||
|
|
||
| export function GetBytesB64(b64string: string): Uint8Array { | ||
| return Uint8Array.from(window.atob(b64string), c => c.charCodeAt(0)); | ||
| } |
Oops, something went wrong.