From 3fe27fe207eed52d89f29af20599bf688eb83f72 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sun, 22 Dec 2024 23:37:48 +0100 Subject: [PATCH] Switch back to the WEB client for the local API with potokens --- _scripts/dev-runner.js | 15 ++- _scripts/injectAllowedPaths.mjs | 2 + _scripts/webpack.botGuardScript.config.js | 23 ++++ package.json | 4 +- src/botGuardScript.js | 37 ++++++ src/constants.js | 4 +- src/main/index.js | 26 ++-- src/main/poTokenGenerator.js | 140 ++++++++++++++++++++++ src/renderer/helpers/api/local.js | 112 +++++++---------- src/renderer/views/Watch/Watch.js | 27 ++--- yarn.lock | 5 + 11 files changed, 283 insertions(+), 112 deletions(-) create mode 100644 _scripts/webpack.botGuardScript.config.js create mode 100644 src/botGuardScript.js create mode 100644 src/main/poTokenGenerator.js diff --git a/_scripts/dev-runner.js b/_scripts/dev-runner.js index fa5d16db56c0d..15d9b356b842e 100644 --- a/_scripts/dev-runner.js +++ b/_scripts/dev-runner.js @@ -18,12 +18,14 @@ const web = process.argv.indexOf('--web') !== -1 let mainConfig let rendererConfig +let botGuardScriptConfig let webConfig let SHAKA_LOCALES_TO_BE_BUNDLED if (!web) { mainConfig = require('./webpack.main.config') rendererConfig = require('./webpack.renderer.config') + botGuardScriptConfig = require('./webpack.botGuardScript.config') SHAKA_LOCALES_TO_BE_BUNDLED = rendererConfig.SHAKA_LOCALES_TO_BE_BUNDLED delete rendererConfig.SHAKA_LOCALES_TO_BE_BUNDLED @@ -98,6 +100,14 @@ function setupNotifyLocaleUpdate(compiler, devServer) { }) } +function startBotGuardScript() { + webpack(botGuardScriptConfig, (err) => { + if (err) console.error(err) + + console.log(`\nCompiled ${botGuardScriptConfig.name} script!`) + }) +} + function startMain() { const compiler = webpack(mainConfig) const { name } = compiler @@ -196,7 +206,10 @@ function startWeb () { }) } if (!web) { - startRenderer(startMain) + startRenderer(() => { + startBotGuardScript() + startMain() + }) } else { startWeb() } diff --git a/_scripts/injectAllowedPaths.mjs b/_scripts/injectAllowedPaths.mjs index 2310bf49839b1..e5c1ce2293232 100644 --- a/_scripts/injectAllowedPaths.mjs +++ b/_scripts/injectAllowedPaths.mjs @@ -21,6 +21,8 @@ const paths = readdirSync(distDirectory, { // disallow the renderer process/browser windows to read the main.js file dirent.name !== 'main.js' && dirent.name !== 'main.js.LICENSE.txt' && + // disallow the renderer process/browser windows to read the botGuardScript.js file + dirent.name !== 'botGuardScript.js' && // filter out any web build files, in case the dist directory contains a web build !dirent.parentPath.startsWith(webDirectory) }) diff --git a/_scripts/webpack.botGuardScript.config.js b/_scripts/webpack.botGuardScript.config.js new file mode 100644 index 0000000000000..9b66ffc370f21 --- /dev/null +++ b/_scripts/webpack.botGuardScript.config.js @@ -0,0 +1,23 @@ +const path = require('path') + +/** @type {import('webpack').Configuration} */ +module.exports = { + name: 'botGuardScript', + // Always use production mode, as we use the output as a function body and the debug output doesn't work for that + mode: 'production', + devtool: false, + target: 'web', + entry: { + botGuardScript: path.join(__dirname, '../src/botGuardScript.js'), + }, + output: { + filename: '[name].js', + path: path.join(__dirname, '../dist'), + library: { + type: 'modern-module' + } + }, + experiments: { + outputModule: true + } +} diff --git a/package.json b/package.json index ad4f7970f7ab8..d2836a5bf7924 100644 --- a/package.json +++ b/package.json @@ -43,10 +43,11 @@ "lint-style": "stylelint \"**/*.{css,scss}\"", "lint-style-fix": "stylelint --fix \"**/*.{css,scss}\"", "lint-yml": "eslint --config eslint.config.mjs \"**/*.yml\" \"**/*.yaml\"", - "pack": "run-p pack:main pack:renderer && node _scripts/injectAllowedPaths.mjs", + "pack": "run-p pack:main pack:renderer pack:botGuardScript && node _scripts/injectAllowedPaths.mjs", "pack:main": "webpack --mode=production --node-env=production --config _scripts/webpack.main.config.js", "pack:renderer": "webpack --mode=production --node-env=production --config _scripts/webpack.renderer.config.js", "pack:web": "webpack --mode=production --node-env=production --config _scripts/webpack.web.config.js", + "pack:botGuardScript": "webpack --config _scripts/webpack.botGuardScript.config.js", "postinstall": "run-s --silent rebuild:electron patch-shaka", "prettier": "prettier --write \"{src,_scripts}/**/*.{js,vue}\"", "rebuild:electron": "electron-builder install-app-deps", @@ -61,6 +62,7 @@ "@fortawesome/vue-fontawesome": "^2.0.10", "@seald-io/nedb": "^4.0.4", "autolinker": "^4.0.0", + "bgutils-js": "^3.1.0", "electron-context-menu": "^4.0.4", "lodash.debounce": "^4.0.8", "marked": "^15.0.4", diff --git a/src/botGuardScript.js b/src/botGuardScript.js new file mode 100644 index 0000000000000..11da8855a6472 --- /dev/null +++ b/src/botGuardScript.js @@ -0,0 +1,37 @@ +import { BG } from 'bgutils-js' + +// This script has it's own webpack config, as it gets passed as a string to Electron's evaluateJavaScript function +// in src/main/poTokenGenerator.js +export default async function(visitorData) { + const requestKey = 'O43z0dpjhgX20SCx4KAo' + + const bgConfig = { + fetch: (input, init) => fetch(input, init), + requestKey, + globalObj: window, + identifier: visitorData + } + + const challenge = await BG.Challenge.create(bgConfig) + + if (!challenge) { + throw new Error('Could not get challenge') + } + + const interpreterJavascript = challenge.interpreterJavascript.privateDoNotAccessOrElseSafeScriptWrappedValue + + if (interpreterJavascript) { + // eslint-disable-next-line no-new-func + new Function(interpreterJavascript)() + } else { + console.warn('Unable to load VM.') + } + + const poTokenResult = await BG.PoToken.generate({ + program: challenge.program, + globalName: challenge.globalName, + bgConfig + }) + + return poTokenResult.poToken +} diff --git a/src/constants.js b/src/constants.js index 97be9daf6fe9f..13294dddd90fa 100644 --- a/src/constants.js +++ b/src/constants.js @@ -40,7 +40,9 @@ const IpcChannels = { PLAYER_CACHE_GET: 'player-cache-get', PLAYER_CACHE_SET: 'player-cache-set', - SET_INVIDIOUS_AUTHORIZATION: 'set-invidious-authorization' + SET_INVIDIOUS_AUTHORIZATION: 'set-invidious-authorization', + + GENERATE_PO_TOKEN: 'generate-po-token', } const DBActions = { diff --git a/src/main/index.js b/src/main/index.js index 806a99f0feef8..739ff9ee78f47 100644 --- a/src/main/index.js +++ b/src/main/index.js @@ -22,6 +22,7 @@ import { brotliDecompress } from 'zlib' import contextMenu from 'electron-context-menu' import packageDetails from '../../package.json' +import { generatePoToken } from './poTokenGenerator' const brotliDecompressAsync = promisify(brotliDecompress) @@ -427,24 +428,9 @@ function runApp() { requestHeaders.Referer = 'https://www.youtube.com/' requestHeaders.Origin = 'https://www.youtube.com' - // Make iOS requests work and look more realistic - if (requestHeaders['x-youtube-client-name'] === '5') { - delete requestHeaders.Referer - delete requestHeaders.Origin - delete requestHeaders['Sec-Fetch-Site'] - delete requestHeaders['Sec-Fetch-Mode'] - delete requestHeaders['Sec-Fetch-Dest'] - delete requestHeaders['sec-ch-ua'] - delete requestHeaders['sec-ch-ua-mobile'] - delete requestHeaders['sec-ch-ua-platform'] - - requestHeaders['User-Agent'] = requestHeaders['x-user-agent'] - delete requestHeaders['x-user-agent'] - } else { - requestHeaders['Sec-Fetch-Site'] = 'same-origin' - requestHeaders['Sec-Fetch-Mode'] = 'same-origin' - requestHeaders['X-Youtube-Bootstrap-Logged-In'] = 'false' - } + requestHeaders['Sec-Fetch-Site'] = 'same-origin' + requestHeaders['Sec-Fetch-Mode'] = 'same-origin' + requestHeaders['X-Youtube-Bootstrap-Logged-In'] = 'false' } else if (urlObj.origin.endsWith('.googlevideo.com') && urlObj.pathname === '/videoplayback') { requestHeaders.Referer = 'https://www.youtube.com/' requestHeaders.Origin = 'https://www.youtube.com' @@ -884,6 +870,10 @@ function runApp() { }) }) + ipcMain.handle(IpcChannels.GENERATE_PO_TOKEN, (_, visitorData) => { + return generatePoToken(visitorData) + }) + ipcMain.on(IpcChannels.ENABLE_PROXY, (_, url) => { session.defaultSession.setProxy({ proxyRules: url diff --git a/src/main/poTokenGenerator.js b/src/main/poTokenGenerator.js new file mode 100644 index 0000000000000..5871d7483975d --- /dev/null +++ b/src/main/poTokenGenerator.js @@ -0,0 +1,140 @@ +import { session, WebContentsView } from 'electron' +import { readFile } from 'fs/promises' +import { join } from 'path' + +/** + * Generates a poToken (proof of origin token) using `bgutils-js`. + * The script to generate it is `src/botGuardScript.js` + * + * This is intentionally split out into it's own thing, with it's own temporary in-memory session, + * as the BotGuard stuff accesses the global `document` and `window` objects and also requires making some requests. + * So we definitely don't want it running in the same places as the rest of the FreeTube code with the user data. + * @param {string} visitorData + * @returns {Promise} + */ +export async function generatePoToken(visitorData) { + const sessionUuid = crypto.randomUUID() + + const theSession = session.fromPartition(`potoken-${sessionUuid}`, { cache: false }) + + theSession.setPermissionCheckHandler(() => false) + // eslint-disable-next-line n/no-callback-literal + theSession.setPermissionRequestHandler((webContents, permission, callback) => callback(false)) + + theSession.setUserAgent( + theSession.getUserAgent() + .split(' ') + .filter(part => !part.includes('Electron')) + .join(' ') + ) + + const webContentsView = new WebContentsView({ + webPreferences: { + backgroundThrottling: false, + safeDialogs: true, + sandbox: true, + v8CacheOptions: 'none', + session: theSession, + offscreen: true + } + }) + + webContentsView.webContents.setWindowOpenHandler(() => ({ action: 'deny' })) + + webContentsView.webContents.setAudioMuted(true) + webContentsView.setBounds({ + x: 0, + y: 0, + width: 1920, + height: 1080 + }) + + webContentsView.webContents.debugger.attach() + + await webContentsView.webContents.loadURL('data:text/html,', { + baseURLForDataURL: 'https://www.youtube.com' + }) + + await webContentsView.webContents.debugger.sendCommand('Emulation.setUserAgentOverride', { + userAgent: theSession.getUserAgent(), + acceptLanguage: 'en-US', + platform: 'Win32', + userAgentMetadata: { + brands: [ + { + brand: 'Not/A)Brand', + version: '99' + }, + { + brand: 'Chromium', + version: process.versions.chrome.split('.')[0] + } + ], + fullVersionList: [ + { + brand: 'Not/A)Brand', + version: '99.0.0.0' + }, + { + brand: 'Chromium', + version: process.versions.chrome + } + ], + platform: 'Windows', + platformVersion: '10.0.0', + architecture: 'x86', + model: '', + mobile: false, + bitness: '64', + wow64: false + } + }) + + await webContentsView.webContents.debugger.sendCommand('Emulation.setDeviceMetricsOverride', { + width: 1920, + height: 1080, + deviceScaleFactor: 1, + mobile: false, + screenWidth: 1920, + screenHeight: 1080, + positionX: 0, + positionY: 0, + screenOrientation: { + type: 'landscapePrimary', + angle: 0 + } + }) + + const script = await getScript(visitorData) + + const response = await webContentsView.webContents.executeJavaScript(script) + + webContentsView.webContents.close({ waitForBeforeUnload: false }) + await theSession.closeAllConnections() + + return response +} + +let cachedScript + +/** + * @param {string} visitorData + */ +async function getScript(visitorData) { + if (!cachedScript) { + const pathToScript = process.env.NODE_ENV === 'development' + ? join(__dirname, '../../dist/botGuardScript.js') + /* eslint-disable-next-line n/no-path-concat */ + : `${__dirname}/botGuardScript.js` + + const content = await readFile(pathToScript, 'utf-8') + + const match = content.match(/export{(\w+) as default};/) + + const functionName = match[1] + + cachedScript = content.replace(match[0], `;${functionName}("FT_VISITOR_DATA")`) + } + + return cachedScript.replace('FT_VISITOR_DATA', visitorData) +} diff --git a/src/renderer/helpers/api/local.js b/src/renderer/helpers/api/local.js index d5ad4aeb1bf76..bd245a5c6f396 100644 --- a/src/renderer/helpers/api/local.js +++ b/src/renderer/helpers/api/local.js @@ -1,6 +1,6 @@ -import { ClientType, Innertube, Misc, Parser, UniversalCache, Utils, YT, YTNodes } from 'youtubei.js' +import { Innertube, Misc, Mixins, Parser, UniversalCache, Utils, YT, YTNodes } from 'youtubei.js' import Autolinker from 'autolinker' -import { SEARCH_CHAR_LIMIT } from '../../../constants' +import { IpcChannels, SEARCH_CHAR_LIMIT } from '../../../constants' import { PlayerCache } from './PlayerCache' import { @@ -10,7 +10,6 @@ import { extractNumberFromString, getChannelPlaylistId, getRelativeTimeFromDate, - randomArrayItem, } from '../utils' const TRACKING_PARAM_NAMES = [ @@ -21,25 +20,6 @@ const TRACKING_PARAM_NAMES = [ 'utm_content', ] -const IOS_VERSIONS = [ - '17.5.1', - '17.5', - '17.4.1', - '17.4', - '17.3.1', - '17.3', -] - -const YOUTUBE_IOS_CLIENT_VERSIONS = [ - '19.29.1', - '19.28.1', - '19.26.5', - '19.25.4', - '19.25.3', - '19.24.3', - '19.24.2', -] - /** * Creates a lightweight Innertube instance, which is faster to create or * an instance that can decode the streaming URLs, which is slower to create @@ -77,32 +57,7 @@ async function createInnertube({ withPlayer = false, location = undefined, safet client_type: clientType, // use browser fetch - fetch: (input, init) => { - // Make iOS requests work and look more realistic - if (init?.headers instanceof Headers && init.headers.get('x-youtube-client-name') === '5') { - // Use a random iOS version and YouTube iOS client version to make the requests look less suspicious - const clientVersion = randomArrayItem(YOUTUBE_IOS_CLIENT_VERSIONS) - const iosVersion = randomArrayItem(IOS_VERSIONS) - - init.headers.set('x-youtube-client-version', clientVersion) - - // We can't set the user-agent here, but in the main process we take the x-user-agent and set it as the user-agent - init.headers.delete('user-agent') - init.headers.set('x-user-agent', `com.google.ios.youtube/${clientVersion} (iPhone16,2; CPU iOS ${iosVersion.replaceAll('.', '_')} like Mac OS X; en_US)`) - - const bodyJson = JSON.parse(init.body) - - const client = bodyJson.context.client - - client.clientVersion = clientVersion - client.deviceModel = 'iPhone16,2' // iPhone 15 Pro Max - client.osVersion = iosVersion - - init.body = JSON.stringify(bodyJson) - } - - return fetch(input, init) - }, + fetch: (input, init) => fetch(input, init), cache, generate_session_locally: !!generateSessionLocally }) @@ -242,46 +197,61 @@ export async function getLocalSearchContinuation(continuationData) { export async function getLocalVideoInfo(id) { const webInnertube = await createInnertube({ withPlayer: true, generateSessionLocally: false }) + let poToken + + if (process.env.IS_ELECTRON) { + const { ipcRenderer } = require('electron') + + try { + poToken = await ipcRenderer.invoke(IpcChannels.GENERATE_PO_TOKEN, webInnertube.session.context.client.visitorData) + + webInnertube.session.po_token = poToken + webInnertube.session.player.po_token = poToken + } catch (error) { + console.error('Local API, poToken generation failed', error) + throw error + } + } + const info = await webInnertube.getInfo(id) const hasTrailer = info.has_trailer const trailerIsAgeRestricted = info.getTrailerInfo() === null - if (hasTrailer) { - /** @type {import('youtubei.js').YTNodes.PlayerLegacyDesktopYpcTrailer} */ - const trailerScreen = info.playability_status.error_screen - id = trailerScreen.video_id - } - if ((info.playability_status.status === 'UNPLAYABLE' && (!hasTrailer || trailerIsAgeRestricted)) || info.playability_status.status === 'LOGIN_REQUIRED') { return info } - const iosInnertube = await createInnertube({ clientType: ClientType.IOS }) - - const iosInfo = await iosInnertube.getBasicInfo(id, 'iOS') - if (hasTrailer) { - info.playability_status = iosInfo.playability_status - info.streaming_data = iosInfo.streaming_data - info.basic_info.start_timestamp = iosInfo.basic_info.start_timestamp - info.basic_info.duration = iosInfo.basic_info.duration - info.captions = iosInfo.captions - info.storyboards = iosInfo.storyboards - } else if (iosInfo.streaming_data) { - info.streaming_data.adaptive_formats = iosInfo.streaming_data.adaptive_formats - info.streaming_data.hls_manifest_url = iosInfo.streaming_data.hls_manifest_url + /** @type {import('youtubei.js').YTNodes.PlayerLegacyDesktopYpcTrailer} */ + const trailerScreen = info.playability_status.error_screen - // Use the legacy formats from the original web response as the iOS client doesn't have any legacy formats + const trailerInfo = new Mixins.MediaInfo([{ data: trailerScreen.trailer.player_response }]) - for (const format of info.streaming_data.adaptive_formats) { - format.freeTubeUrl = format.url - } + info.playability_status = trailerInfo.playability_status + info.streaming_data = trailerInfo.streaming_data + info.basic_info.start_timestamp = trailerInfo.basic_info.start_timestamp + info.basic_info.duration = trailerInfo.basic_info.duration + info.captions = trailerInfo.captions + info.storyboards = trailerInfo.storyboards } if (info.streaming_data) { decipherFormats(info.streaming_data.formats, webInnertube.actions.session.player) + decipherFormats(info.streaming_data.adaptive_formats, webInnertube.actions.session.player) + + if (info.streaming_data.dash_manifest_url) { + let url = info.streaming_data.dash_manifest_url + + if (url.includes('?')) { + url += `&pot=${encodeURIComponent(poToken)}&mpd_version=7` + } else { + url += `${url.endsWith('/') ? '' : '/'}pot/${encodeURIComponent(poToken)}/mpd_version/7` + } + + info.streaming_data.dash_manifest_url = url + } } return info diff --git a/src/renderer/views/Watch/Watch.js b/src/renderer/views/Watch/Watch.js index 91e7017ce42b0..0b43332b71716 100644 --- a/src/renderer/views/Watch/Watch.js +++ b/src/renderer/views/Watch/Watch.js @@ -566,26 +566,13 @@ export default defineComponent({ } if (useRemoteManifest) { - // The live DASH manifest is currently unusable it is not available on the iOS client - // but the web ones returns 403s after 1 minute of playback so we have to use the HLS one for now. - // Leaving the code here commented out in case we can use it again in the future - - // if (result.streaming_data.dash_manifest_url) { - // let src = result.streaming_data.dash_manifest_url - - // if (src.includes('?')) { - // src += '&mpd_version=7' - // } else { - // src += `${src.endsWith('/') ? '' : '/'}mpd_version/7` - // } - - // this.manifestSrc = src - // this.manifestMimeType = MANIFEST_TYPE_DASH - // } else { - - this.manifestSrc = result.streaming_data.hls_manifest_url - this.manifestMimeType = MANIFEST_TYPE_HLS - // } + if (result.streaming_data.dash_manifest_url) { + this.manifestSrc = result.streaming_data.dash_manifest_url + this.manifestMimeType = MANIFEST_TYPE_DASH + } else { + this.manifestSrc = result.streaming_data.hls_manifest_url + this.manifestMimeType = MANIFEST_TYPE_HLS + } } this.streamingDataExpiryDate = result.streaming_data.expires diff --git a/yarn.lock b/yarn.lock index ef7cb998d9b7f..3072a278862fd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2521,6 +2521,11 @@ batch@0.6.1: resolved "https://registry.yarnpkg.com/batch/-/batch-0.6.1.tgz#dc34314f4e679318093fc760272525f94bf25c16" integrity sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY= +bgutils-js@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/bgutils-js/-/bgutils-js-3.1.0.tgz#b06484e40a653fb42bd75c261f31e499b18e865a" + integrity sha512-2S80c/B4OQFubJLD5ddRRp74utrvjA70x9U0RsIVK7gJaDnaPrbw+bnXWxnEnc0euLznmO9jxOtTTC7FxGmv6w== + big.js@^5.2.2: version "5.2.2" resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328"