From 70961822098cabcd3c926b75bda0f090185a141c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?WangLiang/=E7=8E=8B=E8=89=AF?= <841369634@qq.com> Date: Tue, 26 Mar 2024 12:08:47 +0800 Subject: [PATCH] =?UTF-8?q?optimize:=20=E4=BC=98=E5=8C=96=20`util.match.js?= =?UTF-8?q?`=EF=BC=8C=E7=AE=80=E5=8C=96=E9=85=8D=E7=BD=AE=EF=BC=8C?= =?UTF-8?q?=E5=90=8C=E6=97=B6=E4=BF=AE=E5=A4=8D=E5=8C=B9=E9=85=8D=E9=94=99?= =?UTF-8?q?=E4=B9=B1=E7=9A=84=E9=97=AE=E9=A2=98=E3=80=82=20(#279)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/src/config.js | 14 +- packages/core/src/config/index.js | 33 ++-- .../core/src/modules/plugin/node/index.js | 2 +- .../src/modules/plugin/overwall/config.js | 43 ++--- packages/core/src/modules/proxy/index.js | 2 +- packages/core/src/modules/server/index.js | 12 +- .../src/shell/scripts/extra-path/index.js | 2 +- .../shell/scripts/set-system-proxy/index.js | 9 +- packages/core/src/shell/shell.js | 7 +- packages/core/src/shell/test.js | 10 +- packages/core/src/utils/util.proxy.js | 3 - .../src/lib/interceptor/impl/res/cacheRes.js | 1 - .../src/lib/proxy/middleware/overwall.js | 22 ++- .../proxy/mitmproxy/createConnectHandler.js | 4 +- packages/mitmproxy/src/options.js | 15 +- packages/mitmproxy/src/utils/util.match.js | 155 ++++++++++++++++-- .../mitmproxy/src/utils/util.match.old.js | 53 ++++++ packages/mitmproxy/test/matchUtilTest.js | 53 ++++++ 18 files changed, 336 insertions(+), 104 deletions(-) delete mode 100644 packages/core/src/utils/util.proxy.js create mode 100644 packages/mitmproxy/src/utils/util.match.old.js create mode 100644 packages/mitmproxy/test/matchUtilTest.js diff --git a/packages/core/src/config.js b/packages/core/src/config.js index c544f4714..00c40c888 100644 --- a/packages/core/src/config.js +++ b/packages/core/src/config.js @@ -29,7 +29,7 @@ function _getConfigPath () { if (!fs.existsSync(dir)) { fs.mkdirSync(dir) } - return dir + '/config.json' + return path.join(dir, 'config.json') } let timer @@ -43,7 +43,7 @@ const configApi = { await configApi.downloadRemoteConfig() configApi.reload() } catch (e) { - log.error(e) + log.error('定时下载远程配置并重载配置失败', e) } } await download() @@ -56,10 +56,10 @@ const configApi = { const remoteConfigUrl = get().app.remoteConfig.url // eslint-disable-next-line handle-callback-err return new Promise((resolve, reject) => { - log.info('下载远程配置：', remoteConfigUrl) + log.info('开始下载远程配置:', remoteConfigUrl) request(remoteConfigUrl, (error, response, body) => { if (error) { - log.error('下载远程配置失败', error) + log.error('下载远程配置失败, error:', error, ', response:', response, ', body:', body) reject(error) return } @@ -73,7 +73,7 @@ const configApi = { try { remoteConfig = JSON5.parse(body) } catch (e) { - log.error('远程配置内容格式不正确:', body) + log.error(`远程配置内容格式不正确, url: ${remoteConfigUrl}, body: ${body}`) remoteConfig = null } @@ -96,8 +96,8 @@ const configApi = { if (get().app.remoteConfig.enabled !== true) { return {} } + const path = _getRemoteSavePath() try { - const path = _getRemoteSavePath() if (fs.existsSync(path)) { log.info('读取远程配置文件:', path) const file = fs.readFileSync(path) @@ -106,7 +106,7 @@ const configApi = { log.warn('远程配置文件不存在:', path) } } catch (e) { - log.warn('远程配置读取失败:', e) + log.warn('远程配置读取失败:', path, ', error:', e) } return {} diff --git a/packages/core/src/config/index.js b/packages/core/src/config/index.js index 940def0a6..f1b8d78ae 100644 --- a/packages/core/src/config/index.js +++ b/packages/core/src/config/index.js @@ -229,17 +229,11 @@ module.exports = { '*.cn': true, 'cn.*': true, '*china*': true, - 'dingtalk.com': true, '*.dingtalk.com': true, - 'apple.com': true, '*.apple.com': true, - 'microsoft.com': true, '*.microsoft.com': true, - 'alipay.com': true, '*.alipay.com': true, - 'qq.com': true, '*.qq.com': true, - 'baidu.com': true, '*.baidu.com': true }, sniList: { @@ -269,21 +263,24 @@ module.exports = { } }, mapping: { - '*github*.com': 'quad9', - '*github.io': 'quad9', - '*stackoverflow.com': 'quad9', + '*.github.com': 'quad9', + '*.*github*.com': 'quad9', + '*.github.io': 'quad9', + '*.docker.com': 'quad9', + '*.docker*.com': 'quad9', + '*.stackoverflow.com': 'quad9', '*.electronjs.org': 'quad9', - '*amazonaws.com': 'quad9', - '*yarnpkg.com': 'quad9', - '*cloudfront.net': 'quad9', - '*cloudflare.com': 'quad9', + '*.amazonaws.com': 'quad9', + '*.yarnpkg.com': 'quad9', + '*.cloudfront.net': 'quad9', + '*.cloudflare.com': 'quad9', 'img.shields.io': 'quad9', '*.vuepress.vuejs.org': 'quad9', - 'gh.docmirror.top': 'quad9', - '*v2ex.com': 'quad9', - '*pypi.org': 'quad9', - '*jetbrains.com': 'quad9', - '*azureedge.net': 'quad9' + '*.gh.docmirror.top': 'quad9', + '*.v2ex.com': 'quad9', + '*.pypi.org': 'quad9', + '*.jetbrains.com': 'quad9', + '*.azureedge.net': 'quad9' }, speedTest: { enabled: true, diff --git a/packages/core/src/modules/plugin/node/index.js b/packages/core/src/modules/plugin/node/index.js index 2380b0319..fa7fc0fe3 100644 --- a/packages/core/src/modules/plugin/node/index.js +++ b/packages/core/src/modules/plugin/node/index.js @@ -6,7 +6,7 @@ const NodePlugin = function (context) { try { await nodeApi.setVariables() } catch (err) { - log.warn('set variables error', err) + log.warn('set variables error:', err) } const ip = '127.0.0.1' diff --git a/packages/core/src/modules/plugin/overwall/config.js b/packages/core/src/modules/plugin/overwall/config.js index 62e6b8e61..edbafb981 100644 --- a/packages/core/src/modules/plugin/overwall/config.js +++ b/packages/core/src/modules/plugin/overwall/config.js @@ -10,30 +10,31 @@ module.exports = { } }, targets: { + '*.github.com': true, '*github*.com': true, - '*wikimedia.org': true, - 'v2ex.com': true, - '*azureedge.net': true, - '*cloudfront.net': true, - '*bing.com': true, - '*discourse-cdn.com': true, - '*gravatar.com': true, - '*docker.com': true, - '*vueuse.org': true, - '*elastic.co': true, - '*optimizely.com': true, - '*stackpathcdn.com': true, - '*fastly.net': true, - '*cloudflare.com': true, - '*233v2.com': true, - '*v2fly.org': true, - '*telegram.org': true, - '*amazon.com': true, - '*googleapis.com': true, + '*.wikimedia.org': true, + '*.v2ex.com': true, + '*.azureedge.net': true, + '*.cloudfront.net': true, + '*.bing.com': true, + '*.discourse-cdn.com': true, + '*.gravatar.com': true, + '*.docker.com': true, + '*.vueuse.org': true, + '*.elastic.co': true, + '*.optimizely.com': true, + '*.stackpathcdn.com': true, + '*.fastly.net': true, + '*.cloudflare.com': true, + '*.233v2.com': true, + '*.v2fly.org': true, + '*.telegram.org': true, + '*.amazon.com': true, + '*.googleapis.com': true, '*.google-analytics.com': true, - '*cloudflareinsights.com': true, + '*.cloudflareinsights.com': true, '*.intlify.dev': true, - '*segment.io': true, + '*.segment.io': true, '*.shields.io': true, '*.jsdelivr.net': true }, diff --git a/packages/core/src/modules/proxy/index.js b/packages/core/src/modules/proxy/index.js index 6c4bfa9ac..3854fffe6 100644 --- a/packages/core/src/modules/proxy/index.js +++ b/packages/core/src/modules/proxy/index.js @@ -34,7 +34,7 @@ const ProxyPlugin = function (context) { log.info('关闭系统代理成功') return true } catch (err) { - log.error('关闭系统代理失败', err) + log.error('关闭系统代理失败:', err) return false } }, diff --git a/packages/core/src/modules/server/index.js b/packages/core/src/modules/server/index.js index 18cc11e90..4fb519ecc 100644 --- a/packages/core/src/modules/server/index.js +++ b/packages/core/src/modules/server/index.js @@ -86,16 +86,16 @@ const serverApi = { } } serverProcess.on('beforeExit', (code) => { - log.warn('server process beforeExit', code) + log.warn('server process beforeExit, code:', code) }) serverProcess.on('SIGPIPE', (code, signal) => { - log.warn('server process SIGPIPE', code, signal) + log.warn(`server process SIGPIPE, code: ${code}, signal:`, signal) }) serverProcess.on('exit', (code, signal) => { - log.warn('server process exit', code, signal) + log.warn(`server process exit, code: ${code}, signal:`, signal) }) serverProcess.on('uncaughtException', (err, origin) => { - log.error('server process uncaughtException', err) + log.error('server process uncaughtException:', err) }) serverProcess.on('message', function (msg) { log.info('收到子进程消息', msg.type, msg.event.key, msg.message) @@ -130,13 +130,13 @@ const serverApi = { // fireStatus('ing')// 关闭中 server.close((err) => { if (err) { - log.warn('close error', err, ',', err.code, ',', err.message, ',', err.errno) + log.warn('close error:', err) if (err.code === 'ERR_SERVER_NOT_RUNNING') { log.info('代理服务关闭成功') resolve() return } - log.warn('代理服务关闭失败', err) + log.warn('代理服务关闭失败:', err) reject(err) } else { log.info('代理服务关闭成功') diff --git a/packages/core/src/shell/scripts/extra-path/index.js b/packages/core/src/shell/scripts/extra-path/index.js index eb890d2fb..959d4409a 100644 --- a/packages/core/src/shell/scripts/extra-path/index.js +++ b/packages/core/src/shell/scripts/extra-path/index.js @@ -3,7 +3,7 @@ const path = require('path') function getExtraPath () { let extraPath = process.env.DS_EXTRA_PATH - log.info('extraPath', extraPath) + log.info('extraPath:', extraPath) if (!extraPath) { extraPath = __dirname } diff --git a/packages/core/src/shell/scripts/set-system-proxy/index.js b/packages/core/src/shell/scripts/set-system-proxy/index.js index bd3ece910..4cb259fa4 100644 --- a/packages/core/src/shell/scripts/set-system-proxy/index.js +++ b/packages/core/src/shell/scripts/set-system-proxy/index.js @@ -49,7 +49,10 @@ async function _winSetProxy (exec, ip, port, setEnv) { } const proxyPath = extraPath.getProxyExePath() - await execFile(proxyPath, ['global', `http=http://${ip}:${port};https=http://${ip}:${port}`, excludeIpStr]) + const execFun = 'global' + const proxyAddr = `http=http://${ip}:${port};https=http://${ip}:${port}` + log.info(`执行“设置系统代理”的程序: ${proxyPath} ${execFun} ${proxyAddr} ......(省略排除IP列表)`) + await execFile(proxyPath, [execFun, proxyAddr, excludeIpStr]) if (setEnv) { log.info('同时设置 https_proxy') @@ -71,11 +74,11 @@ const executor = { const { ip, port, setEnv } = params if (ip == null) { // 清空代理 - log.info('关闭代理') + log.info('关闭windows系统代理') return _winUnsetProxy(exec, setEnv) } else { // 设置代理 - log.info('设置代理:', ip, port, setEnv) + log.info('设置windows系统代理:', ip, port, setEnv) return _winSetProxy(exec, ip, port, setEnv) } }, diff --git a/packages/core/src/shell/shell.js b/packages/core/src/shell/shell.js index 80699c7d7..9cee8ebda 100644 --- a/packages/core/src/shell/shell.js +++ b/packages/core/src/shell/shell.js @@ -1,9 +1,6 @@ -const util = require('util') const os = require('os') const childProcess = require('child_process') -const _exec = childProcess.exec const _execFile = childProcess.execFile -const exec = util.promisify(_exec) const PowerShell = require('node-powershell') const log = require('../utils/util.log') const fixPath = require('fix-path') @@ -96,8 +93,8 @@ function _childExec (composeCmds, options = {}) { function childExec (composeCmds) { return new Promise((resolve, reject) => { - var encoding = 'cp936' - var binaryEncoding = 'binary' + const encoding = 'cp936' + const binaryEncoding = 'binary' const childProcess = require('child_process') log.info('shell:', composeCmds) diff --git a/packages/core/src/shell/test.js b/packages/core/src/shell/test.js index df9c3b4bc..6217899e6 100644 --- a/packages/core/src/shell/test.js +++ b/packages/core/src/shell/test.js @@ -10,18 +10,18 @@ // } // ) -// var process = require('child_process') +// const process = require('child_process') // -// var cmd = 'set' +// const cmd = 'set' // process.exec(cmd, function (error, stdout, stderr) { // console.log('error:' + error) // console.log('stdout:' + stdout) // console.log('stderr:' + stderr) // }) -// var HttpsProxyAgent = require('https-proxy-agent') -// var proxy = 'http://user:pass@xxx.com:port' -// var agent = new HttpsProxyAgent(proxy) +// const HttpsProxyAgent = require('https-proxy-agent') +// const proxy = 'http://user:pass@xxx.com:port' +// const agent = new HttpsProxyAgent(proxy) // const https = require('https') // https.get('https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js', (res) => { // console.log('状态码:', res.statusCode) diff --git a/packages/core/src/utils/util.proxy.js b/packages/core/src/utils/util.proxy.js deleted file mode 100644 index 8dd2cbd50..000000000 --- a/packages/core/src/utils/util.proxy.js +++ /dev/null @@ -1,3 +0,0 @@ -const proxyConfig = require('@docmirror/mitmproxy/config.js') -module.exports = { -} diff --git a/packages/mitmproxy/src/lib/interceptor/impl/res/cacheRes.js b/packages/mitmproxy/src/lib/interceptor/impl/res/cacheRes.js index a62a28894..2fa911e92 100644 --- a/packages/mitmproxy/src/lib/interceptor/impl/res/cacheRes.js +++ b/packages/mitmproxy/src/lib/interceptor/impl/res/cacheRes.js @@ -81,7 +81,6 @@ module.exports = { } res.setHeader('Dev-Sidecar-Cache-Response-Interceptor', 'cacheRes:maxAge=' + maxAge) - log.info('[cacheRes]', 'maxAge=' + maxAge) }, is (interceptOpt) { const maxAge = cacheReq.getMaxAge(interceptOpt) diff --git a/packages/mitmproxy/src/lib/proxy/middleware/overwall.js b/packages/mitmproxy/src/lib/proxy/middleware/overwall.js index 22af3a6d8..57bf3790c 100644 --- a/packages/mitmproxy/src/lib/proxy/middleware/overwall.js +++ b/packages/mitmproxy/src/lib/proxy/middleware/overwall.js @@ -1,10 +1,12 @@ const url = require('url') +const lodash = require('lodash') const pac = require('./source/pac') const matchUtil = require('../../../utils/util.match') +const log = require('../../../utils/util.log') let pacClient = null -function matched (hostname, regexpMap) { - const ret1 = matchUtil.matchHostname(regexpMap, hostname) +function matched (hostname, overWallTargetMap) { + const ret1 = matchUtil.matchHostname(overWallTargetMap, hostname, 'matched overwall') if (ret1) { return true } @@ -13,9 +15,12 @@ function matched (hostname, regexpMap) { } const ret = pacClient.FindProxyForURL('https://' + hostname, hostname) if (ret && ret.indexOf('PROXY ') === 0) { + log.info(`matchHostname: matched overwall: '${hostname}' -> '${ret}' in pac.txt`) return true + } else { + // log.debug(`matchHostname: matched overwall: Not-Matched '${hostname}' -> '${ret}' in pac.txt`) + return false } - return false } module.exports = function createOverWallIntercept (overWallConfig) { @@ -36,11 +41,11 @@ module.exports = function createOverWallIntercept (overWallConfig) { if (keys.length === 0) { return null } - const regexpMap = matchUtil.domainMapRegexply(overWallConfig.targets) + const overWallTargetMap = matchUtil.domainMapRegexply(overWallConfig.targets) return { sslConnectInterceptor: (req, cltSocket, head) => { const hostname = req.url.split(':')[0] - return matched(hostname, regexpMap) + return matched(hostname, overWallTargetMap) }, requestIntercept (context, req, res, ssl, next) { const { rOptions, log, RequestCounter } = context @@ -48,7 +53,7 @@ module.exports = function createOverWallIntercept (overWallConfig) { return } const hostname = rOptions.hostname - if (!matched(hostname, regexpMap)) { + if (!matched(hostname, overWallTargetMap)) { return } const cacheKey = '__over_wall_proxy__' @@ -81,6 +86,9 @@ module.exports = function createOverWallIntercept (overWallConfig) { const proxy = proxyTarget.indexOf('http') === 0 ? proxyTarget : (rOptions.protocol + '//' + proxyTarget) // eslint-disable-next-line node/no-deprecated-api const URL = url.parse(proxy) + rOptions.origional = lodash.cloneDeep(rOptions) // 备份原始请求参数 + delete rOptions.origional.agent + delete rOptions.origional.headers rOptions.protocol = URL.protocol rOptions.hostname = URL.host rOptions.host = URL.host @@ -92,7 +100,7 @@ module.exports = function createOverWallIntercept (overWallConfig) { if (URL.port == null) { rOptions.port = port || (rOptions.protocol === 'https:' ? 443 : 80) } - log.info('OverWall:', rOptions.hostname, proxyTarget) + log.info('OverWall:', rOptions.hostname, '➜', proxyTarget) if (context.requestCount) { log.debug('OverWall choice:', JSON.stringify(context.requestCount)) } diff --git a/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js b/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js index a3c7e00e7..4c0158d23 100644 --- a/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js +++ b/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js @@ -1,13 +1,11 @@ const net = require('net') const url = require('url') const log = require('../../../utils/util.log') -// const colors = require('colors') const DnsUtil = require('../../dns/index') const localIP = '127.0.0.1' const defaultDns = require('dns') const matchUtil = require('../../../utils/util.match') const speedTest = require('../../speed/index.js') -const sniExtract = require('../tls/sniUtil.js') function isSslConnect (sslConnectInterceptors, req, cltSocket, head) { for (const intercept of sslConnectInterceptors) { const ret = intercept(req, cltSocket, head) @@ -57,7 +55,7 @@ function connect (req, cltSocket, head, hostname, port, dnsConfig, sniRegexpMap) // log.info('connect:', hostname, port) const start = new Date().getTime() let isDnsIntercept = null - // const replaceSni = matchUtil.matchHostname(sniRegexpMap, hostname) + // const replaceSni = matchUtil.matchHostname(sniRegexpMap, hostname, 'sni') try { const options = { port, diff --git a/packages/mitmproxy/src/options.js b/packages/mitmproxy/src/options.js index a95bb7dba..1a1cbe0e9 100644 --- a/packages/mitmproxy/src/options.js +++ b/packages/mitmproxy/src/options.js @@ -24,17 +24,22 @@ module.exports = (config) => { if (!overwallConfig.pac.pacFileAbsolutePath) { overwallConfig.pac.pacFileAbsolutePath = path.join(setting.rootDir, overwallConfig.pac.pacFilePath) } - const overwallMiddleware = createOverwallMiddleware(overwallConfig) + + // 插件列表 const middlewares = [] + + // 梯子插件：如果启用了，则添加到插件列表中 + const overwallMiddleware = createOverwallMiddleware(overwallConfig) if (overwallMiddleware) { middlewares.push(overwallMiddleware) } + const options = { host: serverConfig.host, port: serverConfig.port, dnsConfig: { providers: dnsUtil.initDNS(serverConfig.dns.providers), - mapping: dnsMapping, + mapping: matchUtil.domainMapRegexply(dnsMapping), speedTest: config.dns.speedTest }, setting, @@ -42,13 +47,13 @@ module.exports = (config) => { middlewares, sslConnectInterceptor: (req, cltSocket, head) => { const hostname = req.url.split(':')[0] - const inWhiteList = matchUtil.matchHostname(whiteList, hostname) != null + const inWhiteList = matchUtil.matchHostname(whiteList, hostname, 'in whiteList') != null if (inWhiteList) { log.info('白名单域名，不拦截', hostname) return false // 所有都不拦截 } // 配置了拦截的域名，将会被代理 - const matched = !!matchUtil.matchHostname(intercepts, hostname) + const matched = !!matchUtil.matchHostname(intercepts, hostname, 'matched intercepts') if (matched === true) { return matched // 拦截 } @@ -57,7 +62,7 @@ module.exports = (config) => { createIntercepts: (context) => { const rOptions = context.rOptions const hostname = rOptions.hostname - const interceptOpts = matchUtil.matchHostname(intercepts, hostname) + const interceptOpts = matchUtil.matchHostname(intercepts, hostname, 'get interceptOpts') if (!interceptOpts) { // 该域名没有配置拦截器，直接过 return } diff --git a/packages/mitmproxy/src/utils/util.match.js b/packages/mitmproxy/src/utils/util.match.js index eaaed0872..e5ebd6d43 100644 --- a/packages/mitmproxy/src/utils/util.match.js +++ b/packages/mitmproxy/src/utils/util.match.js @@ -1,51 +1,172 @@ const lodash = require('lodash') +const log = require('./util.log') + function isMatched (url, regexp) { return url.match(regexp) } function domainRegexply (target) { - return target.replace(/\./g, '\\.').replace(/\*/g, '.*') + return '^' + target.replace(/\./g, '\\.').replace(/\*/g, '.*') + '$' } function domainMapRegexply (hostMap) { const regexpMap = {} + const origin = {} // 用于快速匹配，见matchHostname、matchHostnameAll方法 if (hostMap == null) { return regexpMap } lodash.each(hostMap, (value, domain) => { - if (domain.indexOf('*') >= 0) { - const regDomain = domainRegexply(domain) + if (domain.indexOf('*') >= 0 || domain[0] === '^') { + const regDomain = domain[0] !== '^' ? domainRegexply(domain) : domain regexpMap[regDomain] = value + + if (domain.indexOf('*') === 0 && domain.lastIndexOf('*') === 0) { + origin[domain] = value + } } else { - regexpMap[domain] = value + origin[domain] = value } }) + regexpMap.origin = origin return regexpMap } -function matchHostname (hostMap, hostname) { +function matchHostname (hostMap, hostname, action) { + // log.error('matchHostname:', action, hostMap) + if (hostMap == null) { + log.warn(`matchHostname: ${action}: '${hostname}' Not-Matched, hostMap is null`) return null } - const value = hostMap[hostname] + if (hostMap.origin == null) { + log.warn(`matchHostname: ${action}: '${hostname}' Not-Matched, hostMap.origin is null`) + return null + } + + // 域名快速匹配：直接匹配 或者 两种前缀通配符匹配 + let value = hostMap.origin[hostname] if (value) { - return value + log.info(`matchHostname: ${action}: '${hostname}' -> '${hostname}': ${JSON.stringify(value)}`) + return value // 快速匹配成功 } - if (!value) { - for (const target in hostMap) { - if (target.indexOf('*') < 0) { - continue - } - // 正则表达式匹配 - if (hostname.match(target)) { - return hostMap[target] - } + value = hostMap.origin['*' + hostname] + if (value) { + log.info(`matchHostname: ${action}: '${hostname}' -> '*${hostname}': ${JSON.stringify(value)}`) + return value // 快速匹配成功 + } + value = hostMap.origin['*.' + hostname] + if (value) { + log.info(`matchHostname: ${action}: '${hostname}' -> '*.${hostname}': ${JSON.stringify(value)}`) + return value // 快速匹配成功 + } + + // 通配符匹配 或 正则表达式匹配 + for (const target in hostMap) { + if (target === 'origin') { + continue + } + + // if (target.indexOf('*') < 0 && target[0] !== '^') { + // continue // 不是通配符匹配串，也不是正则表达式，跳过 + // } + + // 如果是通配符匹配串，转换为正则表达式 + let regexp = target + // if (target[0] !== '^') { + // regexp = domainRegexply(regexp) + // } + + // 正则表达式匹配 + if (hostname.match(regexp)) { + value = hostMap[target] + log.info(`matchHostname: ${action}: '${hostname}' -> '${target}': ${JSON.stringify(value)}`) + return value } } + + log.debug(`matchHostname: ${action}: '${hostname}' Not-Matched`) } + +function merge (oldObj, newObj) { + return lodash.mergeWith(oldObj, newObj, function (objValue, srcValue) { + if (lodash.isArray(objValue)) { + return srcValue + } + }) +} + +function matchHostnameAll (hostMap, hostname, action) { + // log.debug('matchHostnameAll:', action, hostMap) + + if (hostMap == null) { + log.warn(`matchHostnameAll: ${action}: '${hostname}', hostMap is null`) + return null + } + if (hostMap.origin == null) { + log.warn(`matchHostnameAll: ${action}: '${hostname}', hostMap.origin is null`) + return null + } + + let values = {} + let hasValue = false + + // 域名快速匹配：直接匹配 或者 两种前缀通配符匹配 + let value = hostMap.origin[hostname] + if (value) { + log.info(`matchHostnameAll: ${action}: '${hostname}' -> '${hostname}': ${JSON.stringify(value)}`) + values = merge(values, value) + hasValue = true + } + value = hostMap.origin['*' + hostname] + if (value) { + log.info(`matchHostnameAll: ${action}: '${hostname}' -> '*${hostname}': ${JSON.stringify(value)}`) + values = merge(values, value) + hasValue = true + } + value = hostMap.origin['*.' + hostname] + if (value) { + log.info(`matchHostnameAll: ${action}: '${hostname}' -> '*.${hostname}': ${JSON.stringify(value)}`) + values = merge(values, value) + hasValue = true + } + + // 通配符匹配 或 正则表达式匹配 + for (const target in hostMap) { + if (target === 'origin') { + continue + } + + // if (target.indexOf('*') < 0 && target[0] !== '^') { + // continue // 不是通配符匹配串，也不是正则表达式，跳过 + // } + + // 如果是通配符匹配串，转换为正则表达式 + let regexp = target + // if (target[0] !== '^') { + // regexp = domainRegexply(regexp) + // } + + // 正则表达式匹配 + if (hostname.match(regexp)) { + value = hostMap[target] + // log.info(`matchHostname: ${action}: '${hostname}' -> '${target}': ${JSON.stringify(value)}`) + values = merge(values, value) + hasValue = true + } + } + + if (hasValue) { + log.info(`*matchHostnameAll*: ${action}: '${hostname}':`, JSON.stringify(values)) + return values + } else { + log.debug(`*matchHostnameAll*: ${action}: '${hostname}' Not-Matched`) + } +} + module.exports = { isMatched, domainRegexply, domainMapRegexply, - matchHostname + matchHostname, + matchHostnameAll } diff --git a/packages/mitmproxy/src/utils/util.match.old.js b/packages/mitmproxy/src/utils/util.match.old.js new file mode 100644 index 000000000..ff2b1fc7d --- /dev/null +++ b/packages/mitmproxy/src/utils/util.match.old.js @@ -0,0 +1,53 @@ +// 警告：此文件不再使用，仅用于测试，可在 test/matchUtilTest.js 中比对新逻辑与旧逻辑的效果差异 + +const lodash = require('lodash') +function isMatched (url, regexp) { + return url.match(regexp) +} + +function domainRegexply (target) { + return target.replace(/\./g, '\\.').replace(/\*/g, '.*') +} + +function domainMapRegexply (hostMap) { + const regexpMap = {} + if (hostMap == null) { + return regexpMap + } + lodash.each(hostMap, (value, domain) => { + if (domain.indexOf('*') >= 0) { + const regDomain = domainRegexply(domain) + regexpMap[regDomain] = value + } else { + regexpMap[domain] = value + } + }) + return regexpMap +} + +function matchHostname (hostMap, hostname) { + if (hostMap == null) { + return null + } + const value = hostMap[hostname] + if (value) { + return value + } + if (!value) { + for (const target in hostMap) { + if (target.indexOf('*') < 0) { + continue + } + // 正则表达式匹配 + if (hostname.match(target)) { + return hostMap[target] + } + } + } +} +module.exports = { + isMatched, + domainRegexply, + domainMapRegexply, + matchHostname +} diff --git a/packages/mitmproxy/test/matchUtilTest.js b/packages/mitmproxy/test/matchUtilTest.js new file mode 100644 index 000000000..1d05b37e9 --- /dev/null +++ b/packages/mitmproxy/test/matchUtilTest.js @@ -0,0 +1,53 @@ +const matchUtil = require('../src/utils/util.match') + +const hostMap = matchUtil.domainMapRegexply({ + 'aaa.com': true, + '*bbb.com': true, + '*.ccc.com': true, + '^.{1,3}ddd.com$': true, + '*.cn': true +}) + +console.log(hostMap) + +console.log('test1: aaa.com') +const value11 = matchUtil.matchHostname(hostMap, 'aaa.com', 'test1.1') +const value12 = matchUtil.matchHostname(hostMap, 'aaaa.com', 'test1.2') +const value13 = matchUtil.matchHostname(hostMap, 'aaaa.comx', 'test1.3') +console.log(value11) // true +console.log(value12) // undefined +console.log(value13) // undefined + +console.log('test2: *bbb.com') +const value21 = matchUtil.matchHostname(hostMap, 'bbb.com', 'test2.1') +const value22 = matchUtil.matchHostname(hostMap, 'xbbb.com', 'test2.2') +const value23 = matchUtil.matchHostname(hostMap, 'bbb.comx', 'test2.3') +const value24 = matchUtil.matchHostname(hostMap, 'x.bbb.com', 'test2.4') +console.log(value21) // true +console.log(value22) // true +console.log(value23) // undefined +console.log(value24) // true + +console.log('test3: *.ccc.com') +const value31 = matchUtil.matchHostname(hostMap, 'ccc.com', 'test3.1') +const value32 = matchUtil.matchHostname(hostMap, 'x.ccc.com', 'test3.2') +const value33 = matchUtil.matchHostname(hostMap, 'xccc.com', 'test3.3') +console.log(value31) // true +console.log(value32) // true +console.log(value33) // undefined + +console.log('test4: ^.{1,3}ddd.com$') +const value41 = matchUtil.matchHostname(hostMap, 'ddd.com', 'test4.1') +const value42 = matchUtil.matchHostname(hostMap, 'x.ddd.com', 'test4.2') +const value43 = matchUtil.matchHostname(hostMap, 'xddd.com', 'test4.3') +console.log(value41) // undefined +console.log(value42) // true +console.log(value43) // true + +console.log('test5: *.cn') +const value51 = matchUtil.matchHostname(hostMap, 'eee.cn', 'test5.1') +const value52 = matchUtil.matchHostname(hostMap, 'x.eee.cn', 'test5.2') +const value53 = matchUtil.matchHostname(hostMap, 'aaaa.cnet.com', 'test5.3') +console.log(value51) // true +console.log(value52) // true +console.log(value53) // undefined