optimize: 优化 util.match.js，简化配置，同时修复匹配错乱的问题。 (docmirror#279)

tanxingsong · Mar 28, 2024 · 7096182 · 7096182
1 parent cf65f34
commit 7096182
Show file tree

Hide file tree

Showing 18 changed files with 336 additions and 104 deletions.
diff --git a/packages/core/src/config.js b/packages/core/src/config.js
@@ -29,7 +29,7 @@ function _getConfigPath () {
   if (!fs.existsSync(dir)) {
     fs.mkdirSync(dir)
   }
-  return dir + '/config.json'
+  return path.join(dir, 'config.json')
 }
 
 let timer
@@ -43,7 +43,7 @@ const configApi = {
         await configApi.downloadRemoteConfig()
         configApi.reload()
       } catch (e) {
-        log.error(e)
+        log.error('定时下载远程配置并重载配置失败', e)
       }
     }
     await download()
@@ -56,10 +56,10 @@ const configApi = {
     const remoteConfigUrl = get().app.remoteConfig.url
     // eslint-disable-next-line handle-callback-err
     return new Promise((resolve, reject) => {
-      log.info('下载远程配置：', remoteConfigUrl)
+      log.info('开始下载远程配置:', remoteConfigUrl)
       request(remoteConfigUrl, (error, response, body) => {
         if (error) {
-          log.error('下载远程配置失败', error)
+          log.error('下载远程配置失败, error:', error, ', response:', response, ', body:', body)
           reject(error)
           return
         }
@@ -73,7 +73,7 @@ const configApi = {
           try {
             remoteConfig = JSON5.parse(body)
           } catch (e) {
-            log.error('远程配置内容格式不正确:', body)
+            log.error(`远程配置内容格式不正确, url: ${remoteConfigUrl}, body: ${body}`)
             remoteConfig = null
           }
 
@@ -96,8 +96,8 @@ const configApi = {
     if (get().app.remoteConfig.enabled !== true) {
       return {}
     }
+    const path = _getRemoteSavePath()
     try {
-      const path = _getRemoteSavePath()
       if (fs.existsSync(path)) {
         log.info('读取远程配置文件:', path)
         const file = fs.readFileSync(path)
@@ -106,7 +106,7 @@ const configApi = {
         log.warn('远程配置文件不存在:', path)
       }
     } catch (e) {
-      log.warn('远程配置读取失败:', e)
+      log.warn('远程配置读取失败:', path, ', error:', e)
     }
 
     return {}

diff --git a/packages/core/src/config/index.js b/packages/core/src/config/index.js
@@ -229,17 +229,11 @@ module.exports = {
       '*.cn': true,
       'cn.*': true,
       '*china*': true,
-      'dingtalk.com': true,
       '*.dingtalk.com': true,
-      'apple.com': true,
       '*.apple.com': true,
-      'microsoft.com': true,
       '*.microsoft.com': true,
-      'alipay.com': true,
       '*.alipay.com': true,
-      'qq.com': true,
       '*.qq.com': true,
-      'baidu.com': true,
       '*.baidu.com': true
     },
     sniList: {
@@ -269,21 +263,24 @@ module.exports = {
         }
       },
       mapping: {
-        '*github*.com': 'quad9',
-        '*github.io': 'quad9',
-        '*stackoverflow.com': 'quad9',
+        '*.github.com': 'quad9',
+        '*.*github*.com': 'quad9',
+        '*.github.io': 'quad9',
+        '*.docker.com': 'quad9',
+        '*.docker*.com': 'quad9',
+        '*.stackoverflow.com': 'quad9',
         '*.electronjs.org': 'quad9',
-        '*amazonaws.com': 'quad9',
-        '*yarnpkg.com': 'quad9',
-        '*cloudfront.net': 'quad9',
-        '*cloudflare.com': 'quad9',
+        '*.amazonaws.com': 'quad9',
+        '*.yarnpkg.com': 'quad9',
+        '*.cloudfront.net': 'quad9',
+        '*.cloudflare.com': 'quad9',
         'img.shields.io': 'quad9',
         '*.vuepress.vuejs.org': 'quad9',
-        'gh.docmirror.top': 'quad9',
-        '*v2ex.com': 'quad9',
-        '*pypi.org': 'quad9',
-        '*jetbrains.com': 'quad9',
-        '*azureedge.net': 'quad9'
+        '*.gh.docmirror.top': 'quad9',
+        '*.v2ex.com': 'quad9',
+        '*.pypi.org': 'quad9',
+        '*.jetbrains.com': 'quad9',
+        '*.azureedge.net': 'quad9'
       },
       speedTest: {
         enabled: true,

diff --git a/packages/core/src/modules/plugin/node/index.js b/packages/core/src/modules/plugin/node/index.js
@@ -6,7 +6,7 @@ const NodePlugin = function (context) {
       try {
         await nodeApi.setVariables()
       } catch (err) {
-        log.warn('set variables error', err)
+        log.warn('set variables error:', err)
       }
 
       const ip = '127.0.0.1'

diff --git a/packages/core/src/modules/plugin/overwall/config.js b/packages/core/src/modules/plugin/overwall/config.js
@@ -10,30 +10,31 @@ module.exports = {
     }
   },
   targets: {
+    '*.github.com': true,
     '*github*.com': true,
-    '*wikimedia.org': true,
-    'v2ex.com': true,
-    '*azureedge.net': true,
-    '*cloudfront.net': true,
-    '*bing.com': true,
-    '*discourse-cdn.com': true,
-    '*gravatar.com': true,
-    '*docker.com': true,
-    '*vueuse.org': true,
-    '*elastic.co': true,
-    '*optimizely.com': true,
-    '*stackpathcdn.com': true,
-    '*fastly.net': true,
-    '*cloudflare.com': true,
-    '*233v2.com': true,
-    '*v2fly.org': true,
-    '*telegram.org': true,
-    '*amazon.com': true,
-    '*googleapis.com': true,
+    '*.wikimedia.org': true,
+    '*.v2ex.com': true,
+    '*.azureedge.net': true,
+    '*.cloudfront.net': true,
+    '*.bing.com': true,
+    '*.discourse-cdn.com': true,
+    '*.gravatar.com': true,
+    '*.docker.com': true,
+    '*.vueuse.org': true,
+    '*.elastic.co': true,
+    '*.optimizely.com': true,
+    '*.stackpathcdn.com': true,
+    '*.fastly.net': true,
+    '*.cloudflare.com': true,
+    '*.233v2.com': true,
+    '*.v2fly.org': true,
+    '*.telegram.org': true,
+    '*.amazon.com': true,
+    '*.googleapis.com': true,
     '*.google-analytics.com': true,
-    '*cloudflareinsights.com': true,
+    '*.cloudflareinsights.com': true,
     '*.intlify.dev': true,
-    '*segment.io': true,
+    '*.segment.io': true,
     '*.shields.io': true,
     '*.jsdelivr.net': true
   },

diff --git a/packages/core/src/modules/proxy/index.js b/packages/core/src/modules/proxy/index.js
@@ -34,7 +34,7 @@ const ProxyPlugin = function (context) {
         log.info('关闭系统代理成功')
         return true
       } catch (err) {
-        log.error('关闭系统代理失败', err)
+        log.error('关闭系统代理失败:', err)
         return false
       }
     },

diff --git a/packages/core/src/modules/server/index.js b/packages/core/src/modules/server/index.js
@@ -86,16 +86,16 @@ const serverApi = {
       }
     }
     serverProcess.on('beforeExit', (code) => {
-      log.warn('server process beforeExit', code)
+      log.warn('server process beforeExit, code:', code)
     })
     serverProcess.on('SIGPIPE', (code, signal) => {
-      log.warn('server process SIGPIPE', code, signal)
+      log.warn(`server process SIGPIPE, code: ${code}, signal:`, signal)
     })
     serverProcess.on('exit', (code, signal) => {
-      log.warn('server process exit', code, signal)
+      log.warn(`server process exit, code: ${code}, signal:`, signal)
     })
     serverProcess.on('uncaughtException', (err, origin) => {
-      log.error('server process uncaughtException', err)
+      log.error('server process uncaughtException:', err)
     })
     serverProcess.on('message', function (msg) {
       log.info('收到子进程消息', msg.type, msg.event.key, msg.message)
@@ -130,13 +130,13 @@ const serverApi = {
         // fireStatus('ing')// 关闭中
         server.close((err) => {
           if (err) {
-            log.warn('close error', err, ',', err.code, ',', err.message, ',', err.errno)
+            log.warn('close error:', err)
             if (err.code === 'ERR_SERVER_NOT_RUNNING') {
               log.info('代理服务关闭成功')
               resolve()
               return
             }
-            log.warn('代理服务关闭失败', err)
+            log.warn('代理服务关闭失败:', err)
             reject(err)
           } else {
             log.info('代理服务关闭成功')

diff --git a/packages/core/src/shell/scripts/extra-path/index.js b/packages/core/src/shell/scripts/extra-path/index.js
@@ -3,7 +3,7 @@ const path = require('path')
 
 function getExtraPath () {
   let extraPath = process.env.DS_EXTRA_PATH
-  log.info('extraPath', extraPath)
+  log.info('extraPath:', extraPath)
   if (!extraPath) {
     extraPath = __dirname
   }

diff --git a/packages/core/src/shell/scripts/set-system-proxy/index.js b/packages/core/src/shell/scripts/set-system-proxy/index.js
@@ -49,7 +49,10 @@ async function _winSetProxy (exec, ip, port, setEnv) {
   }
 
   const proxyPath = extraPath.getProxyExePath()
-  await execFile(proxyPath, ['global', `http=http://${ip}:${port};https=http://${ip}:${port}`, excludeIpStr])
+  const execFun = 'global'
+  const proxyAddr = `http=http://${ip}:${port};https=http://${ip}:${port}`
+  log.info(`执行“设置系统代理”的程序: ${proxyPath} ${execFun} ${proxyAddr} ......(省略排除IP列表)`)
+  await execFile(proxyPath, [execFun, proxyAddr, excludeIpStr])
 
   if (setEnv) {
     log.info('同时设置 https_proxy')
@@ -71,11 +74,11 @@ const executor = {
     const { ip, port, setEnv } = params
     if (ip == null) {
       // 清空代理
-      log.info('关闭代理')
+      log.info('关闭windows系统代理')
       return _winUnsetProxy(exec, setEnv)
     } else {
       // 设置代理
-      log.info('设置代理:', ip, port, setEnv)
+      log.info('设置windows系统代理:', ip, port, setEnv)
       return _winSetProxy(exec, ip, port, setEnv)
     }
   },

diff --git a/packages/core/src/shell/shell.js b/packages/core/src/shell/shell.js
@@ -1,9 +1,6 @@
-const util = require('util')
 const os = require('os')
 const childProcess = require('child_process')
-const _exec = childProcess.exec
 const _execFile = childProcess.execFile
-const exec = util.promisify(_exec)
 const PowerShell = require('node-powershell')
 const log = require('../utils/util.log')
 const fixPath = require('fix-path')
@@ -96,8 +93,8 @@ function _childExec (composeCmds, options = {}) {
 
 function childExec (composeCmds) {
   return new Promise((resolve, reject) => {
-    var encoding = 'cp936'
-    var binaryEncoding = 'binary'
+    const encoding = 'cp936'
+    const binaryEncoding = 'binary'
 
     const childProcess = require('child_process')
     log.info('shell:', composeCmds)

diff --git a/packages/core/src/shell/test.js b/packages/core/src/shell/test.js
@@ -10,18 +10,18 @@
 //   }
 // )
 
-// var process = require('child_process')
+// const process = require('child_process')
 //
-// var cmd = 'set'
+// const cmd = 'set'
 // process.exec(cmd, function (error, stdout, stderr) {
 //   console.log('error:' + error)
 //   console.log('stdout:' + stdout)
 //   console.log('stderr:' + stderr)
 // })
 
-// var HttpsProxyAgent = require('https-proxy-agent')
-// var proxy = 'http://user:[email protected]:port'
-// var agent = new HttpsProxyAgent(proxy)
+// const HttpsProxyAgent = require('https-proxy-agent')
+// const proxy = 'http://user:[email protected]:port'
+// const agent = new HttpsProxyAgent(proxy)
 // const https = require('https')
 // https.get('https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js', (res) => {
 //   console.log('状态码:', res.statusCode)

diff --git a/packages/core/src/utils/util.proxy.js b/packages/core/src/utils/util.proxy.js
diff --git a/packages/mitmproxy/src/lib/interceptor/impl/res/cacheRes.js b/packages/mitmproxy/src/lib/interceptor/impl/res/cacheRes.js
@@ -81,7 +81,6 @@ module.exports = {
     }
 
     res.setHeader('Dev-Sidecar-Cache-Response-Interceptor', 'cacheRes:maxAge=' + maxAge)
-    log.info('[cacheRes]', 'maxAge=' + maxAge)
   },
   is (interceptOpt) {
     const maxAge = cacheReq.getMaxAge(interceptOpt)

diff --git a/packages/mitmproxy/src/lib/proxy/middleware/overwall.js b/packages/mitmproxy/src/lib/proxy/middleware/overwall.js
@@ -1,10 +1,12 @@
 const url = require('url')
+const lodash = require('lodash')
 const pac = require('./source/pac')
 const matchUtil = require('../../../utils/util.match')
+const log = require('../../../utils/util.log')
 let pacClient = null
 
-function matched (hostname, regexpMap) {
-  const ret1 = matchUtil.matchHostname(regexpMap, hostname)
+function matched (hostname, overWallTargetMap) {
+  const ret1 = matchUtil.matchHostname(overWallTargetMap, hostname, 'matched overwall')
   if (ret1) {
     return true
   }
@@ -13,9 +15,12 @@ function matched (hostname, regexpMap) {
   }
   const ret = pacClient.FindProxyForURL('https://' + hostname, hostname)
   if (ret && ret.indexOf('PROXY ') === 0) {
+    log.info(`matchHostname: matched overwall: '${hostname}' -> '${ret}' in pac.txt`)
     return true
+  } else {
+    // log.debug(`matchHostname: matched overwall: Not-Matched '${hostname}' -> '${ret}' in pac.txt`)
+    return false
   }
-  return false
 }
 
 module.exports = function createOverWallIntercept (overWallConfig) {
@@ -36,19 +41,19 @@ module.exports = function createOverWallIntercept (overWallConfig) {
   if (keys.length === 0) {
     return null
   }
-  const regexpMap = matchUtil.domainMapRegexply(overWallConfig.targets)
+  const overWallTargetMap = matchUtil.domainMapRegexply(overWallConfig.targets)
   return {
     sslConnectInterceptor: (req, cltSocket, head) => {
       const hostname = req.url.split(':')[0]
-      return matched(hostname, regexpMap)
+      return matched(hostname, overWallTargetMap)
     },
     requestIntercept (context, req, res, ssl, next) {
       const { rOptions, log, RequestCounter } = context
       if (rOptions.protocol === 'http:') {
         return
       }
       const hostname = rOptions.hostname
-      if (!matched(hostname, regexpMap)) {
+      if (!matched(hostname, overWallTargetMap)) {
         return
       }
       const cacheKey = '__over_wall_proxy__'
@@ -81,6 +86,9 @@ module.exports = function createOverWallIntercept (overWallConfig) {
       const proxy = proxyTarget.indexOf('http') === 0 ? proxyTarget : (rOptions.protocol + '//' + proxyTarget)
       // eslint-disable-next-line node/no-deprecated-api
       const URL = url.parse(proxy)
+      rOptions.origional = lodash.cloneDeep(rOptions) // 备份原始请求参数
+      delete rOptions.origional.agent
+      delete rOptions.origional.headers
       rOptions.protocol = URL.protocol
       rOptions.hostname = URL.host
       rOptions.host = URL.host
@@ -92,7 +100,7 @@ module.exports = function createOverWallIntercept (overWallConfig) {
       if (URL.port == null) {
         rOptions.port = port || (rOptions.protocol === 'https:' ? 443 : 80)
       }
-      log.info('OverWall:', rOptions.hostname, proxyTarget)
+      log.info('OverWall:', rOptions.hostname, '➜', proxyTarget)
       if (context.requestCount) {
         log.debug('OverWall choice:', JSON.stringify(context.requestCount))
       }