diff --git a/node-proxy/app.ts b/node-proxy/app.ts index 388f408..fac3b85 100644 --- a/node-proxy/app.ts +++ b/node-proxy/app.ts @@ -4,6 +4,7 @@ import http from 'http' import Koa from 'koa' import serve from 'koa-static' +import { port } from '@/config' import { logger } from '@/common/logger' import alisRouter from '@/router/alist' import otherRouter from '@/router/other' @@ -25,7 +26,7 @@ app.use(otherRouter.routes()) const server = http.createServer(app.callback()) server.maxConnections = 1000 -server.listen(5343, () => logger.info('服务启动成功: ' + 5343)) +server.listen(port, () => logger.info('服务启动成功: ' + port)) setInterval(() => { logger.debug('server_connections', server.connections, Date.now()) diff --git a/node-proxy/src/@types/index.d.ts b/node-proxy/src/@types/index.d.ts index bee15c1..9899601 100644 --- a/node-proxy/src/@types/index.d.ts +++ b/node-proxy/src/@types/index.d.ts @@ -91,12 +91,11 @@ declare global { //preProxy后将以下属性添加到ctx.state中供middleware使用 type ProxiedState = { isWebdav: boolean + selfHost: string urlAddr: string serverAddr: string serverConfig: T - selfHost: string origin: string fileSize: number - passwdInfo?: PasswdInfo } } diff --git a/node-proxy/src/@types/webdav.d.ts b/node-proxy/src/@types/webdav.d.ts new file mode 100644 index 0000000..0679bab --- /dev/null +++ b/node-proxy/src/@types/webdav.d.ts @@ -0,0 +1,20 @@ +declare namespace webdav { + type FileInfo = { + href: string + propstat: { + prop: { + displayname: string + getcontentlength?: number + } + status: string + } + } + + type PropfindResp = { + multistatus: { + response: T + } + } +} + +export { webdav } diff --git a/node-proxy/src/dao/fileDao.ts b/node-proxy/src/dao/fileDao.ts index 8fddcfb..c0ed0f3 100644 --- a/node-proxy/src/dao/fileDao.ts +++ b/node-proxy/src/dao/fileDao.ts @@ -1,6 +1,7 @@ import nedb from '@/dao/levelDB' import type { alist } from '@/@types/alist' +import { logger } from '@/common/logger' export const fileInfoTable = 'fileInfoTable_' @@ -11,11 +12,19 @@ const cacheTime = 60 * 24 export async function cacheFileInfo(fileInfo: alist.FileInfo | WebdavFileInfo) { fileInfo.path = decodeURIComponent(fileInfo.path) const pathKey = fileInfoTable + fileInfo.path + logger.info(`FileDao 保存文件信息: ${pathKey}`) await nedb.setExpire(pathKey, fileInfo, 1000 * 60 * cacheTime) } // 获取文件信息，偶尔要清理一下缓存 -export async function getFileInfo(path: string) { +export async function getFileInfo(path: string): Promise { const pathKey = decodeURIComponent(fileInfoTable + path) + logger.info(`FileDao 获取文件信息: ${pathKey}`) return await nedb.getValue(pathKey) } + +export async function deleteFileInfo(path: string) { + const pathKey = decodeURIComponent(fileInfoTable + path) + logger.info(`FileDao 删除文件信息: ${pathKey}`) + await nedb.datastore.removeMany({ key: pathKey }, {}) +} diff --git a/node-proxy/src/dao/levelDB.ts b/node-proxy/src/dao/levelDB.ts index 7421859..10d214c 100644 --- a/node-proxy/src/dao/levelDB.ts +++ b/node-proxy/src/dao/levelDB.ts @@ -43,7 +43,7 @@ class Nedb { //存值，无过期时间 async setValue(key: string, value: Value) { await this.datastore.removeMany({ key }, {}) - logger.info('存储键值(无过期时间)', key, JSON.stringify(value)) + logger.trace('存储键值(无过期时间)', key, JSON.stringify(value)) await this.datastore.insert({ key, expire: -1, value }) } @@ -51,7 +51,7 @@ class Nedb { async setExpire(key: string, value: Value, second = 6 * 10) { await this.datastore.removeMany({ key }, {}) const expire = Date.now() + second * 1000 - logger.info(`存储键值(过期时间${expire})`, key, JSON.stringify(value)) + logger.trace(`存储键值(过期时间${expire})`, key, JSON.stringify(value)) await this.datastore.insert({ key, expire, value }) } diff --git a/node-proxy/src/middleware/common.ts b/node-proxy/src/middleware/common.ts index f802e0a..08dcb89 100644 --- a/node-proxy/src/middleware/common.ts +++ b/node-proxy/src/middleware/common.ts @@ -87,11 +87,14 @@ export const exceptionMiddleware: Middleware = async ( try { await next() if (ctx.status === 404) { - ctx.throw(404, '请求资源未找到!') + if (ctx.state?.isWebdav) { + logger.warn(ctx.state?.urlAddr, '404') + } else { + ctx.throw(404, '请求资源未找到!') + } } } catch (err) { - logger.error('@@err') - console.trace(err) + logger.error(err) const status = err.status || 500 // 生产环境时 500 错误的详细错误内容不返回给客户端，因为可能包含敏感信息 @@ -111,29 +114,28 @@ export const exceptionMiddleware: Middleware = async ( } export const proxyHandler: Middleware = async (ctx: ParameterizedContext>) => { - const { state, req: request, res: response } = ctx - - const { headers } = request + const state = ctx.state + const { method, headers } = ctx.req // 要定位请求文件的位置 bytes=98304- const range = headers.range const start = range ? Number(range.replace('bytes=', '').split('-')[0]) : 0 + const urlPath = new URL(state.urlAddr).pathname // 检查路径是否满足加密要求，要拦截的路径可能有中文 - const { passwdInfo } = pathFindPasswd(state.serverConfig.passwdList, decodeURIComponent(request.url)) + const { passwdInfo } = pathFindPasswd(state.serverConfig.passwdList, decodeURIComponent(urlPath)) logger.info('匹配密码信息', passwdInfo === null ? '无密码' : passwdInfo.password) let encryptTransform: Transform, decryptTransform: Transform // fix webdav move file - if (request.method.toLocaleUpperCase() === 'MOVE' && headers.destination) { + if (method.toLocaleUpperCase() === 'MOVE' && headers.destination) { let destination = flat(headers.destination) - destination = state.serverAddr + destination.substring(destination.indexOf(path.dirname(request.url)), destination.length) - request.headers.destination = destination + destination = state.serverAddr + destination.substring(destination.indexOf(path.dirname(urlPath)), destination.length) + headers.destination = destination } // 如果是上传文件，那么进行流加密，目前只支持webdav上传，如果alist页面有上传功能，那么也可以兼容进来 - if (request.method.toLocaleUpperCase() === 'PUT' && passwdInfo) { + if (method.toLocaleUpperCase() === 'PUT' && passwdInfo) { // 兼容macos的webdav客户端x-expected-entity-length - ctx.state.fileSize = Number(headers['content-length'] || flat(headers['x-expected-entity-length']) || 0) // 需要知道文件长度，等于0 说明不用加密，这个来自webdav奇怪的请求 if (ctx.state.fileSize !== 0) { encryptTransform = new FlowEnc(passwdInfo.password, passwdInfo.encType, ctx.state.fileSize).encryptTransform() @@ -141,7 +143,7 @@ export const proxyHandler: Middleware = async , ParsedContext const encrypted = passwdInfo && passwdInfo.encName - logger.info(`文件路径: ${filePath} 是否被加密 ${encrypted}`) + logger.info(`文件路径: ${filePath} 是否被加密 ${Boolean(encrypted)}`) if (encrypted) { // reset content-length length @@ -155,7 +155,6 @@ alistRouter.put, EmptyObj>('/fs/put', emptyMiddleware, const state = ctx.state const uploadPath = headers['file-path'] ? decodeURIComponent(headers['file-path'] as string) : '/-' const { passwdInfo } = pathFindPasswd(state.serverConfig.passwdList, uploadPath) - state.fileSize = Number(headers['content-length'] || 0) const encrypted = passwdInfo && passwdInfo.encName logger.info(`上传文件: ${uploadPath} 加密${Boolean(encrypted)}`) @@ -219,6 +218,7 @@ alistRouter.all, ParsedContext, ParsedContext { }) otherRouter.all, EmptyObj>(new RegExp(alistServer.path), preProxy(alistServer, false), async (ctx) => { - let respBody = await httpClient({ + const body = await httpClient({ urlAddr: ctx.state.urlAddr, reqBody: JSON.stringify(ctx.request.body), request: ctx.req, response: ctx.res, }) - respBody = respBody.replace( + ctx.body = body.replace( '', `
@@ -113,7 +112,6 @@ otherRouter.all, EmptyObj>(new RegExp(alistServer.path
` ) - ctx.body = respBody }) export default otherRouter diff --git a/node-proxy/src/router/webdav/index.ts b/node-proxy/src/router/webdav/index.ts index f7e4cf9..c791412 100644 --- a/node-proxy/src/router/webdav/index.ts +++ b/node-proxy/src/router/webdav/index.ts @@ -2,19 +2,19 @@ import Router from 'koa-router' import { preProxy } from '@/utils/common' import { alistServer, webdavServer } from '@/config' -import { encDavMiddleware } from '@/router/webdav/middlewares' -import { compose, proxyHandler } from '@/middleware/common' +import { webdavHookMiddleware } from '@/router/webdav/middlewares' const webdavRouter = new Router() // 初始化webdav路由 webdavServer.forEach((webdavConfig) => { if (webdavConfig.enable) { - webdavRouter.all(new RegExp(webdavConfig.path), compose(preProxy(webdavConfig, true), encDavMiddleware), proxyHandler) + webdavRouter.all, EmptyObj>(new RegExp(webdavConfig.path), preProxy(webdavConfig, true), webdavHookMiddleware) } }) // 单独处理alist的所有/dav -webdavRouter.all(/^\/dav\/*/, compose(preProxy(alistServer, false), encDavMiddleware), proxyHandler) +// webdavRouter.all(/^\/dav\/*/, compose(preProxy(alistServer, false), encDavMiddleware), proxyHandler) +webdavRouter.all, EmptyObj>(/^\/dav\/*/, preProxy(alistServer, true), webdavHookMiddleware) export default webdavRouter diff --git a/node-proxy/src/router/webdav/middlewares.ts b/node-proxy/src/router/webdav/middlewares.ts index 53f90f4..4578507 100644 --- a/node-proxy/src/router/webdav/middlewares.ts +++ b/node-proxy/src/router/webdav/middlewares.ts @@ -1,144 +1,271 @@ import path from 'path' import { XMLParser } from 'fast-xml-parser' -import type { Middleware } from 'koa' +import type { Context, Middleware } from 'koa' -import { flat, sleep } from '@/utils/common' +import FlowEnc from '@/utils/flowEnc' +import { flat } from '@/utils/common' import { logger } from '@/common/logger' -import { httpClient } from '@/utils/httpClient' -import { cacheFileInfo, getFileInfo } from '@/dao/fileDao' +import { getWebdavFileInfo } from '@/utils/webdavClient' +import { httpClient, httpFlowClient } from '@/utils/httpClient' +import { cacheFileInfo, deleteFileInfo, getFileInfo } from '@/dao/fileDao' import { convertRealName, convertShowName, pathFindPasswd } from '@/utils/cryptoUtil' import { cacheWebdavFileInfo, getFileNameForShow } from '@/router/webdav/utils' +import { webdav } from '@/@types/webdav' const parser = new XMLParser({ removeNSPrefix: true }) -export const encDavMiddleware: Middleware> = async (ctx, next) => { - const request = ctx.req - const state = ctx.state +const headHook = async (ctx: Context, state: ProxiedState, passwdInfo: PasswdInfo) => { + const urlPath = new URL(state.urlAddr).pathname + const realFileName = convertRealName(passwdInfo.password, passwdInfo.encType, urlPath) + const urlAddr = path.dirname(state.urlAddr) + '/' + realFileName + + ctx.body = await httpClient({ + urlAddr, + request: ctx.req, + response: ctx.res, + }) +} + +const propfindHook = async (ctx: Context, state: ProxiedState, passwdInfo: PasswdInfo) => { + const urlPath = new URL(state.urlAddr).pathname + const realFileName = convertRealName(passwdInfo.password, passwdInfo.encType, urlPath) + const sourceUrl = path.dirname(urlPath) + '/' + realFileName + logger.info(`webdav PROPFIND: ${sourceUrl}`) + + const sourceFileInfo = await getFileInfo(sourceUrl) + let urlAddr: string + + if (sourceFileInfo && !sourceFileInfo.is_dir) { + logger.info(`非文件夹，需要转换url: ${state.urlAddr} -> ${sourceUrl}`) + urlAddr = path.dirname(state.urlAddr) + '/' + realFileName + } else { + logger.info('为文件夹，无需转换url，直接请求') + urlAddr = state.urlAddr + } + + let respBody = await httpClient({ + urlAddr, + request: ctx.req, + response: ctx.res, + }) + + const respData: webdav.PropfindResp = parser.parse(respBody) + + if (respData.multistatus) { + const response = respData.multistatus.response + const files = response instanceof Array ? response : [response] + + await Promise.all( + files.map(async (fileInfo) => { + await cacheWebdavFileInfo(fileInfo) + + const { fileName, showName } = await getFileNameForShow(fileInfo, passwdInfo) - const { passwdInfo } = pathFindPasswd(state.serverConfig.passwdList, decodeURIComponent(request.url)) - - if (ctx.method.toLocaleUpperCase() === 'PROPFIND' && passwdInfo && passwdInfo.encName) { - // check dir, convert url - const url = request.url - if (passwdInfo && passwdInfo.encName) { - // check dir, convert url - const reqFileName = path.basename(url) - // cache source file info, realName has execute encodeUrl()，this '(' '+' can't encodeUrl. - const realName = convertRealName(passwdInfo.password, passwdInfo.encType, url) - // when the name contain the '+,!' , - const sourceUrl = path.dirname(url) + '/' + realName - const sourceFileInfo = await getFileInfo(sourceUrl) - logger.debug('@@@sourceFileInfo', sourceFileInfo, reqFileName, realName, url, sourceUrl) - // it is file, convert file name - if (sourceFileInfo && !sourceFileInfo.is_dir) { - request.url = path.dirname(request.url) + '/' + realName - ctx.state.urlAddr = path.dirname(ctx.state.urlAddr) + '/' + realName - } - } - // decrypt file name - let respBody = await httpClient({ - urlAddr: state.urlAddr, - request: ctx.req, - response: ctx.res, - }) - const respData = parser.parse(respBody) - // convert file name for show - if (respData.multistatus) { - const respJson = respData.multistatus.response - if (respJson instanceof Array) { - // console.log('@@respJsonArray', respJson) - for (const fileInfo of respJson) { - await cacheWebdavFileInfo(fileInfo) - if (passwdInfo && passwdInfo.encName) { - const { fileName, showName } = await getFileNameForShow(fileInfo, passwdInfo) - // logger.debug('@@getFileNameForShow1 list', passwdInfo.password, fileName, decodeURI(fileName), showName) - if (fileName) { - const showXmlName = showName.replace(/&/g, '&').replace(/`, `${encodeURI(showXmlName)}`) - respBody = respBody.replace(`${decodeURI(fileName)}`, `${decodeURI(showXmlName)}`) - } - } - } - // waiting cacheWebdavFileInfo a moment - await sleep(50) - } else if (passwdInfo && passwdInfo.encName) { - const { fileName, showName } = await getFileNameForShow(respJson, passwdInfo) - // logger.debug('@@getFileNameForShow2 file', fileName, showName, url, respJson.propstat) if (fileName) { const showXmlName = showName.replace(/&/g, '&').replace(/`, `${encodeURI(showXmlName)}`) respBody = respBody.replace(`${decodeURI(fileName)}`, `${decodeURI(showXmlName)}`) } + }) + ) + } + + if (ctx.res.statusCode === 404) { + // fix rclone propfind 404 ，because rclone copy will get error 501 + ctx.res.end(respBody) + return + } + + // fix webdav 401 bug，群晖遇到401不能使用 ctx.res.end(respBody)，而rclone遇到404只能使用ctx.res.end(respBody),神奇的bug + ctx.status = ctx.res.statusCode + ctx.body = respBody +} + +const getHook = async (ctx: Context, state: ProxiedState, passwdInfo: PasswdInfo) => { + const urlPath = new URL(state.urlAddr).pathname + const realFileName = convertRealName(passwdInfo.password, passwdInfo.encType, urlPath) + + const headers = ctx.request.headers + const start = headers.range ? Number(headers.range.replace('bytes=', '').split('-')[0]) : 0 + const urlAddr = path.dirname(state.urlAddr) + '/' + realFileName + + // maybe from aliyun drive, check this req url while get file list from enc folder + if (urlPath.endsWith('/')) { + let respBody = await httpClient({ + urlAddr, + request: ctx.req, + response: ctx.res, + }) + + const aurlArr = respBody.match(/href="[^"]*"/g) + // logger.debug('@@aurlArr', aurlArr) + if (aurlArr && aurlArr.length) { + for (let urlStr of aurlArr) { + urlStr = urlStr.replace('href="', '').replace('"', '') + const aurl = decodeURIComponent(urlStr.replace('href="', '').replace('"', '')) + const baseUrl = decodeURIComponent(urlPath) + if (aurl.includes(baseUrl)) { + const fileName = path.basename(aurl) + const showName = convertShowName(passwdInfo.password, passwdInfo.encType, fileName) + logger.debug('@@aurl', urlStr, showName) + respBody = respBody.replace(path.basename(urlStr), encodeURI(showName)).replace(fileName, showName) + } } } - // 检查数据兼容的问题，优先XML对比。 - // logger.debug('@@respJsxml', respBody, ctx.headers) - // const resultBody = parser.parse(respBody) - // logger.debug('@@respJSONData2', ctx.res.statusCode, JSON.stringify(resultBody)) - - if (ctx.res.statusCode === 404) { - // fix rclone propfind 404 ，because rclone copy will get error 501 - ctx.res.end(respBody) - return - } - // fix webdav 401 bug，群晖遇到401不能使用 ctx.res.end(respBody)，而rclone遇到404只能使用ctx.res.end(respBody),神奇的bug - ctx.status = ctx.res.statusCode - ctx.body = respBody + + ctx.res.end(respBody) return } - if ('COPY,MOVE'.includes(request.method.toLocaleUpperCase()) && passwdInfo && passwdInfo.encName) { - const url = request.url - const realName = convertRealName(passwdInfo.password, passwdInfo.encType, url) - request.headers.destination = path.dirname(flat(request.headers.destination)) + '/' + encodeURI(realName) - request.url = path.dirname(request.url) + '/' + encodeURI(realName) - state.urlAddr = path.dirname(state.urlAddr) + '/' + encodeURI(realName) + const webdavFileInfo = await getWebdavFileInfo(urlAddr, headers.authorization) + + if (!webdavFileInfo) { + ctx.status = 404 + return } - // upload file - if ('GET,PUT,DELETE'.includes(request.method.toLocaleUpperCase()) && passwdInfo && passwdInfo.encName) { - const url = request.url - // check dir, convert url - const fileName = path.basename(url) - const realName = convertRealName(passwdInfo.password, passwdInfo.encType, url) - // maybe from aliyundrive, check this req url while get file list from enc folder - if (url.endsWith('/') && 'GET,DELETE'.includes(request.method.toLocaleUpperCase())) { - let respBody = await httpClient({ - urlAddr: state.urlAddr, - request: ctx.req, - response: ctx.res, - }) - if (request.method.toLocaleUpperCase() === 'GET') { - const aurlArr = respBody.match(/href="[^"]*"/g) - // logger.debug('@@aurlArr', aurlArr) - if (aurlArr && aurlArr.length) { - for (let urlStr of aurlArr) { - urlStr = urlStr.replace('href="', '').replace('"', '') - const aurl = decodeURIComponent(urlStr.replace('href="', '').replace('"', '')) - const baseUrl = decodeURIComponent(url) - if (aurl.includes(baseUrl)) { - const fileName = path.basename(aurl) - const showName = convertShowName(passwdInfo.password, passwdInfo.encType, fileName) - logger.debug('@@aurl', urlStr, showName) - respBody = respBody.replace(path.basename(urlStr), encodeURI(showName)).replace(fileName, showName) - } - } - } - } - ctx.res.end(respBody) - return + //更新缓存 + await cacheFileInfo(webdavFileInfo) + + const flowEnc = new FlowEnc(passwdInfo.password, passwdInfo.encType, webdavFileInfo.size) + if (start) await flowEnc.setPosition(start) + + await httpFlowClient({ + urlAddr, + passwdInfo, + fileSize: state.fileSize, + request: ctx.req, + response: ctx.res, + encryptTransform: undefined, + decryptTransform: flowEnc.decryptTransform(), + }) +} + +const putHook = async (ctx: Context, state: ProxiedState, passwdInfo: PasswdInfo) => { + const urlPath = new URL(state.urlAddr).pathname + const realFileName = convertRealName(passwdInfo.password, passwdInfo.encType, urlPath) + const headers = ctx.request.headers + + const urlAddr = path.dirname(state.urlAddr) + '/' + realFileName + logger.info(`webdav上传: ${urlAddr}`) + + const encryptTransform = state.fileSize === 0 ? undefined : new FlowEnc(passwdInfo.password, passwdInfo.encType, state.fileSize).encryptTransform() + + const body = await httpFlowClient({ + urlAddr, + passwdInfo, + fileSize: state.fileSize, + request: ctx.req, + response: ctx.res, + encryptTransform, + decryptTransform: undefined, + }) + + const webdavFileInfo = await getWebdavFileInfo(urlAddr, headers.authorization) + + //缓存文件信息 + await cacheFileInfo(webdavFileInfo) + + ctx.body = body +} + +const deleteHook = async (ctx: Context, state: ProxiedState, passwdInfo: PasswdInfo) => { + const urlPath = new URL(state.urlAddr).pathname + const realFileName = convertRealName(passwdInfo.password, passwdInfo.encType, urlPath) + + const urlAddr = path.dirname(state.urlAddr) + '/' + realFileName + logger.info(`webdav删除: ${urlAddr}`) + + //删除缓存 + await deleteFileInfo(new URL(urlAddr).pathname) + + ctx.body = await httpClient({ + urlAddr, + request: ctx.req, + response: ctx.res, + }) +} + +const copyOrMoveHook = async (ctx: Context, state: ProxiedState, passwdInfo: PasswdInfo) => { + const isDir = await getFileInfo(new URL(state.urlAddr).pathname) + + if (isDir) { + ctx.req.headers.destination = flat(ctx.req.headers.destination).replace(state.selfHost, state.serverAddr) + ctx.body = await httpClient({ urlAddr: state.urlAddr, request: ctx.req, response: ctx.res }) + return + } + + const urlPath = new URL(state.urlAddr).pathname + const realFileName = convertRealName(passwdInfo.password, passwdInfo.encType, urlPath) + + const urlAddr = path.dirname(state.urlAddr) + '/' + encodeURI(realFileName) + const destination = flat(ctx.req.headers.destination) + const destinationDir = path.dirname(destination).replace(state.selfHost, state.serverAddr) + const destinationRealFileName = convertRealName(passwdInfo.password, passwdInfo.encType, flat(ctx.req.headers.destination)) + + //将headers.destination与原文件的path转换为未加密状态 + ctx.req.headers.destination = destinationDir + '/' + destinationRealFileName + + logger.info(`webdav移动/复制文件: ${urlAddr} -> ${ctx.req.headers.destination}`) + + //删除旧缓存 + const fileInfo = await getFileInfo(new URL(urlAddr).pathname) + await deleteFileInfo(fileInfo.path) + + //保存新缓存 + fileInfo.path = new URL(ctx.req.headers.destination).pathname + await cacheFileInfo(fileInfo) + + ctx.body = await httpClient({ urlAddr: urlAddr, request: ctx.req, response: ctx.res }) +} + +export const webdavHookMiddleware: Middleware> = async (ctx) => { + const method = ctx.req.method.toLocaleUpperCase() + const state = ctx.state + + logger.info('开始转换webdav请求', method, decodeURIComponent(state.urlAddr), ctx.headers.authorization) + + const { passwdInfo } = pathFindPasswd(state.serverConfig.passwdList, decodeURIComponent(state.urlAddr)) + + const encrypted = passwdInfo?.encName + + if (!encrypted || !'HEAD,PROPFIND,GET,PUT,COPY,MOVE,DELETE'.includes(method)) { + logger.info('无需转换的webdav请求') + + if (method === 'MOVE') { + ctx.req.headers.destination = flat(ctx.req.headers.destination).replace(state.selfHost, state.serverAddr) } - // console.log('@@convert file name', fileName, realName) - request.url = path.dirname(request.url) + '/' + realName - state.urlAddr = path.dirname(state.urlAddr) + '/' + realName - // cache file before upload in next(), rclone cmd 'copy' will PROPFIND this file when the file upload success right now - const contentLength = Number(flat(request.headers['content-length'] || request.headers['x-expected-entity-length']) || 0) - const fileDetail = { path: url, name: fileName, is_dir: false, size: contentLength } - logger.info('@@@put url', url) - // 在页面上传文件，rclone会重复上传，所以要进行缓存文件信息，也不能在next() 因为rclone copy命令会出异常 - await cacheFileInfo(fileDetail) + await httpFlowClient({ + urlAddr: state.urlAddr, + passwdInfo, + fileSize: state.fileSize, + request: ctx.req, + response: ctx.res, + }) + return + } + + switch (method) { + case 'HEAD': + await headHook(ctx, state, passwdInfo) + return + case 'PROPFIND': + await propfindHook(ctx, state, passwdInfo) + return + case 'GET': + await getHook(ctx, state, passwdInfo) + return + case 'PUT': + await putHook(ctx, state, passwdInfo) + return + case 'DELETE': + await deleteHook(ctx, state, passwdInfo) + return + case 'COPY': + case 'MOVE': + await copyOrMoveHook(ctx, state, passwdInfo) + return } - await next() } diff --git a/node-proxy/src/router/webdav/utils.ts b/node-proxy/src/router/webdav/utils.ts index afe0377..40c7c83 100644 --- a/node-proxy/src/router/webdav/utils.ts +++ b/node-proxy/src/router/webdav/utils.ts @@ -1,8 +1,9 @@ import path from 'path' import { cacheFileInfo } from '@/dao/fileDao' import { convertShowName } from '@/utils/cryptoUtil' +import { webdav } from '@/@types/webdav' -export const cacheWebdavFileInfo = async (fileInfo) => { +export const cacheWebdavFileInfo = async (fileInfo: webdav.FileInfo) => { let contentLength = -1 const href = fileInfo.href const fileName = path.basename(href) @@ -11,8 +12,7 @@ export const cacheWebdavFileInfo = async (fileInfo) => { } else if (fileInfo.propstat.prop) { contentLength = fileInfo.propstat.prop.getcontentlength } - // logger.debug('@@@cacheWebdavFileInfo', href, fileName) - // it is a file + if (contentLength !== undefined && contentLength > -1) { const fileDetail = { path: href, name: fileName, is_dir: false, size: contentLength } await cacheFileInfo(fileDetail) @@ -24,7 +24,7 @@ export const cacheWebdavFileInfo = async (fileInfo) => { return fileDetail } -export const getFileNameForShow = async (fileInfo, passwdInfo: PasswdInfo) => { +export const getFileNameForShow = async (fileInfo: webdav.FileInfo, passwdInfo: PasswdInfo) => { let contentLength = -1 const href = fileInfo.href const fileName = path.basename(href) diff --git a/node-proxy/src/utils/common.ts b/node-proxy/src/utils/common.ts index 5ec3148..d89af08 100644 --- a/node-proxy/src/utils/common.ts +++ b/node-proxy/src/utils/common.ts @@ -6,8 +6,6 @@ export const flat = (value: T | T[]): T => { } //存储请求的部分原始信息，供代理使用 -export function preProxy(serverConfig: WebdavServer, isWebdav: true): Middleware -export function preProxy(serverConfig: AlistServer, isWebdav: false): Middleware export function preProxy(serverConfig: WebdavServer | AlistServer, isWebdav: boolean): Middleware { return async (ctx, next) => { const { serverHost, serverPort, https } = serverConfig @@ -21,12 +19,13 @@ export function preProxy(serverConfig: WebdavServer | AlistServer, isWebdav: boo const request = ctx.request const protocol = https ? 'https' : 'http' - ctx.state.selfHost = request.headers.host // 原来的host保留，以后可能会用到 + ctx.state.selfHost = `${protocol}://${request.headers.host}` // 原来的host保留，以后可能会用到 ctx.state.origin = request.headers.origin request.headers.host = serverHost + ':' + serverPort ctx.state.urlAddr = `${protocol}://${request.headers.host}${request.url}` ctx.state.serverAddr = `${protocol}://${request.headers.host}` ctx.state.serverConfig = serverConfig + ctx.state.fileSize = Number(request.headers['content-length'] || flat(request.headers['x-expected-entity-length']) || 0) await next() } diff --git a/node-proxy/src/utils/cryptoUtil.ts b/node-proxy/src/utils/cryptoUtil.ts index 381ab29..e32c1b2 100644 --- a/node-proxy/src/utils/cryptoUtil.ts +++ b/node-proxy/src/utils/cryptoUtil.ts @@ -4,8 +4,8 @@ import { pathToRegexp } from 'path-to-regexp' import Crcn from './crypto/crc6-8' import MixBase64 from './crypto/mixBase64' -import { getPassWdOutward } from './flowEnc' import { logger } from '@/common/logger' +import { getPassWdOutward } from './flowEnc' const crc6 = new Crcn(6) const origPrefix = 'orig_' diff --git a/node-proxy/src/utils/httpClient.ts b/node-proxy/src/utils/httpClient.ts index 04c0655..fa902cc 100644 --- a/node-proxy/src/utils/httpClient.ts +++ b/node-proxy/src/utils/httpClient.ts @@ -1,34 +1,117 @@ import path from 'path' import http from 'http' import https from 'node:https' -import { Transform } from 'stream' -import crypto, { randomUUID } from 'crypto' +import crypto from 'crypto' +import type { Transform } from 'stream' import { flat } from '@/utils/common' -import levelDB from '@/dao/levelDB' -import { decodeName } from '@/utils/cryptoUtil' +import nedb from '@/dao/levelDB' import { logger } from '@/common/logger' +import { decodeName } from '@/utils/cryptoUtil' // 默认maxFreeSockets=256 const httpsAgent = new https.Agent({ keepAlive: true }) const httpAgent = new http.Agent({ keepAlive: true }) +const defaultOnMessageCallback: ({ + urlAddr, + reqBody, + request, + response, + message, +}: { + urlAddr: string + reqBody?: string + request: http.IncomingMessage + response: http.ServerResponse + message: http.IncomingMessage +}) => Promise = async (params) => { + const { response, message } = params + + response.statusCode = message.statusCode + + for (const key in message.headers) { + response.setHeader(key, message.headers[key]) + } +} + +const defaultOnFlowMessageCallback: ({ + urlAddr, + passwdInfo, + fileSize, + request, + response, + encryptTransform, + decryptTransform, + message, +}: { + urlAddr: string + passwdInfo?: PasswdInfo + fileSize: number + request: http.IncomingMessage + response: http.ServerResponse + encryptTransform?: Transform + decryptTransform?: Transform + message: http.IncomingMessage +}) => Promise = async (params) => { + const { urlAddr, passwdInfo, fileSize, request, response, message, decryptTransform } = params + + response.statusCode = message.statusCode + + if (response.statusCode % 300 < 5) { + // 可能出现304，redirectUrl = undefined + const redirectUrl = message.headers.location || '-' + // 百度云盘不是https，坑爹，因为天翼云会多次302，所以这里要保持，跳转后的路径保持跟上次一致，经过本服务器代理就可以解密 + if (decryptTransform && passwdInfo?.enable) { + const key = crypto.randomUUID() + + await nedb.setExpire(key, { redirectUrl, passwdInfo, fileSize }, 60 * 60 * 72) // 缓存起来，默认3天，足够下载和观看了 + // 、Referer + message.headers.location = `/redirect/${key}?decode=1&lastUrl=${encodeURIComponent(urlAddr)}` + } + logger.info(`httpFlow重定向到:`, redirectUrl) + } else if (message.headers['content-range'] && message.statusCode === 200) { + response.statusCode = 206 + } + + // 设置headers + for (const key in message.headers) { + response.setHeader(key, message.headers[key]) + } + + // 下载时解密文件名 + if (request.method === 'GET' && response.statusCode === 200 && passwdInfo && passwdInfo.enable && passwdInfo.encName) { + let fileName = decodeURIComponent(path.basename(urlAddr)) + fileName = decodeName(passwdInfo.password, passwdInfo.encType, fileName.replace(path.extname(fileName), '')) + + if (fileName) { + let disposition = flat(response.getHeader('content-disposition')).toString() + disposition = disposition ? disposition.replace(/filename\*?=[^=;]*;?/g, '') : '' + + logger.info(`httpFlow解密后文件名:`, fileName) + response.setHeader('content-disposition', disposition + `filename*=UTF-8''${encodeURIComponent(fileName)};`) + } + } +} + export async function httpClient({ urlAddr, reqBody, request, response, + onMessage = defaultOnMessageCallback, }: { urlAddr: string reqBody?: string request: http.IncomingMessage - response?: http.ServerResponse + response: http.ServerResponse + onMessage?: typeof defaultOnMessageCallback | null }): Promise { - const { method, headers, url } = request - logger.info('代理请求发起: ', method, urlAddr) - logger.trace('http请求头: ', headers) + //如果传入ctx.res，那么当代理请求返回响应头时，ctx.res也会立即返回响应头。若不传入则当代理请求完全完成时再手动处理ctx.res + const { method, headers } = request + logger.info('代理http请求 发起: ', method, urlAddr) + // 创建请求 - const httpRequest = ~urlAddr.indexOf('https') ? https : http const options = { method, headers, @@ -36,41 +119,55 @@ export async function httpClient({ rejectUnauthorized: false, } - return new Promise((resolve) => { + let result = '' + return new Promise((resolve, reject) => { // 处理重定向的请求，让下载的流量经过代理服务器 - const httpReq = httpRequest.request(urlAddr, options, async (httpResp) => { - logger.info('http响应接收: 状态码', httpResp.statusCode) - logger.trace('http响应头:', httpResp.headers) - if (response) { - response.statusCode = httpResp.statusCode - for (const key in httpResp.headers) { - response.setHeader(key, httpResp.headers[key]) + const proxy = (~urlAddr.indexOf('https') ? https : http).request(urlAddr, options, async (message) => { + logger.info(`代理http响应 接收: 状态码 ${message.statusCode}`) + + //默认的回调函数，设置response的响应码和响应头 + onMessage && (await onMessage({ urlAddr, reqBody, request, response, message })) + + message.on('data', (chunk) => { + result += chunk + }) + + message.on('end', () => { + resolve(result) + }) + + message.on('close', () => { + logger.info('代理http响应 结束') + }) + + message.on('error', (err) => { + if (err.message === 'aborted') { + logger.warn(`代理http响应 提前终止读取`) + } else { + logger.error('代理http响应 获取时出错: ', err) + reject(err) } - } - - let result = '' - httpResp - .on('data', (chunk) => { - result += chunk - }) - .on('end', () => { - resolve(result) - logger.info('代理请求结束结束: ', url) - }) + }) + }) + + proxy.on('error', (err) => { + logger.error('代理http请求 发起时出错: ', err) + reject(err) }) - httpReq.on('error', (err) => { - logger.error('http请求出错: ', err) + proxy.on('close', () => { + logger.info('代理http请求 结束 ') }) - // check request type if (!reqBody) { - url ? request.pipe(httpReq) : httpReq.end() - return + // 如果没有传入请求体，将原请求的请求体转发 + // 当ctx.request的请求体end时，proxy会自动end + request.pipe(proxy) + } else { + //如果传入了请求体，将它作为转发请求的请求体 + proxy.write(reqBody) + proxy.end() } - - httpReq.write(reqBody) - httpReq.end() }) } @@ -82,20 +179,20 @@ export async function httpFlowClient({ response, encryptTransform, decryptTransform, + onMessage = defaultOnFlowMessageCallback, }: { urlAddr: string - passwdInfo: PasswdInfo + passwdInfo?: PasswdInfo fileSize: number request: http.IncomingMessage - response?: http.ServerResponse + response: http.ServerResponse encryptTransform?: Transform decryptTransform?: Transform + onMessage?: typeof defaultOnFlowMessageCallback }) { - const { method, headers, url } = request - const reqId = randomUUID() - logger.info(`代理请求(${reqId})发起: `, method, urlAddr) - logger.info(`httpFlow(${reqId})加解密: `, `流加密${!!encryptTransform}`, `流解密${!!decryptTransform}`) - logger.trace(`httpFlow(${reqId})请求头: `, headers) + const { method, headers } = request + + logger.info(`代理httpFlow请求 发起: `, method, urlAddr, `流加密${!!encryptTransform}`, `流解密${!!decryptTransform}`) // 创建请求 const options = { method, @@ -103,67 +200,61 @@ export async function httpFlowClient({ agent: ~urlAddr.indexOf('https') ? httpsAgent : httpAgent, rejectUnauthorized: false, } - const httpRequest = ~urlAddr.indexOf('https') ? https : http - return new Promise((resolve) => { - // 处理重定向的请求，让下载的流量经过代理服务器 - const httpReq = httpRequest.request(urlAddr, options, async (httpResp) => { - logger.info(`httpFlow(${reqId})响应接收: 状态码`, httpResp.statusCode) - logger.trace(`httpFlow(${reqId})响应头: `, httpResp.headers) - response.statusCode = httpResp.statusCode - if (response.statusCode % 300 < 5) { - // 可能出现304，redirectUrl = undefined - const redirectUrl = httpResp.headers.location || '-' - // 百度云盘不是https，坑爹，因为天翼云会多次302，所以这里要保持，跳转后的路径保持跟上次一致，经过本服务器代理就可以解密 - if (decryptTransform && passwdInfo.enable) { - const key = crypto.randomUUID() - - await levelDB.setExpire(key, { redirectUrl, passwdInfo, fileSize }, 60 * 60 * 72) // 缓存起来，默认3天，足够下载和观看了 - // 、Referer - httpResp.headers.location = `/redirect/${key}?decode=1&lastUrl=${encodeURIComponent(url)}` - } - logger.info(`httpFlow(${reqId})重定向到:`, redirectUrl) - } else if (httpResp.headers['content-range'] && httpResp.statusCode === 200) { - response.statusCode = 206 - } - // 设置headers - for (const key in httpResp.headers) { - response.setHeader(key, httpResp.headers[key]) - } - // 下载时解密文件名 - if (method === 'GET' && response.statusCode === 200 && passwdInfo && passwdInfo.enable && passwdInfo.encName) { - let fileName = decodeURIComponent(path.basename(url)) - fileName = decodeName(passwdInfo.password, passwdInfo.encType, fileName.replace(path.extname(fileName), '')) - - if (fileName) { - let cd = response.getHeader('content-disposition') - cd = flat(cd) - cd = cd ? cd.toString().replace(/filename\*?=[^=;]*;?/g, '') : '' - logger.info(`httpFlow(${reqId})解密后文件名:`, fileName) - response.setHeader('content-disposition', cd + `filename*=UTF-8''${encodeURIComponent(fileName)};`) + + return new Promise((resolve, reject) => { + const proxy = (~urlAddr.indexOf('https') ? https : http).request(urlAddr, options, async (message) => { + logger.info(`代理httpFlow响应 接收: 状态码 ${message.statusCode}`) + + //默认的回调函数，设置response的响应码和响应头 + await onMessage({ + urlAddr, + passwdInfo, + fileSize, + request, + response, + encryptTransform, + decryptTransform, + message, + }) + + message.on('end', () => { + resolve() + }) + + message.on('close', () => { + logger.info(`代理httpFlow响应 结束`) + if (decryptTransform) decryptTransform.destroy() + }) + + message.on('error', (err) => { + if (err.message === 'aborted') { + logger.warn(`代理httpFlow响应 提前终止读取`) + } else { + logger.error(`代理httpFlow响应 获取时出错: `, err) + reject(err) } - } - - httpResp - .on('end', () => { - resolve() - }) - .on('close', () => { - logger.info(`代理请求(${reqId})结束:`, urlAddr) - // response.destroy() - if (decryptTransform) decryptTransform.destroy() - }) - // 是否需要解密 - decryptTransform ? httpResp.pipe(decryptTransform).pipe(response) : httpResp.pipe(response) + }) + + // 根据是否需要解密，将message的数据传递给response返回 + decryptTransform ? message.pipe(decryptTransform).pipe(response) : message.pipe(response) + }) + + proxy.on('error', (err) => { + logger.error(`代理httpFlow请求 发起时出错: `, err) + reject(err) }) - httpReq.on('error', (err) => { - logger.error(`httpFlow(${reqId})请求出错:`, urlAddr, err) + + proxy.on('close', () => { + logger.info(`代理httpFlow请求 结束`) + if (encryptTransform) encryptTransform.destroy() }) - // 是否需要加密 - encryptTransform ? request.pipe(encryptTransform).pipe(httpReq) : request.pipe(httpReq) + // 重定向的请求 关闭时 关闭被重定向的请求 response.on('close', () => { - logger.trace(`响应(${reqId})关闭: `, url) - httpReq.destroy() + proxy.destroy() }) + + // 是否需要加密，将request的数据传递给proxy转发 + encryptTransform ? request.pipe(encryptTransform).pipe(proxy) : request.pipe(proxy) }) } diff --git a/node-proxy/src/utils/webdavClient.ts b/node-proxy/src/utils/webdavClient.ts index e34a34f..69b4a4e 100644 --- a/node-proxy/src/utils/webdavClient.ts +++ b/node-proxy/src/utils/webdavClient.ts @@ -1,38 +1,63 @@ -import http from 'http' - import { XMLParser } from 'fast-xml-parser' -import { httpClient } from '@/utils/httpClient' +import https from 'node:https' +import http from 'http' +import { webdav } from '@/@types/webdav' import { logger } from '@/common/logger' -// get file info from webdav -export async function getWebdavFileInfo(urlAddr: string, authorization: string): Promise { - //eslint-disable-next-line - //@ts-ignore - const request: http.IncomingMessage = { - method: 'PROPFIND', - headers: { - depth: '1', - authorization, - }, - } +const httpsAgent = new https.Agent({ keepAlive: true }) +const httpAgent = new http.Agent({ keepAlive: true }) +const parser = new XMLParser({ removeNSPrefix: true }) - const parser = new XMLParser({ removeNSPrefix: true }) +const webdavPropfind = async (urlAddr: string, authorization: string): Promise> => { + logger.info(`webdav获取文件信息: ${decodeURIComponent(urlAddr)}`) - try { - const XMLData = await httpClient({ + let result = '' + + return new Promise((resolve, reject) => { + const request = (~urlAddr.indexOf('https') ? https : http).request( urlAddr, - request, - }) - const respBody = parser.parse(XMLData) + { + method: 'PROPFIND', + headers: { + depth: '1', + authorization, + }, + agent: ~urlAddr.indexOf('https') ? httpsAgent : httpAgent, + rejectUnauthorized: false, + }, + async (message) => { + logger.info('webdav接收文件信息') + + if (message.statusCode === 404) { + reject('404') + } + + message.on('data', (chunk) => { + result += chunk + }) + + message.on('end', () => { + resolve(parser.parse(result)) + }) + } + ) + + request.end() + }) +} + +// get file info from webdav +export async function getWebdavFileInfo(urlAddr: string, authorization: string): Promise { + try { + const respBody = await webdavPropfind(urlAddr, authorization) const res = respBody.multistatus.response - const size = Number(res.propstat.prop.getcontentlength || 0) + const size = res.propstat.prop.getcontentlength const name = res.propstat.prop.displayname - return { size, name, is_dir: size === 0, path: res.href } + return { size, name, is_dir: size === undefined, path: res.href } } catch (e) { - logger.error('@@webdavFileInfo_error:', urlAddr) + if (e === '404') return null } - return null } diff --git a/node-proxy/tsconfig.json b/node-proxy/tsconfig.json index 248ce3b..e67da05 100644 --- a/node-proxy/tsconfig.json +++ b/node-proxy/tsconfig.json @@ -12,7 +12,7 @@ ] }, // "allowImportingTsExtensions": true, - "allowJs": false, + "allowJs": true, "outDir": "./dist" }, "include": [ diff --git a/node-proxy/webpack.config.ts b/node-proxy/webpack.config.ts index 5b5932b..4d91a03 100644 --- a/node-proxy/webpack.config.ts +++ b/node-proxy/webpack.config.ts @@ -10,7 +10,7 @@ const output = { } export default () => { return { - entry: { index: path.resolve('./app.ts'), PRGAThreadCom: path.resolve('./src/utils/PRGAThreadCom.js') }, + entry: { index: path.resolve('./app.ts'), PRGAThreadCom: path.resolve('./src/utils/PRGAThreadCom.ts') }, output, module: { rules: [