From e1a8b158e04ad567d92d8daf3cc0898ee18f1a2e Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Fri, 3 Sep 2021 21:00:44 +0900 Subject: [PATCH] Tune download (#2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * s2-2 * allowedPrivateNetworks * style * Proxyの間にあると誤解しそうなのでconfigの記述順を変更 * Fix error handler --- .config/example.yml | 4 ++ package.json | 2 + src/config/types.ts | 2 + src/misc/download-url.ts | 67 ++++++++++++++++++++++-------- src/server/file/send-drive-file.ts | 6 +-- src/server/proxy/proxy-media.ts | 4 +- yarn.lock | 44 ++++++++++++++++++++ 7 files changed, 106 insertions(+), 23 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index d5ea4ba607cc..2e13b63977a5 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -157,3 +157,7 @@ id: 'aid' # Sign to ActivityPub GET request (default: false) #signToActivityPubGet: true + +#allowedPrivateNetworks: [ +# '127.0.0.1/32' +#] diff --git a/package.json b/package.json index 1f9baa3ca27d..53dead5ef75d 100644 --- a/package.json +++ b/package.json @@ -148,6 +148,7 @@ "http-signature": "1.3.5", "idb-keyval": "5.1.3", "insert-text-at-cursor": "0.3.0", + "ip-cidr": "3.0.4", "is-svg": "4.3.1", "js-yaml": "4.1.0", "jsdom": "16.7.0", @@ -184,6 +185,7 @@ "postcss": "8.3.6", "postcss-loader": "6.1.1", "prismjs": "1.24.1", + "private-ip": "2.2.1", "probe-image-size": "7.2.1", "promise-limit": "2.7.0", "pug": "3.0.2", diff --git a/src/config/types.ts b/src/config/types.ts index 8084be1864e4..55beac6f556b 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -37,6 +37,8 @@ export type Source = { proxySmtp?: string; proxyBypassHosts?: string[]; + allowedPrivateNetworks?: string[]; + accesslog?: string; clusterLimit?: number; diff --git a/src/misc/download-url.ts b/src/misc/download-url.ts index 43e061c71595..463fb555bbb3 100644 --- a/src/misc/download-url.ts +++ b/src/misc/download-url.ts @@ -1,13 +1,13 @@ import * as fs from 'fs'; import * as stream from 'stream'; import * as util from 'util'; -import { URL } from 'url'; -import fetch from 'node-fetch'; -import { getAgentByUrl } from './fetch'; -import { AbortController } from 'abort-controller'; +import got, * as Got from 'got'; +import { httpAgent, httpsAgent } from './fetch'; import config from '@/config/index'; import * as chalk from 'chalk'; import Logger from '@/services/logger'; +import * as IPCIDR from 'ip-cidr'; +const PrivateIp = require('private-ip'); const pipeline = util.promisify(stream.pipeline); @@ -15,26 +15,57 @@ export async function downloadUrl(url: string, path: string) { const logger = new Logger('download'); logger.info(`Downloading ${chalk.cyan(url)} ...`); - const controller = new AbortController(); - setTimeout(() => { - controller.abort(); - }, 60 * 1000); - const response = await fetch(new URL(url).href, { + const timeout = 30 * 1000; + const operationTimeout = 60 * 1000; + + const req = got.stream(url, { headers: { 'User-Agent': config.userAgent }, - timeout: 10 * 1000, - signal: controller.signal, - agent: getAgentByUrl, + timeout: { + lookup: timeout, + connect: timeout, + secureConnect: timeout, + socket: timeout, // read timeout + response: timeout, + send: timeout, + request: operationTimeout, // whole operation timeout + }, + agent: { + http: httpAgent, + https: httpsAgent, + }, + retry: 0, + }).on('response', (res: Got.Response) => { + if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) { + if (isPrivateIp(res.ip)) { + logger.warn(`Blocked address: ${res.ip}`); + req.destroy(); + } + } + }).on('error', (e: any) => { + if (e.name === 'HTTPError') { + const statusCode = e.response?.statusCode; + const statusMessage = e.response?.statusMessage; + e.name = `StatusError`; + e.statusCode = statusCode; + e.message = `${statusCode} ${statusMessage}`; + } }); - if (!response.ok) { - logger.error(`Got ${response.status} (${url})`); - throw response.status; - } - - await pipeline(response.body, fs.createWriteStream(path)); + await pipeline(req, fs.createWriteStream(path)); logger.succ(`Download finished: ${chalk.cyan(url)}`); } + +function isPrivateIp(ip: string) { + for (const net of config.allowedPrivateNetworks || []) { + const cidr = new IPCIDR(net); + if (cidr.contains(ip)) { + return false; + } + } + + return PrivateIp(ip); +} diff --git a/src/server/file/send-drive-file.ts b/src/server/file/send-drive-file.ts index c455de6447d3..a73164ed21c7 100644 --- a/src/server/file/send-drive-file.ts +++ b/src/server/file/send-drive-file.ts @@ -83,10 +83,10 @@ export default async function(ctx: Koa.Context) { ctx.set('Content-Type', image.type); ctx.set('Cache-Control', 'max-age=31536000, immutable'); } catch (e) { - serverLogger.error(e); + serverLogger.error(e.statusCode); - if (typeof e == 'number' && e >= 400 && e < 500) { - ctx.status = e; + if (typeof e.statusCode === 'number' && e.statusCode >= 400 && e.statusCode < 500) { + ctx.status = e.statusCode; ctx.set('Cache-Control', 'max-age=86400'); } else { ctx.status = 500; diff --git a/src/server/proxy/proxy-media.ts b/src/server/proxy/proxy-media.ts index fb38a5c07bad..3bd65dfe67ee 100644 --- a/src/server/proxy/proxy-media.ts +++ b/src/server/proxy/proxy-media.ts @@ -39,8 +39,8 @@ export async function proxyMedia(ctx: Koa.Context) { } catch (e) { serverLogger.error(e); - if (typeof e == 'number' && e >= 400 && e < 500) { - ctx.status = e; + if (typeof e.statusCode === 'number' && e.statusCode >= 400 && e.statusCode < 500) { + ctx.status = e.statusCode; } else { ctx.status = 500; } diff --git a/yarn.lock b/yarn.lock index ccfee76557af..8151dfe089e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5742,6 +5742,27 @@ ioredis@^4.27.0: redis-parser "^3.0.0" standard-as-callback "^2.1.0" +ip-address@^7.1.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-7.1.0.tgz#4a9c699e75b51cbeb18b38de8ed216efa1a490c5" + integrity sha512-V9pWC/VJf2lsXqP7IWJ+pe3P1/HCYGBMZrrnT62niLGjAfCbeiwXMUxaeHvnVlz19O27pvXP4azs+Pj/A0x+SQ== + dependencies: + jsbn "1.1.0" + sprintf-js "1.1.2" + +ip-cidr@3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/ip-cidr/-/ip-cidr-3.0.4.tgz#a915c47e00f47ea8d5f8ed662ea6161471c44375" + integrity sha512-pKNiqmBlTvEkhaLAa3+FOmYSY0/jjADVxxjA3NbujZZTT8mjLI90Q+6mwg6kd0fNm0RuAOkWJ1u1a/ETmlrPNQ== + dependencies: + ip-address "^7.1.0" + jsbn "^1.1.0" + +ip-regex@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-4.3.0.tgz#687275ab0f57fa76978ff8f4dddc8a23d5990db5" + integrity sha512-B9ZWJxHHOHUhUjCPrMpLD4xEq35bUTClHM1S6CBU5ixQnkZmwipwgc96vAd7AAGM9TGHvJR+Uss+/Ak6UphK+Q== + ip@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a" @@ -6238,6 +6259,11 @@ js-yaml@~3.7.0: argparse "^1.0.7" esprima "^2.6.0" +jsbn@1.1.0, jsbn@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040" + integrity sha1-sBMHyym2GKHtJux56RH4A8TaAEA= + jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" @@ -7440,6 +7466,11 @@ nested-property@4.0.0: resolved "https://registry.yarnpkg.com/nested-property/-/nested-property-4.0.0.tgz#a67b5a31991e701e03cdbaa6453bc5b1011bb88d" integrity sha512-yFehXNWRs4cM0+dz7QxCd06hTbWbSkV0ISsqBfkntU6TOY4Qm3Q88fRRLOddkGh2Qq6dZvnKVAahfhjcUvLnyA== +netmask@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/netmask/-/netmask-2.0.2.tgz#8b01a07644065d536383835823bc52004ebac5e7" + integrity sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg== + next-line@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/next-line/-/next-line-1.1.0.tgz#fcae57853052b6a9bae8208e40dd7d3c2d304603" @@ -8861,6 +8892,14 @@ prismjs@1.24.1: resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.24.1.tgz#c4d7895c4d6500289482fa8936d9cdd192684036" integrity sha512-mNPsedLuk90RVJioIky8ANZEwYm5w9LcvCXrxHlwf4fNVSn8jEipMybMkWUyyF0JhnC+C4VcOVSBuHRKs1L5Ow== +private-ip@2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/private-ip/-/private-ip-2.2.1.tgz#4fe167d04e12eca5c67cdcbd3224e86b38c79253" + integrity sha512-jN1WT/br/VNW9xEcwHr6DjtOKxQ5qOIqmh7o+co2TWgq56pZJw99iO3UT1tWdfgsQiyK9FqG4ji3ykwpjFqITA== + dependencies: + ip-regex "^4.3.0" + netmask "^2.0.2" + probe-image-size@7.2.1: version "7.2.1" resolved "https://registry.yarnpkg.com/probe-image-size/-/probe-image-size-7.2.1.tgz#df0c924e67e247bc94f8fcb0fad7f0081061fc44" @@ -10103,6 +10142,11 @@ split@^1.0.0: dependencies: through "2" +sprintf-js@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.2.tgz#da1765262bf8c0f571749f2ad6c26300207ae673" + integrity sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug== + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"