-
-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* s2-2 * allowedPrivateNetworks * style * Proxyの間にあると誤解しそうなのでconfigの記述順を変更 * Fix error handler
- Loading branch information
Showing
7 changed files
with
106 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,40 +1,71 @@ | ||
import * as fs from 'fs'; | ||
import * as stream from 'stream'; | ||
import * as util from 'util'; | ||
import { URL } from 'url'; | ||
import fetch from 'node-fetch'; | ||
import { getAgentByUrl } from './fetch'; | ||
import { AbortController } from 'abort-controller'; | ||
import got, * as Got from 'got'; | ||
import { httpAgent, httpsAgent } from './fetch'; | ||
import config from '@/config/index'; | ||
import * as chalk from 'chalk'; | ||
import Logger from '@/services/logger'; | ||
import * as IPCIDR from 'ip-cidr'; | ||
const PrivateIp = require('private-ip'); | ||
|
||
const pipeline = util.promisify(stream.pipeline); | ||
|
||
export async function downloadUrl(url: string, path: string) { | ||
const logger = new Logger('download'); | ||
|
||
logger.info(`Downloading ${chalk.cyan(url)} ...`); | ||
const controller = new AbortController(); | ||
setTimeout(() => { | ||
controller.abort(); | ||
}, 60 * 1000); | ||
|
||
const response = await fetch(new URL(url).href, { | ||
const timeout = 30 * 1000; | ||
const operationTimeout = 60 * 1000; | ||
|
||
const req = got.stream(url, { | ||
headers: { | ||
'User-Agent': config.userAgent | ||
}, | ||
timeout: 10 * 1000, | ||
signal: controller.signal, | ||
agent: getAgentByUrl, | ||
timeout: { | ||
lookup: timeout, | ||
connect: timeout, | ||
secureConnect: timeout, | ||
socket: timeout, // read timeout | ||
response: timeout, | ||
send: timeout, | ||
request: operationTimeout, // whole operation timeout | ||
}, | ||
agent: { | ||
http: httpAgent, | ||
https: httpsAgent, | ||
}, | ||
retry: 0, | ||
}).on('response', (res: Got.Response) => { | ||
if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) { | ||
if (isPrivateIp(res.ip)) { | ||
logger.warn(`Blocked address: ${res.ip}`); | ||
req.destroy(); | ||
} | ||
} | ||
}).on('error', (e: any) => { | ||
if (e.name === 'HTTPError') { | ||
const statusCode = e.response?.statusCode; | ||
const statusMessage = e.response?.statusMessage; | ||
e.name = `StatusError`; | ||
e.statusCode = statusCode; | ||
e.message = `${statusCode} ${statusMessage}`; | ||
} | ||
}); | ||
|
||
if (!response.ok) { | ||
logger.error(`Got ${response.status} (${url})`); | ||
throw response.status; | ||
} | ||
|
||
await pipeline(response.body, fs.createWriteStream(path)); | ||
await pipeline(req, fs.createWriteStream(path)); | ||
|
||
logger.succ(`Download finished: ${chalk.cyan(url)}`); | ||
} | ||
|
||
function isPrivateIp(ip: string) { | ||
for (const net of config.allowedPrivateNetworks || []) { | ||
const cidr = new IPCIDR(net); | ||
if (cidr.contains(ip)) { | ||
return false; | ||
} | ||
} | ||
|
||
return PrivateIp(ip); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5742,6 +5742,27 @@ ioredis@^4.27.0: | |
redis-parser "^3.0.0" | ||
standard-as-callback "^2.1.0" | ||
|
||
ip-address@^7.1.0: | ||
version "7.1.0" | ||
resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-7.1.0.tgz#4a9c699e75b51cbeb18b38de8ed216efa1a490c5" | ||
integrity sha512-V9pWC/VJf2lsXqP7IWJ+pe3P1/HCYGBMZrrnT62niLGjAfCbeiwXMUxaeHvnVlz19O27pvXP4azs+Pj/A0x+SQ== | ||
dependencies: | ||
jsbn "1.1.0" | ||
sprintf-js "1.1.2" | ||
|
||
[email protected]: | ||
version "3.0.4" | ||
resolved "https://registry.yarnpkg.com/ip-cidr/-/ip-cidr-3.0.4.tgz#a915c47e00f47ea8d5f8ed662ea6161471c44375" | ||
integrity sha512-pKNiqmBlTvEkhaLAa3+FOmYSY0/jjADVxxjA3NbujZZTT8mjLI90Q+6mwg6kd0fNm0RuAOkWJ1u1a/ETmlrPNQ== | ||
dependencies: | ||
ip-address "^7.1.0" | ||
jsbn "^1.1.0" | ||
|
||
ip-regex@^4.3.0: | ||
version "4.3.0" | ||
resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-4.3.0.tgz#687275ab0f57fa76978ff8f4dddc8a23d5990db5" | ||
integrity sha512-B9ZWJxHHOHUhUjCPrMpLD4xEq35bUTClHM1S6CBU5ixQnkZmwipwgc96vAd7AAGM9TGHvJR+Uss+/Ak6UphK+Q== | ||
|
||
ip@^1.1.5: | ||
version "1.1.5" | ||
resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a" | ||
|
@@ -6238,6 +6259,11 @@ js-yaml@~3.7.0: | |
argparse "^1.0.7" | ||
esprima "^2.6.0" | ||
|
||
[email protected], jsbn@^1.1.0: | ||
version "1.1.0" | ||
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040" | ||
integrity sha1-sBMHyym2GKHtJux56RH4A8TaAEA= | ||
|
||
jsbn@~0.1.0: | ||
version "0.1.1" | ||
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" | ||
|
@@ -7440,6 +7466,11 @@ [email protected]: | |
resolved "https://registry.yarnpkg.com/nested-property/-/nested-property-4.0.0.tgz#a67b5a31991e701e03cdbaa6453bc5b1011bb88d" | ||
integrity sha512-yFehXNWRs4cM0+dz7QxCd06hTbWbSkV0ISsqBfkntU6TOY4Qm3Q88fRRLOddkGh2Qq6dZvnKVAahfhjcUvLnyA== | ||
|
||
netmask@^2.0.2: | ||
version "2.0.2" | ||
resolved "https://registry.yarnpkg.com/netmask/-/netmask-2.0.2.tgz#8b01a07644065d536383835823bc52004ebac5e7" | ||
integrity sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg== | ||
|
||
next-line@^1.1.0: | ||
version "1.1.0" | ||
resolved "https://registry.yarnpkg.com/next-line/-/next-line-1.1.0.tgz#fcae57853052b6a9bae8208e40dd7d3c2d304603" | ||
|
@@ -8861,6 +8892,14 @@ [email protected]: | |
resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.24.1.tgz#c4d7895c4d6500289482fa8936d9cdd192684036" | ||
integrity sha512-mNPsedLuk90RVJioIky8ANZEwYm5w9LcvCXrxHlwf4fNVSn8jEipMybMkWUyyF0JhnC+C4VcOVSBuHRKs1L5Ow== | ||
|
||
[email protected]: | ||
version "2.2.1" | ||
resolved "https://registry.yarnpkg.com/private-ip/-/private-ip-2.2.1.tgz#4fe167d04e12eca5c67cdcbd3224e86b38c79253" | ||
integrity sha512-jN1WT/br/VNW9xEcwHr6DjtOKxQ5qOIqmh7o+co2TWgq56pZJw99iO3UT1tWdfgsQiyK9FqG4ji3ykwpjFqITA== | ||
dependencies: | ||
ip-regex "^4.3.0" | ||
netmask "^2.0.2" | ||
|
||
[email protected]: | ||
version "7.2.1" | ||
resolved "https://registry.yarnpkg.com/probe-image-size/-/probe-image-size-7.2.1.tgz#df0c924e67e247bc94f8fcb0fad7f0081061fc44" | ||
|
@@ -10103,6 +10142,11 @@ split@^1.0.0: | |
dependencies: | ||
through "2" | ||
|
||
[email protected]: | ||
version "1.1.2" | ||
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.2.tgz#da1765262bf8c0f571749f2ad6c26300207ae673" | ||
integrity sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug== | ||
|
||
sprintf-js@~1.0.2: | ||
version "1.0.3" | ||
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" | ||
|