calckey/packages/backend/src/misc/download-url.ts

90 lines
2.4 KiB
TypeScript

import * as fs from 'node:fs';
import * as stream from 'node:stream';
import * as util from 'node:util';
import got, * as Got from 'got';
import { httpAgent, httpsAgent, StatusError } from './fetch.js';
import config from '@/config/index.js';
import chalk from 'chalk';
import Logger from '@/services/logger.js';
import IPCIDR from 'ip-cidr';
import PrivateIp from 'private-ip';
const pipeline = util.promisify(stream.pipeline);
export async function downloadUrl(url: string, path: string): Promise<void> {
const logger = new Logger('download');
logger.info(`Downloading ${chalk.cyan(url)} ...`);
const timeout = 30 * 1000;
const operationTimeout = 60 * 1000;
const maxSize = config.maxFileSize || 262144000;
const req = got.stream(url, {
headers: {
'User-Agent': config.userAgent,
},
timeout: {
lookup: timeout,
connect: timeout,
secureConnect: timeout,
socket: timeout, // read timeout
response: timeout,
send: timeout,
request: operationTimeout, // whole operation timeout
},
agent: {
http: httpAgent,
https: httpsAgent,
},
http2: false, // default
retry: {
limit: 0,
},
}).on('response', (res: Got.Response) => {
if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) {
if (isPrivateIp(res.ip)) {
logger.warn(`Blocked address: ${res.ip}`);
req.destroy();
}
}
const contentLength = res.headers['content-length'];
if (contentLength != null) {
const size = Number(contentLength);
if (size > maxSize) {
logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
req.destroy();
}
}
}).on('downloadProgress', (progress: Got.Progress) => {
if (progress.transferred > maxSize) {
logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
req.destroy();
}
});
try {
await pipeline(req, fs.createWriteStream(path));
} catch (e) {
if (e instanceof Got.HTTPError) {
throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
} else {
throw e;
}
}
logger.succ(`Download finished: ${chalk.cyan(url)}`);
}
function isPrivateIp(ip: string): boolean {
for (const net of config.allowedPrivateNetworks || []) {
const cidr = new IPCIDR(net);
if (cidr.contains(ip)) {
return false;
}
}
return PrivateIp(ip);
}