use gunzip-maybe tar-stream to replace exec
This commit is contained in:
parent
324d1a932d
commit
809d418018
|
@ -65,6 +65,7 @@
|
|||
"file-type": "17.1.6",
|
||||
"fluent-ffmpeg": "2.1.2",
|
||||
"got": "12.5.3",
|
||||
"gunzip-maybe": "^1.4.2",
|
||||
"hpagent": "0.1.2",
|
||||
"ioredis": "5.3.2",
|
||||
"ip-cidr": "3.1.0",
|
||||
|
@ -125,6 +126,7 @@
|
|||
"summaly": "2.7.0",
|
||||
"syslog-pro": "1.0.0",
|
||||
"systeminformation": "5.17.17",
|
||||
"tar-stream": "^3.1.6",
|
||||
"tesseract.js": "^3.0.3",
|
||||
"tinycolor2": "1.5.2",
|
||||
"tmp": "0.2.1",
|
||||
|
|
|
@ -3,10 +3,11 @@ import Logger from "@/services/logger.js";
|
|||
import { createTemp, createTempDir } from "./create-temp.js";
|
||||
import { downloadUrl } from "./download-url.js";
|
||||
import { addFile } from "@/services/drive/add-file.js";
|
||||
import { exec } from "node:child_process";
|
||||
import { Users } from "@/models/index.js";
|
||||
import * as tar from 'tar-stream';
|
||||
import gunzip from "gunzip-maybe";
|
||||
|
||||
const logger = new Logger("download-text-file");
|
||||
const logger = new Logger("process-masto-notes");
|
||||
|
||||
export async function processMastoNotes(
|
||||
url: string,
|
||||
|
@ -32,27 +33,69 @@ export async function processMastoNotes(
|
|||
function processMastoFile(fn: string, dir: string, uid: string) {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
const user = await Users.findOneBy({ id: uid });
|
||||
exec(
|
||||
`tar -xf ${fn} -C ${dir}`,
|
||||
async (error: any, stdout: string, stderr: string) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
}
|
||||
const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`));
|
||||
for (const note of outbox.orderedItems) {
|
||||
for (const attachment of note.object.attachment) {
|
||||
const url = attachment.url.replace("..", "");
|
||||
try {
|
||||
const fpath = `${dir}${url}`;
|
||||
const driveFile = await addFile({ user: user, path: fpath });
|
||||
attachment.driveFile = driveFile;
|
||||
} catch (e) {
|
||||
logger.error(`Skipped adding file to drive: ${url}`);
|
||||
}
|
||||
try{
|
||||
logger.info(`Start unzip ${fn}`);
|
||||
await unzipTarGz(fn, dir);
|
||||
logger.info(`Unzip to ${dir}`);
|
||||
const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`));
|
||||
for (const note of outbox.orderedItems) {
|
||||
for (const attachment of note.object.attachment) {
|
||||
const url = attachment.url.replace("..", "");
|
||||
try {
|
||||
const fpath = `${dir}${url}`;
|
||||
const driveFile = await addFile({ user: user, path: fpath });
|
||||
attachment.driveFile = driveFile;
|
||||
} catch (e) {
|
||||
logger.error(`Skipped adding file to drive: ${url}`);
|
||||
}
|
||||
}
|
||||
resolve(outbox);
|
||||
},
|
||||
);
|
||||
}
|
||||
resolve(outbox);
|
||||
}catch(e){
|
||||
logger.error(`Error on extract masto note package: ${fn}`);
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function createFileDir(fn: string){
|
||||
if(!fs.existsSync(fn)){
|
||||
fs.mkdirSync(fn, {recursive: true});
|
||||
fs.rmdirSync(fn);
|
||||
}
|
||||
}
|
||||
|
||||
function unzipTarGz(fn: string, dir: string){
|
||||
return new Promise(async (resolve, reject) => {
|
||||
const onErr = (err: any) => {
|
||||
logger.error(`pipe broken: ${err}`);
|
||||
reject();
|
||||
}
|
||||
try{
|
||||
const extract = tar.extract().on('error', onErr);
|
||||
dir = dir.endsWith("/") ? dir : dir + "/";
|
||||
const ls: string[] = [];
|
||||
extract.on('entry', function (header: any, stream: any, next: any) {
|
||||
try{
|
||||
ls.push(dir + header.name);
|
||||
createFileDir(dir + header.name);
|
||||
stream.on('error', onErr).pipe(fs.createWriteStream(dir + header.name)).on('error', onErr);
|
||||
next();
|
||||
}catch(e){
|
||||
logger.error(`create dir error:${e}`);
|
||||
reject();
|
||||
}
|
||||
});
|
||||
|
||||
extract.on('finish', function () {
|
||||
resolve(ls);
|
||||
});
|
||||
|
||||
fs.createReadStream(fn).on('error', onErr).pipe(gunzip()).on('error', onErr).pipe(extract).on('error', onErr);
|
||||
|
||||
}catch(e){
|
||||
logger.error(`unzipTarGz error: ${e}`);
|
||||
reject();
|
||||
}
|
||||
});
|
||||
}
|
Loading…
Reference in New Issue