natty
/
calckey
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

use gunzip-maybe tar-stream to replace exec

This commit is contained in:
CGsama 2023-07-16 01:41:57 -04:00
parent 324d1a932d
commit 809d418018
2 changed files with 67 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/backend/package.json
View File

@ -65,6 +65,7 @@
"file-type": "17.1.6",
"fluent-ffmpeg": "2.1.2",
"got": "12.5.3",
"gunzip-maybe": "^1.4.2",
"hpagent": "0.1.2",
"ioredis": "5.3.2",
"ip-cidr": "3.1.0",
@ -125,6 +126,7 @@
"summaly": "2.7.0",
"syslog-pro": "1.0.0",
"systeminformation": "5.17.17",
"tar-stream": "^3.1.6",
"tesseract.js": "^3.0.3",
"tinycolor2": "1.5.2",
"tmp": "0.2.1",

87
packages/backend/src/misc/process-masto-notes.ts
View File

@ -3,10 +3,11 @@ import Logger from "@/services/logger.js";
import { createTemp, createTempDir } from "./create-temp.js";
import { downloadUrl } from "./download-url.js";
import { addFile } from "@/services/drive/add-file.js";
import { exec } from "node:child_process";
import { Users } from "@/models/index.js";
import * as tar from 'tar-stream';
import gunzip from "gunzip-maybe";
const logger = new Logger("download-text-file");
const logger = new Logger("process-masto-notes");
export async function processMastoNotes(
url: string,
@ -32,27 +33,69 @@ export async function processMastoNotes(
function processMastoFile(fn: string, dir: string, uid: string) {
return new Promise(async (resolve, reject) => {
const user = await Users.findOneBy({ id: uid });
exec(
`tar -xf ${fn} -C ${dir}`,
async (error: any, stdout: string, stderr: string) => {
if (error) {
reject(error);
}
const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`));
for (const note of outbox.orderedItems) {
for (const attachment of note.object.attachment) {
const url = attachment.url.replace("..", "");
try {
const fpath = `${dir}${url}`;
const driveFile = await addFile({ user: user, path: fpath });
attachment.driveFile = driveFile;
} catch (e) {
logger.error(`Skipped adding file to drive: ${url}`);
}
try{
logger.info(`Start unzip ${fn}`);
await unzipTarGz(fn, dir);
logger.info(`Unzip to ${dir}`);
const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`));
for (const note of outbox.orderedItems) {
for (const attachment of note.object.attachment) {
const url = attachment.url.replace("..", "");
try {
const fpath = `${dir}${url}`;
const driveFile = await addFile({ user: user, path: fpath });
attachment.driveFile = driveFile;
} catch (e) {
logger.error(`Skipped adding file to drive: ${url}`);
}
}
resolve(outbox);
},
);
}
resolve(outbox);
}catch(e){
logger.error(`Error on extract masto note package: ${fn}`);
reject(e);
}
});
}
function createFileDir(fn: string){
if(!fs.existsSync(fn)){
fs.mkdirSync(fn, {recursive: true});
fs.rmdirSync(fn);
}
}
function unzipTarGz(fn: string, dir: string){
return new Promise(async (resolve, reject) => {
const onErr = (err: any) => {
logger.error(`pipe broken: ${err}`);
reject();
}
try{
const extract = tar.extract().on('error', onErr);
dir = dir.endsWith("/") ? dir : dir + "/";
const ls: string[] = [];
extract.on('entry', function (header: any, stream: any, next: any) {
try{
ls.push(dir + header.name);
createFileDir(dir + header.name);
stream.on('error', onErr).pipe(fs.createWriteStream(dir + header.name)).on('error', onErr);
next();
}catch(e){
logger.error(`create dir error:${e}`);
reject();
}
});
extract.on('finish', function () {
resolve(ls);
});
fs.createReadStream(fn).on('error', onErr).pipe(gunzip()).on('error', onErr).pipe(extract).on('error', onErr);
}catch(e){
logger.error(`unzipTarGz error: ${e}`);
reject();
}
});
}