handle import mastodon package

This commit is contained in:
CGsama 2023-07-15 23:02:00 -04:00
parent cae6ba0edb
commit 324d1a932d
3 changed files with 95 additions and 13 deletions

View File

@ -0,0 +1,58 @@
import * as fs from "node:fs";
import Logger from "@/services/logger.js";
import { createTemp, createTempDir } from "./create-temp.js";
import { downloadUrl } from "./download-url.js";
import { addFile } from "@/services/drive/add-file.js";
import { exec } from "node:child_process";
import { Users } from "@/models/index.js";
const logger = new Logger("download-text-file");
export async function processMastoNotes(
url: string,
uid: string,
): Promise<any> {
// Create temp file
const [path, cleanup] = await createTemp();
const [unzipPath, unzipCleanup] = await createTempDir();
logger.info(`Temp file is ${path}`);
try {
// write content at URL to temp file
await downloadUrl(url, path);
return await processMastoFile(path, unzipPath, uid);
} finally {
cleanup();
unzipCleanup();
}
}
function processMastoFile(fn: string, dir: string, uid: string) {
return new Promise(async (resolve, reject) => {
const user = await Users.findOneBy({ id: uid });
exec(
`tar -xf ${fn} -C ${dir}`,
async (error: any, stdout: string, stderr: string) => {
if (error) {
reject(error);
}
const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`));
for (const note of outbox.orderedItems) {
for (const attachment of note.object.attachment) {
const url = attachment.url.replace("..", "");
try {
const fpath = `${dir}${url}`;
const driveFile = await addFile({ user: user, path: fpath });
attachment.driveFile = driveFile;
} catch (e) {
logger.error(`Skipped adding file to drive: ${url}`);
}
}
}
resolve(outbox);
},
);
});
}

View File

@ -45,19 +45,26 @@ export async function importMastoPost(
throw e; throw e;
} }
job.progress(80); job.progress(80);
const urls = post.object.attachment
.map((x: any) => x.url) let files: DriveFile[] = (post.object.attachment || [])
.filter((x: String) => x.startsWith("http")); .map((x: any) => x?.driveFile)
const files: DriveFile[] = []; .filter((x: any) => x);
for (const url of urls) {
try { if (files.length == 0) {
const file = await uploadFromUrl({ const urls = post.object.attachment
url: url, .map((x: any) => x.url)
user: user, .filter((x: String) => x.startsWith("http"));
}); files = [];
files.push(file); for (const url of urls) {
} catch (e) { try {
logger.error(`Skipped adding file to drive: ${url}`); const file = await uploadFromUrl({
url: url,
user: user,
});
files.push(file);
} catch (e) {
logger.error(`Skipped adding file to drive: ${url}`);
}
} }
} }

View File

@ -1,4 +1,5 @@
import { downloadTextFile } from "@/misc/download-text-file.js"; import { downloadTextFile } from "@/misc/download-text-file.js";
import { processMastoNotes } from "@/misc/process-masto-notes.js";
import { Users, DriveFiles } from "@/models/index.js"; import { Users, DriveFiles } from "@/models/index.js";
import type { DbUserImportPostsJobData } from "@/queue/types.js"; import type { DbUserImportPostsJobData } from "@/queue/types.js";
import { queueLogger } from "../../logger.js"; import { queueLogger } from "../../logger.js";
@ -30,6 +31,22 @@ export async function importPosts(
return; return;
} }
if (file.name.endsWith("tar.gz")) {
try {
logger.info("Parsing animal style posts in package");
const outbox = await processMastoNotes(file.url, job.data.user.id);
for (const post of outbox.orderedItems) {
createImportMastoPostJob(job.data.user, post, job.data.signatureCheck);
}
} catch (e) {
// handle error
logger.warn(`Error reading: ${e}`);
}
logger.succ("Imported");
done();
return;
}
const json = await downloadTextFile(file.url); const json = await downloadTextFile(file.url);
try { try {