From 324d1a932d313f524181df89603d95cf254d9629 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sat, 15 Jul 2023 23:02:00 -0400 Subject: [PATCH] handle import mastodon package --- .../backend/src/misc/process-masto-notes.ts | 58 +++++++++++++++++++ .../queue/processors/db/import-masto-post.ts | 33 ++++++----- .../src/queue/processors/db/import-posts.ts | 17 ++++++ 3 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 packages/backend/src/misc/process-masto-notes.ts diff --git a/packages/backend/src/misc/process-masto-notes.ts b/packages/backend/src/misc/process-masto-notes.ts new file mode 100644 index 0000000000..1310748d2a --- /dev/null +++ b/packages/backend/src/misc/process-masto-notes.ts @@ -0,0 +1,58 @@ +import * as fs from "node:fs"; +import Logger from "@/services/logger.js"; +import { createTemp, createTempDir } from "./create-temp.js"; +import { downloadUrl } from "./download-url.js"; +import { addFile } from "@/services/drive/add-file.js"; +import { exec } from "node:child_process"; +import { Users } from "@/models/index.js"; + +const logger = new Logger("download-text-file"); + +export async function processMastoNotes( + url: string, + uid: string, +): Promise { + // Create temp file + const [path, cleanup] = await createTemp(); + + const [unzipPath, unzipCleanup] = await createTempDir(); + + logger.info(`Temp file is ${path}`); + + try { + // write content at URL to temp file + await downloadUrl(url, path); + return await processMastoFile(path, unzipPath, uid); + } finally { + cleanup(); + unzipCleanup(); + } +} + +function processMastoFile(fn: string, dir: string, uid: string) { + return new Promise(async (resolve, reject) => { + const user = await Users.findOneBy({ id: uid }); + exec( + `tar -xf ${fn} -C ${dir}`, + async (error: any, stdout: string, stderr: string) => { + if (error) { + reject(error); + } + const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`)); + for (const note of outbox.orderedItems) { + for (const attachment of note.object.attachment) { + const url = attachment.url.replace("..", ""); + try { + const fpath = `${dir}${url}`; + const driveFile = await addFile({ user: user, path: fpath }); + attachment.driveFile = driveFile; + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } + } + } + resolve(outbox); + }, + ); + }); +} diff --git a/packages/backend/src/queue/processors/db/import-masto-post.ts b/packages/backend/src/queue/processors/db/import-masto-post.ts index 05166b0858..1d18008a0f 100644 --- a/packages/backend/src/queue/processors/db/import-masto-post.ts +++ b/packages/backend/src/queue/processors/db/import-masto-post.ts @@ -45,19 +45,26 @@ export async function importMastoPost( throw e; } job.progress(80); - const urls = post.object.attachment - .map((x: any) => x.url) - .filter((x: String) => x.startsWith("http")); - const files: DriveFile[] = []; - for (const url of urls) { - try { - const file = await uploadFromUrl({ - url: url, - user: user, - }); - files.push(file); - } catch (e) { - logger.error(`Skipped adding file to drive: ${url}`); + + let files: DriveFile[] = (post.object.attachment || []) + .map((x: any) => x?.driveFile) + .filter((x: any) => x); + + if (files.length == 0) { + const urls = post.object.attachment + .map((x: any) => x.url) + .filter((x: String) => x.startsWith("http")); + files = []; + for (const url of urls) { + try { + const file = await uploadFromUrl({ + url: url, + user: user, + }); + files.push(file); + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } } } diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index f92a5f710a..54061f8bf9 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -1,4 +1,5 @@ import { downloadTextFile } from "@/misc/download-text-file.js"; +import { processMastoNotes } from "@/misc/process-masto-notes.js"; import { Users, DriveFiles } from "@/models/index.js"; import type { DbUserImportPostsJobData } from "@/queue/types.js"; import { queueLogger } from "../../logger.js"; @@ -30,6 +31,22 @@ export async function importPosts( return; } + if (file.name.endsWith("tar.gz")) { + try { + logger.info("Parsing animal style posts in package"); + const outbox = await processMastoNotes(file.url, job.data.user.id); + for (const post of outbox.orderedItems) { + createImportMastoPostJob(job.data.user, post, job.data.signatureCheck); + } + } catch (e) { + // handle error + logger.warn(`Error reading: ${e}`); + } + logger.succ("Imported"); + done(); + return; + } + const json = await downloadTextFile(file.url); try {