From ede47e45c8162674a02d28862f256dd072d9570f Mon Sep 17 00:00:00 2001 From: April John Date: Mon, 8 May 2023 22:06:30 +0200 Subject: [PATCH 1/3] try fetching replies --- packages/backend/src/queue/processors/db/import-posts.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index a0a916b41e..cd24029fbd 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -9,6 +9,7 @@ import type { DbUserImportPostsJobData } from "@/queue/types.js"; import { queueLogger } from "../../logger.js"; import type Bull from "bull"; import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js"; +import { resolveNote } from "@/remote/activitypub/models/note.js"; const logger = queueLogger.createSubLogger("import-posts"); @@ -81,8 +82,9 @@ export async function importPosts( for (const post of parsed.orderedItems) { try { linenum++; + let reply = null; if (post.object.inReplyTo != null) { - continue; + reply = await resolveNote(post.object.inReplyTo); } if (post.directMessage) { continue; @@ -105,7 +107,7 @@ export async function importPosts( files: undefined, poll: undefined, text: text || undefined, - reply: null, + reply, renote: null, cw: post.sensitive, localOnly: false, From ecf263bd91753bc4cfdb0bce4f47e1d2a187834a Mon Sep 17 00:00:00 2001 From: April John Date: Mon, 8 May 2023 22:08:29 +0200 Subject: [PATCH 2/3] import replies too --- packages/backend/src/queue/processors/db/import-posts.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index cd24029fbd..47096d414b 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -10,6 +10,7 @@ import { queueLogger } from "../../logger.js"; import type Bull from "bull"; import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js"; import { resolveNote } from "@/remote/activitypub/models/note.js"; +import { Note } from "@/models/entities/note.js"; const logger = queueLogger.createSubLogger("import-posts"); @@ -82,7 +83,7 @@ export async function importPosts( for (const post of parsed.orderedItems) { try { linenum++; - let reply = null; + let reply: Note | null = null; if (post.object.inReplyTo != null) { reply = await resolveNote(post.object.inReplyTo); } From 3bedd1b34799d969cd218b4a1a67651532a204e6 Mon Sep 17 00:00:00 2001 From: April John Date: Mon, 8 May 2023 22:30:40 +0200 Subject: [PATCH 3/3] fix: run post import async --- .../src/queue/processors/db/import-posts.ts | 76 ++++++++++--------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index 47096d414b..2aa02a80b1 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -81,46 +81,48 @@ export async function importPosts( } else if (parsed instanceof Object) { logger.info("Parsing animal style posts"); for (const post of parsed.orderedItems) { - try { - linenum++; - let reply: Note | null = null; - if (post.object.inReplyTo != null) { - reply = await resolveNote(post.object.inReplyTo); - } - if (post.directMessage) { - continue; - } - if (job.data.signatureCheck) { - if (!post.signature) { - continue; - } - } - let text; + async () =>{ try { - text = htmlToMfm(post.object.content, post.object.tag); + linenum++; + let reply: Note | null = null; + if (post.object.inReplyTo != null) { + reply = await resolveNote(post.object.inReplyTo); + } + if (post.directMessage) { + return; + } + if (job.data.signatureCheck) { + if (!post.signature) { + return; + } + } + let text; + try { + text = htmlToMfm(post.object.content, post.object.tag); + } catch (e) { + return; + } + logger.info(`Posting[${linenum}] ...`); + + const note = await create(user, { + createdAt: new Date(post.object.published), + files: undefined, + poll: undefined, + text: text || undefined, + reply, + renote: null, + cw: post.sensitive, + localOnly: false, + visibility: "hidden", + visibleUsers: [], + channel: null, + apMentions: new Array(0), + apHashtags: undefined, + apEmojis: undefined, + }); } catch (e) { - continue; + logger.warn(`Error in line:${linenum} ${e}`); } - logger.info(`Posting[${linenum}] ...`); - - const note = await create(user, { - createdAt: new Date(post.object.published), - files: undefined, - poll: undefined, - text: text || undefined, - reply, - renote: null, - cw: post.sensitive, - localOnly: false, - visibility: "hidden", - visibleUsers: [], - channel: null, - apMentions: new Array(0), - apHashtags: undefined, - apEmojis: undefined, - }); - } catch (e) { - logger.warn(`Error in line:${linenum} ${e}`); } } }