Feat: move post imports to new jobs

packages/backend/src/queue/index.ts

@@ -333,6 +333,44 @@ export function createImportPostsJob(
 	);
 }
 
+export function createImportMastoPostJob(
+	user: ThinUser,
+	post: any,
+	signatureCheck: boolean,
+) {
+	return dbQueue.add(
+		"importMastoPost",
+		{
+			user: user,
+			post: post,
+			signatureCheck: signatureCheck,
+		},
+		{
+			removeOnComplete: true,
+			removeOnFail: true,
+		},
+	);
+}
+
+export function createImportCkPostJob(
+	user: ThinUser,
+	post: any,
+	signatureCheck: boolean,
+) {
+	return dbQueue.add(
+		"importCkPost",
+		{
+			user: user,
+			post: post,
+			signatureCheck: signatureCheck,
+		},
+		{
+			removeOnComplete: true,
+			removeOnFail: true,
+		},
+	);
+}
+
 export function createImportMutingJob(user: ThinUser, fileId: DriveFile["id"]) {
 	return dbQueue.add(
 		"importMuting",

packages/backend/src/queue/processors/db/import-posts.ts

@@ -1,16 +1,9 @@
-import { IsNull } from "typeorm";
-import follow from "@/services/following/create.js";
-
-import * as Post from "@/misc/post.js";
-import create from "@/services/note/create.js";
 import { downloadTextFile } from "@/misc/download-text-file.js";
 import { Users, DriveFiles } from "@/models/index.js";
 import type { DbUserImportPostsJobData } from "@/queue/types.js";
 import { queueLogger } from "../../logger.js";
 import type Bull from "bull";
-import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js";
+import { createImportCkPostJob, createImportMastoPostJob } from "@/queue/index.js";
-import { resolveNote } from "@/remote/activitypub/models/note.js";
-import { Note } from "@/models/entities/note.js";
 
 const logger = queueLogger.createSubLogger("import-posts");
 
@@ -36,94 +29,17 @@ export async function importPosts(
 
 	const json = await downloadTextFile(file.url);
 
-	let linenum = 0;
-
 	try {
 		const parsed = JSON.parse(json);
 		if (parsed instanceof Array) {
 			logger.info("Parsing key style posts");
 			for (const post of JSON.parse(json)) {
-				try {
+				createImportCkPostJob(job.data.user, post, job.data.signatureCheck);
-					linenum++;
-					if (post.replyId != null) {
-						continue;
-					}
-					if (post.renoteId != null) {
-						continue;
-					}
-					if (post.visibility !== "public") {
-						continue;
-					}
-					const { text, cw, localOnly, createdAt } = Post.parse(post);
-
-					logger.info(`Posting[${linenum}] ...`);
-
-					const note = await create(user, {
-						createdAt: createdAt,
-						files: undefined,
-						poll: undefined,
-						text: text || undefined,
-						reply: null,
-						renote: null,
-						cw: cw,
-						localOnly,
-						visibility: "hidden",
-						visibleUsers: [],
-						channel: null,
-						apMentions: new Array(0),
-						apHashtags: undefined,
-						apEmojis: undefined,
-					});
-				} catch (e) {
-					logger.warn(`Error in line:${linenum} ${e}`);
-				}
 			}
 		} else if (parsed instanceof Object) {
 			logger.info("Parsing animal style posts");
 			for (const post of parsed.orderedItems) {
-				async () => {
+				createImportMastoPostJob(job.data.user, post, job.data.signatureCheck);
-					try {
-						linenum++;
-						let reply: Note | null = null;
-						if (post.object.inReplyTo != null) {
-							reply = await resolveNote(post.object.inReplyTo);
-						}
-						if (post.directMessage) {
-							return;
-						}
-						if (job.data.signatureCheck) {
-							if (!post.signature) {
-								return;
-							}
-						}
-						let text;
-						try {
-							text = htmlToMfm(post.object.content, post.object.tag);
-						} catch (e) {
-							return;
-						}
-						logger.info(`Posting[${linenum}] ...`);
-
-						const note = await create(user, {
-							createdAt: new Date(post.object.published),
-							files: undefined,
-							poll: undefined,
-							text: text || undefined,
-							reply,
-							renote: null,
-							cw: post.sensitive,
-							localOnly: false,
-							visibility: "hidden",
-							visibleUsers: [],
-							channel: null,
-							apMentions: new Array(0),
-							apHashtags: undefined,
-							apEmojis: undefined,
-						});
-					} catch (e) {
-						logger.warn(`Error in line:${linenum} ${e}`);
-					}
-				};
 			}
 		}
 	} catch (e) {

packages/backend/src/queue/processors/db/index.ts

@@ -12,6 +12,8 @@ import { importUserLists } from "./import-user-lists.js";
 import { deleteAccount } from "./delete-account.js";
 import { importMuting } from "./import-muting.js";
 import { importPosts } from "./import-posts.js";
+import { importMastoPost } from "./import-masto-post.js";
+import { importCkPost } from "./import-calckey-post.js";
 import { importBlocking } from "./import-blocking.js";
 import { importCustomEmojis } from "./import-custom-emojis.js";
 
@@ -28,6 +30,8 @@ const jobs = {
 	importBlocking,
 	importUserLists,
 	importPosts,
+	importMastoPost,
+	importCkPost,
 	importCustomEmojis,
 	deleteAccount,
 } as Record<

packages/backend/src/queue/types.ts

@@ -23,7 +23,8 @@ export type DbJobData =
 	| DbUserJobData
 	| DbUserImportPostsJobData
 	| DbUserImportJobData
-	| DbUserDeleteJobData;
+	| DbUserDeleteJobData
+	| DbUserImportMastoPostJobData;
 
 export type DbUserJobData = {
 	user: ThinUser;
@@ -47,6 +48,12 @@ export type DbUserImportPostsJobData = {
 	signatureCheck: boolean;
 };
 
+export type DbUserImportMastoPostJobData = {
+	user: ThinUser;
+	post: any;
+	signatureCheck: boolean;
+};
+
 export type ObjectStorageJobData =
 	| ObjectStorageFileJobData
 	| Record<string, unknown>;