From d5c80e74de0a97a7820cc24a6df4d8bad2e59700 Mon Sep 17 00:00:00 2001 From: cutestnekoaqua Date: Tue, 28 Mar 2023 23:29:47 +0200 Subject: [PATCH 1/3] feat: experimental post import --- packages/backend/src/misc/post.ts | 14 ++++++ packages/backend/src/queue/index.ts | 17 ++++++++ .../backend/src/queue/processors/db/index.ts | 2 + packages/backend/src/server/api/endpoints.ts | 2 + .../server/api/endpoints/i/import-posts.ts | 43 +++++++++++++++++++ .../src/pages/settings/import-export.vue | 10 +++++ 6 files changed, 88 insertions(+) create mode 100644 packages/backend/src/misc/post.ts create mode 100644 packages/backend/src/server/api/endpoints/i/import-posts.ts diff --git a/packages/backend/src/misc/post.ts b/packages/backend/src/misc/post.ts new file mode 100644 index 0000000000..e14aa3444f --- /dev/null +++ b/packages/backend/src/misc/post.ts @@ -0,0 +1,14 @@ +export type Post = { + text: string | null; + cw: string | null; + localOnly: boolean; + createdAt: Date; +}; + +export function parse(acct: any): Post { + return { text: acct.text, cw: acct.cw, localOnly: acct.localOnly, createdAt: new Date(acct.createdAt) }; +} + +export function toJson(acct: Post): string { + return { text: acct.text, cw: acct.cw, localOnly: acct.localOnly }.toString(); +} diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index c387efe927..d5b42d637f 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -314,6 +314,23 @@ export function createImportFollowingJob( ); } +export function createImportPostsJob( + user: ThinUser, + fileId: DriveFile["id"], +) { + return dbQueue.add( + "importPosts", + { + user: user, + fileId: fileId, + }, + { + removeOnComplete: true, + removeOnFail: true, + }, + ); +} + export function createImportMutingJob(user: ThinUser, fileId: DriveFile["id"]) { return dbQueue.add( "importMuting", diff --git a/packages/backend/src/queue/processors/db/index.ts b/packages/backend/src/queue/processors/db/index.ts index 90173053fb..22b55a3683 100644 --- a/packages/backend/src/queue/processors/db/index.ts +++ b/packages/backend/src/queue/processors/db/index.ts @@ -11,6 +11,7 @@ import { importFollowing } from "./import-following.js"; import { importUserLists } from "./import-user-lists.js"; import { deleteAccount } from "./delete-account.js"; import { importMuting } from "./import-muting.js"; +import { importPosts } from "./import-posts.js"; import { importBlocking } from "./import-blocking.js"; import { importCustomEmojis } from "./import-custom-emojis.js"; @@ -26,6 +27,7 @@ const jobs = { importMuting, importBlocking, importUserLists, + importPosts, importCustomEmojis, deleteAccount, } as Record< diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index ba0e721b9e..920f871995 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -182,6 +182,7 @@ import * as ep___i_exportBlocking from "./endpoints/i/export-blocking.js"; import * as ep___i_exportFollowing from "./endpoints/i/export-following.js"; import * as ep___i_exportMute from "./endpoints/i/export-mute.js"; import * as ep___i_exportNotes from "./endpoints/i/export-notes.js"; +import * as ep___i_importPosts from "./endpoints/i/import-posts.js"; import * as ep___i_exportUserLists from "./endpoints/i/export-user-lists.js"; import * as ep___i_favorites from "./endpoints/i/favorites.js"; import * as ep___i_gallery_likes from "./endpoints/i/gallery/likes.js"; @@ -527,6 +528,7 @@ const eps = [ ["i/export-following", ep___i_exportFollowing], ["i/export-mute", ep___i_exportMute], ["i/export-notes", ep___i_exportNotes], + ["i/import-posts", ep___i_importPosts], ["i/export-user-lists", ep___i_exportUserLists], ["i/favorites", ep___i_favorites], ["i/gallery/likes", ep___i_gallery_likes], diff --git a/packages/backend/src/server/api/endpoints/i/import-posts.ts b/packages/backend/src/server/api/endpoints/i/import-posts.ts new file mode 100644 index 0000000000..254e37d8a4 --- /dev/null +++ b/packages/backend/src/server/api/endpoints/i/import-posts.ts @@ -0,0 +1,43 @@ +import define from "../../define.js"; +import { createImportPostsJob } from "@/queue/index.js"; +import { ApiError } from "../../error.js"; +import { DriveFiles } from "@/models/index.js"; +import { DAY } from "@/const.js"; + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: DAY, + max: 1, + }, + errors: { + noSuchFile: { + message: "No such file.", + code: "NO_SUCH_FILE", + id: "e674141e-bd2a-ba85-e616-aefb187c9c2a", + }, + + emptyFile: { + message: "That file is empty.", + code: "EMPTY_FILE", + id: "d2f12af1-e7b4-feac-86a3-519548f2728e", + }, + }, +} as const; + +export const paramDef = { + type: "object", + properties: { + fileId: { type: "string", format: "misskey:id" }, + }, + required: ["fileId"], +} as const; + +export default define(meta, paramDef, async (ps, user) => { + const file = await DriveFiles.findOneBy({ id: ps.fileId }); + + if (file == null) throw new ApiError(meta.errors.noSuchFile); + if (file.size === 0) throw new ApiError(meta.errors.emptyFile); + createImportPostsJob(user, file.id); +}); diff --git a/packages/client/src/pages/settings/import-export.vue b/packages/client/src/pages/settings/import-export.vue index 64c3cb4f98..4822ccff06 100644 --- a/packages/client/src/pages/settings/import-export.vue +++ b/packages/client/src/pages/settings/import-export.vue @@ -7,6 +7,11 @@ {{ i18n.ts.export }} + + + + {{ i18n.ts.import }} + @@ -108,6 +113,11 @@ const exportNotes = () => { os.api('i/export-notes', {}).then(onExportSuccess).catch(onError); }; +const importPosts = async (ev) => { + const file = await selectFile(ev.currentTarget ?? ev.target); + os.api('i/import-posts', { fileId: file.id }).then(onImportSuccess).catch(onError); +}; + const exportFollowing = () => { os.api('i/export-following', { excludeMuting: excludeMutingUsers.value, From 99b471d43b5d1cc29b5098fe68cdfd6f29babe81 Mon Sep 17 00:00:00 2001 From: cutestnekoaqua Date: Tue, 28 Mar 2023 23:48:27 +0200 Subject: [PATCH 2/3] why is this git ignored? --- .../src/queue/processors/db/import-posts.ts | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 packages/backend/src/queue/processors/db/import-posts.ts diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts new file mode 100644 index 0000000000..a5c5c36923 --- /dev/null +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -0,0 +1,76 @@ +import { IsNull } from "typeorm"; +import follow from "@/services/following/create.js"; + +import * as Post from "@/misc/posts.js"; +import create from "@/services/note/create.js"; +import { downloadTextFile } from "@/misc/download-text-file.js"; +import { Users, DriveFiles } from "@/models/index.js"; +import type { DbUserImportJobData } from "@/queue/types.js"; +import { queueLogger } from "../../logger.js"; +import type Bull from "bull"; + +const logger = queueLogger.createSubLogger("import-posts"); + +export async function importPosts( + job: Bull.Job, + done: any, +): Promise { + logger.info(`Importing following of ${job.data.user.id} ...`); + + const user = await Users.findOneBy({ id: job.data.user.id }); + if (user == null) { + done(); + return; + } + + const file = await DriveFiles.findOneBy({ + id: job.data.fileId, + }); + if (file == null) { + done(); + return; + } + + const csv = await downloadTextFile(file.url); + + let linenum = 0; + + if (file.type.endsWith("json")) { + for (const post of JSON.parse(csv)) { + try { + if (post.replyId != null) { + continue; + } + if (post.renoteId != null) { + continue; + } + if (post.visibility !== "public") continue; + const { text, cw, localOnly, createdAt } = Post.parse(post); + + logger.info(`Posting[${linenum}] ...`); + + const note = await create(user, { + createdAt: createdAt, + files: undefined, + poll: undefined, + text: text || undefined, + reply: null, + renote: null, + cw: cw, + localOnly, + visibility: "public", + visibleUsers: [], + channel: null, + apMentions: undefined, + apHashtags: undefined, + apEmojis: undefined, + }); + } catch (e) { + logger.warn(`Error in line:${linenum} ${e}`); + } + } + } + + logger.succ("Imported"); + done(); +} From 450f798f12e40653d756f1782a7cb2b78fd50f94 Mon Sep 17 00:00:00 2001 From: cutestnekoaqua Date: Tue, 28 Mar 2023 23:48:27 +0200 Subject: [PATCH 3/3] why is this git ignored? --- packages/backend/src/queue/processors/db/import-posts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index a5c5c36923..a28350c757 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -1,7 +1,7 @@ import { IsNull } from "typeorm"; import follow from "@/services/following/create.js"; -import * as Post from "@/misc/posts.js"; +import * as Post from "@/misc/post.js"; import create from "@/services/note/create.js"; import { downloadTextFile } from "@/misc/download-text-file.js"; import { Users, DriveFiles } from "@/models/index.js";