diff --git a/packages/backend/src/misc/post.ts b/packages/backend/src/misc/post.ts new file mode 100644 index 0000000000..e14aa3444f --- /dev/null +++ b/packages/backend/src/misc/post.ts @@ -0,0 +1,14 @@ +export type Post = { + text: string | null; + cw: string | null; + localOnly: boolean; + createdAt: Date; +}; + +export function parse(acct: any): Post { + return { text: acct.text, cw: acct.cw, localOnly: acct.localOnly, createdAt: new Date(acct.createdAt) }; +} + +export function toJson(acct: Post): string { + return { text: acct.text, cw: acct.cw, localOnly: acct.localOnly }.toString(); +} diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index c387efe927..d5b42d637f 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -314,6 +314,23 @@ export function createImportFollowingJob( ); } +export function createImportPostsJob( + user: ThinUser, + fileId: DriveFile["id"], +) { + return dbQueue.add( + "importPosts", + { + user: user, + fileId: fileId, + }, + { + removeOnComplete: true, + removeOnFail: true, + }, + ); +} + export function createImportMutingJob(user: ThinUser, fileId: DriveFile["id"]) { return dbQueue.add( "importMuting", diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts new file mode 100644 index 0000000000..f4948269c5 --- /dev/null +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -0,0 +1,132 @@ +import { IsNull } from "typeorm"; +import follow from "@/services/following/create.js"; + +import * as Post from "@/misc/post.js"; +import create from "@/services/note/create.js"; +import { downloadTextFile } from "@/misc/download-text-file.js"; +import { Users, DriveFiles } from "@/models/index.js"; +import type { DbUserImportJobData } from "@/queue/types.js"; +import { queueLogger } from "../../logger.js"; +import type Bull from "bull"; +import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js"; + +const logger = queueLogger.createSubLogger("import-posts"); + +export async function importPosts( + job: Bull.Job, + done: any, +): Promise { + logger.info(`Importing posts of ${job.data.user.id} ...`); + + const user = await Users.findOneBy({ id: job.data.user.id }); + if (user == null) { + done(); + return; + } + + const file = await DriveFiles.findOneBy({ + id: job.data.fileId, + }); + if (file == null) { + done(); + return; + } + + const json = await downloadTextFile(file.url); + + let linenum = 0; + + try { + const parsed = JSON.parse(json); + if (parsed instanceof Array) { + logger.info("Parsing key style posts"); + for (const post of JSON.parse(json)) { + try { + linenum++; + if (post.replyId != null) { + logger.info(`Is reply, skip [${linenum}] ...`); + continue; + } + if (post.renoteId != null) { + logger.info(`Is boost, skip [${linenum}] ...`); + continue; + } + if (post.visibility !== "public") { + logger.info(`Is non-public, skip [${linenum}] ...`); + continue; + } + const { text, cw, localOnly, createdAt } = Post.parse(post); + + logger.info(`Posting[${linenum}] ...`); + + const note = await create(user, { + createdAt: createdAt, + files: undefined, + poll: undefined, + text: text || undefined, + reply: null, + renote: null, + cw: cw, + localOnly, + visibility: "public", + visibleUsers: [], + channel: null, + apMentions: null, + apHashtags: undefined, + apEmojis: undefined, + }); + } catch (e) { + logger.warn(`Error in line:${linenum} ${e}`); + } + } + } else if (parsed instanceof Object) { + logger.info("Parsing animal style posts"); + for (const post of parsed.orderedItems) { + try { + linenum++; + if (post.inReplyTo != null) { + logger.info(`Is reply, skip [${linenum}] ...`); + continue; + } + if (post.directMessage) { + logger.info(`Is dm, skip [${linenum}] ...`); + continue; + } + let text; + try { + text = htmlToMfm(post.content, post.tag); + } catch (e) { + logger.warn(`Error while parsing text in line ${linenum}: ${e}`); + continue; + } + logger.info(`Posting[${linenum}] ...`); + + const note = await create(user, { + createdAt: new Date(post.published), + files: undefined, + poll: undefined, + text: text || undefined, + reply: null, + renote: null, + cw: post.sensitive, + localOnly: false, + visibility: "public", + visibleUsers: [], + channel: null, + apMentions: null, + apHashtags: undefined, + apEmojis: undefined, + }); + } catch (e) { + logger.warn(`Error in line:${linenum} ${e}`); + } + } + } + } catch (e) { + // handle error + logger.warn(`Error reading: ${e}`); + } + + logger.succ("Imported"); + done(); +} diff --git a/packages/backend/src/queue/processors/db/index.ts b/packages/backend/src/queue/processors/db/index.ts index 90173053fb..22b55a3683 100644 --- a/packages/backend/src/queue/processors/db/index.ts +++ b/packages/backend/src/queue/processors/db/index.ts @@ -11,6 +11,7 @@ import { importFollowing } from "./import-following.js"; import { importUserLists } from "./import-user-lists.js"; import { deleteAccount } from "./delete-account.js"; import { importMuting } from "./import-muting.js"; +import { importPosts } from "./import-posts.js"; import { importBlocking } from "./import-blocking.js"; import { importCustomEmojis } from "./import-custom-emojis.js"; @@ -26,6 +27,7 @@ const jobs = { importMuting, importBlocking, importUserLists, + importPosts, importCustomEmojis, deleteAccount, } as Record< diff --git a/packages/backend/src/remote/activitypub/models/note.ts b/packages/backend/src/remote/activitypub/models/note.ts index a0945ae7b1..033157b081 100644 --- a/packages/backend/src/remote/activitypub/models/note.ts +++ b/packages/backend/src/remote/activitypub/models/note.ts @@ -112,13 +112,13 @@ export async function createNote( const note: IPost = object; if (note.id && !note.id.startsWith("https://")) { - throw new Error(`unexpected shcema of note.id: ${note.id}`); + throw new Error(`unexpected schema of note.id: ${note.id}`); } const url = getOneApHrefNullable(note.url); if (url && !url.startsWith("https://")) { - throw new Error(`unexpected shcema of note url: ${url}`); + throw new Error(`unexpected schema of note url: ${url}`); } logger.debug(`Note fetched: ${JSON.stringify(note, null, 2)}`); diff --git a/packages/backend/src/remote/activitypub/renderer/announce.ts b/packages/backend/src/remote/activitypub/renderer/announce.ts index cff79a3f72..bb04a7f3d7 100644 --- a/packages/backend/src/remote/activitypub/renderer/announce.ts +++ b/packages/backend/src/remote/activitypub/renderer/announce.ts @@ -13,6 +13,9 @@ export default (object: any, note: Note) => { } else if (note.visibility === "home") { to = [`${attributedTo}/followers`]; cc = ["https://www.w3.org/ns/activitystreams#Public"]; + } else if (note.visibility === 'followers') { + to = [`${attributedTo}/followers`]; + cc = []; } else { return null; } diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index ba0e721b9e..920f871995 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -182,6 +182,7 @@ import * as ep___i_exportBlocking from "./endpoints/i/export-blocking.js"; import * as ep___i_exportFollowing from "./endpoints/i/export-following.js"; import * as ep___i_exportMute from "./endpoints/i/export-mute.js"; import * as ep___i_exportNotes from "./endpoints/i/export-notes.js"; +import * as ep___i_importPosts from "./endpoints/i/import-posts.js"; import * as ep___i_exportUserLists from "./endpoints/i/export-user-lists.js"; import * as ep___i_favorites from "./endpoints/i/favorites.js"; import * as ep___i_gallery_likes from "./endpoints/i/gallery/likes.js"; @@ -527,6 +528,7 @@ const eps = [ ["i/export-following", ep___i_exportFollowing], ["i/export-mute", ep___i_exportMute], ["i/export-notes", ep___i_exportNotes], + ["i/import-posts", ep___i_importPosts], ["i/export-user-lists", ep___i_exportUserLists], ["i/favorites", ep___i_favorites], ["i/gallery/likes", ep___i_gallery_likes], diff --git a/packages/backend/src/server/api/endpoints/i/import-posts.ts b/packages/backend/src/server/api/endpoints/i/import-posts.ts new file mode 100644 index 0000000000..16517960fe --- /dev/null +++ b/packages/backend/src/server/api/endpoints/i/import-posts.ts @@ -0,0 +1,43 @@ +import define from "../../define.js"; +import { createImportPostsJob } from "@/queue/index.js"; +import { ApiError } from "../../error.js"; +import { DriveFiles } from "@/models/index.js"; +import { DAY } from "@/const.js"; + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: DAY, + max: 9999999, + }, + errors: { + noSuchFile: { + message: "No such file.", + code: "NO_SUCH_FILE", + id: "e674141e-bd2a-ba85-e616-aefb187c9c2a", + }, + + emptyFile: { + message: "That file is empty.", + code: "EMPTY_FILE", + id: "d2f12af1-e7b4-feac-86a3-519548f2728e", + }, + }, +} as const; + +export const paramDef = { + type: "object", + properties: { + fileId: { type: "string", format: "misskey:id" }, + }, + required: ["fileId"], +} as const; + +export default define(meta, paramDef, async (ps, user) => { + const file = await DriveFiles.findOneBy({ id: ps.fileId }); + + if (file == null) throw new ApiError(meta.errors.noSuchFile); + if (file.size === 0) throw new ApiError(meta.errors.emptyFile); + createImportPostsJob(user, file.id); +}); diff --git a/packages/client/src/pages/admin/index.vue b/packages/client/src/pages/admin/index.vue index 81a27cf38d..3633bef989 100644 --- a/packages/client/src/pages/admin/index.vue +++ b/packages/client/src/pages/admin/index.vue @@ -24,7 +24,7 @@