From 9c2264fb8e476dcb686a7751e75ec0f3ce40d3b3 Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 15 Jul 2023 00:19:09 -0700 Subject: [PATCH 1/5] =?UTF-8?q?fix:=20=F0=9F=90=9B=20allow=20up=20to=20102?= =?UTF-8?q?4=20chars=20for=20SMTP=20login?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #10472 --- .../migration/1678426061773-tweak-varchar-length.js | 10 ++++++++++ packages/backend/src/models/entities/meta.ts | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 packages/backend/migration/1678426061773-tweak-varchar-length.js diff --git a/packages/backend/migration/1678426061773-tweak-varchar-length.js b/packages/backend/migration/1678426061773-tweak-varchar-length.js new file mode 100644 index 0000000000..8833745991 --- /dev/null +++ b/packages/backend/migration/1678426061773-tweak-varchar-length.js @@ -0,0 +1,10 @@ +export class tweakVarcharLength1678426061773 { + name = 'tweakVarcharLength1678426061773' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpUser" TYPE character varying(1024)`, undefined); + await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpPass" TYPE character varying(1024)`, undefined); + } + + async down(queryRunner) {} +} diff --git a/packages/backend/src/models/entities/meta.ts b/packages/backend/src/models/entities/meta.ts index d3de7ba53b..e90a5269a1 100644 --- a/packages/backend/src/models/entities/meta.ts +++ b/packages/backend/src/models/entities/meta.ts @@ -326,13 +326,13 @@ export class Meta { public smtpPort: number | null; @Column("varchar", { - length: 128, + length: 1024, nullable: true, }) public smtpUser: string | null; @Column("varchar", { - length: 128, + length: 1024, nullable: true, }) public smtpPass: string | null; From 247c5ec760c37968c8f35867861cba3459a996fd Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 15 Jul 2023 14:28:44 -0700 Subject: [PATCH 2/5] dev78 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b38da39fa7..a30d85b3d3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "calckey", - "version": "14.0.0-dev77", + "version": "14.0.0-dev78", "codename": "aqua", "repository": { "type": "git", From 49b0a52f9f345aa7b81db9eec404afc1864b5ee8 Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 15 Jul 2023 14:50:23 -0700 Subject: [PATCH 3/5] fix: :wrench: max post length 100000 Pleroma doesn't accept >100K (ref: https://blob.cat/objects/82f33d96-534f-45ee-902a-f77c2723db8b) --- .config/example.yml | 2 +- packages/backend/src/const.ts | 7 +++++-- packages/backend/src/misc/hard-limits.ts | 3 +-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 9716915b67..f73f4f1d79 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -121,7 +121,7 @@ redis: # ┌─────────────────────┐ #───┘ Other configuration └───────────────────────────────────── -# Maximum length of a post (default 3000, max 250000000) +# Maximum length of a post (default 3000, max 100000) #maxNoteLength: 3000 # Maximum length of an image caption (default 1500, max 8192) diff --git a/packages/backend/src/const.ts b/packages/backend/src/const.ts index 1f1e63292f..49f012c5ea 100644 --- a/packages/backend/src/const.ts +++ b/packages/backend/src/const.ts @@ -1,7 +1,10 @@ import config from "@/config/index.js"; -import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js"; +import { DB_MAX_NOTE_TEXT_LENGTH, DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js"; -export const MAX_NOTE_TEXT_LENGTH = config.maxNoteLength ?? 3000; +export const MAX_NOTE_TEXT_LENGTH = Math.min( + config.maxNoteLength ?? 3000, + DB_MAX_NOTE_TEXT_LENGTH, +); export const MAX_CAPTION_TEXT_LENGTH = Math.min( config.maxCaptionLength ?? 1500, DB_MAX_IMAGE_COMMENT_LENGTH, diff --git a/packages/backend/src/misc/hard-limits.ts b/packages/backend/src/misc/hard-limits.ts index f1a58a4259..62671a706a 100644 --- a/packages/backend/src/misc/hard-limits.ts +++ b/packages/backend/src/misc/hard-limits.ts @@ -3,9 +3,8 @@ /** * Maximum note text length that can be stored in DB. * Surrogate pairs count as one - * DEPRECARTED: use const/MAX_NOTE_TEXT_LENGTH instead */ -// export const DB_MAX_NOTE_TEXT_LENGTH = 8192; +export const DB_MAX_NOTE_TEXT_LENGTH = 100000; /** * Maximum image description length that can be stored in DB. From bd57b95a97098c72a1cb24891f1a3d0ea6086dfc Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 15 Jul 2023 15:32:39 -0700 Subject: [PATCH 4/5] docs: :bulb: going past DB_MAX_NOTE_TEXT_LENGTH --- packages/backend/src/misc/hard-limits.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/backend/src/misc/hard-limits.ts b/packages/backend/src/misc/hard-limits.ts index 62671a706a..5ce3e0ac9a 100644 --- a/packages/backend/src/misc/hard-limits.ts +++ b/packages/backend/src/misc/hard-limits.ts @@ -3,6 +3,11 @@ /** * Maximum note text length that can be stored in DB. * Surrogate pairs count as one + * + * NOTE: this can hypothetically be pushed further + * (up to 250000000), but will likely cause truncations + * and incompatibilities with other servers, + * as well as potential performance issues. */ export const DB_MAX_NOTE_TEXT_LENGTH = 100000; From cae6ba0edbf8a11f7a6e22bec23d213f27d1682b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=82=B3=E3=83=AB=E3=82=BB=E3=83=83=E3=83=88=E5=A7=AB=40?= =?UTF-8?q?=E3=81=8C=E3=82=93=E3=81=B0=E3=82=89=E3=81=AA=E3=81=84?= Date: Sat, 15 Jul 2023 22:56:09 +0000 Subject: [PATCH 5/5] =?UTF-8?q?feat:=20=E2=9C=A8=20Add=20media=20to=20Mast?= =?UTF-8?q?odon=20and=20Calckey=20post=20imports=20(#10496)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What does this PR do? Adding files fields in the export notes option, and corresponding import notes Current the mastodon import does not import any attachments, this pr will use the "upload from url" feature to include medias if its a valid URL. There are many way to convert the outbox.json file, can be simple as upload media_attachments to any web hosting and do string replace on the json file. I also create a tool that upload the tar.gz file with auto convert and host the media as simplify the process at https://tempfile.moegirl.live Detail example can be found at https://fedi.moegirl.live/notes/9h76gtqnp2gwl5dz https://r2temp.moegirl.live/2023/7/15/15356683-050f-423a-b331-c9a05561f52a/shana-settings-_-meng-zhai-le-yuan-xyou-yu-ou-xiang-de-luo-ke-ke-wu-yan-moe-otaku-elysian-x-gloomily-idol-s-rococo-luncheon----mozilla-firefox-private-browsing-2023-07-15-18-36-37.mp4 Co-authored-by: CGsama Reviewed-on: https://codeberg.org/calckey/calckey/pulls/10496 Co-authored-by: コルセット姫@がんばらない Co-committed-by: コルセット姫@がんばらない --- .../src/queue/processors/db/export-notes.ts | 9 +++++---- .../processors/db/import-calckey-post.ts | 19 +++++++++++++++++- .../queue/processors/db/import-masto-post.ts | 20 ++++++++++++++++++- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts index de8fac05b4..bf53f83603 100644 --- a/packages/backend/src/queue/processors/db/export-notes.ts +++ b/packages/backend/src/queue/processors/db/export-notes.ts @@ -4,7 +4,7 @@ import * as fs from "node:fs"; import { queueLogger } from "../../logger.js"; import { addFile } from "@/services/drive/add-file.js"; import { format as dateFormat } from "date-fns"; -import { Users, Notes, Polls } from "@/models/index.js"; +import { Users, Notes, Polls, DriveFiles } from "@/models/index.js"; import { MoreThan } from "typeorm"; import type { Note } from "@/models/entities/note.js"; import type { Poll } from "@/models/entities/poll.js"; @@ -75,7 +75,7 @@ export async function exportNotes( if (note.hasPoll) { poll = await Polls.findOneByOrFail({ noteId: note.id }); } - const content = JSON.stringify(serialize(note, poll)); + const content = JSON.stringify(await serialize(note, poll)); const isFirst = exportedNotesCount === 0; await write(isFirst ? content : ",\n" + content); exportedNotesCount++; @@ -112,15 +112,16 @@ export async function exportNotes( done(); } -function serialize( +async function serialize( note: Note, poll: Poll | null = null, -): Record { +): Promise> { return { id: note.id, text: note.text, createdAt: note.createdAt, fileIds: note.fileIds, + files: await DriveFiles.packMany(note.fileIds), replyId: note.replyId, renoteId: note.renoteId, poll: poll, diff --git a/packages/backend/src/queue/processors/db/import-calckey-post.ts b/packages/backend/src/queue/processors/db/import-calckey-post.ts index 28e794aa0c..945c50d637 100644 --- a/packages/backend/src/queue/processors/db/import-calckey-post.ts +++ b/packages/backend/src/queue/processors/db/import-calckey-post.ts @@ -3,6 +3,8 @@ import create from "@/services/note/create.js"; import { Users } from "@/models/index.js"; import type { DbUserImportMastoPostJobData } from "@/queue/types.js"; import { queueLogger } from "../../logger.js"; +import { uploadFromUrl } from "@/services/drive/upload-from-url.js"; +import type { DriveFile } from "@/models/entities/drive-file.js"; import type Bull from "bull"; const logger = queueLogger.createSubLogger("import-calckey-post"); @@ -29,10 +31,25 @@ export async function importCkPost( done(); return; } + const urls = (post.files || []) + .map((x: any) => x.url) + .filter((x: String) => x.startsWith("http")); + const files: DriveFile[] = []; + for (const url of urls) { + try { + const file = await uploadFromUrl({ + url: url, + user: user, + }); + files.push(file); + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } + } const { text, cw, localOnly, createdAt } = Post.parse(post); const note = await create(user, { createdAt: createdAt, - files: undefined, + files: files.length == 0 ? undefined : files, poll: undefined, text: text || undefined, reply: null, diff --git a/packages/backend/src/queue/processors/db/import-masto-post.ts b/packages/backend/src/queue/processors/db/import-masto-post.ts index efa4adf3f3..05166b0858 100644 --- a/packages/backend/src/queue/processors/db/import-masto-post.ts +++ b/packages/backend/src/queue/processors/db/import-masto-post.ts @@ -6,6 +6,8 @@ import type Bull from "bull"; import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js"; import { resolveNote } from "@/remote/activitypub/models/note.js"; import { Note } from "@/models/entities/note.js"; +import { uploadFromUrl } from "@/services/drive/upload-from-url.js"; +import type { DriveFile } from "@/models/entities/drive-file.js"; const logger = queueLogger.createSubLogger("import-masto-post"); @@ -43,9 +45,25 @@ export async function importMastoPost( throw e; } job.progress(80); + const urls = post.object.attachment + .map((x: any) => x.url) + .filter((x: String) => x.startsWith("http")); + const files: DriveFile[] = []; + for (const url of urls) { + try { + const file = await uploadFromUrl({ + url: url, + user: user, + }); + files.push(file); + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } + } + const note = await create(user, { createdAt: new Date(post.object.published), - files: undefined, + files: files.length == 0 ? undefined : files, poll: undefined, text: text || undefined, reply,