diff --git a/.config/example.yml b/.config/example.yml index 9716915b67..f73f4f1d79 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -121,7 +121,7 @@ redis: # ┌─────────────────────┐ #───┘ Other configuration └───────────────────────────────────── -# Maximum length of a post (default 3000, max 250000000) +# Maximum length of a post (default 3000, max 100000) #maxNoteLength: 3000 # Maximum length of an image caption (default 1500, max 8192) diff --git a/package.json b/package.json index b38da39fa7..a30d85b3d3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "calckey", - "version": "14.0.0-dev77", + "version": "14.0.0-dev78", "codename": "aqua", "repository": { "type": "git", diff --git a/packages/backend/migration/1678426061773-tweak-varchar-length.js b/packages/backend/migration/1678426061773-tweak-varchar-length.js new file mode 100644 index 0000000000..8833745991 --- /dev/null +++ b/packages/backend/migration/1678426061773-tweak-varchar-length.js @@ -0,0 +1,10 @@ +export class tweakVarcharLength1678426061773 { + name = 'tweakVarcharLength1678426061773' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpUser" TYPE character varying(1024)`, undefined); + await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpPass" TYPE character varying(1024)`, undefined); + } + + async down(queryRunner) {} +} diff --git a/packages/backend/src/const.ts b/packages/backend/src/const.ts index 1f1e63292f..49f012c5ea 100644 --- a/packages/backend/src/const.ts +++ b/packages/backend/src/const.ts @@ -1,7 +1,10 @@ import config from "@/config/index.js"; -import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js"; +import { DB_MAX_NOTE_TEXT_LENGTH, DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js"; -export const MAX_NOTE_TEXT_LENGTH = config.maxNoteLength ?? 3000; +export const MAX_NOTE_TEXT_LENGTH = Math.min( + config.maxNoteLength ?? 3000, + DB_MAX_NOTE_TEXT_LENGTH, +); export const MAX_CAPTION_TEXT_LENGTH = Math.min( config.maxCaptionLength ?? 1500, DB_MAX_IMAGE_COMMENT_LENGTH, diff --git a/packages/backend/src/misc/hard-limits.ts b/packages/backend/src/misc/hard-limits.ts index f1a58a4259..5ce3e0ac9a 100644 --- a/packages/backend/src/misc/hard-limits.ts +++ b/packages/backend/src/misc/hard-limits.ts @@ -3,9 +3,13 @@ /** * Maximum note text length that can be stored in DB. * Surrogate pairs count as one - * DEPRECARTED: use const/MAX_NOTE_TEXT_LENGTH instead + * + * NOTE: this can hypothetically be pushed further + * (up to 250000000), but will likely cause truncations + * and incompatibilities with other servers, + * as well as potential performance issues. */ -// export const DB_MAX_NOTE_TEXT_LENGTH = 8192; +export const DB_MAX_NOTE_TEXT_LENGTH = 100000; /** * Maximum image description length that can be stored in DB. diff --git a/packages/backend/src/models/entities/meta.ts b/packages/backend/src/models/entities/meta.ts index d3de7ba53b..e90a5269a1 100644 --- a/packages/backend/src/models/entities/meta.ts +++ b/packages/backend/src/models/entities/meta.ts @@ -326,13 +326,13 @@ export class Meta { public smtpPort: number | null; @Column("varchar", { - length: 128, + length: 1024, nullable: true, }) public smtpUser: string | null; @Column("varchar", { - length: 128, + length: 1024, nullable: true, }) public smtpPass: string | null; diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts index de8fac05b4..bf53f83603 100644 --- a/packages/backend/src/queue/processors/db/export-notes.ts +++ b/packages/backend/src/queue/processors/db/export-notes.ts @@ -4,7 +4,7 @@ import * as fs from "node:fs"; import { queueLogger } from "../../logger.js"; import { addFile } from "@/services/drive/add-file.js"; import { format as dateFormat } from "date-fns"; -import { Users, Notes, Polls } from "@/models/index.js"; +import { Users, Notes, Polls, DriveFiles } from "@/models/index.js"; import { MoreThan } from "typeorm"; import type { Note } from "@/models/entities/note.js"; import type { Poll } from "@/models/entities/poll.js"; @@ -75,7 +75,7 @@ export async function exportNotes( if (note.hasPoll) { poll = await Polls.findOneByOrFail({ noteId: note.id }); } - const content = JSON.stringify(serialize(note, poll)); + const content = JSON.stringify(await serialize(note, poll)); const isFirst = exportedNotesCount === 0; await write(isFirst ? content : ",\n" + content); exportedNotesCount++; @@ -112,15 +112,16 @@ export async function exportNotes( done(); } -function serialize( +async function serialize( note: Note, poll: Poll | null = null, -): Record { +): Promise> { return { id: note.id, text: note.text, createdAt: note.createdAt, fileIds: note.fileIds, + files: await DriveFiles.packMany(note.fileIds), replyId: note.replyId, renoteId: note.renoteId, poll: poll, diff --git a/packages/backend/src/queue/processors/db/import-calckey-post.ts b/packages/backend/src/queue/processors/db/import-calckey-post.ts index 28e794aa0c..945c50d637 100644 --- a/packages/backend/src/queue/processors/db/import-calckey-post.ts +++ b/packages/backend/src/queue/processors/db/import-calckey-post.ts @@ -3,6 +3,8 @@ import create from "@/services/note/create.js"; import { Users } from "@/models/index.js"; import type { DbUserImportMastoPostJobData } from "@/queue/types.js"; import { queueLogger } from "../../logger.js"; +import { uploadFromUrl } from "@/services/drive/upload-from-url.js"; +import type { DriveFile } from "@/models/entities/drive-file.js"; import type Bull from "bull"; const logger = queueLogger.createSubLogger("import-calckey-post"); @@ -29,10 +31,25 @@ export async function importCkPost( done(); return; } + const urls = (post.files || []) + .map((x: any) => x.url) + .filter((x: String) => x.startsWith("http")); + const files: DriveFile[] = []; + for (const url of urls) { + try { + const file = await uploadFromUrl({ + url: url, + user: user, + }); + files.push(file); + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } + } const { text, cw, localOnly, createdAt } = Post.parse(post); const note = await create(user, { createdAt: createdAt, - files: undefined, + files: files.length == 0 ? undefined : files, poll: undefined, text: text || undefined, reply: null, diff --git a/packages/backend/src/queue/processors/db/import-masto-post.ts b/packages/backend/src/queue/processors/db/import-masto-post.ts index efa4adf3f3..05166b0858 100644 --- a/packages/backend/src/queue/processors/db/import-masto-post.ts +++ b/packages/backend/src/queue/processors/db/import-masto-post.ts @@ -6,6 +6,8 @@ import type Bull from "bull"; import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js"; import { resolveNote } from "@/remote/activitypub/models/note.js"; import { Note } from "@/models/entities/note.js"; +import { uploadFromUrl } from "@/services/drive/upload-from-url.js"; +import type { DriveFile } from "@/models/entities/drive-file.js"; const logger = queueLogger.createSubLogger("import-masto-post"); @@ -43,9 +45,25 @@ export async function importMastoPost( throw e; } job.progress(80); + const urls = post.object.attachment + .map((x: any) => x.url) + .filter((x: String) => x.startsWith("http")); + const files: DriveFile[] = []; + for (const url of urls) { + try { + const file = await uploadFromUrl({ + url: url, + user: user, + }); + files.push(file); + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } + } + const note = await create(user, { createdAt: new Date(post.object.published), - files: undefined, + files: files.length == 0 ? undefined : files, poll: undefined, text: text || undefined, reply,