Merge branch 'develop' of https://codeberg.org/calckey/calckey into upstream

This commit is contained in:
freeplay 2023-07-16 13:42:21 -04:00
commit 48c52c3ee8
9 changed files with 67 additions and 14 deletions

View File

@ -121,7 +121,7 @@ redis:
# ┌─────────────────────┐ # ┌─────────────────────┐
#───┘ Other configuration └───────────────────────────────────── #───┘ Other configuration └─────────────────────────────────────
# Maximum length of a post (default 3000, max 250000000) # Maximum length of a post (default 3000, max 100000)
#maxNoteLength: 3000 #maxNoteLength: 3000
# Maximum length of an image caption (default 1500, max 8192) # Maximum length of an image caption (default 1500, max 8192)

View File

@ -1,6 +1,6 @@
{ {
"name": "calckey", "name": "calckey",
"version": "14.0.0-dev77", "version": "14.0.0-dev78",
"codename": "aqua", "codename": "aqua",
"repository": { "repository": {
"type": "git", "type": "git",

View File

@ -0,0 +1,10 @@
export class tweakVarcharLength1678426061773 {
name = 'tweakVarcharLength1678426061773'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpUser" TYPE character varying(1024)`, undefined);
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpPass" TYPE character varying(1024)`, undefined);
}
async down(queryRunner) {}
}

View File

@ -1,7 +1,10 @@
import config from "@/config/index.js"; import config from "@/config/index.js";
import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js"; import { DB_MAX_NOTE_TEXT_LENGTH, DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
export const MAX_NOTE_TEXT_LENGTH = config.maxNoteLength ?? 3000; export const MAX_NOTE_TEXT_LENGTH = Math.min(
config.maxNoteLength ?? 3000,
DB_MAX_NOTE_TEXT_LENGTH,
);
export const MAX_CAPTION_TEXT_LENGTH = Math.min( export const MAX_CAPTION_TEXT_LENGTH = Math.min(
config.maxCaptionLength ?? 1500, config.maxCaptionLength ?? 1500,
DB_MAX_IMAGE_COMMENT_LENGTH, DB_MAX_IMAGE_COMMENT_LENGTH,

View File

@ -3,9 +3,13 @@
/** /**
* Maximum note text length that can be stored in DB. * Maximum note text length that can be stored in DB.
* Surrogate pairs count as one * Surrogate pairs count as one
* DEPRECARTED: use const/MAX_NOTE_TEXT_LENGTH instead *
* NOTE: this can hypothetically be pushed further
* (up to 250000000), but will likely cause truncations
* and incompatibilities with other servers,
* as well as potential performance issues.
*/ */
// export const DB_MAX_NOTE_TEXT_LENGTH = 8192; export const DB_MAX_NOTE_TEXT_LENGTH = 100000;
/** /**
* Maximum image description length that can be stored in DB. * Maximum image description length that can be stored in DB.

View File

@ -326,13 +326,13 @@ export class Meta {
public smtpPort: number | null; public smtpPort: number | null;
@Column("varchar", { @Column("varchar", {
length: 128, length: 1024,
nullable: true, nullable: true,
}) })
public smtpUser: string | null; public smtpUser: string | null;
@Column("varchar", { @Column("varchar", {
length: 128, length: 1024,
nullable: true, nullable: true,
}) })
public smtpPass: string | null; public smtpPass: string | null;

View File

@ -4,7 +4,7 @@ import * as fs from "node:fs";
import { queueLogger } from "../../logger.js"; import { queueLogger } from "../../logger.js";
import { addFile } from "@/services/drive/add-file.js"; import { addFile } from "@/services/drive/add-file.js";
import { format as dateFormat } from "date-fns"; import { format as dateFormat } from "date-fns";
import { Users, Notes, Polls } from "@/models/index.js"; import { Users, Notes, Polls, DriveFiles } from "@/models/index.js";
import { MoreThan } from "typeorm"; import { MoreThan } from "typeorm";
import type { Note } from "@/models/entities/note.js"; import type { Note } from "@/models/entities/note.js";
import type { Poll } from "@/models/entities/poll.js"; import type { Poll } from "@/models/entities/poll.js";
@ -75,7 +75,7 @@ export async function exportNotes(
if (note.hasPoll) { if (note.hasPoll) {
poll = await Polls.findOneByOrFail({ noteId: note.id }); poll = await Polls.findOneByOrFail({ noteId: note.id });
} }
const content = JSON.stringify(serialize(note, poll)); const content = JSON.stringify(await serialize(note, poll));
const isFirst = exportedNotesCount === 0; const isFirst = exportedNotesCount === 0;
await write(isFirst ? content : ",\n" + content); await write(isFirst ? content : ",\n" + content);
exportedNotesCount++; exportedNotesCount++;
@ -112,15 +112,16 @@ export async function exportNotes(
done(); done();
} }
function serialize( async function serialize(
note: Note, note: Note,
poll: Poll | null = null, poll: Poll | null = null,
): Record<string, unknown> { ): Promise<Record<string, unknown>> {
return { return {
id: note.id, id: note.id,
text: note.text, text: note.text,
createdAt: note.createdAt, createdAt: note.createdAt,
fileIds: note.fileIds, fileIds: note.fileIds,
files: await DriveFiles.packMany(note.fileIds),
replyId: note.replyId, replyId: note.replyId,
renoteId: note.renoteId, renoteId: note.renoteId,
poll: poll, poll: poll,

View File

@ -3,6 +3,8 @@ import create from "@/services/note/create.js";
import { Users } from "@/models/index.js"; import { Users } from "@/models/index.js";
import type { DbUserImportMastoPostJobData } from "@/queue/types.js"; import type { DbUserImportMastoPostJobData } from "@/queue/types.js";
import { queueLogger } from "../../logger.js"; import { queueLogger } from "../../logger.js";
import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
import type Bull from "bull"; import type Bull from "bull";
const logger = queueLogger.createSubLogger("import-calckey-post"); const logger = queueLogger.createSubLogger("import-calckey-post");
@ -29,10 +31,25 @@ export async function importCkPost(
done(); done();
return; return;
} }
const urls = (post.files || [])
.map((x: any) => x.url)
.filter((x: String) => x.startsWith("http"));
const files: DriveFile[] = [];
for (const url of urls) {
try {
const file = await uploadFromUrl({
url: url,
user: user,
});
files.push(file);
} catch (e) {
logger.error(`Skipped adding file to drive: ${url}`);
}
}
const { text, cw, localOnly, createdAt } = Post.parse(post); const { text, cw, localOnly, createdAt } = Post.parse(post);
const note = await create(user, { const note = await create(user, {
createdAt: createdAt, createdAt: createdAt,
files: undefined, files: files.length == 0 ? undefined : files,
poll: undefined, poll: undefined,
text: text || undefined, text: text || undefined,
reply: null, reply: null,

View File

@ -6,6 +6,8 @@ import type Bull from "bull";
import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js"; import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js";
import { resolveNote } from "@/remote/activitypub/models/note.js"; import { resolveNote } from "@/remote/activitypub/models/note.js";
import { Note } from "@/models/entities/note.js"; import { Note } from "@/models/entities/note.js";
import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
const logger = queueLogger.createSubLogger("import-masto-post"); const logger = queueLogger.createSubLogger("import-masto-post");
@ -43,9 +45,25 @@ export async function importMastoPost(
throw e; throw e;
} }
job.progress(80); job.progress(80);
const urls = post.object.attachment
.map((x: any) => x.url)
.filter((x: String) => x.startsWith("http"));
const files: DriveFile[] = [];
for (const url of urls) {
try {
const file = await uploadFromUrl({
url: url,
user: user,
});
files.push(file);
} catch (e) {
logger.error(`Skipped adding file to drive: ${url}`);
}
}
const note = await create(user, { const note = await create(user, {
createdAt: new Date(post.object.published), createdAt: new Date(post.object.published),
files: undefined, files: files.length == 0 ? undefined : files,
poll: undefined, poll: undefined,
text: text || undefined, text: text || undefined,
reply, reply,