From 3ddcffd1690b0ad1b213ef0585a7ac307553a457 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Fri, 19 May 2023 22:26:13 -0400 Subject: [PATCH] enhance: emoji width and height --- .../migration/1684494870830-EmojiSize.js | 19 +++++++ packages/backend/package.json | 2 + .../backend/src/@types/probe-image-size.d.ts | 37 -------------- packages/backend/src/misc/emoji-meta.ts | 50 +++++++++++++++++++ packages/backend/src/misc/get-file-info.ts | 2 +- packages/backend/src/misc/populate-emojis.ts | 12 ++++- packages/backend/src/models/entities/emoji.ts | 11 ++++ .../backend/src/models/repositories/emoji.ts | 2 + packages/backend/src/models/schema/emoji.ts | 10 ++++ packages/backend/src/queue/index.ts | 6 +++ .../processors/db/import-custom-emojis.ts | 8 ++- .../src/queue/processors/system/index.ts | 2 + .../processors/system/local-emoji-size.ts | 42 ++++++++++++++++ .../src/remote/activitypub/models/note.ts | 20 +++++++- .../server/api/endpoints/admin/emoji/add.ts | 10 ++++ .../server/api/endpoints/admin/emoji/copy.ts | 10 ++++ .../api/endpoints/admin/emoji/list-remote.ts | 10 ++++ .../server/api/endpoints/admin/emoji/list.ts | 10 ++++ pnpm-lock.yaml | 25 ++++++++++ 19 files changed, 247 insertions(+), 41 deletions(-) create mode 100644 packages/backend/migration/1684494870830-EmojiSize.js delete mode 100644 packages/backend/src/@types/probe-image-size.d.ts create mode 100644 packages/backend/src/misc/emoji-meta.ts create mode 100644 packages/backend/src/queue/processors/system/local-emoji-size.ts diff --git a/packages/backend/migration/1684494870830-EmojiSize.js b/packages/backend/migration/1684494870830-EmojiSize.js new file mode 100644 index 0000000000..6d42f1a614 --- /dev/null +++ b/packages/backend/migration/1684494870830-EmojiSize.js @@ -0,0 +1,19 @@ +export class EmojiSize1684494870830 { + name = "EmojiSize1684494870830"; + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "emoji" ADD "width" integer`); + await queryRunner.query( + `COMMENT ON COLUMN "emoji"."width" IS 'Image width'`, + ); + await queryRunner.query(`ALTER TABLE "emoji" ADD "height" integer`); + await queryRunner.query( + `COMMENT ON COLUMN "emoji"."height" IS 'Image height'`, + ); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "emoji" DROP COLUMN "height"`); + await queryRunner.query(`ALTER TABLE "emoji" DROP COLUMN "width"`); + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index 68fa6915e2..e38e531f34 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -41,6 +41,7 @@ "ajv": "8.11.2", "archiver": "5.3.1", "argon2": "^0.30.3", + "async-mutex": "^0.4.0", "autobind-decorator": "2.4.0", "autolinker": "4.0.0", "autwh": "0.1.0", @@ -161,6 +162,7 @@ "@types/node-fetch": "3.0.3", "@types/nodemailer": "6.4.7", "@types/oauth": "0.9.1", + "@types/probe-image-size": "^7.2.0", "@types/pug": "2.0.6", "@types/punycode": "2.1.0", "@types/qrcode": "1.5.0", diff --git a/packages/backend/src/@types/probe-image-size.d.ts b/packages/backend/src/@types/probe-image-size.d.ts deleted file mode 100644 index 4ed13df7fa..0000000000 --- a/packages/backend/src/@types/probe-image-size.d.ts +++ /dev/null @@ -1,37 +0,0 @@ -declare module "probe-image-size" { - import type { ReadStream } from "node:fs"; - - type ProbeOptions = { - retries: 1; - timeout: 30000; - }; - - type ProbeResult = { - width: number; - height: number; - length?: number; - type: string; - mime: string; - wUnits: "in" | "mm" | "cm" | "pt" | "pc" | "px" | "em" | "ex"; - hUnits: "in" | "mm" | "cm" | "pt" | "pc" | "px" | "em" | "ex"; - url?: string; - }; - - function probeImageSize( - src: string | ReadStream, - options?: ProbeOptions, - ): Promise; - function probeImageSize( - src: string | ReadStream, - callback: (err: Error | null, result?: ProbeResult) => void, - ): void; - function probeImageSize( - src: string | ReadStream, - options: ProbeOptions, - callback: (err: Error | null, result?: ProbeResult) => void, - ): void; - - namespace probeImageSize {} // Hack - - export = probeImageSize; -} diff --git a/packages/backend/src/misc/emoji-meta.ts b/packages/backend/src/misc/emoji-meta.ts new file mode 100644 index 0000000000..fd9d9baa5c --- /dev/null +++ b/packages/backend/src/misc/emoji-meta.ts @@ -0,0 +1,50 @@ +import probeImageSize from "probe-image-size"; +import { Mutex, withTimeout } from "async-mutex"; + +import { FILE_TYPE_BROWSERSAFE } from "@/const.js"; +import Logger from "@/services/logger.js"; +import { Cache } from "./cache.js"; + +export type Size = { + width: number; + height: number; +}; + +const cache = new Cache(1000 * 60 * 10); // once every 10 minutes for the same url +const mutex = withTimeout(new Mutex(), 1000); + +export async function getEmojiSize(url: string): Promise { + const logger = new Logger("emoji"); + + await mutex.runExclusive(() => { + const attempted = cache.get(url); + if (!attempted) { + cache.set(url, true); + } else { + logger.warn(`Attempt limit exceeded: ${url}`); + throw new Error("Too many attempts"); + } + }); + + try { + logger.info(`Retrieving emoji size from ${url}`); + const { width, height, mime } = await probeImageSize(url, { + timeout: 5000, + }); + if (!(mime.startsWith("image/") && FILE_TYPE_BROWSERSAFE.includes(mime))) { + throw new Error("Unsupported image type"); + } + return { width, height }; + } catch (e) { + throw new Error(`Unable to retrieve metadata: ${e}`); + } +} + +export function getNormalSize( + { width, height }: Size, + orientation?: number, +): Size { + return (orientation || 0) >= 5 + ? { width: height, height: width } + : { width, height }; +} diff --git a/packages/backend/src/misc/get-file-info.ts b/packages/backend/src/misc/get-file-info.ts index a63de286ea..76964890e7 100644 --- a/packages/backend/src/misc/get-file-info.ts +++ b/packages/backend/src/misc/get-file-info.ts @@ -5,9 +5,9 @@ import * as stream from "node:stream"; import * as util from "node:util"; import { FSWatcher } from "chokidar"; import { fileTypeFromFile } from "file-type"; +import probeImageSize from "probe-image-size"; import FFmpeg from "fluent-ffmpeg"; import isSvg from "is-svg"; -import probeImageSize from "probe-image-size"; import { type predictionType } from "nsfwjs"; import sharp from "sharp"; import { encode } from "blurhash"; diff --git a/packages/backend/src/misc/populate-emojis.ts b/packages/backend/src/misc/populate-emojis.ts index 3f20f9f10d..7aee4ec253 100644 --- a/packages/backend/src/misc/populate-emojis.ts +++ b/packages/backend/src/misc/populate-emojis.ts @@ -16,6 +16,8 @@ const cache = new Cache(1000 * 60 * 60 * 12); type PopulatedEmoji = { name: string; url: string; + width: number | null; + height: number | null; }; function normalizeHost( @@ -68,7 +70,13 @@ export async function populateEmoji( host: host ?? IsNull(), })) || null; - const emoji = await cache.fetch(`${name} ${host}`, queryOrNull); + const cacheKey = `${name} ${host}`; + let emoji = await cache.fetch(cacheKey, queryOrNull); + + if (emoji && !(emoji.width && emoji.height)) { + emoji = await queryOrNull(); + cache.set(cacheKey, emoji); + } if (emoji == null) return null; @@ -83,6 +91,8 @@ export async function populateEmoji( return { name: emojiName, url, + width: emoji.width, + height: emoji.height, }; } diff --git a/packages/backend/src/models/entities/emoji.ts b/packages/backend/src/models/entities/emoji.ts index 2315686968..773265d91c 100644 --- a/packages/backend/src/models/entities/emoji.ts +++ b/packages/backend/src/models/entities/emoji.ts @@ -46,6 +46,7 @@ export class Emoji { public uri: string | null; // publicUrlの方のtypeが入る + // (mime) @Column('varchar', { length: 64, nullable: true, }) @@ -60,4 +61,14 @@ export class Emoji { length: 1024, nullable: true, }) public license: string | null; + + @Column('integer', { + nullable: true, comment: 'Image width', + }) + public width: number | null; + + @Column('integer', { + nullable: true, comment: "Image height", + }) + public height: number | null; } diff --git a/packages/backend/src/models/repositories/emoji.ts b/packages/backend/src/models/repositories/emoji.ts index 6eabfe9558..e9a940f958 100644 --- a/packages/backend/src/models/repositories/emoji.ts +++ b/packages/backend/src/models/repositories/emoji.ts @@ -16,6 +16,8 @@ export const EmojiRepository = db.getRepository(Emoji).extend({ // || emoji.originalUrl してるのは後方互換性のため url: emoji.publicUrl || emoji.originalUrl, license: emoji.license, + width: emoji.width, + height: emoji.height, }; }, diff --git a/packages/backend/src/models/schema/emoji.ts b/packages/backend/src/models/schema/emoji.ts index 8994381b31..8b01f09ab2 100644 --- a/packages/backend/src/models/schema/emoji.ts +++ b/packages/backend/src/models/schema/emoji.ts @@ -45,5 +45,15 @@ export const packedEmojiSchema = { optional: false, nullable: true, }, + width: { + type: "number", + optional: false, + nullable: true, + }, + height: { + type: "number", + optional: false, + nullable: true, + }, }, } as const; diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index 78696c4e2f..1b50f35287 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -567,6 +567,12 @@ export default function () { }, ); + systemQueue.add( + "setLocalEmojiSizes", + {}, + { removeOnComplete: true, removeOnFail: true }, + ); + processSystemQueue(systemQueue); } diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index 8549fea446..e2454405fd 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -10,6 +10,7 @@ import type { DbUserImportJobData } from "@/queue/types.js"; import { addFile } from "@/services/drive/add-file.js"; import { genId } from "@/misc/gen-id.js"; import { db } from "@/db/postgre.js"; +import probeImageSize from "probe-image-size"; const logger = queueLogger.createSubLogger("import-custom-emojis"); @@ -66,7 +67,10 @@ export async function importCustomEmojis( name: record.fileName, force: true, }); - const emoji = await Emojis.insert({ + const file = fs.createReadStream(emojiPath); + const size = await probeImageSize(file); + file.destroy(); + await Emojis.insert({ id: genId(), updatedAt: new Date(), name: emojiInfo.name, @@ -77,6 +81,8 @@ export async function importCustomEmojis( publicUrl: driveFile.webpublicUrl ?? driveFile.url, type: driveFile.webpublicType ?? driveFile.type, license: emojiInfo.license, + width: size.width || null, + height: size.height || null, }).then((x) => Emojis.findOneByOrFail(x.identifiers[0])); } diff --git a/packages/backend/src/queue/processors/system/index.ts b/packages/backend/src/queue/processors/system/index.ts index 68833d76f4..53321de5f9 100644 --- a/packages/backend/src/queue/processors/system/index.ts +++ b/packages/backend/src/queue/processors/system/index.ts @@ -4,6 +4,7 @@ import { resyncCharts } from "./resync-charts.js"; import { cleanCharts } from "./clean-charts.js"; import { checkExpiredMutings } from "./check-expired-mutings.js"; import { clean } from "./clean.js"; +import { setLocalEmojiSizes } from "./local-emoji-size.js"; const jobs = { tickCharts, @@ -11,6 +12,7 @@ const jobs = { cleanCharts, checkExpiredMutings, clean, + setLocalEmojiSizes, } as Record< string, | Bull.ProcessCallbackFunction> diff --git a/packages/backend/src/queue/processors/system/local-emoji-size.ts b/packages/backend/src/queue/processors/system/local-emoji-size.ts new file mode 100644 index 0000000000..d696bbd863 --- /dev/null +++ b/packages/backend/src/queue/processors/system/local-emoji-size.ts @@ -0,0 +1,42 @@ +import type Bull from "bull"; +import { IsNull } from "typeorm"; +import { Emojis } from "@/models/index.js"; + +import { queueLogger } from "../../logger.js"; +import { getEmojiSize } from "@/misc/emoji-meta.js"; + +const logger = queueLogger.createSubLogger("local-emoji-size"); + +export async function setLocalEmojiSizes( + _job: Bull.Job>, + done: any, +): Promise { + logger.info("Setting sizes of local emojis..."); + + const emojis = await Emojis.findBy([ + { host: IsNull(), width: IsNull(), height: IsNull() }, + ]); + logger.info(`${emojis.length} emojis need to be fetched.`); + + for (let i = 0; i < emojis.length; i++) { + try { + const size = await getEmojiSize(emojis[i].publicUrl); + await Emojis.update(emojis[i].id, { + width: size.width || null, + height: size.height || null, + }); + } catch (e) { + logger.error( + `Unable to set emoji size (${i + 1}/${emojis.length}): ${e}`, + ); + /* skip if any error happens */ + } finally { + // wait for 1sec so that this would not overwhelm the object storage. + await new Promise((resolve) => setTimeout(resolve, 1000)); + if (i % 10 === 9) logger.succ(`fetched ${i + 1}/${emojis.length} emojis`); + } + } + + logger.succ("Done."); + done(); +} diff --git a/packages/backend/src/remote/activitypub/models/note.ts b/packages/backend/src/remote/activitypub/models/note.ts index f37af6cb4f..3fff212433 100644 --- a/packages/backend/src/remote/activitypub/models/note.ts +++ b/packages/backend/src/remote/activitypub/models/note.ts @@ -52,6 +52,7 @@ import { UserProfiles } from "@/models/index.js"; import { In } from "typeorm"; import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js"; import { truncate } from "@/misc/truncate.js"; +import { type Size, getEmojiSize } from "@/misc/emoji-meta.js"; const logger = apLogger; @@ -472,8 +473,15 @@ export async function extractEmojis( (tag.updated != null && exists.updatedAt != null && new Date(tag.updated) > exists.updatedAt) || - tag.icon!.url !== exists.originalUrl + tag.icon!.url !== exists.originalUrl || + !(exists.width && exists.height) ) { + let size: Size = { width: 0, height: 0 }; + try { + size = await getEmojiSize(tag.icon!.url); + } catch { + /* skip if any error happens */ + } await Emojis.update( { host, @@ -484,6 +492,8 @@ export async function extractEmojis( originalUrl: tag.icon!.url, publicUrl: tag.icon!.url, updatedAt: new Date(), + width: size.width || null, + height: size.height || null, }, ); @@ -498,6 +508,12 @@ export async function extractEmojis( logger.info(`register emoji host=${host}, name=${name}`); + let size: Size = { width: 0, height: 0 }; + try { + size = await getEmojiSize(tag.icon!.url); + } catch { + /* skip if any error happens */ + } return await Emojis.insert({ id: genId(), host, @@ -507,6 +523,8 @@ export async function extractEmojis( publicUrl: tag.icon!.url, updatedAt: new Date(), aliases: [], + width: size.width || null, + height: size.height || null, } as Partial).then((x) => Emojis.findOneByOrFail(x.identifiers[0]), ); diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts index bfc025834f..7d40816135 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts @@ -6,6 +6,7 @@ import { ApiError } from "../../../error.js"; import rndstr from "rndstr"; import { publishBroadcastStream } from "@/services/stream.js"; import { db } from "@/db/postgre.js"; +import { type Size, getEmojiSize } from "@/misc/emoji-meta.js"; export const meta = { tags: ["admin"], @@ -39,6 +40,13 @@ export default define(meta, paramDef, async (ps, me) => { ? file.name.split(".")[0] : `_${rndstr("a-z0-9", 8)}_`; + let size: Size = { width: 0, height: 0 }; + try { + size = await getEmojiSize(file.url); + } catch { + /* skip if any error happens */ + } + const emoji = await Emojis.insert({ id: genId(), updatedAt: new Date(), @@ -50,6 +58,8 @@ export default define(meta, paramDef, async (ps, me) => { publicUrl: file.webpublicUrl ?? file.url, type: file.webpublicType ?? file.type, license: null, + width: size.width || null, + height: size.height || null, }).then((x) => Emojis.findOneByOrFail(x.identifiers[0])); await db.queryResultCache!.remove(["meta_emojis"]); diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts index 951158f7d4..45cb9464db 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts @@ -6,6 +6,7 @@ import type { DriveFile } from "@/models/entities/drive-file.js"; import { uploadFromUrl } from "@/services/drive/upload-from-url.js"; import { publishBroadcastStream } from "@/services/stream.js"; import { db } from "@/db/postgre.js"; +import { type Size, getEmojiSize } from "@/misc/emoji-meta.js"; export const meta = { tags: ["admin"], @@ -64,6 +65,13 @@ export default define(meta, paramDef, async (ps, me) => { throw new ApiError(); } + let size: Size = { width: 0, height: 0 }; + try { + size = await getEmojiSize(driveFile.url); + } catch { + /* skip if any error happens */ + } + const copied = await Emojis.insert({ id: genId(), updatedAt: new Date(), @@ -74,6 +82,8 @@ export default define(meta, paramDef, async (ps, me) => { publicUrl: driveFile.webpublicUrl ?? driveFile.url, type: driveFile.webpublicType ?? driveFile.type, license: emoji.license, + width: size.width || null, + height: size.height || null, }).then((x) => Emojis.findOneByOrFail(x.identifiers[0])); await db.queryResultCache!.remove(["meta_emojis"]); diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts index fae986dd96..07d70365d6 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts @@ -60,6 +60,16 @@ export const meta = { optional: false, nullable: true, }, + width: { + type: "number", + optional: false, + nullable: true, + }, + height: { + type: "number", + optional: false, + nullable: true, + }, }, }, }, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts index aa49f14803..9390dfb202 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts @@ -60,6 +60,16 @@ export const meta = { optional: false, nullable: true, }, + width: { + type: "number", + optional: false, + nullable: true, + }, + height: { + type: "number", + optional: false, + nullable: true, + }, }, }, }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3d647bd147..cf1ceb4985 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -131,6 +131,9 @@ importers: argon2: specifier: ^0.30.3 version: 0.30.3 + async-mutex: + specifier: ^0.4.0 + version: 0.4.0 autobind-decorator: specifier: 2.4.0 version: 2.4.0 @@ -493,6 +496,9 @@ importers: '@types/oauth': specifier: 0.9.1 version: 0.9.1 + '@types/probe-image-size': + specifier: ^7.2.0 + version: 7.2.0 '@types/pug': specifier: 2.0.6 version: 2.0.6 @@ -3409,6 +3415,12 @@ packages: resolution: {integrity: sha512-Z61JK7DKDtdKTWwLeElSEBcWGRLY8g95ic5FoQqI9CMx0ns/Ghep3B4DfcEimiKMvtamNVULVNKEsiwV3aQmXw==} dev: true + /@types/needle@3.2.0: + resolution: {integrity: sha512-6XzvzEyJ2ozFNfPajFmqH9JOt0Hp+9TawaYpJT59iIP/zR0U37cfWCRwosyIeEBBZBi021Osq4jGAD3AOju5fg==} + dependencies: + '@types/node': 18.11.18 + dev: true + /@types/node-fetch@2.6.2: resolution: {integrity: sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==} dependencies: @@ -3464,6 +3476,13 @@ packages: resolution: {integrity: sha512-KufADq8uQqo1pYKVIYzfKbJfBAc0sOeXqGbFaSpv8MRmC/zXgowNZmFcbngndGk922QDmOASEXUZCaY48gs4cg==} dev: true + /@types/probe-image-size@7.2.0: + resolution: {integrity: sha512-R5H3vw62gHNHrn+JGZbKejb+Z2D/6E5UNVlhCzIaBBLroMQMOFqy5Pap2gM+ZZHdqBtVU0/cx/M6to+mOJcoew==} + dependencies: + '@types/needle': 3.2.0 + '@types/node': 18.11.18 + dev: true + /@types/pug@2.0.6: resolution: {integrity: sha512-SnHmG9wN1UVmagJOnyo/qkk0Z7gejYxOYYmaAwr5u2yFYfsupN3sg10kyzN8Hep/2zbHxCnsumxOoRIRMBwKCg==} dev: true @@ -4466,6 +4485,12 @@ packages: stream-exhaust: 1.0.2 dev: true + /async-mutex@0.4.0: + resolution: {integrity: sha512-eJFZ1YhRR8UN8eBLoNzcDPcy/jqjsg6I1AP+KvWQX80BqOSW1oJPJXDylPUEeMr2ZQvHgnQ//Lp6f3RQ1zI7HA==} + dependencies: + tslib: 2.4.1 + dev: false + /async-settle@1.0.0: resolution: {integrity: sha512-VPXfB4Vk49z1LHHodrEQ6Xf7W4gg1w0dAPROHngx7qgDjqmIQ+fXmwgGXTW/ITLai0YLSvWepJOP9EVpMnEAcw==} engines: {node: '>= 0.10'}