From 88e3d3e8cbe26c0280308e819965a64a91491f90 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 11 Mar 2023 14:11:40 +0900 Subject: [PATCH] =?UTF-8?q?enhance(server):=20=E7=94=BB=E5=83=8F=E5=9C=A7?= =?UTF-8?q?=E7=B8=AE=E5=91=A8=E3=82=8A=EF=BC=88=E4=B8=BB=E3=81=AB=E3=82=B5?= =?UTF-8?q?=E3=83=A0=E3=83=8D=E3=82=A4=E3=83=AB=E3=81=AE=E4=BB=95=E6=A7=98?= =?UTF-8?q?=EF=BC=89=E3=81=AE=E5=A4=89=E6=9B=B4=20(#10287)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * DriveService, is-mime-image * static, previewをavifに, アニメーション画像でもthumbnailを生成 * fallback * animated: true * fix * avatarはwebp * revert ?? file.url --------- Co-authored-by: syuilo --- packages/backend/src/core/DriveService.ts | 37 +++--- .../src/core/ImageProcessingService.ts | 111 ++++++++++-------- .../core/entities/DriveFileEntityService.ts | 4 +- packages/backend/src/misc/is-mime-image.ts | 8 +- .../backend/src/server/FileServerService.ts | 10 +- .../src/server/web/UrlPreviewService.ts | 2 +- .../frontend/src/components/MkMediaImage.vue | 2 +- .../frontend/src/pages/user/index.photos.vue | 2 +- packages/frontend/src/scripts/media-proxy.ts | 7 +- .../frontend/src/widgets/WidgetPhotos.vue | 2 +- 10 files changed, 101 insertions(+), 84 deletions(-) diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index dfaacffc1..7eccf4b3b 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -2,6 +2,7 @@ import * as fs from 'node:fs'; import { Inject, Injectable } from '@nestjs/common'; import { v4 as uuid } from 'uuid'; import sharp from 'sharp'; +import { sharpBmp } from 'sharp-read-bmp'; import { IsNull } from 'typeorm'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository } from '@/models/index.js'; @@ -34,6 +35,7 @@ import { FileInfoService } from '@/core/FileInfoService.js'; import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; import { correctFilename } from '@/misc/correct-filename.js'; +import { isMimeImage } from '@/misc/is-mime-image.js'; import type S3 from 'aws-sdk/clients/s3.js'; type AddFileArgs = { @@ -274,8 +276,8 @@ export class DriveService { } } - if (!['image/jpeg', 'image/png', 'image/webp', 'image/avif', 'image/svg+xml'].includes(type)) { - this.registerLogger.debug('web image and thumbnail not created (not an required file)'); + if (!isMimeImage(type, 'sharp-convertible-image-with-bmp')) { + this.registerLogger.debug('web image and thumbnail not created (cannot convert by sharp)'); return { webpublic: null, thumbnail: null, @@ -284,22 +286,16 @@ export class DriveService { let img: sharp.Sharp | null = null; let satisfyWebpublic: boolean; + let isAnimated: boolean; try { - img = sharp(path); + img = await sharpBmp(path, type); const metadata = await img.metadata(); - const isAnimated = metadata.pages && metadata.pages > 1; - - // skip animated - if (isAnimated) { - return { - webpublic: null, - thumbnail: null, - }; - } + isAnimated = !!(metadata.pages && metadata.pages > 1); satisfyWebpublic = !!( - type !== 'image/svg+xml' && type !== 'image/avif' && + type !== 'image/svg+xml' && // security reason + type !== 'image/avif' && // not supported by Mastodon !(metadata.exif ?? metadata.iptc ?? metadata.xmp ?? metadata.tifftagPhotoshop) && metadata.width && metadata.width <= 2048 && metadata.height && metadata.height <= 2048 @@ -315,15 +311,13 @@ export class DriveService { // #region webpublic let webpublic: IImage | null = null; - if (generateWeb && !satisfyWebpublic) { + if (generateWeb && !satisfyWebpublic && !isAnimated) { this.registerLogger.info('creating web image'); try { - if (type === 'image/jpeg') { - webpublic = await this.imageProcessingService.convertSharpToJpeg(img, 2048, 2048); - } else if (['image/webp', 'image/avif'].includes(type)) { + if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) { webpublic = await this.imageProcessingService.convertSharpToWebp(img, 2048, 2048); - } else if (['image/png', 'image/svg+xml'].includes(type)) { + } else if (['image/png', 'image/bmp', 'image/svg+xml'].includes(type)) { webpublic = await this.imageProcessingService.convertSharpToPng(img, 2048, 2048); } else { this.registerLogger.debug('web image not created (not an required image)'); @@ -333,6 +327,7 @@ export class DriveService { } } else { if (satisfyWebpublic) this.registerLogger.info('web image not created (original satisfies webpublic)'); + else if (isAnimated) this.registerLogger.info('web image not created (animated image)'); else this.registerLogger.info('web image not created (from remote)'); } // #endregion webpublic @@ -341,10 +336,10 @@ export class DriveService { let thumbnail: IImage | null = null; try { - if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(type)) { - thumbnail = await this.imageProcessingService.convertSharpToWebp(img, 498, 280); + if (isAnimated) { + thumbnail = await this.imageProcessingService.convertSharpToWebp(sharp(path, { animated: true }), 374, 317, { alphaQuality: 70 }); } else { - this.registerLogger.debug('thumbnail not created (not an required file)'); + thumbnail = await this.imageProcessingService.convertSharpToAvif(img, 498, 422); } } catch (err) { this.registerLogger.warn('thumbnail not created (an error occured)', err as Error); diff --git a/packages/backend/src/core/ImageProcessingService.ts b/packages/backend/src/core/ImageProcessingService.ts index 7c88f5e9a..3246475d1 100644 --- a/packages/backend/src/core/ImageProcessingService.ts +++ b/packages/backend/src/core/ImageProcessingService.ts @@ -15,15 +15,28 @@ export type IImageStream = { type: string; }; -export type IImageStreamable = IImage | IImageStream; +export type IImageSharp = { + data: sharp.Sharp; + ext: string | null; + type: string; +}; + +export type IImageStreamable = IImage | IImageStream | IImageSharp; export const webpDefault: sharp.WebpOptions = { - quality: 85, + quality: 77, alphaQuality: 95, lossless: false, nearLossless: false, smartSubsample: true, mixed: true, + effort: 2, +}; + +export const avifDefault: sharp.AvifOptions = { + quality: 60, + lossless: false, + effort: 2, }; import { bindThis } from '@/decorators.js'; @@ -37,36 +50,6 @@ export class ImageProcessingService { ) { } - /** - * Convert to JPEG - * with resize, remove metadata, resolve orientation, stop animation - */ - @bindThis - public async convertToJpeg(path: string, width: number, height: number): Promise { - return this.convertSharpToJpeg(await sharp(path), width, height); - } - - @bindThis - public async convertSharpToJpeg(sharp: sharp.Sharp, width: number, height: number): Promise { - const data = await sharp - .resize(width, height, { - fit: 'inside', - withoutEnlargement: true, - }) - .rotate() - .jpeg({ - quality: 85, - progressive: true, - }) - .toBuffer(); - - return { - data, - ext: 'jpg', - type: 'image/jpeg', - }; - } - /** * Convert to WebP * with resize, remove metadata, resolve orientation, stop animation @@ -78,29 +61,22 @@ export class ImageProcessingService { @bindThis public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise { - const data = await sharp - .resize(width, height, { - fit: 'inside', - withoutEnlargement: true, - }) - .rotate() - .webp(options) - .toBuffer(); + const result = this.convertSharpToWebpStream(sharp, width, height, options); return { - data, - ext: 'webp', - type: 'image/webp', + data: await result.data.toBuffer(), + ext: result.ext, + type: result.type, }; } @bindThis - public convertToWebpStream(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageStream { + public convertToWebpStream(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageSharp { return this.convertSharpToWebpStream(sharp(path), width, height, options); } @bindThis - public convertSharpToWebpStream(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageStream { + public convertSharpToWebpStream(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): IImageSharp { const data = sharp .resize(width, height, { fit: 'inside', @@ -115,13 +91,56 @@ export class ImageProcessingService { type: 'image/webp', }; } + + /** + * Convert to Avif + * with resize, remove metadata, resolve orientation, stop animation + */ + @bindThis + public async convertToAvif(path: string, width: number, height: number, options: sharp.AvifOptions = avifDefault): Promise { + return this.convertSharpToAvif(sharp(path), width, height, options); + } + + @bindThis + public async convertSharpToAvif(sharp: sharp.Sharp, width: number, height: number, options: sharp.AvifOptions = avifDefault): Promise { + const result = this.convertSharpToAvifStream(sharp, width, height, options); + + return { + data: await result.data.toBuffer(), + ext: result.ext, + type: result.type, + }; + } + + @bindThis + public convertToAvifStream(path: string, width: number, height: number, options: sharp.AvifOptions = avifDefault): IImageSharp { + return this.convertSharpToAvifStream(sharp(path), width, height, options); + } + + @bindThis + public convertSharpToAvifStream(sharp: sharp.Sharp, width: number, height: number, options: sharp.AvifOptions = avifDefault): IImageSharp { + const data = sharp + .resize(width, height, { + fit: 'inside', + withoutEnlargement: true, + }) + .rotate() + .avif(options); + + return { + data, + ext: 'avif', + type: 'image/avif', + }; + } + /** * Convert to PNG * with resize, remove metadata, resolve orientation, stop animation */ @bindThis public async convertToPng(path: string, width: number, height: number): Promise { - return this.convertSharpToPng(await sharp(path), width, height); + return this.convertSharpToPng(sharp(path), width, height); } @bindThis diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts index 74a0689d8..1a6913b80 100644 --- a/packages/backend/src/core/entities/DriveFileEntityService.ts +++ b/packages/backend/src/core/entities/DriveFileEntityService.ts @@ -76,7 +76,7 @@ export class DriveFileEntityService { @bindThis private getProxiedUrl(url: string, mode?: 'static' | 'avatar'): string { return appendQuery( - `${this.config.mediaProxy}/${mode ?? 'image'}.webp`, + `${this.config.mediaProxy}/${mode ?? 'image'}.${mode === 'avatar' ? 'webp' : 'avif'}`, query({ url, ...(mode ? { [mode]: '1' } : {}), @@ -104,7 +104,7 @@ export class DriveFileEntityService { const url = file.webpublicUrl ?? file.url; - return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? this.getProxiedUrl(url, 'static') : null); + return file.thumbnailUrl ?? (isMimeImage(file.type, 'sharp-convertible-image') ? url : null); } @bindThis diff --git a/packages/backend/src/misc/is-mime-image.ts b/packages/backend/src/misc/is-mime-image.ts index 0b6d147dc..46a66efc0 100644 --- a/packages/backend/src/misc/is-mime-image.ts +++ b/packages/backend/src/misc/is-mime-image.ts @@ -2,10 +2,10 @@ import { FILE_TYPE_BROWSERSAFE } from '@/const.js'; const dictionary = { 'safe-file': FILE_TYPE_BROWSERSAFE, - 'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml'], - 'sharp-animation-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml'], - 'sharp-convertible-image-with-bmp': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'], - 'sharp-animation-convertible-image-with-bmp': ['image/jpeg', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'], + 'sharp-convertible-image': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml'], + 'sharp-animation-convertible-image': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml'], + 'sharp-convertible-image-with-bmp': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'], + 'sharp-animation-convertible-image-with-bmp': ['image/jpeg', 'image/tiff', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/svg+xml', 'image/x-icon', 'image/bmp'], }; export const isMimeImage = (mime: string, type: keyof typeof dictionary): boolean => dictionary[type].includes(mime); diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 6db9a9672..fb1c67f20 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -130,7 +130,7 @@ export class FileServerService { if (isMimeImage(file.mime, 'sharp-convertible-image-with-bmp')) { reply.header('Cache-Control', 'max-age=31536000, immutable'); - const url = new URL(`${this.config.mediaProxy}/static.webp`); + const url = new URL(`${this.config.mediaProxy}/static.avif`); url.searchParams.set('url', file.url); url.searchParams.set('static', '1'); @@ -151,7 +151,7 @@ export class FileServerService { if (['image/svg+xml'].includes(file.mime)) { reply.header('Cache-Control', 'max-age=31536000, immutable'); - const url = new URL(`${this.config.mediaProxy}/svg.webp`); + const url = new URL(`${this.config.mediaProxy}/svg.avif`); url.searchParams.set('url', file.url); file.cleanup(); @@ -291,9 +291,9 @@ export class FileServerService { }; } } else if ('static' in request.query) { - image = this.imageProcessingService.convertSharpToWebpStream(await sharpBmp(file.path, file.mime), 498, 280); + image = this.imageProcessingService.convertSharpToAvifStream(await sharpBmp(file.path, file.mime), 498, 422); } else if ('preview' in request.query) { - image = this.imageProcessingService.convertSharpToWebpStream(await sharpBmp(file.path, file.mime), 200, 200); + image = this.imageProcessingService.convertSharpToAvifStream(await sharpBmp(file.path, file.mime), 200, 200); } else if ('badge' in request.query) { const mask = (await sharpBmp(file.path, file.mime)) .resize(96, 96, { @@ -325,7 +325,7 @@ export class FileServerService { type: 'image/png', }; } else if (file.mime === 'image/svg+xml') { - image = this.imageProcessingService.convertToWebpStream(file.path, 2048, 2048); + image = this.imageProcessingService.convertToAvifStream(file.path, 2048, 2048); } else if (!file.mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(file.mime)) { throw new StatusError('Rejected type', 403, 'Rejected type'); } diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 2ce7293a5..5f4d53d0e 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -33,7 +33,7 @@ export class UrlPreviewService { private wrap(url?: string | null): string | null { return url != null ? url.match(/^https?:\/\//) - ? `${this.config.mediaProxy}/preview.webp?${query({ + ? `${this.config.mediaProxy}/preview.avif?${query({ url, preview: '1', })}` diff --git a/packages/frontend/src/components/MkMediaImage.vue b/packages/frontend/src/components/MkMediaImage.vue index 6091b4001..a4065dcd0 100644 --- a/packages/frontend/src/components/MkMediaImage.vue +++ b/packages/frontend/src/components/MkMediaImage.vue @@ -43,7 +43,7 @@ let darkMode = $ref(defaultStore.state.darkMode); const url = (props.raw || defaultStore.state.loadRawImages) ? props.image.url : defaultStore.state.disableShowingAnimatedImages - ? getStaticImageUrl(props.image.thumbnailUrl) + ? getStaticImageUrl(props.image.url) : props.image.thumbnailUrl; // Plugin:register_note_view_interruptor を使って書き換えられる可能性があるためwatchする diff --git a/packages/frontend/src/pages/user/index.photos.vue b/packages/frontend/src/pages/user/index.photos.vue index 607082c1e..85f6591ee 100644 --- a/packages/frontend/src/pages/user/index.photos.vue +++ b/packages/frontend/src/pages/user/index.photos.vue @@ -41,7 +41,7 @@ let images = $ref<{ function thumbnail(image: misskey.entities.DriveFile): string { return defaultStore.state.disableShowingAnimatedImages - ? getStaticImageUrl(image.thumbnailUrl) + ? getStaticImageUrl(image.url) : image.thumbnailUrl; } diff --git a/packages/frontend/src/scripts/media-proxy.ts b/packages/frontend/src/scripts/media-proxy.ts index 2fe5bdcf8..d0c95e0b7 100644 --- a/packages/frontend/src/scripts/media-proxy.ts +++ b/packages/frontend/src/scripts/media-proxy.ts @@ -10,7 +10,10 @@ export function getProxiedImageUrl(imageUrl: string, type?: 'preview', mustOrigi imageUrl = (new URL(imageUrl)).searchParams.get('url') ?? imageUrl; } - return `${mustOrigin ? localProxy : instance.mediaProxy}/image.webp?${query({ + return `${mustOrigin ? localProxy : instance.mediaProxy}/${ + type === 'preview' ? 'preview.avif' + : 'image.webp' + }?${query({ url: imageUrl, fallback: '1', ...(type ? { [type]: '1' } : {}), @@ -38,7 +41,7 @@ export function getStaticImageUrl(baseUrl: string): string { return u.href; } - return `${instance.mediaProxy}/static.webp?${query({ + return `${instance.mediaProxy}/static.avif?${query({ url: u.href, static: '1', })}`; diff --git a/packages/frontend/src/widgets/WidgetPhotos.vue b/packages/frontend/src/widgets/WidgetPhotos.vue index 562249f09..716bbb427 100644 --- a/packages/frontend/src/widgets/WidgetPhotos.vue +++ b/packages/frontend/src/widgets/WidgetPhotos.vue @@ -67,7 +67,7 @@ const onDriveFileCreated = (file) => { const thumbnail = (image: any): string => { return defaultStore.state.disableShowingAnimatedImages - ? getStaticImageUrl(image.thumbnailUrl) + ? getStaticImageUrl(image.url) : image.thumbnailUrl; };