From 2dfed75402969893ba10f500de254b12434c8f3f Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 4 Feb 2023 13:38:51 +0900 Subject: [PATCH] perf(server): improvement of external mediaProxy (#9787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(server): improvement of external mediaProxy * add a comment * :v: * /filesでsharpの処理を行わずリダイレクトする * fix * thumbnail => static * Fix #9788 * add avatar mode * add url * fix * static.webp * remove encodeURIComponent from media proxy path * remove existance check --- .config/example.yml | 1 + CHANGELOG.md | 3 + packages/backend/src/config.ts | 9 +++ .../backend/src/core/CustomEmojiService.ts | 2 +- .../src/core/entities/ChannelEntityService.ts | 2 +- .../core/entities/DriveFileEntityService.ts | 40 ++++++---- .../src/core/entities/UserEntityService.ts | 8 +- .../backend/src/server/FileServerService.ts | 77 +++++++++++-------- packages/backend/src/server/ServerService.ts | 2 +- .../backend/src/server/api/endpoints/meta.ts | 6 ++ .../src/server/web/UrlPreviewService.ts | 2 +- packages/frontend/src/scripts/media-proxy.ts | 20 +++-- 12 files changed, 110 insertions(+), 62 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 8fe41da15..a19b5d04e 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -130,6 +130,7 @@ proxyBypassHosts: #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 # Media Proxy +# Reference Implementation: https://github.com/misskey-dev/media-proxy #mediaProxy: https://example.com/proxy # Proxy remote files (default: false) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e10edf68..0ad1e3621 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ You should also include the user name that made the change. - syslogのサポートが削除されました ### Improvements +- 外部メディアプロキシへの対応を強化しました + 外部メディアプロキシのFastify実装を作りました + https://github.com/misskey-dev/media-proxy - ロールで広告の非表示が有効になっている場合は最初から広告を非表示にするように ## 13.2.6 (2023/02/01) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 1d4e70065..aa98ef1d2 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -87,6 +87,8 @@ export type Mixin = { userAgent: string; clientEntry: string; clientManifestExists: boolean; + mediaProxy: string; + externalMediaProxyEnabled: boolean; }; export type Config = Source & Mixin; @@ -135,6 +137,13 @@ export function loadConfig() { mixin.clientEntry = clientManifest['src/init.ts']; mixin.clientManifestExists = clientManifestExists; + const externalMediaProxy = config.mediaProxy ? + config.mediaProxy.endsWith('/') ? config.mediaProxy.substring(0, config.mediaProxy.length - 1) : config.mediaProxy + : null; + const internalMediaProxy = `${mixin.scheme}://${mixin.host}/proxy`; + mixin.mediaProxy = externalMediaProxy ?? internalMediaProxy; + mixin.externalMediaProxyEnabled = externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy; + if (!config.redis.prefix) config.redis.prefix = mixin.host; return Object.assign(config, mixin); diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index 39814e1be..63f031944 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -120,7 +120,7 @@ export class CustomEmojiService { const url = isLocal ? emojiUrl : this.config.proxyRemoteFiles - ? `${this.config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}` + ? `${this.config.mediaProxy}/emoji.webp?${query({ url: emojiUrl })}` : emojiUrl; return url; diff --git a/packages/backend/src/core/entities/ChannelEntityService.ts b/packages/backend/src/core/entities/ChannelEntityService.ts index 5e2f019a1..6ce590aa9 100644 --- a/packages/backend/src/core/entities/ChannelEntityService.ts +++ b/packages/backend/src/core/entities/ChannelEntityService.ts @@ -54,7 +54,7 @@ export class ChannelEntityService { name: channel.name, description: channel.description, userId: channel.userId, - bannerUrl: banner ? this.driveFileEntityService.getPublicUrl(banner, false) : null, + bannerUrl: banner ? this.driveFileEntityService.getPublicUrl(banner) : null, usersCount: channel.usersCount, notesCount: channel.notesCount, diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts index 7f54cfdea..efc196f74 100644 --- a/packages/backend/src/core/entities/DriveFileEntityService.ts +++ b/packages/backend/src/core/entities/DriveFileEntityService.ts @@ -71,27 +71,41 @@ export class DriveFileEntityService { } @bindThis - public getPublicUrl(file: DriveFile, thumbnail = false): string | null { + public getPublicUrl(file: DriveFile, mode? : 'static' | 'avatar'): string | null { // static = thumbnail + const proxiedUrl = (url: string) => appendQuery( + `${this.config.mediaProxy}/${mode ?? 'image'}.webp`, + query({ + url, + ...(mode ? { [mode]: '1' } : {}), + }) + ); + // リモートかつメディアプロキシ - if (file.uri != null && file.userHost != null && this.config.mediaProxy != null) { - return appendQuery(this.config.mediaProxy, query({ - url: file.uri, - thumbnail: thumbnail ? '1' : undefined, - })); + if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) { + return proxiedUrl(file.uri); } // リモートかつ期限切れはローカルプロキシを試みる if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) { - const key = thumbnail ? file.thumbnailAccessKey : file.webpublicAccessKey; + const key = mode === 'static' ? file.thumbnailAccessKey : file.webpublicAccessKey; if (key && !key.match('/')) { // 古いものはここにオブジェクトストレージキーが入ってるので除外 - return `${this.config.url}/files/${key}`; + const url = `${this.config.url}/files/${key}`; + if (mode === 'avatar') return proxiedUrl(url); + return url; } } const isImage = file.type && ['image/png', 'image/apng', 'image/gif', 'image/jpeg', 'image/webp', 'image/avif', 'image/svg+xml'].includes(file.type); - return thumbnail ? (file.thumbnailUrl ?? (isImage ? (file.webpublicUrl ?? file.url) : null)) : (file.webpublicUrl ?? file.url); + if (mode === 'static') { + return file.thumbnailUrl ?? (isImage ? (file.webpublicUrl ?? file.url) : null); + } + + const url = file.webpublicUrl ?? file.url; + + if (mode === 'avatar') return proxiedUrl(url); + return url; } @bindThis @@ -166,8 +180,8 @@ export class DriveFileEntityService { isSensitive: file.isSensitive, blurhash: file.blurhash, properties: opts.self ? file.properties : this.getPublicProperties(file), - url: opts.self ? file.url : this.getPublicUrl(file, false), - thumbnailUrl: this.getPublicUrl(file, true), + url: opts.self ? file.url : this.getPublicUrl(file), + thumbnailUrl: this.getPublicUrl(file, 'static'), comment: file.comment, folderId: file.folderId, folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, { @@ -201,8 +215,8 @@ export class DriveFileEntityService { isSensitive: file.isSensitive, blurhash: file.blurhash, properties: opts.self ? file.properties : this.getPublicProperties(file), - url: opts.self ? file.url : this.getPublicUrl(file, false), - thumbnailUrl: this.getPublicUrl(file, true), + url: opts.self ? file.url : this.getPublicUrl(file), + thumbnailUrl: this.getPublicUrl(file, 'static'), comment: file.comment, folderId: file.folderId, folder: opts.detail && file.folderId ? this.driveFolderEntityService.pack(file.folderId, { diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index aaa80033b..ff42c0735 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -314,10 +314,10 @@ export class UserEntityService implements OnModuleInit { @bindThis public async getAvatarUrl(user: User): Promise { if (user.avatar) { - return this.driveFileEntityService.getPublicUrl(user.avatar, true) ?? this.getIdenticonUrl(user.id); + return this.driveFileEntityService.getPublicUrl(user.avatar, 'avatar') ?? this.getIdenticonUrl(user.id); } else if (user.avatarId) { const avatar = await this.driveFilesRepository.findOneByOrFail({ id: user.avatarId }); - return this.driveFileEntityService.getPublicUrl(avatar, true) ?? this.getIdenticonUrl(user.id); + return this.driveFileEntityService.getPublicUrl(avatar, 'avatar') ?? this.getIdenticonUrl(user.id); } else { return this.getIdenticonUrl(user.id); } @@ -326,7 +326,7 @@ export class UserEntityService implements OnModuleInit { @bindThis public getAvatarUrlSync(user: User): string { if (user.avatar) { - return this.driveFileEntityService.getPublicUrl(user.avatar, true) ?? this.getIdenticonUrl(user.id); + return this.driveFileEntityService.getPublicUrl(user.avatar, 'avatar') ?? this.getIdenticonUrl(user.id); } else { return this.getIdenticonUrl(user.id); } @@ -422,7 +422,7 @@ export class UserEntityService implements OnModuleInit { createdAt: user.createdAt.toISOString(), updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null, lastFetchedAt: user.lastFetchedAt ? user.lastFetchedAt.toISOString() : null, - bannerUrl: user.banner ? this.driveFileEntityService.getPublicUrl(user.banner, false) : null, + bannerUrl: user.banner ? this.driveFileEntityService.getPublicUrl(user.banner) : null, bannerBlurhash: user.banner?.blurhash ?? null, isLocked: user.isLocked, isSilenced: this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote), diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 40024270a..39bc4c1d9 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -137,38 +137,38 @@ export class FileServerService { try { if (file.state === 'remote') { - const convertFile = async () => { - if (file.fileRole === 'thumbnail') { - if (['image/jpeg', 'image/webp', 'image/avif', 'image/png', 'image/svg+xml'].includes(file.mime)) { - return this.imageProcessingService.convertToWebpStream( - file.path, - 498, - 280 - ); - } else if (file.mime.startsWith('video/')) { - return await this.videoProcessingService.generateVideoThumbnail(file.path); - } - } + let image: IImageStreamable | null = null; - if (file.fileRole === 'webpublic') { - if (['image/svg+xml'].includes(file.mime)) { - return this.imageProcessingService.convertToWebpStream( - file.path, - 2048, - 2048, - { ...webpDefault, lossless: true } - ) - } - } + if (file.fileRole === 'thumbnail') { + if (isMimeImage(file.mime, 'sharp-convertible-image')) { + reply.header('Cache-Control', 'max-age=31536000, immutable'); - return { + const url = new URL(`${this.config.mediaProxy}/static.webp`); + url.searchParams.set('url', file.url); + url.searchParams.set('static', '1'); + return await reply.redirect(301, url.toString()); + } else if (file.mime.startsWith('video/')) { + image = await this.videoProcessingService.generateVideoThumbnail(file.path); + } + } + + if (file.fileRole === 'webpublic') { + if (['image/svg+xml'].includes(file.mime)) { + reply.header('Cache-Control', 'max-age=31536000, immutable'); + + const url = new URL(`${this.config.mediaProxy}/svg.webp`); + url.searchParams.set('url', file.url); + return await reply.redirect(301, url.toString()); + } + } + + if (!image) { + image = { data: fs.createReadStream(file.path), ext: file.ext, type: file.mime, }; - }; - - const image = await convertFile(); + } if ('pipe' in image.data && typeof image.data.pipe === 'function') { // image.dataがstreamなら、stream終了後にcleanup @@ -180,7 +180,6 @@ export class FileServerService { } reply.header('Content-Type', FILE_TYPE_BROWSERSAFE.includes(image.type) ? image.type : 'application/octet-stream'); - reply.header('Cache-Control', 'max-age=31536000, immutable'); return image.data; } @@ -217,6 +216,23 @@ export class FileServerService { return; } + if (this.config.externalMediaProxyEnabled) { + // 外部のメディアプロキシが有効なら、そちらにリダイレクト + + reply.header('Cache-Control', 'public, max-age=259200'); // 3 days + + const url = new URL(`${this.config.mediaProxy}/${request.params.url || ''}`); + + for (const [key, value] of Object.entries(request.query)) { + url.searchParams.append(key, value); + } + + return await reply.redirect( + 301, + url.toString(), + ); + } + // Create temp file const file = await this.getStreamAndTypeFromUrl(url); if (file === '404') { @@ -236,7 +252,7 @@ export class FileServerService { const isAnimationConvertibleImage = isMimeImage(file.mime, 'sharp-animation-convertible-image'); let image: IImageStreamable | null = null; - if ('emoji' in request.query && isConvertibleImage) { + if (('emoji' in request.query || 'avatar' in request.query) && isConvertibleImage) { if (!isAnimationConvertibleImage && !('static' in request.query)) { image = { data: fs.createReadStream(file.path), @@ -246,7 +262,7 @@ export class FileServerService { } else { const data = sharp(file.path, { animated: !('static' in request.query) }) .resize({ - height: 128, + height: 'emoji' in request.query ? 128 : 320, withoutEnlargement: true, }) .webp(webpDefault); @@ -370,7 +386,7 @@ export class FileServerService { @bindThis private async getFileFromKey(key: string): Promise< - { state: 'remote'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; mime: string; ext: string | null; path: string; cleanup: () => void; } + { state: 'remote'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; url: string; mime: string; ext: string | null; path: string; cleanup: () => void; } | { state: 'stored_internal'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: DriveFile; mime: string; ext: string | null; path: string; } | '404' | '204' @@ -392,6 +408,7 @@ export class FileServerService { const result = await this.downloadAndDetectTypeFromUrl(file.uri); return { ...result, + url: file.uri, fileRole: isThumbnail ? 'thumbnail' : isWebpublic ? 'webpublic' : 'original', file, } diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index beb3a34ec..c7a2c99f9 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -106,7 +106,7 @@ export class ServerService { } } - const url = new URL('/proxy/emoji.webp', this.config.url); + const url = new URL(`${this.config.mediaProxy}/emoji.webp`); // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ） url.searchParams.set('url', emoji.publicUrl || emoji.originalUrl); url.searchParams.set('emoji', '1'); diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index 3baf94532..2fa7a09d4 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -181,6 +181,10 @@ export const meta = { type: 'string', optional: false, nullable: true, }, + mediaProxy: { + type: 'string', + optional: false, nullable: false, + }, features: { type: 'object', optional: true, nullable: false, @@ -307,6 +311,8 @@ export default class extends Endpoint { policies: { ...DEFAULT_POLICIES, ...instance.policies }, + mediaProxy: this.config.mediaProxy, + ...(ps.detail ? { pinnedPages: instance.pinnedPages, pinnedClipId: instance.pinnedClipId, diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 802b404ce..1bf88fe43 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -33,7 +33,7 @@ export class UrlPreviewService { private wrap(url?: string): string | null { return url != null ? url.match(/^https?:\/\//) - ? `${this.config.url}/proxy/preview.webp?${query({ + ? `${this.config.mediaProxy}/preview.webp?${query({ url, preview: '1', })}` diff --git a/packages/frontend/src/scripts/media-proxy.ts b/packages/frontend/src/scripts/media-proxy.ts index bea164e7c..274e96e0a 100644 --- a/packages/frontend/src/scripts/media-proxy.ts +++ b/packages/frontend/src/scripts/media-proxy.ts @@ -1,8 +1,9 @@ import { query, appendQuery } from '@/scripts/url'; import { url } from '@/config'; +import { instance } from '@/instance'; export function getProxiedImageUrl(imageUrl: string, type?: 'preview'): string { - if (imageUrl.startsWith(`${url}/proxy/`) || imageUrl.startsWith('/proxy/')) { + if (imageUrl.startsWith(instance.mediaProxy + '/') || imageUrl.startsWith('/proxy/')) { // もう既にproxyっぽそうだったらsearchParams付けるだけ return appendQuery(imageUrl, query({ fallback: '1', @@ -10,7 +11,7 @@ export function getProxiedImageUrl(imageUrl: string, type?: 'preview'): string { })); } - return `${url}/proxy/image.webp?${query({ + return `${instance.mediaProxy}/image.webp?${query({ url: imageUrl, fallback: '1', ...(type ? { [type]: '1' } : {}), @@ -25,22 +26,19 @@ export function getProxiedImageUrlNullable(imageUrl: string | null | undefined, export function getStaticImageUrl(baseUrl: string): string { const u = baseUrl.startsWith('http') ? new URL(baseUrl) : new URL(baseUrl, url); - if (u.href.startsWith(`${url}/proxy/`)) { - // もう既にproxyっぽそうだったらsearchParams付けるだけ - u.searchParams.set('static', '1'); - return u.href; - } - if (u.href.startsWith(`${url}/emoji/`)) { // もう既にemojiっぽそうだったらsearchParams付けるだけ u.searchParams.set('static', '1'); return u.href; } - // 拡張子がないとキャッシュしてくれないCDNがあるのでダミーの名前を指定する - const dummy = `${encodeURIComponent(`${u.host}${u.pathname}`)}.webp`; + if (u.href.startsWith(instance.mediaProxy + '/')) { + // もう既にproxyっぽそうだったらsearchParams付けるだけ + u.searchParams.set('static', '1'); + return u.href; + } - return `${url}/proxy/${dummy}?${query({ + return `${instance.mediaProxy}/static.webp?${query({ url: u.href, static: '1', })}`;