enhance(backend): make ftt db fallback configurable

CHANGELOG.md

@@ -32,6 +32,7 @@
 - Fix: 特定の条件下でノートがnyaizeされない問題を修正
 
 ### Server
+- Enhance: FTTのデータベースへのフォールバック処理を行うかどうかを設定可能に
 - Fix: トークンのないプラグインをアンインストールするときにエラーが出ないように
 - Fix: 投稿通知がオンでもダイレクト投稿はユーザーに通知されないようにされました
 - Fix: ユーザタイムラインの「ノート」選択時にリノートが混ざり込んでしまうことがある問題の修正 #12306

locales/index.d.ts

@@ -1285,6 +1285,8 @@ export interface Locale {
         "shortName": string;
         "shortNameDescription": string;
         "fanoutTimelineDescription": string;
+        "fanoutTimelineDbFallback": string;
+        "fanoutTimelineDbFallbackDescription": string;
     };
     "_accountMigration": {
         "moveFrom": string;

locales/ja-JP.yml

@@ -1272,6 +1272,8 @@ _serverSettings:
   shortName: "略称"
   shortNameDescription: "サーバーの正式名称が長い場合に、代わりに表示することのできる略称や通称。"
   fanoutTimelineDescription: "有効にすると、各種タイムラインを取得する際のパフォーマンスが大幅に向上し、データベースへの負荷を軽減することが可能です。ただし、Redisのメモリ使用量は増加します。サーバーのメモリ容量が少ない場合、または動作が不安定な場合は無効にすることができます。"
+  fanoutTimelineDbFallback: "データベースへのフォールバック"
+  fanoutTimelineDbFallbackDescription: "有効にすると、タイムラインがキャッシュされていない場合にDBへ追加で問い合わせを行うフォールバック処理を行います。無効にすると、フォールバック処理を行わないことでさらにサーバーの負荷を軽減することができますが、タイムラインが取得できる範囲に制限が生じます。"
 
 _accountMigration:
   moveFrom: "別のアカウントからこのアカウントに移行"

packages/backend/migration/1700096812223-enableFanoutTimelineDbFallback.js

@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+export class EnableFanoutTimelineDbFallback1700096812223 {
+    name = 'EnableFanoutTimelineDbFallback1700096812223'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" ADD "enableFanoutTimelineDbFallback" boolean NOT NULL DEFAULT true`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableFanoutTimelineDbFallback"`);
+    }
+}

packages/backend/src/models/Meta.ts

@@ -494,6 +494,11 @@ export class MiMeta {
 	})
 	public enableFanoutTimeline: boolean;
 
+	@Column('boolean', {
+		default: true,
+	})
+	public enableFanoutTimelineDbFallback: boolean;
+
 	@Column('integer', {
 		default: 300,
 	})

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -295,6 +295,10 @@ export const meta = {
 				type: 'boolean',
 				optional: false, nullable: false,
 			},
+			enableFanoutTimelineDbFallback: {
+				type: 'boolean',
+				optional: false, nullable: false,
+			},
 			perLocalUserUserTimelineCacheMax: {
 				type: 'number',
 				optional: false, nullable: false,
@@ -424,6 +428,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				policies: { ...DEFAULT_POLICIES, ...instance.policies },
 				manifestJsonOverride: instance.manifestJsonOverride,
 				enableFanoutTimeline: instance.enableFanoutTimeline,
+				enableFanoutTimelineDbFallback: instance.enableFanoutTimelineDbFallback,
 				perLocalUserUserTimelineCacheMax: instance.perLocalUserUserTimelineCacheMax,
 				perRemoteUserUserTimelineCacheMax: instance.perRemoteUserUserTimelineCacheMax,
 				perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax,

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -121,6 +121,7 @@ export const paramDef = {
 		preservedUsernames: { type: 'array', items: { type: 'string' } },
 		manifestJsonOverride: { type: 'string' },
 		enableFanoutTimeline: { type: 'boolean' },
+		enableFanoutTimelineDbFallback: { type: 'boolean' },
 		perLocalUserUserTimelineCacheMax: { type: 'integer' },
 		perRemoteUserUserTimelineCacheMax: { type: 'integer' },
 		perUserHomeTimelineCacheMax: { type: 'integer' },
@@ -485,6 +486,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.enableFanoutTimeline = ps.enableFanoutTimeline;
 			}
 
+			if (ps.enableFanoutTimelineDbFallback !== undefined) {
+				set.enableFanoutTimelineDbFallback = ps.enableFanoutTimelineDbFallback;
+			}
+
 			if (ps.perLocalUserUserTimelineCacheMax !== undefined) {
 				set.perLocalUserUserTimelineCacheMax = ps.perLocalUserUserTimelineCacheMax;
 			}

packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts

@@ -93,7 +93,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 
 			const serverSettings = await this.metaService.fetch();
 
-			if (serverSettings.enableFanoutTimeline) {
+			if (!serverSettings.enableFanoutTimeline) {
+				return await this.getFromDb({
+					untilId,
+					sinceId,
+					limit: ps.limit,
+					includeMyRenotes: ps.includeMyRenotes,
+					includeRenotedMyNotes: ps.includeRenotedMyNotes,
+					includeLocalRenotes: ps.includeLocalRenotes,
+					withFiles: ps.withFiles,
+					withReplies: ps.withReplies,
+				}, me);
+			}
+
 			const [
 				userIdsWhoMeMuting,
 				userIdsWhoMeMutingRenotes,
@@ -173,19 +185,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				});
 
 				return await this.noteEntityService.packMany(redisTimeline, me);
-				} else { // fallback to db
-					return await this.getFromDb({
-						untilId,
-						sinceId,
-						limit: ps.limit,
-						includeMyRenotes: ps.includeMyRenotes,
-						includeRenotedMyNotes: ps.includeRenotedMyNotes,
-						includeLocalRenotes: ps.includeLocalRenotes,
-						withFiles: ps.withFiles,
-						withReplies: ps.withReplies,
-					}, me);
-				}
 			} else {
+				if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db
 					return await this.getFromDb({
 						untilId,
 						sinceId,
@@ -196,6 +197,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 						withFiles: ps.withFiles,
 						withReplies: ps.withReplies,
 					}, me);
+				} else {
+					return [];
+				}
 			}
 		});
 	}

packages/backend/src/server/api/endpoints/notes/local-timeline.ts

@@ -84,7 +84,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 
 			const serverSettings = await this.metaService.fetch();
 
-			if (serverSettings.enableFanoutTimeline) {
+			if (!serverSettings.enableFanoutTimeline) {
+				return await this.getFromDb({
+					untilId,
+					sinceId,
+					limit: ps.limit,
+					withFiles: ps.withFiles,
+					withReplies: ps.withReplies,
+				}, me);
+			}
+
 			const [
 				userIdsWhoMeMuting,
 				userIdsWhoMeMutingRenotes,
@@ -152,16 +161,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				});
 
 				return await this.noteEntityService.packMany(redisTimeline, me);
-				} else { // fallback to db
-					return await this.getFromDb({
-						untilId,
-						sinceId,
-						limit: ps.limit,
-						withFiles: ps.withFiles,
-						withReplies: ps.withReplies,
-					}, me);
-				}
 			} else {
+				if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db
 					return await this.getFromDb({
 						untilId,
 						sinceId,
@@ -169,6 +170,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 						withFiles: ps.withFiles,
 						withReplies: ps.withReplies,
 					}, me);
+				} else {
+					return [];
+				}
 			}
 		});
 	}

packages/backend/src/server/api/endpoints/notes/timeline.ts

@@ -76,7 +76,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 
 			const serverSettings = await this.metaService.fetch();
 
-			if (serverSettings.enableFanoutTimeline) {
+			if (!serverSettings.enableFanoutTimeline) {
+				return await this.getFromDb({
+					untilId,
+					sinceId,
+					limit: ps.limit,
+					includeMyRenotes: ps.includeMyRenotes,
+					includeRenotedMyNotes: ps.includeRenotedMyNotes,
+					includeLocalRenotes: ps.includeLocalRenotes,
+					withFiles: ps.withFiles,
+					withRenotes: ps.withRenotes,
+				}, me);
+			}
+
 			const [
 				followings,
 				userIdsWhoMeMuting,
@@ -134,19 +146,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				});
 
 				return await this.noteEntityService.packMany(redisTimeline, me);
-				} else { // fallback to db
-					return await this.getFromDb({
-						untilId,
-						sinceId,
-						limit: ps.limit,
-						includeMyRenotes: ps.includeMyRenotes,
-						includeRenotedMyNotes: ps.includeRenotedMyNotes,
-						includeLocalRenotes: ps.includeLocalRenotes,
-						withFiles: ps.withFiles,
-						withRenotes: ps.withRenotes,
-					}, me);
-				}
 			} else {
+				if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db
 					return await this.getFromDb({
 						untilId,
 						sinceId,
@@ -157,6 +158,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 						withFiles: ps.withFiles,
 						withRenotes: ps.withRenotes,
 					}, me);
+				} else {
+					return [];
+				}
 			}
 		});
 	}

packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts

@@ -4,7 +4,8 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
-import type { MiNote, NotesRepository, UserListMembershipsRepository, UserListsRepository } from '@/models/_.js';
+import { Brackets } from 'typeorm';
+import type { MiNote, MiUserList, NotesRepository, UserListMembershipsRepository, UserListsRepository } from '@/models/_.js';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
 import ActiveUsersChart from '@/core/chart/charts/active-users.js';
@@ -14,8 +15,9 @@ import { IdService } from '@/core/IdService.js';
 import { isUserRelated } from '@/misc/is-user-related.js';
 import { FunoutTimelineService } from '@/core/FunoutTimelineService.js';
 import { QueryService } from '@/core/QueryService.js';
+import { MiLocalUser } from '@/models/User.js';
+import { MetaService } from '@/core/MetaService.js';
 import { ApiError } from '../../error.js';
-import { Brackets } from 'typeorm';
 
 export const meta = {
 	tags: ['notes', 'lists'],
@@ -81,7 +83,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private idService: IdService,
 		private funoutTimelineService: FunoutTimelineService,
 		private queryService: QueryService,
-
+		private metaService: MetaService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
 			const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null);
@@ -96,6 +98,21 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				throw new ApiError(meta.errors.noSuchList);
 			}
 
+			const serverSettings = await this.metaService.fetch();
+
+			if (!serverSettings.enableFanoutTimeline) {
+				return await this.getFromDb(list, {
+					untilId,
+					sinceId,
+					limit: ps.limit,
+					includeMyRenotes: ps.includeMyRenotes,
+					includeRenotedMyNotes: ps.includeRenotedMyNotes,
+					includeLocalRenotes: ps.includeLocalRenotes,
+					withFiles: ps.withFiles,
+					withRenotes: ps.withRenotes,
+				}, me);
+			}
+
 			const [
 				userIdsWhoMeMuting,
 				userIdsWhoMeMutingRenotes,
@@ -145,7 +162,35 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 			if (redisTimeline.length > 0) {
 				this.activeUsersChart.read(me);
 				return await this.noteEntityService.packMany(redisTimeline, me);
-			} else { // fallback to db
+			} else {
+				if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db
+					return await this.getFromDb(list, {
+						untilId,
+						sinceId,
+						limit: ps.limit,
+						includeMyRenotes: ps.includeMyRenotes,
+						includeRenotedMyNotes: ps.includeRenotedMyNotes,
+						includeLocalRenotes: ps.includeLocalRenotes,
+						withFiles: ps.withFiles,
+						withRenotes: ps.withRenotes,
+					}, me);
+				} else {
+					return [];
+				}
+			}
+		});
+	}
+
+	private async getFromDb(list: MiUserList, ps: {
+		untilId: string | null,
+		sinceId: string | null,
+		limit: number,
+		includeMyRenotes: boolean,
+		includeRenotedMyNotes: boolean,
+		includeLocalRenotes: boolean,
+		withFiles: boolean,
+		withRenotes: boolean,
+	}, me: MiLocalUser) {
 		//#region Construct query
 		const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId)
 			.innerJoin(this.userListMembershipsRepository.metadata.targetName, 'userListMemberships', 'userListMemberships.userId = note.userId')
@@ -232,6 +277,4 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 
 		return await this.noteEntityService.packMany(timeline, me);
 	}
-		});
-	}
 }

packages/backend/test/unit/activitypub.ts

@@ -94,6 +94,7 @@ describe('ActivityPub', () => {
 		cacheRemoteFiles: true,
 		cacheRemoteSensitiveFiles: true,
 		enableFanoutTimeline: true,
+		enableFanoutTimelineDbFallback: true,
 		perUserHomeTimelineCacheMax: 100,
 		perLocalUserUserTimelineCacheMax: 100,
 		perRemoteUserUserTimelineCacheMax: 100,

packages/frontend/src/pages/admin/settings.vue

@@ -95,6 +95,11 @@ SPDX-License-Identifier: AGPL-3.0-only
 								<template #caption>{{ i18n.ts._serverSettings.fanoutTimelineDescription }}</template>
 							</MkSwitch>
 
+							<MkSwitch v-model="enableFanoutTimelineDbFallback">
+								<template #label>{{ i18n.ts._serverSettings.fanoutTimelineDbFallback }}</template>
+								<template #caption>{{ i18n.ts._serverSettings.fanoutTimelineDbFallbackDescription }}</template>
+							</MkSwitch>
+
 							<MkInput v-model="perLocalUserUserTimelineCacheMax" type="number">
 								<template #label>perLocalUserUserTimelineCacheMax</template>
 							</MkInput>
@@ -171,6 +176,7 @@ let enableServiceWorker: boolean = $ref(false);
 let swPublicKey: any = $ref(null);
 let swPrivateKey: any = $ref(null);
 let enableFanoutTimeline: boolean = $ref(false);
+let enableFanoutTimelineDbFallback: boolean = $ref(false);
 let perLocalUserUserTimelineCacheMax: number = $ref(0);
 let perRemoteUserUserTimelineCacheMax: number = $ref(0);
 let perUserHomeTimelineCacheMax: number = $ref(0);
@@ -192,6 +198,7 @@ async function init(): Promise<void> {
 	swPublicKey = meta.swPublickey;
 	swPrivateKey = meta.swPrivateKey;
 	enableFanoutTimeline = meta.enableFanoutTimeline;
+	enableFanoutTimelineDbFallback = meta.enableFanoutTimelineDbFallback;
 	perLocalUserUserTimelineCacheMax = meta.perLocalUserUserTimelineCacheMax;
 	perRemoteUserUserTimelineCacheMax = meta.perRemoteUserUserTimelineCacheMax;
 	perUserHomeTimelineCacheMax = meta.perUserHomeTimelineCacheMax;
@@ -214,6 +221,7 @@ async function save(): void {
 		swPublicKey,
 		swPrivateKey,
 		enableFanoutTimeline,
+		enableFanoutTimelineDbFallback,
 		perLocalUserUserTimelineCacheMax,
 		perRemoteUserUserTimelineCacheMax,
 		perUserHomeTimelineCacheMax,