enhance(backend): タイムライン等の有効期限を設定可能に

Resolve #11773
This commit is contained in:
syuilo 2023-09-05 15:03:50 +09:00
parent e7d30c8eb4
commit f53cffaeb2
4 changed files with 128 additions and 45 deletions

View File

@ -23,11 +23,9 @@ type RedisOptionsSource = Partial<RedisOptions> & {
}; };
/** /**
* *
*/ */
export type Source = { type Source = {
repository_url?: string;
feedback_url?: string;
url: string; url: string;
port?: number; port?: number;
socket?: string; socket?: string;
@ -93,12 +91,63 @@ export type Source = {
videoThumbnailGenerator?: string; videoThumbnailGenerator?: string;
signToActivityPubGet?: boolean; signToActivityPubGet?: boolean;
perChannelMaxNoteCacheCount?: number;
perUserNotificationsMaxCount?: number;
}; };
/** export type Config = {
* Misskeyが自動的に() url: string;
*/ port: number;
export type Mixin = { socket: string | undefined;
chmodSocket: string | undefined;
disableHsts: boolean | undefined;
db: {
host: string;
port: number;
db: string;
user: string;
pass: string;
disableCache?: boolean;
extra?: { [x: string]: string };
};
dbReplications: boolean | undefined;
dbSlaves: {
host: string;
port: number;
db: string;
user: string;
pass: string;
}[] | undefined;
meilisearch: {
host: string;
port: string;
apiKey: string;
ssl?: boolean;
index: string;
scope?: 'local' | 'global' | string[];
} | undefined;
proxy: string | undefined;
proxySmtp: string | undefined;
proxyBypassHosts: string[] | undefined;
allowedPrivateNetworks: string[] | undefined;
maxFileSize: number | undefined;
accesslog: string | undefined;
clusterLimit: number | undefined;
id: string;
outgoingAddress: string | undefined;
outgoingAddressFamily: 'ipv4' | 'ipv6' | 'dual' | undefined;
deliverJobConcurrency: number | undefined;
inboxJobConcurrency: number | undefined;
relashionshipJobConcurrency: number | undefined;
deliverJobPerSec: number | undefined;
inboxJobPerSec: number | undefined;
relashionshipJobPerSec: number | undefined;
deliverJobMaxAttempts: number | undefined;
inboxJobMaxAttempts: number | undefined;
proxyRemoteFiles: boolean | undefined;
signToActivityPubGet: boolean | undefined;
version: string; version: string;
host: string; host: string;
hostname: string; hostname: string;
@ -117,10 +166,10 @@ export type Mixin = {
redis: RedisOptions & RedisOptionsSource; redis: RedisOptions & RedisOptionsSource;
redisForPubsub: RedisOptions & RedisOptionsSource; redisForPubsub: RedisOptions & RedisOptionsSource;
redisForJobQueue: RedisOptions & RedisOptionsSource; redisForJobQueue: RedisOptions & RedisOptionsSource;
perChannelMaxNoteCacheCount: number;
perUserNotificationsMaxCount: number;
}; };
export type Config = Source & Mixin;
const _filename = fileURLToPath(import.meta.url); const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename); const _dirname = dirname(_filename);
@ -138,7 +187,7 @@ const path = process.env.MISSKEY_CONFIG_YML
? resolve(dir, 'test.yml') ? resolve(dir, 'test.yml')
: resolve(dir, 'default.yml'); : resolve(dir, 'default.yml');
export function loadConfig() { export function loadConfig(): Config {
const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8')); const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8'));
const clientManifestExists = fs.existsSync(_dirname + '/../../../built/_vite_/manifest.json'); const clientManifestExists = fs.existsSync(_dirname + '/../../../built/_vite_/manifest.json');
const clientManifest = clientManifestExists ? const clientManifest = clientManifestExists ?
@ -146,43 +195,72 @@ export function loadConfig() {
: { 'src/_boot_.ts': { file: 'src/_boot_.ts' } }; : { 'src/_boot_.ts': { file: 'src/_boot_.ts' } };
const config = yaml.load(fs.readFileSync(path, 'utf-8')) as Source; const config = yaml.load(fs.readFileSync(path, 'utf-8')) as Source;
const mixin = {} as Mixin;
const url = tryCreateUrl(config.url); const url = tryCreateUrl(config.url);
const version = meta.version;
config.url = url.origin; const host = url.host;
const hostname = url.hostname;
config.port = config.port ?? parseInt(process.env.PORT ?? '', 10); const scheme = url.protocol.replace(/:$/, '');
const wsScheme = scheme.replace('http', 'ws');
mixin.version = meta.version;
mixin.host = url.host;
mixin.hostname = url.hostname;
mixin.scheme = url.protocol.replace(/:$/, '');
mixin.wsScheme = mixin.scheme.replace('http', 'ws');
mixin.wsUrl = `${mixin.wsScheme}://${mixin.host}`;
mixin.apiUrl = `${mixin.scheme}://${mixin.host}/api`;
mixin.authUrl = `${mixin.scheme}://${mixin.host}/auth`;
mixin.driveUrl = `${mixin.scheme}://${mixin.host}/files`;
mixin.userAgent = `Misskey/${meta.version} (${config.url})`;
mixin.clientEntry = clientManifest['src/_boot_.ts'];
mixin.clientManifestExists = clientManifestExists;
const externalMediaProxy = config.mediaProxy ? const externalMediaProxy = config.mediaProxy ?
config.mediaProxy.endsWith('/') ? config.mediaProxy.substring(0, config.mediaProxy.length - 1) : config.mediaProxy config.mediaProxy.endsWith('/') ? config.mediaProxy.substring(0, config.mediaProxy.length - 1) : config.mediaProxy
: null; : null;
const internalMediaProxy = `${mixin.scheme}://${mixin.host}/proxy`; const internalMediaProxy = `${scheme}://${host}/proxy`;
mixin.mediaProxy = externalMediaProxy ?? internalMediaProxy; const redis = convertRedisOptions(config.redis, host);
mixin.externalMediaProxyEnabled = externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy;
mixin.videoThumbnailGenerator = config.videoThumbnailGenerator ? return {
version,
url: url.origin,
port: config.port ?? parseInt(process.env.PORT ?? '', 10),
socket: config.socket,
chmodSocket: config.chmodSocket,
disableHsts: config.disableHsts,
host,
hostname,
scheme,
wsScheme,
wsUrl: `${wsScheme}://${host}`,
apiUrl: `${scheme}://${host}/api`,
authUrl: `${scheme}://${host}/auth`,
driveUrl: `${scheme}://${host}/files`,
db: config.db,
dbReplications: config.dbReplications,
dbSlaves: config.dbSlaves,
meilisearch: config.meilisearch,
redis,
redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis,
redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis,
id: config.id,
proxy: config.proxy,
proxySmtp: config.proxySmtp,
proxyBypassHosts: config.proxyBypassHosts,
allowedPrivateNetworks: config.allowedPrivateNetworks,
maxFileSize: config.maxFileSize,
accesslog: config.accesslog,
clusterLimit: config.clusterLimit,
outgoingAddress: config.outgoingAddress,
outgoingAddressFamily: config.outgoingAddressFamily,
deliverJobConcurrency: config.deliverJobConcurrency,
inboxJobConcurrency: config.inboxJobConcurrency,
relashionshipJobConcurrency: config.relashionshipJobConcurrency,
deliverJobPerSec: config.deliverJobPerSec,
inboxJobPerSec: config.inboxJobPerSec,
relashionshipJobPerSec: config.relashionshipJobPerSec,
deliverJobMaxAttempts: config.deliverJobMaxAttempts,
inboxJobMaxAttempts: config.inboxJobMaxAttempts,
proxyRemoteFiles: config.proxyRemoteFiles,
signToActivityPubGet: config.signToActivityPubGet,
mediaProxy: externalMediaProxy ?? internalMediaProxy,
externalMediaProxyEnabled: externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy,
videoThumbnailGenerator: config.videoThumbnailGenerator ?
config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator
: null; : null,
userAgent: `Misskey/${version} (${config.url})`,
mixin.redis = convertRedisOptions(config.redis, mixin.host); clientEntry: clientManifest['src/_boot_.ts'],
mixin.redisForPubsub = config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, mixin.host) : mixin.redis; clientManifestExists: clientManifestExists,
mixin.redisForJobQueue = config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, mixin.host) : mixin.redis; perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000,
perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 300,
return Object.assign(config, mixin); };
} }
function tryCreateUrl(url: string) { function tryCreateUrl(url: string) {

View File

@ -334,7 +334,7 @@ export class NoteCreateService implements OnApplicationShutdown {
if (data.channel) { if (data.channel) {
this.redisClient.xadd( this.redisClient.xadd(
`channelTimeline:${data.channel.id}`, `channelTimeline:${data.channel.id}`,
'MAXLEN', '~', '1000', 'MAXLEN', '~', this.config.perChannelMaxNoteCacheCount.toString(),
'*', '*',
'note', note.id); 'note', note.id);
} }

View File

@ -17,12 +17,16 @@ import { PushNotificationService } from '@/core/PushNotificationService.js';
import { NotificationEntityService } from '@/core/entities/NotificationEntityService.js'; import { NotificationEntityService } from '@/core/entities/NotificationEntityService.js';
import { IdService } from '@/core/IdService.js'; import { IdService } from '@/core/IdService.js';
import { CacheService } from '@/core/CacheService.js'; import { CacheService } from '@/core/CacheService.js';
import { Config } from '@/config.js';
@Injectable() @Injectable()
export class NotificationService implements OnApplicationShutdown { export class NotificationService implements OnApplicationShutdown {
#shutdownController = new AbortController(); #shutdownController = new AbortController();
constructor( constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.redis) @Inject(DI.redis)
private redisClient: Redis.Redis, private redisClient: Redis.Redis,
@ -96,7 +100,7 @@ export class NotificationService implements OnApplicationShutdown {
const redisIdPromise = this.redisClient.xadd( const redisIdPromise = this.redisClient.xadd(
`notificationTimeline:${notifieeId}`, `notificationTimeline:${notifieeId}`,
'MAXLEN', '~', '300', 'MAXLEN', '~', this.config.perUserNotificationsMaxCount.toString(),
'*', '*',
'data', JSON.stringify(notification)); 'data', JSON.stringify(notification));

View File

@ -69,7 +69,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
//#region Construct query //#region Construct query
const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'),
ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
.andWhere('note.id > :minId', { minId: this.idService.genId(new Date(Date.now() - (1000 * 60 * 60 * 24 * 10))) }) // 10日前まで // パフォーマンス上の利点が無さそう?
//.andWhere('note.id > :minId', { minId: this.idService.genId(new Date(Date.now() - (1000 * 60 * 60 * 24 * 10))) }) // 10日前まで
.innerJoinAndSelect('note.user', 'user') .innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('note.renote', 'renote')