enhance(backend): migrate bull to bullmq (#10910)

* wip

* wip

* Update QueueService.ts

* wip

* refactor

* ✌️

* fix

* Update QueueStatsService.ts

* refactor

* Update ApNoteService.ts

* Update mock-resolver.ts

* refactor

* Update mock-resolver.ts
This commit is contained in:
syuilo 2023-05-29 11:54:49 +09:00 committed by GitHub
parent 7cbd852fe5
commit fd7b77c542
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
53 changed files with 532 additions and 490 deletions

View File

@ -38,6 +38,7 @@
- fix:ロールタイムラインにて全ての投稿が流れてしまう問題の修正 - fix:ロールタイムラインにて全ての投稿が流れてしまう問題の修正
### Server ### Server
- bullをbull-mqにアップグレードし、ジョブキューのパフォーマンスを改善
- Fix: お知らせの画像URLを空にできない問題を修正 - Fix: お知らせの画像URLを空にできない問題を修正
## 13.12.2 ## 13.12.2

View File

@ -76,7 +76,7 @@
"autwh": "0.1.0", "autwh": "0.1.0",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"blurhash": "2.0.5", "blurhash": "2.0.5",
"bull": "4.10.4", "bullmq": "3.14.1",
"cacheable-lookup": "6.1.0", "cacheable-lookup": "6.1.0",
"cbor": "8.1.0", "cbor": "8.1.0",
"chalk": "5.2.0", "chalk": "5.2.0",
@ -167,7 +167,6 @@
"@types/accepts": "1.3.5", "@types/accepts": "1.3.5",
"@types/archiver": "5.3.2", "@types/archiver": "5.3.2",
"@types/bcryptjs": "2.4.2", "@types/bcryptjs": "2.4.2",
"@types/bull": "4.10.0",
"@types/cbor": "6.0.0", "@types/cbor": "6.0.0",
"@types/color-convert": "2.0.0", "@types/color-convert": "2.0.0",
"@types/content-disposition": "0.5.5", "@types/content-disposition": "0.5.5",

View File

@ -190,6 +190,6 @@ function tryCreateUrl(url: string) {
try { try {
return new URL(url); return new URL(url);
} catch (e) { } catch (e) {
throw `url="${url}" is not a valid URL.`; throw new Error(`url="${url}" is not a valid URL.`);
} }
} }

View File

@ -30,7 +30,7 @@ export class CaptchaService {
}, { throwErrorWhenResponseNotOk: false }); }, { throwErrorWhenResponseNotOk: false });
if (!res.ok) { if (!res.ok) {
throw `${res.status}`; throw new Error(`${res.status}`);
} }
return await res.json() as CaptchaResponse; return await res.json() as CaptchaResponse;
@ -39,48 +39,48 @@ export class CaptchaService {
@bindThis @bindThis
public async verifyRecaptcha(secret: string, response: string | null | undefined): Promise<void> { public async verifyRecaptcha(secret: string, response: string | null | undefined): Promise<void> {
if (response == null) { if (response == null) {
throw 'recaptcha-failed: no response provided'; throw new Error('recaptcha-failed: no response provided');
} }
const result = await this.getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(err => { const result = await this.getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(err => {
throw `recaptcha-request-failed: ${err}`; throw new Error(`recaptcha-request-failed: ${err}`);
}); });
if (result.success !== true) { if (result.success !== true) {
const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : '';
throw `recaptcha-failed: ${errorCodes}`; throw new Error(`recaptcha-failed: ${errorCodes}`);
} }
} }
@bindThis @bindThis
public async verifyHcaptcha(secret: string, response: string | null | undefined): Promise<void> { public async verifyHcaptcha(secret: string, response: string | null | undefined): Promise<void> {
if (response == null) { if (response == null) {
throw 'hcaptcha-failed: no response provided'; throw new Error('hcaptcha-failed: no response provided');
} }
const result = await this.getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(err => { const result = await this.getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(err => {
throw `hcaptcha-request-failed: ${err}`; throw new Error(`hcaptcha-request-failed: ${err}`);
}); });
if (result.success !== true) { if (result.success !== true) {
const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : '';
throw `hcaptcha-failed: ${errorCodes}`; throw new Error(`hcaptcha-failed: ${errorCodes}`);
} }
} }
@bindThis @bindThis
public async verifyTurnstile(secret: string, response: string | null | undefined): Promise<void> { public async verifyTurnstile(secret: string, response: string | null | undefined): Promise<void> {
if (response == null) { if (response == null) {
throw 'turnstile-failed: no response provided'; throw new Error('turnstile-failed: no response provided');
} }
const result = await this.getCaptchaResponse('https://challenges.cloudflare.com/turnstile/v0/siteverify', secret, response).catch(err => { const result = await this.getCaptchaResponse('https://challenges.cloudflare.com/turnstile/v0/siteverify', secret, response).catch(err => {
throw `turnstile-request-failed: ${err}`; throw new Error(`turnstile-request-failed: ${err}`);
}); });
if (result.success !== true) { if (result.success !== true) {
const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : '';
throw `turnstile-failed: ${errorCodes}`; throw new Error(`turnstile-failed: ${errorCodes}`);
} }
} }
} }

View File

@ -116,14 +116,14 @@ export class FetchInstanceMetadataService {
const wellknown = await this.httpRequestService.getJson('https://' + instance.host + '/.well-known/nodeinfo') const wellknown = await this.httpRequestService.getJson('https://' + instance.host + '/.well-known/nodeinfo')
.catch(err => { .catch(err => {
if (err.statusCode === 404) { if (err.statusCode === 404) {
throw 'No nodeinfo provided'; throw new Error('No nodeinfo provided');
} else { } else {
throw err.statusCode ?? err.message; throw err.statusCode ?? err.message;
} }
}) as Record<string, unknown>; }) as Record<string, unknown>;
if (wellknown.links == null || !Array.isArray(wellknown.links)) { if (wellknown.links == null || !Array.isArray(wellknown.links)) {
throw 'No wellknown links'; throw new Error('No wellknown links');
} }
const links = wellknown.links as any[]; const links = wellknown.links as any[];
@ -134,7 +134,7 @@ export class FetchInstanceMetadataService {
const link = lnik2_1 ?? lnik2_0 ?? lnik1_0; const link = lnik2_1 ?? lnik2_0 ?? lnik1_0;
if (link == null) { if (link == null) {
throw 'No nodeinfo link provided'; throw new Error('No nodeinfo link provided');
} }
const info = await this.httpRequestService.getJson(link.href) const info = await this.httpRequestService.getJson(link.href)

View File

@ -510,7 +510,7 @@ export class NoteCreateService implements OnApplicationShutdown {
if (data.poll && data.poll.expiresAt) { if (data.poll && data.poll.expiresAt) {
const delay = data.poll.expiresAt.getTime() - Date.now(); const delay = data.poll.expiresAt.getTime() - Date.now();
this.queueService.endedPollNotificationQueue.add({ this.queueService.endedPollNotificationQueue.add(note.id, {
noteId: note.id, noteId: note.id,
}, { }, {
delay, delay,

View File

@ -1,42 +1,11 @@
import { setTimeout } from 'node:timers/promises'; import { setTimeout } from 'node:timers/promises';
import { Inject, Module, OnApplicationShutdown } from '@nestjs/common'; import { Inject, Module, OnApplicationShutdown } from '@nestjs/common';
import Bull from 'bull'; import * as Bull from 'bullmq';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import { QUEUE, baseQueueOptions } from '@/queue/const.js';
import type { Provider } from '@nestjs/common'; import type { Provider } from '@nestjs/common';
import type { DeliverJobData, InboxJobData, DbJobData, ObjectStorageJobData, EndedPollNotificationJobData, WebhookDeliverJobData, RelationshipJobData, DbJobMap } from '../queue/types.js'; import type { DeliverJobData, InboxJobData, EndedPollNotificationJobData, WebhookDeliverJobData, RelationshipJobData } from '../queue/types.js';
function q<T>(config: Config, name: string, limitPerSec = -1) {
return new Bull<T>(name, {
redis: {
port: config.redisForJobQueue.port,
host: config.redisForJobQueue.host,
family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family,
password: config.redisForJobQueue.pass,
db: config.redisForJobQueue.db ?? 0,
},
prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue` : 'queue',
limiter: limitPerSec > 0 ? {
max: limitPerSec,
duration: 1000,
} : undefined,
settings: {
backoffStrategies: {
apBackoff,
},
},
});
}
// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019
function apBackoff(attemptsMade: number, err: Error) {
const baseDelay = 60 * 1000; // 1min
const maxBackoff = 8 * 60 * 60 * 1000; // 8hours
let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay;
backoff = Math.min(backoff, maxBackoff);
backoff += Math.round(backoff * Math.random() * 0.2);
return backoff;
}
export type SystemQueue = Bull.Queue<Record<string, unknown>>; export type SystemQueue = Bull.Queue<Record<string, unknown>>;
export type EndedPollNotificationQueue = Bull.Queue<EndedPollNotificationJobData>; export type EndedPollNotificationQueue = Bull.Queue<EndedPollNotificationJobData>;
@ -49,49 +18,49 @@ export type WebhookDeliverQueue = Bull.Queue<WebhookDeliverJobData>;
const $system: Provider = { const $system: Provider = {
provide: 'queue:system', provide: 'queue:system',
useFactory: (config: Config) => q(config, 'system'), useFactory: (config: Config) => new Bull.Queue(QUEUE.SYSTEM, baseQueueOptions(config, QUEUE.SYSTEM)),
inject: [DI.config], inject: [DI.config],
}; };
const $endedPollNotification: Provider = { const $endedPollNotification: Provider = {
provide: 'queue:endedPollNotification', provide: 'queue:endedPollNotification',
useFactory: (config: Config) => q(config, 'endedPollNotification'), useFactory: (config: Config) => new Bull.Queue(QUEUE.ENDED_POLL_NOTIFICATION, baseQueueOptions(config, QUEUE.ENDED_POLL_NOTIFICATION)),
inject: [DI.config], inject: [DI.config],
}; };
const $deliver: Provider = { const $deliver: Provider = {
provide: 'queue:deliver', provide: 'queue:deliver',
useFactory: (config: Config) => q(config, 'deliver', config.deliverJobPerSec ?? 128), useFactory: (config: Config) => new Bull.Queue(QUEUE.DELIVER, baseQueueOptions(config, QUEUE.DELIVER)),
inject: [DI.config], inject: [DI.config],
}; };
const $inbox: Provider = { const $inbox: Provider = {
provide: 'queue:inbox', provide: 'queue:inbox',
useFactory: (config: Config) => q(config, 'inbox', config.inboxJobPerSec ?? 16), useFactory: (config: Config) => new Bull.Queue(QUEUE.INBOX, baseQueueOptions(config, QUEUE.INBOX)),
inject: [DI.config], inject: [DI.config],
}; };
const $db: Provider = { const $db: Provider = {
provide: 'queue:db', provide: 'queue:db',
useFactory: (config: Config) => q(config, 'db'), useFactory: (config: Config) => new Bull.Queue(QUEUE.DB, baseQueueOptions(config, QUEUE.DB)),
inject: [DI.config], inject: [DI.config],
}; };
const $relationship: Provider = { const $relationship: Provider = {
provide: 'queue:relationship', provide: 'queue:relationship',
useFactory: (config: Config) => q(config, 'relationship', config.relashionshipJobPerSec ?? 64), useFactory: (config: Config) => new Bull.Queue(QUEUE.RELATIONSHIP, baseQueueOptions(config, QUEUE.RELATIONSHIP)),
inject: [DI.config], inject: [DI.config],
}; };
const $objectStorage: Provider = { const $objectStorage: Provider = {
provide: 'queue:objectStorage', provide: 'queue:objectStorage',
useFactory: (config: Config) => q(config, 'objectStorage'), useFactory: (config: Config) => new Bull.Queue(QUEUE.OBJECT_STORAGE, baseQueueOptions(config, QUEUE.OBJECT_STORAGE)),
inject: [DI.config], inject: [DI.config],
}; };
const $webhookDeliver: Provider = { const $webhookDeliver: Provider = {
provide: 'queue:webhookDeliver', provide: 'queue:webhookDeliver',
useFactory: (config: Config) => q(config, 'webhookDeliver', 64), useFactory: (config: Config) => new Bull.Queue(QUEUE.WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.WEBHOOK_DELIVER)),
inject: [DI.config], inject: [DI.config],
}; };

View File

@ -1,6 +1,5 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { v4 as uuid } from 'uuid'; import { v4 as uuid } from 'uuid';
import Bull from 'bull';
import type { IActivity } from '@/core/activitypub/type.js'; import type { IActivity } from '@/core/activitypub/type.js';
import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { DriveFile } from '@/models/entities/DriveFile.js';
import type { Webhook, webhookEventTypes } from '@/models/entities/Webhook.js'; import type { Webhook, webhookEventTypes } from '@/models/entities/Webhook.js';
@ -11,6 +10,7 @@ import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js';
import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js';
import type { DbJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type { DbJobData, RelationshipJobData, ThinUser } from '../queue/types.js';
import type httpSignature from '@peertube/http-signature'; import type httpSignature from '@peertube/http-signature';
import type * as Bull from 'bullmq';
@Injectable() @Injectable()
export class QueueService { export class QueueService {
@ -26,7 +26,43 @@ export class QueueService {
@Inject('queue:relationship') public relationshipQueue: RelationshipQueue, @Inject('queue:relationship') public relationshipQueue: RelationshipQueue,
@Inject('queue:objectStorage') public objectStorageQueue: ObjectStorageQueue, @Inject('queue:objectStorage') public objectStorageQueue: ObjectStorageQueue,
@Inject('queue:webhookDeliver') public webhookDeliverQueue: WebhookDeliverQueue, @Inject('queue:webhookDeliver') public webhookDeliverQueue: WebhookDeliverQueue,
) {} ) {
this.systemQueue.add('tickCharts', {
}, {
repeat: { pattern: '55 * * * *' },
removeOnComplete: true,
});
this.systemQueue.add('resyncCharts', {
}, {
repeat: { pattern: '0 0 * * *' },
removeOnComplete: true,
});
this.systemQueue.add('cleanCharts', {
}, {
repeat: { pattern: '0 0 * * *' },
removeOnComplete: true,
});
this.systemQueue.add('aggregateRetention', {
}, {
repeat: { pattern: '0 0 * * *' },
removeOnComplete: true,
});
this.systemQueue.add('clean', {
}, {
repeat: { pattern: '0 0 * * *' },
removeOnComplete: true,
});
this.systemQueue.add('checkExpiredMutings', {
}, {
repeat: { pattern: '*/5 * * * *' },
removeOnComplete: true,
});
}
@bindThis @bindThis
public deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean) { public deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean) {
@ -42,11 +78,10 @@ export class QueueService {
isSharedInbox, isSharedInbox,
}; };
return this.deliverQueue.add(data, { return this.deliverQueue.add(to, data, {
attempts: this.config.deliverJobMaxAttempts ?? 12, attempts: this.config.deliverJobMaxAttempts ?? 12,
timeout: 1 * 60 * 1000, // 1min
backoff: { backoff: {
type: 'apBackoff', type: 'custom',
}, },
removeOnComplete: true, removeOnComplete: true,
removeOnFail: true, removeOnFail: true,
@ -60,11 +95,10 @@ export class QueueService {
signature, signature,
}; };
return this.inboxQueue.add(data, { return this.inboxQueue.add('', data, {
attempts: this.config.inboxJobMaxAttempts ?? 8, attempts: this.config.inboxJobMaxAttempts ?? 8,
timeout: 5 * 60 * 1000, // 5min
backoff: { backoff: {
type: 'apBackoff', type: 'custom',
}, },
removeOnComplete: true, removeOnComplete: true,
removeOnFail: true, removeOnFail: true,
@ -212,7 +246,7 @@ export class QueueService {
private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb', D extends DbJobData<T>>(name: T, data: D): { private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb', D extends DbJobData<T>>(name: T, data: D): {
name: string, name: string,
data: D, data: D,
opts: Bull.JobOptions, opts: Bull.JobsOptions,
} { } {
return { return {
name, name,
@ -299,10 +333,10 @@ export class QueueService {
} }
@bindThis @bindThis
private generateRelationshipJobData(name: 'follow' | 'unfollow' | 'block' | 'unblock', data: RelationshipJobData, opts: Bull.JobOptions = {}): { private generateRelationshipJobData(name: 'follow' | 'unfollow' | 'block' | 'unblock', data: RelationshipJobData, opts: Bull.JobsOptions = {}): {
name: string, name: string,
data: RelationshipJobData, data: RelationshipJobData,
opts: Bull.JobOptions, opts: Bull.JobsOptions,
} { } {
return { return {
name, name,
@ -351,11 +385,10 @@ export class QueueService {
eventId: uuid(), eventId: uuid(),
}; };
return this.webhookDeliverQueue.add(data, { return this.webhookDeliverQueue.add(webhook.id, data, {
attempts: 4, attempts: 4,
timeout: 1 * 60 * 1000, // 1min
backoff: { backoff: {
type: 'apBackoff', type: 'custom',
}, },
removeOnComplete: true, removeOnComplete: true,
removeOnFail: true, removeOnFail: true,
@ -367,11 +400,11 @@ export class QueueService {
this.deliverQueue.once('cleaned', (jobs, status) => { this.deliverQueue.once('cleaned', (jobs, status) => {
//deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); //deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`);
}); });
this.deliverQueue.clean(0, 'delayed'); this.deliverQueue.clean(0, Infinity, 'delayed');
this.inboxQueue.once('cleaned', (jobs, status) => { this.inboxQueue.once('cleaned', (jobs, status) => {
//inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); //inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`);
}); });
this.inboxQueue.clean(0, 'delayed'); this.inboxQueue.clean(0, Infinity, 'delayed');
} }
} }

View File

@ -94,7 +94,7 @@ class LdSignature {
@bindThis @bindThis
private getLoader() { private getLoader() {
return async (url: string): Promise<any> => { return async (url: string): Promise<any> => {
if (!url.match('^https?\:\/\/')) throw `Invalid URL ${url}`; if (!url.match('^https?\:\/\/')) throw new Error(`Invalid URL ${url}`);
if (this.preLoad) { if (this.preLoad) {
if (url in CONTEXTS) { if (url in CONTEXTS) {
@ -126,7 +126,7 @@ class LdSignature {
timeout: this.loderTimeout, timeout: this.loderTimeout,
}, { throwErrorWhenResponseNotOk: false }).then(res => { }, { throwErrorWhenResponseNotOk: false }).then(res => {
if (!res.ok) { if (!res.ok) {
throw `${res.status} ${res.statusText}`; throw new Error(`${res.status} ${res.statusText}`);
} else { } else {
return res.json(); return res.json();
} }

View File

@ -18,6 +18,7 @@ import { PollService } from '@/core/PollService.js';
import { StatusError } from '@/misc/status-error.js'; import { StatusError } from '@/misc/status-error.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { checkHttps } from '@/misc/check-https.js';
import { getOneApId, getApId, getOneApHrefNullable, validPost, isEmoji, getApType } from '../type.js'; import { getOneApId, getApId, getOneApHrefNullable, validPost, isEmoji, getApType } from '../type.js';
// eslint-disable-next-line @typescript-eslint/consistent-type-imports // eslint-disable-next-line @typescript-eslint/consistent-type-imports
import { ApLoggerService } from '../ApLoggerService.js'; import { ApLoggerService } from '../ApLoggerService.js';
@ -32,7 +33,6 @@ import { ApQuestionService } from './ApQuestionService.js';
import { ApImageService } from './ApImageService.js'; import { ApImageService } from './ApImageService.js';
import type { Resolver } from '../ApResolverService.js'; import type { Resolver } from '../ApResolverService.js';
import type { IObject, IPost } from '../type.js'; import type { IObject, IPost } from '../type.js';
import { checkHttps } from '@/misc/check-https.js';
@Injectable() @Injectable()
export class ApNoteService { export class ApNoteService {
@ -230,7 +230,7 @@ export class ApNoteService {
quote = results.filter((x): x is { status: 'ok', res: Note | null } => x.status === 'ok').map(x => x.res).find(x => x); quote = results.filter((x): x is { status: 'ok', res: Note | null } => x.status === 'ok').map(x => x.res).find(x => x);
if (!quote) { if (!quote) {
if (results.some(x => x.status === 'temperror')) { if (results.some(x => x.status === 'temperror')) {
throw 'quote resolve failed'; throw new Error('quote resolve failed');
} }
} }
} }
@ -311,7 +311,7 @@ export class ApNoteService {
// ブロックしてたら中断 // ブロックしてたら中断
const meta = await this.metaService.fetch(); const meta = await this.metaService.fetch();
if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.extractDbHost(uri))) throw { statusCode: 451 }; if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.extractDbHost(uri))) throw new StatusError('blocked host', 451);
const unlock = await this.appLockService.getApLock(uri); const unlock = await this.appLockService.getApLock(uri);

View File

@ -1,7 +1,11 @@
import { Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import Xev from 'xev'; import Xev from 'xev';
import * as Bull from 'bullmq';
import { QueueService } from '@/core/QueueService.js'; import { QueueService } from '@/core/QueueService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { QUEUE, baseQueueOptions } from '@/queue/const.js';
import type { OnApplicationShutdown } from '@nestjs/common'; import type { OnApplicationShutdown } from '@nestjs/common';
const ev = new Xev(); const ev = new Xev();
@ -13,6 +17,9 @@ export class QueueStatsService implements OnApplicationShutdown {
private intervalId: NodeJS.Timer; private intervalId: NodeJS.Timer;
constructor( constructor(
@Inject(DI.config)
private config: Config,
private queueService: QueueService, private queueService: QueueService,
) { ) {
} }
@ -31,11 +38,14 @@ export class QueueStatsService implements OnApplicationShutdown {
let activeDeliverJobs = 0; let activeDeliverJobs = 0;
let activeInboxJobs = 0; let activeInboxJobs = 0;
this.queueService.deliverQueue.on('global:active', () => { const deliverQueueEvents = new Bull.QueueEvents(QUEUE.DELIVER, baseQueueOptions(this.config, QUEUE.DELIVER));
const inboxQueueEvents = new Bull.QueueEvents(QUEUE.INBOX, baseQueueOptions(this.config, QUEUE.INBOX));
deliverQueueEvents.on('active', () => {
activeDeliverJobs++; activeDeliverJobs++;
}); });
this.queueService.inboxQueue.on('global:active', () => { inboxQueueEvents.on('active', () => {
activeInboxJobs++; activeInboxJobs++;
}); });

View File

@ -21,7 +21,7 @@ function getNoise(): string {
export function genAid(date: Date): string { export function genAid(date: Date): string {
const t = date.getTime(); const t = date.getTime();
if (isNaN(t)) throw 'Failed to create AID: Invalid Date'; if (isNaN(t)) throw new Error('Failed to create AID: Invalid Date');
counter++; counter++;
return getTime(t) + getNoise(); return getTime(t) + getNoise();
} }

View File

@ -5,15 +5,16 @@ const dateTimeIntervals = {
}; };
export function dateUTC(time: number[]): Date { export function dateUTC(time: number[]): Date {
const d = time.length === 2 ? Date.UTC(time[0], time[1]) const d =
: time.length === 3 ? Date.UTC(time[0], time[1], time[2]) time.length === 2 ? Date.UTC(time[0], time[1])
: time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) : time.length === 3 ? Date.UTC(time[0], time[1], time[2])
: time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3])
: time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4])
: time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5])
: null; : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6])
: null;
if (!d) throw 'wrong number of arguments'; if (!d) throw new Error('wrong number of arguments');
return new Date(d); return new Date(d);
} }

View File

@ -1,10 +1,9 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import * as Bull from 'bullmq';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
import { QueueService } from '@/core/QueueService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { getJobInfo } from './get-job-info.js';
import { WebhookDeliverProcessorService } from './processors/WebhookDeliverProcessorService.js'; import { WebhookDeliverProcessorService } from './processors/WebhookDeliverProcessorService.js';
import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js'; import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js';
import { DeliverProcessorService } from './processors/DeliverProcessorService.js'; import { DeliverProcessorService } from './processors/DeliverProcessorService.js';
@ -35,6 +34,33 @@ import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMu
import { CleanProcessorService } from './processors/CleanProcessorService.js'; import { CleanProcessorService } from './processors/CleanProcessorService.js';
import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js';
import { QueueLoggerService } from './QueueLoggerService.js'; import { QueueLoggerService } from './QueueLoggerService.js';
import { QUEUE, baseQueueOptions } from './const.js';
// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019
function httpRelatedBackoff(attemptsMade: number) {
const baseDelay = 60 * 1000; // 1min
const maxBackoff = 8 * 60 * 60 * 1000; // 8hours
let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay;
backoff = Math.min(backoff, maxBackoff);
backoff += Math.round(backoff * Math.random() * 0.2);
return backoff;
}
function getJobInfo(job: Bull.Job | undefined, increment = false): string {
if (job == null) return '-';
const age = Date.now() - job.timestamp;
const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m`
: age > 10000 ? `${Math.floor(age / 1000)}s`
: `${age}ms`;
// onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする
const currentAttempts = job.attemptsMade + (increment ? 1 : 0);
const maxAttempts = job.opts ? job.opts.attempts : 0;
return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`;
}
@Injectable() @Injectable()
export class QueueProcessorService { export class QueueProcessorService {
@ -45,7 +71,6 @@ export class QueueProcessorService {
private config: Config, private config: Config,
private queueLoggerService: QueueLoggerService, private queueLoggerService: QueueLoggerService,
private queueService: QueueService,
private webhookDeliverProcessorService: WebhookDeliverProcessorService, private webhookDeliverProcessorService: WebhookDeliverProcessorService,
private endedPollNotificationProcessorService: EndedPollNotificationProcessorService, private endedPollNotificationProcessorService: EndedPollNotificationProcessorService,
private deliverProcessorService: DeliverProcessorService, private deliverProcessorService: DeliverProcessorService,
@ -97,146 +122,191 @@ export class QueueProcessorService {
} }
} }
const systemLogger = this.logger.createSubLogger('system'); //#region system
const deliverLogger = this.logger.createSubLogger('deliver'); const systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => {
const webhookLogger = this.logger.createSubLogger('webhook'); switch (job.name) {
const inboxLogger = this.logger.createSubLogger('inbox'); case 'tickCharts': return this.tickChartsProcessorService.process();
const dbLogger = this.logger.createSubLogger('db'); case 'resyncCharts': return this.resyncChartsProcessorService.process();
const relationshipLogger = this.logger.createSubLogger('relationship'); case 'cleanCharts': return this.cleanChartsProcessorService.process();
const objectStorageLogger = this.logger.createSubLogger('objectStorage'); case 'aggregateRetention': return this.aggregateRetentionProcessorService.process();
case 'checkExpiredMutings': return this.checkExpiredMutingsProcessorService.process();
case 'clean': return this.cleanProcessorService.process();
default: throw new Error(`unrecognized job type ${job.name} for system`);
}
}, {
...baseQueueOptions(this.config, QUEUE.SYSTEM),
});
this.queueService.systemQueue const systemLogger = this.logger.createSubLogger('system');
.on('waiting', (jobId) => systemLogger.debug(`waiting id=${jobId}`))
systemQueueWorker
.on('active', (job) => systemLogger.debug(`active id=${job.id}`)) .on('active', (job) => systemLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) .on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
.on('error', (job: any, err: Error) => systemLogger.error(`error ${err}`, { job, e: renderError(err) })) .on('error', (err: Error) => systemLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (job) => systemLogger.warn(`stalled id=${job.id}`)); .on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`));
//#endregion
this.queueService.deliverQueue //#region db
.on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`)) const dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => {
.on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) switch (job.name) {
.on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job);
.on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job);
.on('error', (job: any, err: Error) => deliverLogger.error(`error ${err}`, { job, e: renderError(err) })) case 'exportNotes': return this.exportNotesProcessorService.process(job);
.on('stalled', (job) => deliverLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); case 'exportFavorites': return this.exportFavoritesProcessorService.process(job);
case 'exportFollowing': return this.exportFollowingProcessorService.process(job);
case 'exportMuting': return this.exportMutingProcessorService.process(job);
case 'exportBlocking': return this.exportBlockingProcessorService.process(job);
case 'exportUserLists': return this.exportUserListsProcessorService.process(job);
case 'exportAntennas': return this.exportAntennasProcessorService.process(job);
case 'importFollowing': return this.importFollowingProcessorService.process(job);
case 'importFollowingToDb': return this.importFollowingProcessorService.processDb(job);
case 'importMuting': return this.importMutingProcessorService.process(job);
case 'importBlocking': return this.importBlockingProcessorService.process(job);
case 'importBlockingToDb': return this.importBlockingProcessorService.processDb(job);
case 'importUserLists': return this.importUserListsProcessorService.process(job);
case 'importCustomEmojis': return this.importCustomEmojisProcessorService.process(job);
case 'importAntennas': return this.importAntennasProcessorService.process(job);
case 'deleteAccount': return this.deleteAccountProcessorService.process(job);
default: throw new Error(`unrecognized job type ${job.name} for db`);
}
}, {
...baseQueueOptions(this.config, QUEUE.DB),
});
this.queueService.inboxQueue const dbLogger = this.logger.createSubLogger('db');
.on('waiting', (jobId) => inboxLogger.debug(`waiting id=${jobId}`))
.on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`))
.on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`))
.on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`, { job, e: renderError(err) }))
.on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) }))
.on('stalled', (job) => inboxLogger.warn(`stalled ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`));
this.queueService.dbQueue dbQueueWorker
.on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`))
.on('active', (job) => dbLogger.debug(`active id=${job.id}`)) .on('active', (job) => dbLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
.on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) })) .on('error', (err: Error) => dbLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`)); .on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`));
//#endregion
this.queueService.relationshipQueue //#region deliver
.on('waiting', (jobId) => relationshipLogger.debug(`waiting id=${jobId}`)) const deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => this.deliverProcessorService.process(job), {
.on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) ...baseQueueOptions(this.config, QUEUE.DELIVER),
.on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) concurrency: this.config.deliverJobConcurrency ?? 128,
.on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) limiter: {
.on('error', (job: any, err: Error) => relationshipLogger.error(`error ${err}`, { job, e: renderError(err) })) max: this.config.deliverJobPerSec ?? 128,
.on('stalled', (job) => relationshipLogger.warn(`stalled id=${job.id}`)); duration: 1000,
},
settings: {
backoffStrategy: httpRelatedBackoff,
},
});
this.queueService.objectStorageQueue const deliverLogger = this.logger.createSubLogger('deliver');
.on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`))
.on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) }))
.on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) }))
.on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`));
this.queueService.webhookDeliverQueue deliverQueueWorker
.on('waiting', (jobId) => webhookLogger.debug(`waiting id=${jobId}`)) .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`))
.on('error', (err: Error) => deliverLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (jobId) => deliverLogger.warn(`stalled id=${jobId}`));
//#endregion
//#region inbox
const inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => this.inboxProcessorService.process(job), {
...baseQueueOptions(this.config, QUEUE.INBOX),
concurrency: this.config.inboxJobConcurrency ?? 16,
limiter: {
max: this.config.inboxJobPerSec ?? 16,
duration: 1000,
},
settings: {
backoffStrategy: httpRelatedBackoff,
},
});
const inboxLogger = this.logger.createSubLogger('inbox');
inboxQueueWorker
.on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`))
.on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`))
.on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) }))
.on('error', (err: Error) => inboxLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (jobId) => inboxLogger.warn(`stalled id=${jobId}`));
//#endregion
//#region webhook deliver
const webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => this.webhookDeliverProcessorService.process(job), {
...baseQueueOptions(this.config, QUEUE.WEBHOOK_DELIVER),
concurrency: 64,
limiter: {
max: 64,
duration: 1000,
},
settings: {
backoffStrategy: httpRelatedBackoff,
},
});
const webhookLogger = this.logger.createSubLogger('webhook');
webhookDeliverQueueWorker
.on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) .on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`))
.on('error', (job: any, err: Error) => webhookLogger.error(`error ${err}`, { job, e: renderError(err) })) .on('error', (err: Error) => webhookLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (job) => webhookLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); .on('stalled', (jobId) => webhookLogger.warn(`stalled id=${jobId}`));
//#endregion
this.queueService.systemQueue.add('tickCharts', { //#region relationship
const relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => {
switch (job.name) {
case 'follow': return this.relationshipProcessorService.processFollow(job);
case 'unfollow': return this.relationshipProcessorService.processUnfollow(job);
case 'block': return this.relationshipProcessorService.processBlock(job);
case 'unblock': return this.relationshipProcessorService.processUnblock(job);
default: throw new Error(`unrecognized job type ${job.name} for relationship`);
}
}, { }, {
repeat: { cron: '55 * * * *' }, ...baseQueueOptions(this.config, QUEUE.RELATIONSHIP),
removeOnComplete: true, concurrency: this.config.relashionshipJobConcurrency ?? 16,
limiter: {
max: this.config.relashionshipJobPerSec ?? 64,
duration: 1000,
},
}); });
this.queueService.systemQueue.add('resyncCharts', { const relationshipLogger = this.logger.createSubLogger('relationship');
}, {
repeat: { cron: '0 0 * * *' },
removeOnComplete: true,
});
this.queueService.systemQueue.add('cleanCharts', {
}, {
repeat: { cron: '0 0 * * *' },
removeOnComplete: true,
});
this.queueService.systemQueue.add('aggregateRetention', {
}, {
repeat: { cron: '0 0 * * *' },
removeOnComplete: true,
});
this.queueService.systemQueue.add('clean', {
}, {
repeat: { cron: '0 0 * * *' },
removeOnComplete: true,
});
this.queueService.systemQueue.add('checkExpiredMutings', {
}, {
repeat: { cron: '*/5 * * * *' },
removeOnComplete: true,
});
this.queueService.deliverQueue.process(this.config.deliverJobConcurrency ?? 128, (job) => this.deliverProcessorService.process(job));
this.queueService.inboxQueue.process(this.config.inboxJobConcurrency ?? 16, (job) => this.inboxProcessorService.process(job));
this.queueService.endedPollNotificationQueue.process((job, done) => this.endedPollNotificationProcessorService.process(job, done));
this.queueService.webhookDeliverQueue.process(64, (job) => this.webhookDeliverProcessorService.process(job));
this.queueService.dbQueue.process('deleteDriveFiles', (job, done) => this.deleteDriveFilesProcessorService.process(job, done));
this.queueService.dbQueue.process('exportCustomEmojis', (job, done) => this.exportCustomEmojisProcessorService.process(job, done));
this.queueService.dbQueue.process('exportNotes', (job, done) => this.exportNotesProcessorService.process(job, done));
this.queueService.dbQueue.process('exportFavorites', (job, done) => this.exportFavoritesProcessorService.process(job, done));
this.queueService.dbQueue.process('exportFollowing', (job, done) => this.exportFollowingProcessorService.process(job, done));
this.queueService.dbQueue.process('exportMuting', (job, done) => this.exportMutingProcessorService.process(job, done));
this.queueService.dbQueue.process('exportBlocking', (job, done) => this.exportBlockingProcessorService.process(job, done));
this.queueService.dbQueue.process('exportUserLists', (job, done) => this.exportUserListsProcessorService.process(job, done));
this.queueService.dbQueue.process('exportAntennas', (job, done) => this.exportAntennasProcessorService.process(job, done));
this.queueService.dbQueue.process('importFollowing', (job, done) => this.importFollowingProcessorService.process(job, done));
this.queueService.dbQueue.process('importFollowingToDb', (job) => this.importFollowingProcessorService.processDb(job));
this.queueService.dbQueue.process('importMuting', (job, done) => this.importMutingProcessorService.process(job, done));
this.queueService.dbQueue.process('importBlocking', (job, done) => this.importBlockingProcessorService.process(job, done));
this.queueService.dbQueue.process('importBlockingToDb', (job) => this.importBlockingProcessorService.processDb(job));
this.queueService.dbQueue.process('importUserLists', (job, done) => this.importUserListsProcessorService.process(job, done));
this.queueService.dbQueue.process('importCustomEmojis', (job, done) => this.importCustomEmojisProcessorService.process(job, done));
this.queueService.dbQueue.process('importAntennas', (job, done) => this.importAntennasProcessorService.process(job, done));
this.queueService.dbQueue.process('deleteAccount', (job) => this.deleteAccountProcessorService.process(job));
this.queueService.objectStorageQueue.process('deleteFile', 16, (job) => this.deleteFileProcessorService.process(job));
this.queueService.objectStorageQueue.process('cleanRemoteFiles', 16, (job, done) => this.cleanRemoteFilesProcessorService.process(job, done));
{ relationshipQueueWorker
const maxJobs = this.config.relashionshipJobConcurrency ?? 16; .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`))
this.queueService.relationshipQueue.process('follow', maxJobs, (job) => this.relationshipProcessorService.processFollow(job)); .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`))
this.queueService.relationshipQueue.process('unfollow', maxJobs, (job) => this.relationshipProcessorService.processUnfollow(job)); .on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
this.queueService.relationshipQueue.process('block', maxJobs, (job) => this.relationshipProcessorService.processBlock(job)); .on('error', (err: Error) => relationshipLogger.error(`error ${err}`, { e: renderError(err) }))
this.queueService.relationshipQueue.process('unblock', maxJobs, (job) => this.relationshipProcessorService.processUnblock(job)); .on('stalled', (jobId) => relationshipLogger.warn(`stalled id=${jobId}`));
} //#endregion
this.queueService.systemQueue.process('tickCharts', (job, done) => this.tickChartsProcessorService.process(job, done)); //#region object storage
this.queueService.systemQueue.process('resyncCharts', (job, done) => this.resyncChartsProcessorService.process(job, done)); const objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => {
this.queueService.systemQueue.process('cleanCharts', (job, done) => this.cleanChartsProcessorService.process(job, done)); switch (job.name) {
this.queueService.systemQueue.process('aggregateRetention', (job, done) => this.aggregateRetentionProcessorService.process(job, done)); case 'deleteFile': return this.deleteFileProcessorService.process(job);
this.queueService.systemQueue.process('checkExpiredMutings', (job, done) => this.checkExpiredMutingsProcessorService.process(job, done)); case 'cleanRemoteFiles': return this.cleanRemoteFilesProcessorService.process(job);
this.queueService.systemQueue.process('clean', (job, done) => this.cleanProcessorService.process(job, done)); default: throw new Error(`unrecognized job type ${job.name} for objectStorage`);
}
}, {
...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE),
concurrency: 16,
});
const objectStorageLogger = this.logger.createSubLogger('objectStorage');
objectStorageQueueWorker
.on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
.on('error', (err: Error) => objectStorageLogger.error(`error ${err}`, { e: renderError(err) }))
.on('stalled', (jobId) => objectStorageLogger.warn(`stalled id=${jobId}`));
//#endregion
//#region ended poll notification
const endedPollNotificationWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => this.endedPollNotificationProcessorService.process(job), {
...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION),
});
//#endregion
} }
} }

View File

@ -0,0 +1,26 @@
import { Config } from '@/config.js';
import type * as Bull from 'bullmq';
export const QUEUE = {
DELIVER: 'deliver',
INBOX: 'inbox',
SYSTEM: 'system',
ENDED_POLL_NOTIFICATION: 'endedPollNotification',
DB: 'db',
RELATIONSHIP: 'relationship',
OBJECT_STORAGE: 'objectStorage',
WEBHOOK_DELIVER: 'webhookDeliver',
};
export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof typeof QUEUE]): Bull.QueueOptions {
return {
connection: {
port: config.redisForJobQueue.port,
host: config.redisForJobQueue.host,
family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family,
password: config.redisForJobQueue.pass,
db: config.redisForJobQueue.db ?? 0,
},
prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue:${queueName}` : `queue:${queueName}`,
};
}

View File

@ -1,15 +0,0 @@
import Bull from 'bull';
export function getJobInfo(job: Bull.Job, increment = false) {
const age = Date.now() - job.timestamp;
const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m`
: age > 10000 ? `${Math.floor(age / 1000)}s`
: `${age}ms`;
// onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする
const currentAttempts = job.attemptsMade + (increment ? 1 : 0);
const maxAttempts = job.opts ? job.opts.attempts : 0;
return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`;
}

View File

@ -9,7 +9,7 @@ import { deepClone } from '@/misc/clone.js';
import { IdService } from '@/core/IdService.js'; import { IdService } from '@/core/IdService.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
@Injectable() @Injectable()
export class AggregateRetentionProcessorService { export class AggregateRetentionProcessorService {
@ -32,7 +32,7 @@ export class AggregateRetentionProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> { public async process(): Promise<void> {
this.logger.info('Aggregating retention...'); this.logger.info('Aggregating retention...');
const now = new Date(); const now = new Date();
@ -62,7 +62,6 @@ export class AggregateRetentionProcessorService {
} catch (err) { } catch (err) {
if (isDuplicateKeyValueError(err)) { if (isDuplicateKeyValueError(err)) {
this.logger.succ('Skip because it has already been processed by another worker.'); this.logger.succ('Skip because it has already been processed by another worker.');
done();
return; return;
} }
throw err; throw err;
@ -88,6 +87,5 @@ export class AggregateRetentionProcessorService {
} }
this.logger.succ('Retention aggregated.'); this.logger.succ('Retention aggregated.');
done();
} }
} }

View File

@ -7,7 +7,7 @@ import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { UserMutingService } from '@/core/UserMutingService.js'; import { UserMutingService } from '@/core/UserMutingService.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
@Injectable() @Injectable()
export class CheckExpiredMutingsProcessorService { export class CheckExpiredMutingsProcessorService {
@ -27,7 +27,7 @@ export class CheckExpiredMutingsProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> { public async process(): Promise<void> {
this.logger.info('Checking expired mutings...'); this.logger.info('Checking expired mutings...');
const expired = await this.mutingsRepository.createQueryBuilder('muting') const expired = await this.mutingsRepository.createQueryBuilder('muting')
@ -41,6 +41,5 @@ export class CheckExpiredMutingsProcessorService {
} }
this.logger.succ('All expired mutings checked.'); this.logger.succ('All expired mutings checked.');
done();
} }
} }

View File

@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js';
import ApRequestChart from '@/core/chart/charts/ap-request.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
@Injectable() @Injectable()
export class CleanChartsProcessorService { export class CleanChartsProcessorService {
@ -45,7 +45,7 @@ export class CleanChartsProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> { public async process(): Promise<void> {
this.logger.info('Clean charts...'); this.logger.info('Clean charts...');
await Promise.all([ await Promise.all([
@ -64,6 +64,5 @@ export class CleanChartsProcessorService {
]); ]);
this.logger.succ('All charts successfully cleaned.'); this.logger.succ('All charts successfully cleaned.');
done();
} }
} }

View File

@ -7,7 +7,7 @@ import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js'; import { IdService } from '@/core/IdService.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
@Injectable() @Injectable()
export class CleanProcessorService { export class CleanProcessorService {
@ -36,7 +36,7 @@ export class CleanProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> { public async process(): Promise<void> {
this.logger.info('Cleaning...'); this.logger.info('Cleaning...');
this.userIpsRepository.delete({ this.userIpsRepository.delete({
@ -72,6 +72,5 @@ export class CleanProcessorService {
} }
this.logger.succ('Cleaned.'); this.logger.succ('Cleaned.');
done();
} }
} }

View File

@ -5,9 +5,9 @@ import type { DriveFilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js'; import { DriveService } from '@/core/DriveService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
@Injectable() @Injectable()
export class CleanRemoteFilesProcessorService { export class CleanRemoteFilesProcessorService {
@ -27,7 +27,7 @@ export class CleanRemoteFilesProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> { public async process(job: Bull.Job<Record<string, unknown>>): Promise<void> {
this.logger.info('Deleting cached remote files...'); this.logger.info('Deleting cached remote files...');
let deletedCount = 0; let deletedCount = 0;
@ -47,7 +47,7 @@ export class CleanRemoteFilesProcessorService {
}); });
if (files.length === 0) { if (files.length === 0) {
job.progress(100); job.updateProgress(100);
break; break;
} }
@ -62,10 +62,9 @@ export class CleanRemoteFilesProcessorService {
isLink: false, isLink: false,
}); });
job.progress(deletedCount / total); job.updateProgress(deletedCount / total);
} }
this.logger.succ('All cached remote files has been deleted.'); this.logger.succ('All cached remote files has been deleted.');
done();
} }
} }

View File

@ -8,10 +8,10 @@ import { DriveService } from '@/core/DriveService.js';
import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { DriveFile } from '@/models/entities/DriveFile.js';
import type { Note } from '@/models/entities/Note.js'; import type { Note } from '@/models/entities/Note.js';
import { EmailService } from '@/core/EmailService.js'; import { EmailService } from '@/core/EmailService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserDeleteJobData } from '../types.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserDeleteJobData } from '../types.js';
@Injectable() @Injectable()
export class DeleteAccountProcessorService { export class DeleteAccountProcessorService {

View File

@ -5,10 +5,10 @@ import type { UsersRepository, DriveFilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js'; import { DriveService } from '@/core/DriveService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@Injectable() @Injectable()
export class DeleteDriveFilesProcessorService { export class DeleteDriveFilesProcessorService {
@ -31,12 +31,11 @@ export class DeleteDriveFilesProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Deleting drive files of ${job.data.user.id} ...`); this.logger.info(`Deleting drive files of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -56,7 +55,7 @@ export class DeleteDriveFilesProcessorService {
}); });
if (files.length === 0) { if (files.length === 0) {
job.progress(100); job.updateProgress(100);
break; break;
} }
@ -71,10 +70,9 @@ export class DeleteDriveFilesProcessorService {
userId: user.id, userId: user.id,
}); });
job.progress(deletedCount / total); job.updateProgress(deletedCount / total);
} }
this.logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`); this.logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`);
done();
} }
} }

View File

@ -3,10 +3,10 @@ import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js'; import { DriveService } from '@/core/DriveService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { ObjectStorageFileJobData } from '../types.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { ObjectStorageFileJobData } from '../types.js';
@Injectable() @Injectable()
export class DeleteFileProcessorService { export class DeleteFileProcessorService {

View File

@ -1,4 +1,5 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import * as Bull from 'bullmq';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { DriveFilesRepository, InstancesRepository } from '@/models/index.js'; import type { DriveFilesRepository, InstancesRepository } from '@/models/index.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
@ -16,7 +17,6 @@ import { StatusError } from '@/misc/status-error.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DeliverJobData } from '../types.js'; import type { DeliverJobData } from '../types.js';
@Injectable() @Injectable()
@ -121,15 +121,13 @@ export class DeliverProcessorService {
isSuspended: true, isSuspended: true,
}); });
}); });
return `${host} is gone`; throw new Bull.UnrecoverableError(`${host} is gone`);
} }
// HTTPステータスコード4xxはクライアントエラーであり、それはつまり throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`);
// 何回再送しても成功することはないということなのでエラーにはしないでおく
return `${res.statusCode} ${res.statusMessage}`;
} }
// 5xx etc. // 5xx etc.
throw `${res.statusCode} ${res.statusMessage}`; throw new Error(`${res.statusCode} ${res.statusMessage}`);
} else { } else {
// DNS error, socket error, timeout ... // DNS error, socket error, timeout ...
throw res; throw res;

View File

@ -6,7 +6,7 @@ import type Logger from '@/logger.js';
import { NotificationService } from '@/core/NotificationService.js'; import { NotificationService } from '@/core/NotificationService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
import type { EndedPollNotificationJobData } from '../types.js'; import type { EndedPollNotificationJobData } from '../types.js';
@Injectable() @Injectable()
@ -30,10 +30,9 @@ export class EndedPollNotificationProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<EndedPollNotificationJobData>, done: () => void): Promise<void> { public async process(job: Bull.Job<EndedPollNotificationJobData>): Promise<void> {
const note = await this.notesRepository.findOneBy({ id: job.data.noteId }); const note = await this.notesRepository.findOneBy({ id: job.data.noteId });
if (note == null || !note.hasPoll) { if (note == null || !note.hasPoll) {
done();
return; return;
} }
@ -51,7 +50,5 @@ export class EndedPollNotificationProcessorService {
noteId: note.id, noteId: note.id,
}); });
} }
done();
} }
} }

View File

@ -12,7 +12,7 @@ import { createTemp } from '@/misc/create-temp.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type { DBExportAntennasData } from '../types.js'; import type { DBExportAntennasData } from '../types.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
@Injectable() @Injectable()
export class ExportAntennasProcessorService { export class ExportAntennasProcessorService {
@ -39,10 +39,9 @@ export class ExportAntennasProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DBExportAntennasData>, done: () => void): Promise<void> { public async process(job: Bull.Job<DBExportAntennasData>): Promise<void> {
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
const [path, cleanup] = await createTemp(); const [path, cleanup] = await createTemp();
@ -96,7 +95,6 @@ export class ExportAntennasProcessorService {
this.logger.succ('Exported to: ' + driveFile.id); this.logger.succ('Exported to: ' + driveFile.id);
} finally { } finally {
cleanup(); cleanup();
done();
} }
} }
} }

View File

@ -9,10 +9,10 @@ import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js'; import { DriveService } from '@/core/DriveService.js';
import { createTemp } from '@/misc/create-temp.js'; import { createTemp } from '@/misc/create-temp.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@Injectable() @Injectable()
export class ExportBlockingProcessorService { export class ExportBlockingProcessorService {
@ -36,12 +36,11 @@ export class ExportBlockingProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting blocking of ${job.data.user.id} ...`); this.logger.info(`Exporting blocking of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -69,7 +68,7 @@ export class ExportBlockingProcessorService {
}); });
if (blockings.length === 0) { if (blockings.length === 0) {
job.progress(100); job.updateProgress(100);
break; break;
} }
@ -99,7 +98,7 @@ export class ExportBlockingProcessorService {
blockerId: user.id, blockerId: user.id,
}); });
job.progress(exportedCount / total); job.updateProgress(exportedCount / total);
} }
stream.end(); stream.end();
@ -112,7 +111,5 @@ export class ExportBlockingProcessorService {
} finally { } finally {
cleanup(); cleanup();
} }
done();
} }
} }

View File

@ -13,7 +13,7 @@ import { createTemp, createTempDir } from '@/misc/create-temp.js';
import { DownloadService } from '@/core/DownloadService.js'; import { DownloadService } from '@/core/DownloadService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
@Injectable() @Injectable()
export class ExportCustomEmojisProcessorService { export class ExportCustomEmojisProcessorService {
@ -37,12 +37,11 @@ export class ExportCustomEmojisProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job, done: () => void): Promise<void> { public async process(job: Bull.Job): Promise<void> {
this.logger.info('Exporting custom emojis ...'); this.logger.info('Exporting custom emojis ...');
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -117,24 +116,26 @@ export class ExportCustomEmojisProcessorService {
metaStream.end(); metaStream.end();
// Create archive // Create archive
const [archivePath, archiveCleanup] = await createTemp(); await new Promise<void>(async (resolve) => {
const archiveStream = fs.createWriteStream(archivePath); const [archivePath, archiveCleanup] = await createTemp();
const archive = archiver('zip', { const archiveStream = fs.createWriteStream(archivePath);
zlib: { level: 0 }, const archive = archiver('zip', {
}); zlib: { level: 0 },
archiveStream.on('close', async () => { });
this.logger.succ(`Exported to: ${archivePath}`); archiveStream.on('close', async () => {
this.logger.succ(`Exported to: ${archivePath}`);
const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip';
const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true });
this.logger.succ(`Exported to: ${driveFile.id}`); this.logger.succ(`Exported to: ${driveFile.id}`);
cleanup(); cleanup();
archiveCleanup(); archiveCleanup();
done(); resolve();
});
archive.pipe(archiveStream);
archive.directory(path, false);
archive.finalize();
}); });
archive.pipe(archiveStream);
archive.directory(path, false);
archive.finalize();
} }
} }

View File

@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js';
import type { Note } from '@/models/entities/Note.js'; import type { Note } from '@/models/entities/Note.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js'; import type { DbJobDataWithUser } from '../types.js';
@Injectable() @Injectable()
@ -42,12 +42,11 @@ export class ExportFavoritesProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting favorites of ${job.data.user.id} ...`); this.logger.info(`Exporting favorites of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -91,7 +90,7 @@ export class ExportFavoritesProcessorService {
}) as (NoteFavorite & { note: Note & { user: User } })[]; }) as (NoteFavorite & { note: Note & { user: User } })[];
if (favorites.length === 0) { if (favorites.length === 0) {
job.progress(100); job.updateProgress(100);
break; break;
} }
@ -112,7 +111,7 @@ export class ExportFavoritesProcessorService {
userId: user.id, userId: user.id,
}); });
job.progress(exportedFavoritesCount / total); job.updateProgress(exportedFavoritesCount / total);
} }
await write(']'); await write(']');
@ -127,8 +126,6 @@ export class ExportFavoritesProcessorService {
} finally { } finally {
cleanup(); cleanup();
} }
done();
} }
} }

View File

@ -10,10 +10,10 @@ import { DriveService } from '@/core/DriveService.js';
import { createTemp } from '@/misc/create-temp.js'; import { createTemp } from '@/misc/create-temp.js';
import type { Following } from '@/models/entities/Following.js'; import type { Following } from '@/models/entities/Following.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbExportFollowingData } from '../types.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbExportFollowingData } from '../types.js';
@Injectable() @Injectable()
export class ExportFollowingProcessorService { export class ExportFollowingProcessorService {
@ -40,12 +40,11 @@ export class ExportFollowingProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbExportFollowingData>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbExportFollowingData>): Promise<void> {
this.logger.info(`Exporting following of ${job.data.user.id} ...`); this.logger.info(`Exporting following of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -116,7 +115,5 @@ export class ExportFollowingProcessorService {
} finally { } finally {
cleanup(); cleanup();
} }
done();
} }
} }

View File

@ -9,10 +9,10 @@ import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js'; import { DriveService } from '@/core/DriveService.js';
import { createTemp } from '@/misc/create-temp.js'; import { createTemp } from '@/misc/create-temp.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@Injectable() @Injectable()
export class ExportMutingProcessorService { export class ExportMutingProcessorService {
@ -39,12 +39,11 @@ export class ExportMutingProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting muting of ${job.data.user.id} ...`); this.logger.info(`Exporting muting of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -73,7 +72,7 @@ export class ExportMutingProcessorService {
}); });
if (mutes.length === 0) { if (mutes.length === 0) {
job.progress(100); job.updateProgress(100);
break; break;
} }
@ -103,7 +102,7 @@ export class ExportMutingProcessorService {
muterId: user.id, muterId: user.id,
}); });
job.progress(exportedCount / total); job.updateProgress(exportedCount / total);
} }
stream.end(); stream.end();
@ -116,7 +115,5 @@ export class ExportMutingProcessorService {
} finally { } finally {
cleanup(); cleanup();
} }
done();
} }
} }

View File

@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js';
import type { Note } from '@/models/entities/Note.js'; import type { Note } from '@/models/entities/Note.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js'; import type { DbJobDataWithUser } from '../types.js';
@Injectable() @Injectable()
@ -39,12 +39,11 @@ export class ExportNotesProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting notes of ${job.data.user.id} ...`); this.logger.info(`Exporting notes of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -87,7 +86,7 @@ export class ExportNotesProcessorService {
}) as Note[]; }) as Note[];
if (notes.length === 0) { if (notes.length === 0) {
job.progress(100); job.updateProgress(100);
break; break;
} }
@ -108,7 +107,7 @@ export class ExportNotesProcessorService {
userId: user.id, userId: user.id,
}); });
job.progress(exportedNotesCount / total); job.updateProgress(exportedNotesCount / total);
} }
await write(']'); await write(']');
@ -123,8 +122,6 @@ export class ExportNotesProcessorService {
} finally { } finally {
cleanup(); cleanup();
} }
done();
} }
} }

View File

@ -9,10 +9,10 @@ import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js'; import { DriveService } from '@/core/DriveService.js';
import { createTemp } from '@/misc/create-temp.js'; import { createTemp } from '@/misc/create-temp.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@Injectable() @Injectable()
export class ExportUserListsProcessorService { export class ExportUserListsProcessorService {
@ -39,12 +39,11 @@ export class ExportUserListsProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting user lists of ${job.data.user.id} ...`); this.logger.info(`Exporting user lists of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -92,7 +91,5 @@ export class ExportUserListsProcessorService {
} finally { } finally {
cleanup(); cleanup();
} }
done();
} }
} }

View File

@ -8,7 +8,7 @@ import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import { DBAntennaImportJobData } from '../types.js'; import { DBAntennaImportJobData } from '../types.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
const validate = new Ajv().compile({ const validate = new Ajv().compile({
type: 'object', type: 'object',
@ -59,7 +59,7 @@ export class ImportAntennasProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DBAntennaImportJobData>, done: () => void): Promise<void> { public async process(job: Bull.Job<DBAntennaImportJobData>): Promise<void> {
const now = new Date(); const now = new Date();
try { try {
for (const antenna of job.data.antenna) { for (const antenna of job.data.antenna) {
@ -89,8 +89,6 @@ export class ImportAntennasProcessorService {
} }
} catch (err: any) { } catch (err: any) {
this.logger.error(err); this.logger.error(err);
} finally {
done();
} }
} }
} }

View File

@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js';
import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
import { DownloadService } from '@/core/DownloadService.js'; import { DownloadService } from '@/core/DownloadService.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueService } from '@/core/QueueService.js'; import { QueueService } from '@/core/QueueService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js';
@Injectable() @Injectable()
export class ImportBlockingProcessorService { export class ImportBlockingProcessorService {
@ -34,12 +34,11 @@ export class ImportBlockingProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbUserImportJobData>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing blocking of ${job.data.user.id} ...`); this.logger.info(`Importing blocking of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -47,7 +46,6 @@ export class ImportBlockingProcessorService {
id: job.data.fileId, id: job.data.fileId,
}); });
if (file == null) { if (file == null) {
done();
return; return;
} }
@ -56,7 +54,6 @@ export class ImportBlockingProcessorService {
this.queueService.createImportBlockingToDbJob({ id: user.id }, targets); this.queueService.createImportBlockingToDbJob({ id: user.id }, targets);
this.logger.succ('Import jobs created'); this.logger.succ('Import jobs created');
done();
} }
@bindThis @bindThis
@ -85,7 +82,7 @@ export class ImportBlockingProcessorService {
} }
if (target == null) { if (target == null) {
throw `Unable to resolve user: @${username}@${host}`; throw new Error(`Unable to resolve user: @${username}@${host}`);
} }
// skip myself // skip myself

View File

@ -12,7 +12,7 @@ import { DriveService } from '@/core/DriveService.js';
import { DownloadService } from '@/core/DownloadService.js'; import { DownloadService } from '@/core/DownloadService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
import type { DbUserImportJobData } from '../types.js'; import type { DbUserImportJobData } from '../types.js';
// TODO: 名前衝突時の動作を選べるようにする // TODO: 名前衝突時の動作を選べるようにする
@ -45,14 +45,13 @@ export class ImportCustomEmojisProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbUserImportJobData>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info('Importing custom emojis ...'); this.logger.info('Importing custom emojis ...');
const file = await this.driveFilesRepository.findOneBy({ const file = await this.driveFilesRepository.findOneBy({
id: job.data.fileId, id: job.data.fileId,
}); });
if (file == null) { if (file == null) {
done();
return; return;
} }
@ -116,7 +115,6 @@ export class ImportCustomEmojisProcessorService {
cleanup(); cleanup();
this.logger.succ('Imported'); this.logger.succ('Imported');
done();
}); });
unzipStream.pipe(extractor); unzipStream.pipe(extractor);
this.logger.succ(`Unzipping to ${outputPath}`); this.logger.succ(`Unzipping to ${outputPath}`);

View File

@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js';
import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
import { DownloadService } from '@/core/DownloadService.js'; import { DownloadService } from '@/core/DownloadService.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueService } from '@/core/QueueService.js'; import { QueueService } from '@/core/QueueService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js';
@Injectable() @Injectable()
export class ImportFollowingProcessorService { export class ImportFollowingProcessorService {
@ -34,12 +34,11 @@ export class ImportFollowingProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbUserImportJobData>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing following of ${job.data.user.id} ...`); this.logger.info(`Importing following of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -47,7 +46,6 @@ export class ImportFollowingProcessorService {
id: job.data.fileId, id: job.data.fileId,
}); });
if (file == null) { if (file == null) {
done();
return; return;
} }
@ -56,7 +54,6 @@ export class ImportFollowingProcessorService {
this.queueService.createImportFollowingToDbJob({ id: user.id }, targets); this.queueService.createImportFollowingToDbJob({ id: user.id }, targets);
this.logger.succ('Import jobs created'); this.logger.succ('Import jobs created');
done();
} }
@bindThis @bindThis
@ -85,7 +82,7 @@ export class ImportFollowingProcessorService {
} }
if (target == null) { if (target == null) {
throw `Unable to resolve user: @${username}@${host}`; throw new Error(`Unable to resolve user: @${username}@${host}`);
} }
// skip myself // skip myself

View File

@ -9,10 +9,10 @@ import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
import { DownloadService } from '@/core/DownloadService.js'; import { DownloadService } from '@/core/DownloadService.js';
import { UserMutingService } from '@/core/UserMutingService.js'; import { UserMutingService } from '@/core/UserMutingService.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserImportJobData } from '../types.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserImportJobData } from '../types.js';
@Injectable() @Injectable()
export class ImportMutingProcessorService { export class ImportMutingProcessorService {
@ -38,12 +38,11 @@ export class ImportMutingProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbUserImportJobData>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing muting of ${job.data.user.id} ...`); this.logger.info(`Importing muting of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -51,7 +50,6 @@ export class ImportMutingProcessorService {
id: job.data.fileId, id: job.data.fileId,
}); });
if (file == null) { if (file == null) {
done();
return; return;
} }
@ -83,7 +81,7 @@ export class ImportMutingProcessorService {
} }
if (target == null) { if (target == null) {
throw `cannot resolve user: @${username}@${host}`; throw new Error(`cannot resolve user: @${username}@${host}`);
} }
// skip myself // skip myself
@ -98,6 +96,5 @@ export class ImportMutingProcessorService {
} }
this.logger.succ('Imported'); this.logger.succ('Imported');
done();
} }
} }

View File

@ -12,7 +12,7 @@ import { IdService } from '@/core/IdService.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
import type { DbUserImportJobData } from '../types.js'; import type { DbUserImportJobData } from '../types.js';
@Injectable() @Injectable()
@ -46,12 +46,11 @@ export class ImportUserListsProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<DbUserImportJobData>, done: () => void): Promise<void> { public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing user lists of ${job.data.user.id} ...`); this.logger.info(`Importing user lists of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
done();
return; return;
} }
@ -59,7 +58,6 @@ export class ImportUserListsProcessorService {
id: job.data.fileId, id: job.data.fileId,
}); });
if (file == null) { if (file == null) {
done();
return; return;
} }
@ -109,6 +107,5 @@ export class ImportUserListsProcessorService {
} }
this.logger.succ('Imported'); this.logger.succ('Imported');
done();
} }
} }

View File

@ -1,8 +1,8 @@
import { URL } from 'node:url'; import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import httpSignature from '@peertube/http-signature'; import httpSignature from '@peertube/http-signature';
import * as Bull from 'bullmq';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { InstancesRepository, DriveFilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
import { MetaService } from '@/core/MetaService.js'; import { MetaService } from '@/core/MetaService.js';
@ -23,10 +23,8 @@ import { LdSignatureService } from '@/core/activitypub/LdSignatureService.js';
import { ApInboxService } from '@/core/activitypub/ApInboxService.js'; import { ApInboxService } from '@/core/activitypub/ApInboxService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { InboxJobData } from '../types.js'; import type { InboxJobData } from '../types.js';
// ユーザーのinboxにアクティビティが届いた時の処理
@Injectable() @Injectable()
export class InboxProcessorService { export class InboxProcessorService {
private logger: Logger; private logger: Logger;
@ -35,12 +33,6 @@ export class InboxProcessorService {
@Inject(DI.config) @Inject(DI.config)
private config: Config, private config: Config,
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
private utilityService: UtilityService, private utilityService: UtilityService,
private metaService: MetaService, private metaService: MetaService,
private apInboxService: ApInboxService, private apInboxService: ApInboxService,
@ -93,24 +85,24 @@ export class InboxProcessorService {
try { try {
authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor)); authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor));
} catch (err) { } catch (err) {
// 対象が4xxならスキップ // 対象が4xxならスキップ
if (err instanceof StatusError) { if (err instanceof StatusError) {
if (err.isClientError) { if (err.isClientError) {
return `skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`; throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`);
} }
throw `Error in actor ${activity.actor} - ${err.statusCode ?? err}`; throw new Error(`Error in actor ${activity.actor} - ${err.statusCode ?? err}`);
} }
} }
} }
// それでもわからなければ終了 // それでもわからなければ終了
if (authUser == null) { if (authUser == null) {
return 'skip: failed to resolve user'; throw new Bull.UnrecoverableError('skip: failed to resolve user');
} }
// publicKey がなくても終了 // publicKey がなくても終了
if (authUser.key == null) { if (authUser.key == null) {
return 'skip: failed to resolve user publicKey'; throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey');
} }
// HTTP-Signatureの検証 // HTTP-Signatureの検証
@ -118,10 +110,10 @@ export class InboxProcessorService {
// また、signatureのsignerは、activity.actorと一致する必要がある // また、signatureのsignerは、activity.actorと一致する必要がある
if (!httpSignatureValidated || authUser.user.uri !== activity.actor) { if (!httpSignatureValidated || authUser.user.uri !== activity.actor) {
// 一致しなくても、でもLD-Signatureがありそうならそっちも見る // 一致しなくても、でもLD-Signatureがありそうならそっちも見る
if (activity.signature) { if (activity.signature) {
if (activity.signature.type !== 'RsaSignature2017') { if (activity.signature.type !== 'RsaSignature2017') {
return `skip: unsupported LD-signature type ${activity.signature.type}`; throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${activity.signature.type}`);
} }
// activity.signature.creator: https://example.oom/users/user#main-key // activity.signature.creator: https://example.oom/users/user#main-key
@ -134,32 +126,32 @@ export class InboxProcessorService {
// keyIdからLD-Signatureのユーザーを取得 // keyIdからLD-Signatureのユーザーを取得
authUser = await this.apDbResolverService.getAuthUserFromKeyId(activity.signature.creator); authUser = await this.apDbResolverService.getAuthUserFromKeyId(activity.signature.creator);
if (authUser == null) { if (authUser == null) {
return 'skip: LD-Signatureのユーザーが取得できませんでした'; throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした');
} }
if (authUser.key == null) { if (authUser.key == null) {
return 'skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'; throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした');
} }
// LD-Signature検証 // LD-Signature検証
const ldSignature = this.ldSignatureService.use(); const ldSignature = this.ldSignatureService.use();
const verified = await ldSignature.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); const verified = await ldSignature.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false);
if (!verified) { if (!verified) {
return 'skip: LD-Signatureの検証に失敗しました'; throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました');
} }
// もう一度actorチェック // もう一度actorチェック
if (authUser.user.uri !== activity.actor) { if (authUser.user.uri !== activity.actor) {
return `skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`; throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`);
} }
// ブロックしてたら中断 // ブロックしてたら中断
const ldHost = this.utilityService.extractDbHost(authUser.user.uri); const ldHost = this.utilityService.extractDbHost(authUser.user.uri);
if (this.utilityService.isBlockedHost(meta.blockedHosts, ldHost)) { if (this.utilityService.isBlockedHost(meta.blockedHosts, ldHost)) {
return `Blocked request: ${ldHost}`; throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`);
} }
} else { } else {
return `skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`; throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`);
} }
} }
@ -168,7 +160,7 @@ export class InboxProcessorService {
const signerHost = this.utilityService.extractDbHost(authUser.user.uri!); const signerHost = this.utilityService.extractDbHost(authUser.user.uri!);
const activityIdHost = this.utilityService.extractDbHost(activity.id); const activityIdHost = this.utilityService.extractDbHost(activity.id);
if (signerHost !== activityIdHost) { if (signerHost !== activityIdHost) {
return `skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`; throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`);
} }
} }

View File

@ -1,5 +1,5 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
import { UserFollowingService } from '@/core/UserFollowingService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js';
import { UserBlockingService } from '@/core/UserBlockingService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js';

View File

@ -15,7 +15,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js';
import ApRequestChart from '@/core/chart/charts/ap-request.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
@Injectable() @Injectable()
export class ResyncChartsProcessorService { export class ResyncChartsProcessorService {
@ -43,7 +43,7 @@ export class ResyncChartsProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> { public async process(): Promise<void> {
this.logger.info('Resync charts...'); this.logger.info('Resync charts...');
// TODO: ユーザーごとのチャートも更新する // TODO: ユーザーごとのチャートも更新する
@ -55,6 +55,5 @@ export class ResyncChartsProcessorService {
]); ]);
this.logger.succ('All charts successfully resynced.'); this.logger.succ('All charts successfully resynced.');
done();
} }
} }

View File

@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js';
import ApRequestChart from '@/core/chart/charts/ap-request.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull'; import type * as Bull from 'bullmq';
@Injectable() @Injectable()
export class TickChartsProcessorService { export class TickChartsProcessorService {
@ -45,7 +45,7 @@ export class TickChartsProcessorService {
} }
@bindThis @bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> { public async process(): Promise<void> {
this.logger.info('Tick charts...'); this.logger.info('Tick charts...');
await Promise.all([ await Promise.all([
@ -64,6 +64,5 @@ export class TickChartsProcessorService {
]); ]);
this.logger.succ('All charts successfully ticked.'); this.logger.succ('All charts successfully ticked.');
done();
} }
} }

View File

@ -1,4 +1,5 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import * as Bull from 'bullmq';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { WebhooksRepository } from '@/models/index.js'; import type { WebhooksRepository } from '@/models/index.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
@ -7,7 +8,6 @@ import { HttpRequestService } from '@/core/HttpRequestService.js';
import { StatusError } from '@/misc/status-error.js'; import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { WebhookDeliverJobData } from '../types.js'; import type { WebhookDeliverJobData } from '../types.js';
@Injectable() @Injectable()
@ -66,11 +66,11 @@ export class WebhookDeliverProcessorService {
if (res instanceof StatusError) { if (res instanceof StatusError) {
// 4xx // 4xx
if (res.isClientError) { if (res.isClientError) {
return `${res.statusCode} ${res.statusMessage}`; throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`);
} }
// 5xx etc. // 5xx etc.
throw `${res.statusCode} ${res.statusMessage}`; throw new Error(`${res.statusCode} ${res.statusMessage}`);
} else { } else {
// DNS error, socket error, timeout ... // DNS error, socket error, timeout ...
throw res; throw res;

View File

@ -62,7 +62,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
try { try {
if (new URL(ps.inbox).protocol !== 'https:') throw 'https only'; if (new URL(ps.inbox).protocol !== 'https:') throw new Error('https only');
} catch { } catch {
throw new ApiError(meta.errors.invalidUrl); throw new ApiError(meta.errors.invalidUrl);
} }

View File

@ -82,14 +82,14 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
try { try {
if (ps.tag) { if (ps.tag) {
if (!safeForSql(normalizeForSearch(ps.tag))) throw 'Injection'; if (!safeForSql(normalizeForSearch(ps.tag))) throw new Error('Injection');
query.andWhere(`'{"${normalizeForSearch(ps.tag)}"}' <@ note.tags`); query.andWhere(`'{"${normalizeForSearch(ps.tag)}"}' <@ note.tags`);
} else { } else {
query.andWhere(new Brackets(qb => { query.andWhere(new Brackets(qb => {
for (const tags of ps.query!) { for (const tags of ps.query!) {
qb.orWhere(new Brackets(qb => { qb.orWhere(new Brackets(qb => {
for (const tag of tags) { for (const tag of tags) {
if (!safeForSql(normalizeForSearch(tag))) throw 'Injection'; if (!safeForSql(normalizeForSearch(tag))) throw new Error('Injection');
qb.andWhere(`'{"${normalizeForSearch(tag)}"}' <@ note.tags`); qb.andWhere(`'{"${normalizeForSearch(tag)}"}' <@ note.tags`);
} }
})); }));

View File

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
private redisClient: Redis.Redis, private redisClient: Redis.Redis,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
if (process.env.NODE_ENV !== 'test') throw 'NODE_ENV is not a test'; if (process.env.NODE_ENV !== 'test') throw new Error('NODE_ENV is not a test');
await redisClient.flushdb(); await redisClient.flushdb();
await resetDb(this.db); await resetDb(this.db);

View File

@ -52,11 +52,7 @@ export class MockResolver extends Resolver {
const r = this._rs.get(value); const r = this._rs.get(value);
if (!r) { if (!r) {
throw { throw new Error('Not registed for mock');
name: 'StatusError',
statusCode: 404,
message: 'Not registed for mock',
};
} }
const object = JSON.parse(r.content); const object = JSON.parse(r.content);

View File

@ -478,7 +478,7 @@ export async function testPaginationConsistency<Entity extends { id: string, cre
} }
export async function initTestDb(justBorrow = false, initEntities?: any[]) { export async function initTestDb(justBorrow = false, initEntities?: any[]) {
if (process.env.NODE_ENV !== 'test') throw 'NODE_ENV is not a test'; if (process.env.NODE_ENV !== 'test') throw new Error('NODE_ENV is not a test');
const db = new DataSource({ const db = new DataSource({
type: 'postgres', type: 'postgres',

View File

@ -5,15 +5,16 @@ const dateTimeIntervals = {
}; };
export function dateUTC(time: number[]): Date { export function dateUTC(time: number[]): Date {
const d = time.length === 2 ? Date.UTC(time[0], time[1]) const d =
: time.length === 3 ? Date.UTC(time[0], time[1], time[2]) time.length === 2 ? Date.UTC(time[0], time[1])
: time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) : time.length === 3 ? Date.UTC(time[0], time[1], time[2])
: time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3])
: time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4])
: time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5])
: null; : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6])
: null;
if (!d) throw 'wrong number of arguments'; if (!d) throw new Error('wrong number of arguments');
return new Date(d); return new Date(d);
} }

View File

@ -49,7 +49,7 @@ module.exports = {
'no-multi-spaces': ['error'], 'no-multi-spaces': ['error'],
'no-var': ['error'], 'no-var': ['error'],
'prefer-arrow-callback': ['error'], 'prefer-arrow-callback': ['error'],
'no-throw-literal': ['warn'], 'no-throw-literal': ['error'],
'no-param-reassign': ['warn'], 'no-param-reassign': ['warn'],
'no-constant-condition': ['warn'], 'no-constant-condition': ['warn'],
'no-empty-pattern': ['warn'], 'no-empty-pattern': ['warn'],

View File

@ -145,9 +145,9 @@ importers:
blurhash: blurhash:
specifier: 2.0.5 specifier: 2.0.5
version: 2.0.5 version: 2.0.5
bull: bullmq:
specifier: 4.10.4 specifier: 3.14.1
version: 4.10.4 version: 3.14.1
cacheable-lookup: cacheable-lookup:
specifier: 6.1.0 specifier: 6.1.0
version: 6.1.0 version: 6.1.0
@ -489,9 +489,6 @@ importers:
'@types/bcryptjs': '@types/bcryptjs':
specifier: 2.4.2 specifier: 2.4.2
version: 2.4.2 version: 2.4.2
'@types/bull':
specifier: 4.10.0
version: 4.10.0
'@types/cbor': '@types/cbor':
specifier: 6.0.0 specifier: 6.0.0
version: 6.0.0 version: 6.0.0
@ -3956,6 +3953,7 @@ packages:
/@ioredis/commands@1.2.0: /@ioredis/commands@1.2.0:
resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}
dev: false
/@istanbuljs/load-nyc-config@1.1.0: /@istanbuljs/load-nyc-config@1.1.0:
resolution: {integrity: sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==} resolution: {integrity: sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==}
@ -4364,46 +4362,52 @@ packages:
os-filter-obj: 2.0.0 os-filter-obj: 2.0.0
dev: false dev: false
/@msgpackr-extract/msgpackr-extract-darwin-arm64@2.2.0: /@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2:
resolution: {integrity: sha512-Z9LFPzfoJi4mflGWV+rv7o7ZbMU5oAU9VmzCgL240KnqDW65Y2HFCT3MW06/ITJSnbVLacmcEJA8phywK7JinQ==} resolution: {integrity: sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==}
cpu: [arm64] cpu: [arm64]
os: [darwin] os: [darwin]
requiresBuild: true requiresBuild: true
dev: false
optional: true optional: true
/@msgpackr-extract/msgpackr-extract-darwin-x64@2.2.0: /@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.2:
resolution: {integrity: sha512-vq0tT8sjZsy4JdSqmadWVw6f66UXqUCabLmUVHZwUFzMgtgoIIQjT4VVRHKvlof3P/dMCkbMJ5hB1oJ9OWHaaw==} resolution: {integrity: sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==}
cpu: [x64] cpu: [x64]
os: [darwin] os: [darwin]
requiresBuild: true requiresBuild: true
dev: false
optional: true optional: true
/@msgpackr-extract/msgpackr-extract-linux-arm64@2.2.0: /@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.2:
resolution: {integrity: sha512-hlxxLdRmPyq16QCutUtP8Tm6RDWcyaLsRssaHROatgnkOxdleMTgetf9JsdncL8vLh7FVy/RN9i3XR5dnb9cRA==} resolution: {integrity: sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==}
cpu: [arm64] cpu: [arm64]
os: [linux] os: [linux]
requiresBuild: true requiresBuild: true
dev: false
optional: true optional: true
/@msgpackr-extract/msgpackr-extract-linux-arm@2.2.0: /@msgpackr-extract/msgpackr-extract-linux-arm@3.0.2:
resolution: {integrity: sha512-SaJ3Qq4lX9Syd2xEo9u3qPxi/OB+5JO/ngJKK97XDpa1C587H9EWYO6KD8995DAjSinWvdHKRrCOXVUC5fvGOg==} resolution: {integrity: sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==}
cpu: [arm] cpu: [arm]
os: [linux] os: [linux]
requiresBuild: true requiresBuild: true
dev: false
optional: true optional: true
/@msgpackr-extract/msgpackr-extract-linux-x64@2.2.0: /@msgpackr-extract/msgpackr-extract-linux-x64@3.0.2:
resolution: {integrity: sha512-94y5PJrSOqUNcFKmOl7z319FelCLAE0rz/jPCWS+UtdMZvpa4jrQd+cJPQCLp2Fes1yAW/YUQj/Di6YVT3c3Iw==} resolution: {integrity: sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==}
cpu: [x64] cpu: [x64]
os: [linux] os: [linux]
requiresBuild: true requiresBuild: true
dev: false
optional: true optional: true
/@msgpackr-extract/msgpackr-extract-win32-x64@2.2.0: /@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2:
resolution: {integrity: sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA==} resolution: {integrity: sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==}
cpu: [x64] cpu: [x64]
os: [win32] os: [win32]
requiresBuild: true requiresBuild: true
dev: false
optional: true optional: true
/@mswjs/cookies@0.2.2: /@mswjs/cookies@0.2.2:
@ -6511,14 +6515,6 @@ packages:
resolution: {integrity: sha512-+euflG6ygo4bn0JHtn4pYqcXwRtLvElQ7/nnjDu7iYG56H0+OhCd7d6Ug0IE3WcFpZozBKW2+80FUbv5QGk5AQ==} resolution: {integrity: sha512-+euflG6ygo4bn0JHtn4pYqcXwRtLvElQ7/nnjDu7iYG56H0+OhCd7d6Ug0IE3WcFpZozBKW2+80FUbv5QGk5AQ==}
dev: true dev: true
/@types/bull@4.10.0:
resolution: {integrity: sha512-RkYW8K2H3J76HT6twmHYbzJ0GtLDDotpLP9ah9gtiA7zfF6peBH1l5fEiK0oeIZ3/642M7Jcb9sPmor8Vf4w6g==}
dependencies:
bull: 4.10.4
transitivePeerDependencies:
- supports-color
dev: true
/@types/cacheable-request@6.0.3: /@types/cacheable-request@6.0.3:
resolution: {integrity: sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==} resolution: {integrity: sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==}
dependencies: dependencies:
@ -8708,20 +8704,20 @@ packages:
node-gyp-build: 4.6.0 node-gyp-build: 4.6.0
dev: false dev: false
/bull@4.10.4: /bullmq@3.14.1:
resolution: {integrity: sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA==} resolution: {integrity: sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==}
engines: {node: '>=12'}
dependencies: dependencies:
cron-parser: 4.7.1 cron-parser: 4.8.1
debuglog: 1.0.1 glob: 8.1.0
get-port: 5.1.1
ioredis: 5.3.2 ioredis: 5.3.2
lodash: 4.17.21 lodash: 4.17.21
msgpackr: 1.8.1 msgpackr: 1.9.2
semver: 7.5.1 semver: 7.5.1
uuid: 8.3.2 tslib: 2.5.2
uuid: 9.0.0
transitivePeerDependencies: transitivePeerDependencies:
- supports-color - supports-color
dev: false
/busboy@1.6.0: /busboy@1.6.0:
resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==}
@ -9306,6 +9302,7 @@ packages:
/cluster-key-slot@1.1.2: /cluster-key-slot@1.1.2:
resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
engines: {node: '>=0.10.0'} engines: {node: '>=0.10.0'}
dev: false
/co@4.6.0: /co@4.6.0:
resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==}
@ -9607,11 +9604,12 @@ packages:
readable-stream: 3.6.0 readable-stream: 3.6.0
dev: false dev: false
/cron-parser@4.7.1: /cron-parser@4.8.1:
resolution: {integrity: sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA==} resolution: {integrity: sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==}
engines: {node: '>=12.0.0'} engines: {node: '>=12.0.0'}
dependencies: dependencies:
luxon: 3.2.1 luxon: 3.3.0
dev: false
/cropperjs@2.0.0-beta.2: /cropperjs@2.0.0-beta.2:
resolution: {integrity: sha512-jDRSODDGKmi9vp3p/+WXkxMqV/AE+GpSld1U3cHZDRdLy9UykRzurSe8k1dR0TExn45ygCMrv31qkg+K3EeXXw==} resolution: {integrity: sha512-jDRSODDGKmi9vp3p/+WXkxMqV/AE+GpSld1U3cHZDRdLy9UykRzurSe8k1dR0TExn45ygCMrv31qkg+K3EeXXw==}
@ -9890,9 +9888,6 @@ packages:
ms: 2.1.2 ms: 2.1.2
supports-color: 8.1.1 supports-color: 8.1.1
/debuglog@1.0.1:
resolution: {integrity: sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==}
/decamelize-keys@1.1.1: /decamelize-keys@1.1.1:
resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==}
engines: {node: '>=0.10.0'} engines: {node: '>=0.10.0'}
@ -10084,6 +10079,7 @@ packages:
/denque@2.1.0: /denque@2.1.0:
resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==}
engines: {node: '>=0.10'} engines: {node: '>=0.10'}
dev: false
/depd@1.1.2: /depd@1.1.2:
resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==}
@ -11753,6 +11749,7 @@ packages:
/get-port@5.1.1: /get-port@5.1.1:
resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==} resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==}
engines: {node: '>=8'} engines: {node: '>=8'}
dev: true
/get-stream@3.0.0: /get-stream@3.0.0:
resolution: {integrity: sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==} resolution: {integrity: sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==}
@ -12606,6 +12603,7 @@ packages:
standard-as-callback: 2.1.0 standard-as-callback: 2.1.0
transitivePeerDependencies: transitivePeerDependencies:
- supports-color - supports-color
dev: false
/iota-array@1.0.0: /iota-array@1.0.0:
resolution: {integrity: sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==} resolution: {integrity: sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==}
@ -14288,6 +14286,7 @@ packages:
/lodash.defaults@4.2.0: /lodash.defaults@4.2.0:
resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==}
dev: false
/lodash.difference@4.5.0: /lodash.difference@4.5.0:
resolution: {integrity: sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==} resolution: {integrity: sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==}
@ -14319,6 +14318,7 @@ packages:
/lodash.isarguments@3.1.0: /lodash.isarguments@3.1.0:
resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
dev: false
/lodash.isempty@4.4.0: /lodash.isempty@4.4.0:
resolution: {integrity: sha512-oKMuF3xEeqDltrGMfDxAPGIVMSSRv8tbRSODbrs4KGsRRLEhrW8N8Rd4DRgB2+621hY8A8XwwrTVhXWpxFvMzg==} resolution: {integrity: sha512-oKMuF3xEeqDltrGMfDxAPGIVMSSRv8tbRSODbrs4KGsRRLEhrW8N8Rd4DRgB2+621hY8A8XwwrTVhXWpxFvMzg==}
@ -14472,9 +14472,10 @@ packages:
engines: {node: '>=16.14'} engines: {node: '>=16.14'}
dev: true dev: true
/luxon@3.2.1: /luxon@3.3.0:
resolution: {integrity: sha512-QrwPArQCNLAKGO/C+ZIilgIuDnEnKx5QYODdDtbFaxzsbZcc/a7WFq7MhsVYgRlwawLtvOUESTlfJ+hc/USqPg==} resolution: {integrity: sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==}
engines: {node: '>=12'} engines: {node: '>=12'}
dev: false
/lz-string@1.5.0: /lz-string@1.5.0:
resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==}
@ -14950,24 +14951,27 @@ packages:
engines: {node: '>=12.13'} engines: {node: '>=12.13'}
dev: false dev: false
/msgpackr-extract@2.2.0: /msgpackr-extract@3.0.2:
resolution: {integrity: sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==} resolution: {integrity: sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==}
hasBin: true
requiresBuild: true requiresBuild: true
dependencies: dependencies:
node-gyp-build-optional-packages: 5.0.3 node-gyp-build-optional-packages: 5.0.7
optionalDependencies: optionalDependencies:
'@msgpackr-extract/msgpackr-extract-darwin-arm64': 2.2.0 '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.2
'@msgpackr-extract/msgpackr-extract-darwin-x64': 2.2.0 '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.2
'@msgpackr-extract/msgpackr-extract-linux-arm': 2.2.0 '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.2
'@msgpackr-extract/msgpackr-extract-linux-arm64': 2.2.0 '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.2
'@msgpackr-extract/msgpackr-extract-linux-x64': 2.2.0 '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.2
'@msgpackr-extract/msgpackr-extract-win32-x64': 2.2.0 '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.2
dev: false
optional: true optional: true
/msgpackr@1.8.1: /msgpackr@1.9.2:
resolution: {integrity: sha512-05fT4J8ZqjYlR4QcRDIhLCYKUOHXk7C/xa62GzMKj74l3up9k2QZ3LgFc6qWdsPHl91QA2WLWqWc8b8t7GLNNw==} resolution: {integrity: sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==}
optionalDependencies: optionalDependencies:
msgpackr-extract: 2.2.0 msgpackr-extract: 3.0.2
dev: false
/msw-storybook-addon@1.8.0(msw@1.2.1): /msw-storybook-addon@1.8.0(msw@1.2.1):
resolution: {integrity: sha512-dw3vZwqjixmiur0vouRSOax7wPSu9Og2Hspy9JZFHf49bZRjwDiLF0Pfn2NXEkGviYJOJiGxS1ejoTiUwoSg4A==} resolution: {integrity: sha512-dw3vZwqjixmiur0vouRSOax7wPSu9Og2Hspy9JZFHf49bZRjwDiLF0Pfn2NXEkGviYJOJiGxS1ejoTiUwoSg4A==}
@ -15219,8 +15223,10 @@ packages:
fetch-blob: 3.2.0 fetch-blob: 3.2.0
formdata-polyfill: 4.0.10 formdata-polyfill: 4.0.10
/node-gyp-build-optional-packages@5.0.3: /node-gyp-build-optional-packages@5.0.7:
resolution: {integrity: sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==} resolution: {integrity: sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==}
hasBin: true
dev: false
optional: true optional: true
/node-gyp-build@4.6.0: /node-gyp-build@4.6.0:
@ -17252,6 +17258,7 @@ packages:
/redis-errors@1.2.0: /redis-errors@1.2.0:
resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
engines: {node: '>=4'} engines: {node: '>=4'}
dev: false
/redis-info@3.1.0: /redis-info@3.1.0:
resolution: {integrity: sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==} resolution: {integrity: sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==}
@ -17269,6 +17276,7 @@ packages:
engines: {node: '>=4'} engines: {node: '>=4'}
dependencies: dependencies:
redis-errors: 1.2.0 redis-errors: 1.2.0
dev: false
/redis@4.5.1: /redis@4.5.1:
resolution: {integrity: sha512-oxXSoIqMJCQVBTfxP6BNTCtDMyh9G6Vi5wjdPdV/sRKkufyZslDqCScSGcOr6XGR/reAWZefz7E4leM31RgdBA==} resolution: {integrity: sha512-oxXSoIqMJCQVBTfxP6BNTCtDMyh9G6Vi5wjdPdV/sRKkufyZslDqCScSGcOr6XGR/reAWZefz7E4leM31RgdBA==}
@ -18395,6 +18403,7 @@ packages:
/standard-as-callback@2.1.0: /standard-as-callback@2.1.0:
resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==}
dev: false
/start-server-and-test@2.0.0: /start-server-and-test@2.0.0:
resolution: {integrity: sha512-UqKLw0mJbfrsG1jcRLTUlvuRi9sjNuUiDOLI42r7R5fA9dsFoywAy9DoLXNYys9B886E4RCKb+qM1Gzu96h7DQ==} resolution: {integrity: sha512-UqKLw0mJbfrsG1jcRLTUlvuRi9sjNuUiDOLI42r7R5fA9dsFoywAy9DoLXNYys9B886E4RCKb+qM1Gzu96h7DQ==}