natty
/
calckey
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Resolve #4444

This commit is contained in:
syuilo 2019-03-07 23:07:21 +09:00
parent 00c9f4a2e5
commit c934987b14
No known key found for this signature in database
GPG Key ID: BDC4C49D06AB9D69
13 changed files with 146 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
package.json
View File

@ -32,6 +32,7 @@
"@prezzemolo/rap": "0.1.2", "@prezzemolo/rap": "0.1.2",
"@prezzemolo/zip": "0.0.3", "@prezzemolo/zip": "0.0.3",
"@types/bcryptjs": "2.4.2", "@types/bcryptjs": "2.4.2",
"@types/bull": "3.5.8",
"@types/chai-http": "3.0.5", "@types/chai-http": "3.0.5",
"@types/dateformat": "3.0.0", "@types/dateformat": "3.0.0",
"@types/deep-equal": "1.0.1", "@types/deep-equal": "1.0.1",
@ -100,8 +101,8 @@
"autosize": "4.0.2", "autosize": "4.0.2",
"autwh": "0.1.0", "autwh": "0.1.0",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"bee-queue": "1.2.2",
"bootstrap-vue": "2.0.0-rc.13", "bootstrap-vue": "2.0.0-rc.13",
"bull": "3.7.0",
"cafy": "15.1.0", "cafy": "15.1.0",
"chai": "4.2.0", "chai": "4.2.0",
"chai-http": "4.2.1", "chai-http": "4.2.1",

4
src/argv.ts
View File

@ -5,8 +5,7 @@ program
.version(pkg.version) .version(pkg.version)
.option('--no-daemons', 'Disable daemon processes (for debbuging)') .option('--no-daemons', 'Disable daemon processes (for debbuging)')
.option('--disable-clustering', 'Disable clustering') .option('--disable-clustering', 'Disable clustering')
.option('--disable-queue', 'Disable job queue processing') .option('--only-server', 'Run server only (without job queue processing)')
.option('--only-server', 'Run server only (without job queue)')
.option('--only-queue', 'Pocessing job queue only (without server)') .option('--only-queue', 'Pocessing job queue only (without server)')
.option('--quiet', 'Suppress all logs') .option('--quiet', 'Suppress all logs')
.option('--verbose', 'Enable all logs') .option('--verbose', 'Enable all logs')
@ -15,7 +14,6 @@ program
.option('--color', 'This option is a dummy for some external program\'s (e.g. forever) issue.') .option('--color', 'This option is a dummy for some external program\'s (e.g. forever) issue.')
.parse(process.argv); .parse(process.argv);
/*if (process.env.MK_DISABLE_QUEUE)*/ program.disableQueue = true;
if (process.env.MK_ONLY_QUEUE) program.onlyQueue = true; if (process.env.MK_ONLY_QUEUE) program.onlyQueue = true;
export { program }; export { program };

141
src/queue/index.ts
View File

@ -1,73 +1,64 @@
import * as Queue from 'bee-queue'; import * as Queue from 'bull';
import * as httpSignature from 'http-signature'; import * as httpSignature from 'http-signature';
import config from '../config'; import config from '../config';
import { ILocalUser } from '../models/user'; import { ILocalUser } from '../models/user';
import { program } from '../argv'; import { program } from '../argv';
import handler from './processors';
import { queueLogger } from './logger';
const enableQueue = !program.disableQueue; import processDeliver from './processors/deliver';
const enableQueueProcessing = !program.onlyServer && enableQueue; import processInbox from './processors/process-inbox';
const queueAvailable = config.redis != null; import processDb from './processors/db';
const queue = initializeQueue(); function initializeQueue(name: string) {
return new Queue(name, config.redis != null ? {
function initializeQueue() {
if (queueAvailable && enableQueue) {
return new Queue('misskey-queue', {
redis: { redis: {
port: config.redis.port, port: config.redis.port,
host: config.redis.host, host: config.redis.host,
password: config.redis.pass password: config.redis.pass,
}, db: 1
removeOnSuccess: true,
removeOnFailure: true,
getEvents: false,
sendEvents: false,
storeJobs: false
});
} else {
return null;
} }
} : null);
} }
const deliverQueue = initializeQueue('deliver');
const inboxQueue = initializeQueue('inbox');
const dbQueue = initializeQueue('db');
export function deliver(user: ILocalUser, content: any, to: any) { export function deliver(user: ILocalUser, content: any, to: any) {
if (content == null) return; if (content == null) return null;
const data = { const data = {
type: 'deliver',
user, user,
content, content,
to to
}; };
if (queueAvailable && enableQueueProcessing) { return deliverQueue.add(data, {
return queue.createJob(data) attempts: 4,
.retries(8) backoff: {
.backoff('exponential', 1000) type: 'exponential',
.save(); delay: 1000
} else { },
return handler({ data }, () => {}); removeOnComplete: true,
} removeOnFail: true
});
} }
export function processInbox(activity: any, signature: httpSignature.IParsedSignature) { export function inbox(activity: any, signature: httpSignature.IParsedSignature) {
const data = { const data = {
type: 'processInbox',
activity: activity, activity: activity,
signature signature
}; };
if (queueAvailable && enableQueueProcessing) { return inboxQueue.add(data, {
return queue.createJob(data) attempts: 4,
.retries(3) backoff: {
.backoff('exponential', 500) type: 'exponential',
.save(); delay: 1000
} else { },
return handler({ data }, () => {}); removeOnComplete: true,
} removeOnFail: true
});
} }
export function createDeleteNotesJob(user: ILocalUser) { export function createDeleteNotesJob(user: ILocalUser) {
@ -76,11 +67,10 @@ export function createDeleteNotesJob(user: ILocalUser) {
user: user user: user
}; };
if (queueAvailable && enableQueueProcessing) { return dbQueue.add(data, {
return queue.createJob(data).save(); removeOnComplete: true,
} else { removeOnFail: true
return handler({ data }, () => {}); });
}
} }
export function createDeleteDriveFilesJob(user: ILocalUser) { export function createDeleteDriveFilesJob(user: ILocalUser) {
@ -89,11 +79,10 @@ export function createDeleteDriveFilesJob(user: ILocalUser) {
user: user user: user
}; };
if (queueAvailable && enableQueueProcessing) { return dbQueue.add(data, {
return queue.createJob(data).save(); removeOnComplete: true,
} else { removeOnFail: true
return handler({ data }, () => {}); });
}
} }
export function createExportNotesJob(user: ILocalUser) { export function createExportNotesJob(user: ILocalUser) {
@ -102,11 +91,10 @@ export function createExportNotesJob(user: ILocalUser) {
user: user user: user
}; };
if (queueAvailable && enableQueueProcessing) { return dbQueue.add(data, {
return queue.createJob(data).save(); removeOnComplete: true,
} else { removeOnFail: true
return handler({ data }, () => {}); });
}
} }
export function createExportFollowingJob(user: ILocalUser) { export function createExportFollowingJob(user: ILocalUser) {
@ -115,11 +103,10 @@ export function createExportFollowingJob(user: ILocalUser) {
user: user user: user
}; };
if (queueAvailable && enableQueueProcessing) { return dbQueue.add(data, {
return queue.createJob(data).save(); removeOnComplete: true,
} else { removeOnFail: true
return handler({ data }, () => {}); });
}
} }
export function createExportMuteJob(user: ILocalUser) { export function createExportMuteJob(user: ILocalUser) {
@ -128,11 +115,10 @@ export function createExportMuteJob(user: ILocalUser) {
user: user user: user
}; };
if (queueAvailable && enableQueueProcessing) { return dbQueue.add(data, {
return queue.createJob(data).save(); removeOnComplete: true,
} else { removeOnFail: true
return handler({ data }, () => {}); });
}
} }
export function createExportBlockingJob(user: ILocalUser) { export function createExportBlockingJob(user: ILocalUser) {
@ -141,24 +127,23 @@ export function createExportBlockingJob(user: ILocalUser) {
user: user user: user
}; };
if (queueAvailable && enableQueueProcessing) { return dbQueue.add(data, {
return queue.createJob(data).save(); removeOnComplete: true,
} else { removeOnFail: true
return handler({ data }, () => {}); });
}
} }
export default function() { export default function() {
if (queueAvailable && enableQueueProcessing) { if (!program.onlyServer) {
queue.process(128, handler); deliverQueue.process(processDeliver);
queueLogger.succ('Processing started'); inboxQueue.process(processInbox);
dbQueue.process(processDb);
} }
return queue;
} }
export function destroy() { export function destroy() {
/*
queue.destroy().then(n => { queue.destroy().then(n => {
queueLogger.succ(`All job removed (${n} jobs)`); queueLogger.succ(`All job removed (${n} jobs)`);
}); });*/
} }

16
src/queue/processors/delete-drive-files.ts → src/queue/processors/db/delete-drive-files.ts
View File

@ -1,14 +1,14 @@
import * as bq from 'bee-queue'; import * as Bull from 'bull';
import * as mongo from 'mongodb'; import * as mongo from 'mongodb';
import { queueLogger } from '../logger'; import { queueLogger } from '../../logger';
import User from '../../models/user'; import User from '../../../models/user';
import DriveFile from '../../models/drive-file'; import DriveFile from '../../../models/drive-file';
import deleteFile from '../../services/drive/delete-file'; import deleteFile from '../../../services/drive/delete-file';
const logger = queueLogger.createSubLogger('delete-drive-files'); const logger = queueLogger.createSubLogger('delete-drive-files');
export async function deleteDriveFiles(job: bq.Job, done: any): Promise<void> { export async function deleteDriveFiles(job: Bull.Job, done: any): Promise<void> {
logger.info(`Deleting drive files of ${job.data.user._id} ...`); logger.info(`Deleting drive files of ${job.data.user._id} ...`);
const user = await User.findOne({ const user = await User.findOne({
@ -32,7 +32,7 @@ export async function deleteDriveFiles(job: bq.Job, done: any): Promise<void> {
if (files.length === 0) { if (files.length === 0) {
ended = true; ended = true;
if (job.reportProgress) job.reportProgress(100); job.progress(100);
break; break;
} }
@ -47,7 +47,7 @@ export async function deleteDriveFiles(job: bq.Job, done: any): Promise<void> {
userId: user._id, userId: user._id,
}); });
if (job.reportProgress) job.reportProgress(deletedCount / total); job.progress(deletedCount / total);
} }
logger.succ(`All drive files (${deletedCount}) of ${user._id} has been deleted.`); logger.succ(`All drive files (${deletedCount}) of ${user._id} has been deleted.`);

16
src/queue/processors/delete-notes.ts → src/queue/processors/db/delete-notes.ts
View File

@ -1,14 +1,14 @@
import * as bq from 'bee-queue'; import * as Bull from 'bull';
import * as mongo from 'mongodb'; import * as mongo from 'mongodb';
import { queueLogger } from '../logger'; import { queueLogger } from '../../logger';
import Note from '../../models/note'; import Note from '../../../models/note';
import deleteNote from '../../services/note/delete'; import deleteNote from '../../../services/note/delete';
import User from '../../models/user'; import User from '../../../models/user';
const logger = queueLogger.createSubLogger('delete-notes'); const logger = queueLogger.createSubLogger('delete-notes');
export async function deleteNotes(job: bq.Job, done: any): Promise<void> { export async function deleteNotes(job: Bull.Job, done: any): Promise<void> {
logger.info(`Deleting notes of ${job.data.user._id} ...`); logger.info(`Deleting notes of ${job.data.user._id} ...`);
const user = await User.findOne({ const user = await User.findOne({
@ -32,7 +32,7 @@ export async function deleteNotes(job: bq.Job, done: any): Promise<void> {
if (notes.length === 0) { if (notes.length === 0) {
ended = true; ended = true;
if (job.reportProgress) job.reportProgress(100); job.progress(100);
break; break;
} }
@ -47,7 +47,7 @@ export async function deleteNotes(job: bq.Job, done: any): Promise<void> {
userId: user._id, userId: user._id,
}); });
if (job.reportProgress) job.reportProgress(deletedCount / total); job.progress(deletedCount / total);
} }
logger.succ(`All notes (${deletedCount}) of ${user._id} has been deleted.`); logger.succ(`All notes (${deletedCount}) of ${user._id} has been deleted.`);

18
src/queue/processors/export-blocking.ts → src/queue/processors/db/export-blocking.ts
View File

@ -1,18 +1,18 @@
import * as bq from 'bee-queue'; import * as Bull from 'bull';
import * as tmp from 'tmp'; import * as tmp from 'tmp';
import * as fs from 'fs'; import * as fs from 'fs';
import * as mongo from 'mongodb'; import * as mongo from 'mongodb';
import { queueLogger } from '../logger'; import { queueLogger } from '../../logger';
import addFile from '../../services/drive/add-file'; import addFile from '../../../services/drive/add-file';
import User from '../../models/user'; import User from '../../../models/user';
import dateFormat = require('dateformat'); import dateFormat = require('dateformat');
import Blocking from '../../models/blocking'; import Blocking from '../../../models/blocking';
import config from '../../config'; import config from '../../../config';
const logger = queueLogger.createSubLogger('export-blocking'); const logger = queueLogger.createSubLogger('export-blocking');
export async function exportBlocking(job: bq.Job, done: any): Promise<void> { export async function exportBlocking(job: Bull.Job, done: any): Promise<void> {
logger.info(`Exporting blocking of ${job.data.user._id} ...`); logger.info(`Exporting blocking of ${job.data.user._id} ...`);
const user = await User.findOne({ const user = await User.findOne({
@ -48,7 +48,7 @@ export async function exportBlocking(job: bq.Job, done: any): Promise<void> {
if (blockings.length === 0) { if (blockings.length === 0) {
ended = true; ended = true;
if (job.reportProgress) job.reportProgress(100); job.progress(100);
break; break;
} }
@ -74,7 +74,7 @@ export async function exportBlocking(job: bq.Job, done: any): Promise<void> {
blockerId: user._id, blockerId: user._id,
}); });
if (job.reportProgress) job.reportProgress(exportedCount / total); job.progress(exportedCount / total);
} }
stream.end(); stream.end();

18
src/queue/processors/export-following.ts → src/queue/processors/db/export-following.ts
View File

@ -1,18 +1,18 @@
import * as bq from 'bee-queue'; import * as Bull from 'bull';
import * as tmp from 'tmp'; import * as tmp from 'tmp';
import * as fs from 'fs'; import * as fs from 'fs';
import * as mongo from 'mongodb'; import * as mongo from 'mongodb';
import { queueLogger } from '../logger'; import { queueLogger } from '../../logger';
import addFile from '../../services/drive/add-file'; import addFile from '../../../services/drive/add-file';
import User from '../../models/user'; import User from '../../../models/user';
import dateFormat = require('dateformat'); import dateFormat = require('dateformat');
import Following from '../../models/following'; import Following from '../../../models/following';
import config from '../../config'; import config from '../../../config';
const logger = queueLogger.createSubLogger('export-following'); const logger = queueLogger.createSubLogger('export-following');
export async function exportFollowing(job: bq.Job, done: any): Promise<void> { export async function exportFollowing(job: Bull.Job, done: any): Promise<void> {
logger.info(`Exporting following of ${job.data.user._id} ...`); logger.info(`Exporting following of ${job.data.user._id} ...`);
const user = await User.findOne({ const user = await User.findOne({
@ -48,7 +48,7 @@ export async function exportFollowing(job: bq.Job, done: any): Promise<void> {
if (followings.length === 0) { if (followings.length === 0) {
ended = true; ended = true;
if (job.reportProgress) job.reportProgress(100); job.progress(100);
break; break;
} }
@ -74,7 +74,7 @@ export async function exportFollowing(job: bq.Job, done: any): Promise<void> {
followerId: user._id, followerId: user._id,
}); });
if (job.reportProgress) job.reportProgress(exportedCount / total); job.progress(exportedCount / total);
} }
stream.end(); stream.end();

18
src/queue/processors/export-mute.ts → src/queue/processors/db/export-mute.ts
View File

@ -1,18 +1,18 @@
import * as bq from 'bee-queue'; import * as Bull from 'bull';
import * as tmp from 'tmp'; import * as tmp from 'tmp';
import * as fs from 'fs'; import * as fs from 'fs';
import * as mongo from 'mongodb'; import * as mongo from 'mongodb';
import { queueLogger } from '../logger'; import { queueLogger } from '../../logger';
import addFile from '../../services/drive/add-file'; import addFile from '../../../services/drive/add-file';
import User from '../../models/user'; import User from '../../../models/user';
import dateFormat = require('dateformat'); import dateFormat = require('dateformat');
import Mute from '../../models/mute'; import Mute from '../../../models/mute';
import config from '../../config'; import config from '../../../config';
const logger = queueLogger.createSubLogger('export-mute'); const logger = queueLogger.createSubLogger('export-mute');
export async function exportMute(job: bq.Job, done: any): Promise<void> { export async function exportMute(job: Bull.Job, done: any): Promise<void> {
logger.info(`Exporting mute of ${job.data.user._id} ...`); logger.info(`Exporting mute of ${job.data.user._id} ...`);
const user = await User.findOne({ const user = await User.findOne({
@ -48,7 +48,7 @@ export async function exportMute(job: bq.Job, done: any): Promise<void> {
if (mutes.length === 0) { if (mutes.length === 0) {
ended = true; ended = true;
if (job.reportProgress) job.reportProgress(100); job.progress(100);
break; break;
} }
@ -74,7 +74,7 @@ export async function exportMute(job: bq.Job, done: any): Promise<void> {
muterId: user._id, muterId: user._id,
}); });
if (job.reportProgress) job.reportProgress(exportedCount / total); job.progress(exportedCount / total);
} }
stream.end(); stream.end();

16
src/queue/processors/export-notes.ts → src/queue/processors/db/export-notes.ts
View File

@ -1,17 +1,17 @@
import * as bq from 'bee-queue'; import * as Bull from 'bull';
import * as tmp from 'tmp'; import * as tmp from 'tmp';
import * as fs from 'fs'; import * as fs from 'fs';
import * as mongo from 'mongodb'; import * as mongo from 'mongodb';
import { queueLogger } from '../logger'; import { queueLogger } from '../../logger';
import Note, { INote } from '../../models/note'; import Note, { INote } from '../../../models/note';
import addFile from '../../services/drive/add-file'; import addFile from '../../../services/drive/add-file';
import User from '../../models/user'; import User from '../../../models/user';
import dateFormat = require('dateformat'); import dateFormat = require('dateformat');
const logger = queueLogger.createSubLogger('export-notes'); const logger = queueLogger.createSubLogger('export-notes');
export async function exportNotes(job: bq.Job, done: any): Promise<void> { export async function exportNotes(job: Bull.Job, done: any): Promise<void> {
logger.info(`Exporting notes of ${job.data.user._id} ...`); logger.info(`Exporting notes of ${job.data.user._id} ...`);
const user = await User.findOne({ const user = await User.findOne({
@ -58,7 +58,7 @@ export async function exportNotes(job: bq.Job, done: any): Promise<void> {
if (notes.length === 0) { if (notes.length === 0) {
ended = true; ended = true;
if (job.reportProgress) job.reportProgress(100); job.progress(100);
break; break;
} }
@ -83,7 +83,7 @@ export async function exportNotes(job: bq.Job, done: any): Promise<void> {
userId: user._id, userId: user._id,
}); });
if (job.reportProgress) job.reportProgress(exportedNotesCount / total); job.progress(exportedNotesCount / total);
} }
await new Promise((res, rej) => { await new Promise((res, rej) => {

23
src/queue/processors/index.ts → src/queue/processors/db/index.ts
View File

@ -1,31 +1,20 @@
import deliver from './http/deliver'; import * as Bull from 'bull';
import processInbox from './http/process-inbox';
import { deleteNotes } from './delete-notes'; import { deleteNotes } from './delete-notes';
import { deleteDriveFiles } from './delete-drive-files'; import { deleteDriveFiles } from './delete-drive-files';
import { exportNotes } from './export-notes'; import { exportNotes } from './export-notes';
import { exportFollowing } from './export-following'; import { exportFollowing } from './export-following';
import { exportMute } from './export-mute'; import { exportMute } from './export-mute';
import { exportBlocking } from './export-blocking'; import { exportBlocking } from './export-blocking';
import { queueLogger } from '../logger';
const handlers: any = { const jobs = {
deliver,
processInbox,
deleteNotes, deleteNotes,
deleteDriveFiles, deleteDriveFiles,
exportNotes, exportNotes,
exportFollowing, exportFollowing,
exportMute, exportMute,
exportBlocking, exportBlocking,
}; } as any;
export default (job: any, done: any) => { export default function(job: Bull.Job, done: any) {
const handler = handlers[job.data.type]; jobs[job.data.type](job, done);
}
if (handler) {
handler(job, done);
} else {
queueLogger.error(`Unknown job: ${job.data.type}`);
done();
}
};

15
src/queue/processors/http/deliver.ts → src/queue/processors/deliver.ts
View File

@ -1,14 +1,13 @@
import * as bq from 'bee-queue'; import * as Bull from 'bull';
import request from '../../remote/activitypub/request';
import request from '../../../remote/activitypub/request'; import { queueLogger } from '../logger';
import { queueLogger } from '../../logger'; import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-instance-doc';
import { registerOrFetchInstanceDoc } from '../../../services/register-or-fetch-instance-doc'; import Instance from '../../models/instance';
import Instance from '../../../models/instance'; import instanceChart from '../../services/chart/instance';
import instanceChart from '../../../services/chart/instance';
let latest: string = null; let latest: string = null;
export default async (job: bq.Job, done: any): Promise<void> => { export default async (job: Bull.Job, done: any): Promise<void> => {
const { host } = new URL(job.data.to); const { host } = new URL(job.data.to);
try { try {

22
src/queue/processors/http/process-inbox.ts → src/queue/processors/process-inbox.ts
View File

@ -1,21 +1,21 @@
import * as bq from 'bee-queue'; import * as Bull from 'bull';
import * as httpSignature from 'http-signature'; import * as httpSignature from 'http-signature';
import parseAcct from '../../../misc/acct/parse'; import parseAcct from '../../misc/acct/parse';
import User, { IRemoteUser } from '../../../models/user'; import User, { IRemoteUser } from '../../models/user';
import perform from '../../../remote/activitypub/perform'; import perform from '../../remote/activitypub/perform';
import { resolvePerson, updatePerson } from '../../../remote/activitypub/models/person'; import { resolvePerson, updatePerson } from '../../remote/activitypub/models/person';
import { toUnicode } from 'punycode'; import { toUnicode } from 'punycode';
import { URL } from 'url'; import { URL } from 'url';
import { publishApLogStream } from '../../../services/stream'; import { publishApLogStream } from '../../services/stream';
import Logger from '../../../services/logger'; import Logger from '../../services/logger';
import { registerOrFetchInstanceDoc } from '../../../services/register-or-fetch-instance-doc'; import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-instance-doc';
import Instance from '../../../models/instance'; import Instance from '../../models/instance';
import instanceChart from '../../../services/chart/instance'; import instanceChart from '../../services/chart/instance';
const logger = new Logger('inbox'); const logger = new Logger('inbox');
// ユーザーのinboxにアクティビティが届いた時の処理 // ユーザーのinboxにアクティビティが届いた時の処理
export default async (job: bq.Job, done: any): Promise<void> => { export default async (job: Bull.Job, done: any): Promise<void> => {
const signature = job.data.signature; const signature = job.data.signature;
const activity = job.data.activity; const activity = job.data.activity;

2
src/server/activitypub.ts
View File

@ -16,7 +16,7 @@ import Followers from './activitypub/followers';
import Following from './activitypub/following'; import Following from './activitypub/following';
import Featured from './activitypub/featured'; import Featured from './activitypub/featured';
import renderQuestion from '../remote/activitypub/renderer/question'; import renderQuestion from '../remote/activitypub/renderer/question';
import { processInbox } from '../queue'; import { inbox as processInbox } from '../queue';
// Init router // Init router
const router = new Router(); const router = new Router();