diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml
index f1517bf0c1..43f6651b24 100644
--- a/locales/ja-JP.yml
+++ b/locales/ja-JP.yml
@@ -557,6 +557,9 @@ common/views/components/profile-editor.vue:
email-address: "メールアドレス"
email-verified: "メールアドレスが確認されました"
email-not-verified: "メールアドレスが確認されていません。メールボックスをご確認ください。"
+ export: "エクスポート"
+ export-notes: "すべての投稿のエクスポート"
+ export-requested: "エクスポートをリクエストしました。これには時間がかかる場合があります。エクスポートが終わると、ドライブにファイルが追加されます。"
common/views/components/user-list-editor.vue:
users: "ユーザー"
diff --git a/src/client/app/common/views/components/profile-editor.vue b/src/client/app/common/views/components/profile-editor.vue
index feffd25437..d745e7d291 100644
--- a/src/client/app/common/views/components/profile-editor.vue
+++ b/src/client/app/common/views/components/profile-editor.vue
@@ -87,6 +87,14 @@
{{ $t('save') }}
+
+
+
+
+
+ {{ $t('export-notes') }}
+
+
@@ -252,6 +260,15 @@ export default Vue.extend({
email: this.email == '' ? null : this.email
});
});
+ },
+
+ exportNotes() {
+ this.$root.api('i/export-notes', {});
+
+ this.$root.dialog({
+ type: 'info',
+ text: this.$t('export-requested')
+ });
}
}
});
diff --git a/src/queue/index.ts b/src/queue/index.ts
index 65c52d864c..cf8af17a48 100644
--- a/src/queue/index.ts
+++ b/src/queue/index.ts
@@ -1,9 +1,9 @@
import * as Queue from 'bee-queue';
import config from '../config';
-import http from './processors/http';
+
import { ILocalUser } from '../models/user';
-import Logger from '../misc/logger';
import { program } from '../argv';
+import handler from './processors';
const enableQueue = config.redis != null && !program.disableQueue;
@@ -36,7 +36,7 @@ export function createHttpJob(data: any) {
.backoff('exponential', 16384) // 16s
.save();
} else {
- return http({ data }, () => {});
+ return handler({ data }, () => {});
}
}
@@ -51,10 +51,18 @@ export function deliver(user: ILocalUser, content: any, to: any) {
});
}
-export const queueLogger = new Logger('queue');
+export function createExportNotesJob(user: ILocalUser) {
+ if (!enableQueue) throw 'queue disabled';
+
+ return queue.createJob({
+ type: 'exportNotes',
+ user: user
+ })
+ .save();
+}
export default function() {
if (enableQueue) {
- queue.process(128, http);
+ queue.process(128, handler);
}
}
diff --git a/src/queue/logger.ts b/src/queue/logger.ts
new file mode 100644
index 0000000000..99d88bd63b
--- /dev/null
+++ b/src/queue/logger.ts
@@ -0,0 +1,3 @@
+import Logger from '../misc/logger';
+
+export const queueLogger = new Logger('queue', 'orange');
diff --git a/src/queue/processors/export-notes.ts b/src/queue/processors/export-notes.ts
new file mode 100644
index 0000000000..52845a5a9c
--- /dev/null
+++ b/src/queue/processors/export-notes.ts
@@ -0,0 +1,128 @@
+import * as bq from 'bee-queue';
+import * as tmp from 'tmp';
+import * as fs from 'fs';
+import * as mongo from 'mongodb';
+
+import { queueLogger } from '../logger';
+import Note, { INote } from '../../models/note';
+import addFile from '../../services/drive/add-file';
+import User from '../../models/user';
+import dateFormat = require('dateformat');
+
+const logger = queueLogger.createSubLogger('export-notes');
+
+export async function exportNotes(job: bq.Job, done: any): Promise {
+ logger.info(`Exporting notes of ${job.data.user._id} ...`);
+
+ const user = await User.findOne({
+ _id: new mongo.ObjectID(job.data.user._id.toString())
+ });
+
+ // Create temp file
+ const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
+ tmp.file((e, path, fd, cleanup) => {
+ if (e) return rej(e);
+ res([path, cleanup]);
+ });
+ });
+
+ logger.info(`Temp file is ${path}`);
+
+ const stream = fs.createWriteStream(path, { flags: 'a' });
+
+ await new Promise((res, rej) => {
+ stream.write('[', err => {
+ if (err) {
+ logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+
+ let exportedNotesCount = 0;
+ let ended = false;
+ let cursor: any = null;
+
+ while (!ended) {
+ const notes = await Note.find({
+ userId: user._id,
+ ...(cursor ? { _id: { $gt: cursor } } : {})
+ }, {
+ limit: 100,
+ sort: {
+ _id: 1
+ }
+ });
+
+ if (notes.length === 0) {
+ ended = true;
+ job.reportProgress(100);
+ break;
+ }
+
+ cursor = notes[notes.length - 1]._id;
+
+ for (const note of notes) {
+ const content = JSON.stringify(serialize(note));
+ await new Promise((res, rej) => {
+ stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => {
+ if (err) {
+ logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ exportedNotesCount++;
+ }
+
+ const total = await Note.count({
+ userId: user._id,
+ });
+
+ job.reportProgress(exportedNotesCount / total);
+ }
+
+ await new Promise((res, rej) => {
+ stream.write(']', err => {
+ if (err) {
+ logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+
+ stream.end();
+ logger.succ(`Exported to: ${path}`);
+
+ const fileName = dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json';
+ const driveFile = await addFile(user, path, fileName);
+
+ logger.succ(`Exported to: ${driveFile._id}`);
+ cleanup();
+ done();
+}
+
+function serialize(note: INote): any {
+ return {
+ id: note._id,
+ text: note.text,
+ createdAt: note.createdAt,
+ fileIds: note.fileIds,
+ replyId: note.replyId,
+ renoteId: note.renoteId,
+ poll: note.poll,
+ cw: note.cw,
+ viaMobile: note.viaMobile,
+ visibility: note.visibility,
+ visibleUserIds: note.visibleUserIds,
+ appId: note.appId,
+ geo: note.geo,
+ localOnly: note.localOnly
+ };
+}
diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts
index d8d90a2773..d1dad55cd7 100644
--- a/src/queue/processors/http/deliver.ts
+++ b/src/queue/processors/http/deliver.ts
@@ -1,7 +1,7 @@
import * as bq from 'bee-queue';
import request from '../../../remote/activitypub/request';
-import { queueLogger } from '../..';
+import { queueLogger } from '../../logger';
export default async (job: bq.Job, done: any): Promise => {
try {
diff --git a/src/queue/processors/http/index.ts b/src/queue/processors/index.ts
similarity index 57%
rename from src/queue/processors/http/index.ts
rename to src/queue/processors/index.ts
index 74ed723bd3..3f08fe29fb 100644
--- a/src/queue/processors/http/index.ts
+++ b/src/queue/processors/index.ts
@@ -1,10 +1,12 @@
-import deliver from './deliver';
-import processInbox from './process-inbox';
-import { queueLogger } from '../..';
+import deliver from './http/deliver';
+import processInbox from './http/process-inbox';
+import { exportNotes } from './export-notes';
+import { queueLogger } from '../logger';
const handlers: any = {
deliver,
processInbox,
+ exportNotes,
};
export default (job: any, done: any) => {
diff --git a/src/server/api/endpoints/i/export-notes.ts b/src/server/api/endpoints/i/export-notes.ts
new file mode 100644
index 0000000000..6da6e68b9d
--- /dev/null
+++ b/src/server/api/endpoints/i/export-notes.ts
@@ -0,0 +1,18 @@
+import define from '../../define';
+import { createExportNotesJob } from '../../../../queue';
+import ms = require('ms');
+
+export const meta = {
+ secure: true,
+ requireCredential: true,
+ limit: {
+ duration: ms('1day'),
+ max: 1,
+ },
+};
+
+export default define(meta, (ps, user) => new Promise(async (res, rej) => {
+ createExportNotesJob(user);
+
+ res();
+}));