natty
/
calckey
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Resolve #4259

This commit is contained in:
syuilo 2019-03-11 19:43:58 +09:00
parent bb6ede2b8f
commit c3d34bda37
No known key found for this signature in database
GPG Key ID: BDC4C49D06AB9D69
6 changed files with 243 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
locales/ja-JP.yml
View File

@ -701,6 +701,8 @@ common/views/components/profile-editor.vue:
email-verified: "メールアドレスが確認されました"
email-not-verified: "メールアドレスが確認されていません。メールボックスをご確認ください。"
export: "エクスポート"
import: "インポート"
export-and-import: "エクスポートとインポート"
export-targets:
all-notes: "すべての投稿データ"
following-list: "フォロー"
@ -708,6 +710,7 @@ common/views/components/profile-editor.vue:
blocking-list: "ブロック"
user-lists: "リスト"
export-requested: "エクスポートをリクエストしました。これには時間がかかる場合があります。エクスポートが終わると、ドライブにファイルが追加されます。"
import-requested: "インポートをリクエストしました。これには時間がかかる場合があります。"
enter-password: "パスワードを入力してください"
danger-zone: "危険な設定"
delete-account: "アカウントを削除"

26
src/client/app/common/views/components/settings/profile.vue
View File

@ -89,7 +89,7 @@
</section>
<section>
<header>{{ $t('export') }}</header>
<header>{{ $t('export-and-import') }}</header>
<div>
<ui-select v-model="exportTarget">
@ -99,7 +99,10 @@
<option value="blocking">{{ $t('export-targets.blocking-list') }}</option>
<option value="user-lists">{{ $t('export-targets.user-lists') }}</option>
</ui-select>
<ui-button @click="doExport()"><fa :icon="faDownload"/> {{ $t('export') }}</ui-button>
<ui-horizon-group class="fit-bottom">
<ui-button @click="doExport()"><fa :icon="faDownload"/> {{ $t('export') }}</ui-button>
<ui-button @click="doImport()" :disabled="!['user-lists'].includes(exportTarget)"><fa :icon="faUpload"/> {{ $t('import') }}</ui-button>
</ui-horizon-group>
</div>
</section>
@ -119,7 +122,7 @@ import { apiUrl, host } from '../../../../config';
import { toUnicode } from 'punycode';
import langmap from 'langmap';
import { unique } from '../../../../../../prelude/array';
import { faDownload } from '@fortawesome/free-solid-svg-icons';
import { faDownload, faUpload } from '@fortawesome/free-solid-svg-icons';
export default Vue.extend({
i18n: i18n('common/views/components/profile-editor.vue'),
@ -148,7 +151,7 @@ export default Vue.extend({
avatarUploading: false,
bannerUploading: false,
exportTarget: 'notes',
faDownload
faDownload, faUpload
};
},
@ -294,6 +297,21 @@ export default Vue.extend({
});
},
doImport() {
this.$chooseDriveFile().then(file => {
this.$root.api(
this.exportTarget == 'user-lists' ? 'i/import-user-lists' :
null, {
fileId: file.id
});
this.$root.dialog({
type: 'info',
text: this.$t('import-requested')
});
});
},
async deleteAccount() {
const { canceled: canceled, result: password } = await this.$root.dialog({
title: this.$t('enter-password'),

11
src/queue/index.ts
View File

@ -9,6 +9,7 @@ import processDeliver from './processors/deliver';
import processInbox from './processors/inbox';
import processDb from './processors/db';
import { queueLogger } from './logger';
import { IDriveFile } from '../models/drive-file';
function initializeQueue(name: string) {
return new Queue(name, config.redis != null ? {
@ -145,6 +146,16 @@ export function createExportUserListsJob(user: ILocalUser) {
});
}
export function createImportUserListsJob(user: ILocalUser, fileId: IDriveFile['_id']) {
return dbQueue.add('importUserLists', {
user: user,
fileId: fileId
}, {
removeOnComplete: true,
removeOnFail: true
});
}
export default function() {
if (!program.onlyServer) {
deliverQueue.process(128, processDeliver);

140
src/queue/processors/db/import-user-lists.ts Normal file
View File

@ -0,0 +1,140 @@
import * as Bull from 'bull';
import * as tmp from 'tmp';
import * as fs from 'fs';
import * as util from 'util';
import * as mongo from 'mongodb';
import * as request from 'request';
import { queueLogger } from '../../logger';
import User from '../../../models/user';
import config from '../../../config';
import UserList from '../../../models/user-list';
import DriveFile from '../../../models/drive-file';
import chalk from 'chalk';
import { getOriginalUrl } from '../../../misc/get-drive-file-url';
import parseAcct from '../../../misc/acct/parse';
import resolveUser from '../../../remote/resolve-user';
const logger = queueLogger.createSubLogger('import-user-lists');
export async function importUserLists(job: Bull.Job, done: any): Promise<void> {
logger.info(`Importing user lists of ${job.data.user._id} ...`);
const user = await User.findOne({
_id: new mongo.ObjectID(job.data.user._id.toString())
});
const file = await DriveFile.findOne({
_id: new mongo.ObjectID(job.data.fileId.toString())
});
const url = getOriginalUrl(file);
// Create temp file
const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
logger.info(`Temp file is ${path}`);
// write content at URL to temp file
await new Promise((res, rej) => {
logger.info(`Downloading ${chalk.cyan(url)} ...`);
const writable = fs.createWriteStream(path);
writable.on('finish', () => {
logger.succ(`Download finished: ${chalk.cyan(url)}`);
res();
});
writable.on('error', error => {
logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
rej(error);
});
const requestUrl = new URL(url).pathname.match(/[^\u0021-\u00ff]/) ? encodeURI(url) : url;
const req = request({
url: requestUrl,
proxy: config.proxy,
timeout: 10 * 1000,
headers: {
'User-Agent': config.userAgent
}
});
req.pipe(writable);
req.on('response', response => {
if (response.statusCode !== 200) {
logger.error(`Got ${response.statusCode} (${url})`);
writable.close();
rej(response.statusCode);
}
});
req.on('error', error => {
logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
writable.close();
rej(error);
});
});
logger.succ(`Downloaded to: ${path}`);
const csv = await util.promisify(fs.readFile)(path, 'utf8');
for (const line of csv.trim().split('\n')) {
const listName = line.split(',')[0].trim();
const { username, host } = parseAcct(line.split(',')[1].trim());
let list = await UserList.findOne({
userId: user._id,
title: listName
});
if (list == null) {
list = await UserList.insert({
createdAt: new Date(),
userId: user._id,
title: listName,
userIds: []
});
}
let target = host === config.host ? await User.findOne({
host: null,
usernameLower: username.toLowerCase()
}) : await User.findOne({
host: host,
usernameLower: username.toLowerCase()
});
if (host == null && target == null) continue;
if (list.userIds.some(id => id.equals(target._id))) continue;
if (target == null) {
target = await resolveUser(username, host);
}
await UserList.update({ _id: list._id }, {
$push: {
userIds: target._id
}
});
}
logger.succ('Imported');
cleanup();
done();
}

4
src/queue/processors/db/index.ts
View File

@ -6,6 +6,7 @@ import { exportFollowing } from './export-following';
import { exportMute } from './export-mute';
import { exportBlocking } from './export-blocking';
import { exportUserLists } from './export-user-lists';
import { importUserLists } from './import-user-lists';
const jobs = {
deleteNotes,
@ -14,7 +15,8 @@ const jobs = {
exportFollowing,
exportMute,
exportBlocking,
exportUserLists
exportUserLists,
importUserLists
} as any;
export default function(dbQueue: Bull.Queue) {

64
src/server/api/endpoints/i/import-user-lists.ts Normal file
View File

@ -0,0 +1,64 @@
import $ from 'cafy';
import ID, { transform } from '../../../../misc/cafy-id';
import define from '../../define';
import { createImportUserListsJob } from '../../../../queue';
import ms = require('ms');
import DriveFile from '../../../../models/drive-file';
import { ApiError } from '../../error';
export const meta = {
secure: true,
requireCredential: true,
limit: {
duration: ms('1hour'),
max: 1,
},
params: {
fileId: {
validator: $.type(ID),
transform: transform,
}
},
errors: {
noSuchFile: {
message: 'No such file.',
code: 'NO_SUCH_FILE',
id: 'ea9cc34f-c415-4bc6-a6fe-28ac40357049'
},
unexpectedFileType: {
message: 'We need csv file.',
code: 'UNEXPECTED_FILE_TYPE',
id: 'a3c9edda-dd9b-4596-be6a-150ef813745c'
},
tooBigFile: {
message: 'That file is too big.',
code: 'TOO_BIG_FILE',
id: 'ae6e7a22-971b-4b52-b2be-fc0b9b121fe9'
},
emptyFile: {
message: 'That file is empty.',
code: 'EMPTY_FILE',
id: '99efe367-ce6e-4d44-93f8-5fae7b040356'
},
}
};
export default define(meta, async (ps, user) => {
const file = await DriveFile.findOne({
_id: ps.fileId
});
if (file == null) throw new ApiError(meta.errors.noSuchFile);
//if (!file.contentType.endsWith('/csv')) throw new ApiError(meta.errors.unexpectedFileType);
if (file.length > 30000) throw new ApiError(meta.errors.tooBigFile);
if (file.length === 0) throw new ApiError(meta.errors.emptyFile);
createImportUserListsJob(user, file._id);
return;
});