natty
/
misskey-awawa
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: emojis import

This commit is contained in:
syuilo 2022-01-13 00:47:40 +09:00
parent 1f2dab0a83
commit 7e5e763584
6 changed files with 205 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/backend/package.json
View File

@ -180,6 +180,7 @@
"typeorm": "0.2.39",
"typescript": "4.4.4",
"ulid": "2.3.0",
"unzipper": "0.10.11",
"uuid": "8.3.2",
"web-push": "3.4.5",
"websocket": "1.0.34",

12
packages/backend/src/queue/processors/db/export-custom-emojis.ts
View File

@ -52,7 +52,7 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi
});
};
await writeMeta(`{"metaVersion":1,"emojis":[`);
await writeMeta(`{"metaVersion":2,"emojis":[`);
const customEmojis = await Emojis.find({
where: {
@ -64,9 +64,9 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi
});
for (const emoji of customEmojis) {
const exportId = ulid().toLowerCase();
const ext = mime.extension(emoji.type);
const emojiPath = path + '/' + exportId + (ext ? '.' + ext : '');
const fileName = emoji.name + (ext ? '.' + ext : '');
const emojiPath = path + '/' + fileName;
fs.writeFileSync(emojiPath, '', 'binary');
let downloaded = false;
@ -77,8 +77,12 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi
logger.error(e);
}
if (!downloaded) {
fs.unlinkSync(emojiPath);
}
const content = JSON.stringify({
id: exportId,
fileName: fileName,
downloaded: downloaded,
emoji: emoji,
});

84
packages/backend/src/queue/processors/db/import-custom-emojis.ts Normal file
View File

@ -0,0 +1,84 @@
import * as Bull from 'bull';
import * as tmp from 'tmp';
import * as fs from 'fs';
const unzipper = require('unzipper');
import { getConnection } from 'typeorm';
import { queueLogger } from '../../logger';
import { downloadUrl } from '@/misc/download-url';
import { DriveFiles, Emojis } from '@/models/index';
import { DbUserImportJobData } from '@/queue/types';
import addFile from '@/services/drive/add-file';
import { genId } from '@/misc/gen-id';
const logger = queueLogger.createSubLogger('import-custom-emojis');
// TODO: 名前衝突時の動作を選べるようにする
export async function importCustomEmojis(job: Bull.Job<DbUserImportJobData>, done: any): Promise<void> {
logger.info(`Importing custom emojis ...`);
const file = await DriveFiles.findOne({
id: job.data.fileId,
});
if (file == null) {
done();
return;
}
// Create temp dir
const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => {
tmp.dir((e, path, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
logger.info(`Temp dir is ${path}`);
const destPath = path + '/emojis.zip';
try {
fs.writeFileSync(destPath, '', 'binary');
await downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行
logger.error(e);
throw e;
}
const outputPath = path + '/emojis';
const unzipStream = fs.createReadStream(destPath);
const extractor = unzipper.Extract({ path: outputPath });
extractor.on('close', async () => {
const metaRaw = fs.readFileSync(outputPath + '/meta.json', 'utf-8');
const meta = JSON.parse(metaRaw);
for (const record of meta.emojis) {
if (!record.downloaded) continue;
const emojiInfo = record.emoji;
const emojiPath = outputPath + '/' + record.fileName;
await Emojis.delete({
name: emojiInfo.name,
});
const driveFile = await addFile(null, emojiPath, record.fileName, null, null, true);
const emoji = await Emojis.insert({
id: genId(),
updatedAt: new Date(),
name: emojiInfo.name,
category: emojiInfo.category,
host: null,
aliases: emojiInfo.aliases,
url: driveFile.url,
type: driveFile.type,
}).then(x => Emojis.findOneOrFail(x.identifiers[0]));
}
await getConnection().queryResultCache!.remove(['meta_emojis']);
cleanup();
logger.succ('Imported');
done();
});
unzipStream.pipe(extractor);
logger.succ(`Unzipping to ${outputPath}`);
}

2
packages/backend/src/queue/processors/db/index.ts
View File

@ -12,6 +12,7 @@ import { importUserLists } from './import-user-lists';
import { deleteAccount } from './delete-account';
import { importMuting } from './import-muting';
import { importBlocking } from './import-blocking';
import { importCustomEmojis } from './import-custom-emojis';
const jobs = {
deleteDriveFiles,
@ -25,6 +26,7 @@ const jobs = {
importMuting,
importBlocking,
importUserLists,
importCustomEmojis,
deleteAccount,
} as Record<string, Bull.ProcessCallbackFunction<DbJobData> | Bull.ProcessPromiseFunction<DbJobData>>;

95
packages/backend/yarn.lock
View File

@ -1522,6 +1522,11 @@ big-integer@^1.6.16:
resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.48.tgz#8fd88bd1632cba4a1c8c3e3d7159f08bb95b4b9e"
integrity sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==
big-integer@^1.6.17:
version "1.6.51"
resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.51.tgz#0df92a5d9880560d3ff2d5fd20245c889d130686"
integrity sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==
big.js@^5.2.2:
version "5.2.2"
resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328"
@ -1532,6 +1537,14 @@ binary-extensions@^2.0.0:
resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.0.0.tgz#23c0df14f6a88077f5f986c0d167ec03c3d5537c"
integrity sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==
binary@~0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/binary/-/binary-0.3.0.tgz#9f60553bc5ce8c3386f3b553cff47462adecaa79"
integrity sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=
dependencies:
buffers "~0.1.1"
chainsaw "~0.1.0"
bl@^4.0.1, bl@^4.0.3:
version "4.0.3"
resolved "https://registry.yarnpkg.com/bl/-/bl-4.0.3.tgz#12d6287adc29080e22a705e5764b2a9522cdc489"
@ -1546,6 +1559,11 @@ bluebird@^3.7.2:
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
bluebird@~3.4.1:
version "3.4.7"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3"
integrity sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=
blurhash@1.1.4:
version "1.1.4"
resolved "https://registry.yarnpkg.com/blurhash/-/blurhash-1.1.4.tgz#a7010ceb3019cd2c9809b17c910ebf6175d29244"
@ -1677,6 +1695,11 @@ buffer-from@^1.0.0, buffer-from@^1.1.1:
resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
buffer-indexof-polyfill@~1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz#d2732135c5999c64b277fcf9b1abe3498254729c"
integrity sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==
buffer-writer@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-2.0.0.tgz#ce7eb81a38f7829db09c873f2fbb792c0c98ec04"
@ -1707,6 +1730,11 @@ buffer@^6.0.3:
base64-js "^1.3.1"
ieee754 "^1.2.1"
buffers@~0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/buffers/-/buffers-0.1.1.tgz#b24579c3bed4d6d396aeee6d9a8ae7f5482ab7bb"
integrity sha1-skV5w77U1tOWru5tmorn9Ugqt7s=
bufferutil@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/bufferutil/-/bufferutil-4.0.1.tgz#3a177e8e5819a1243fe16b63a199951a7ad8d4a7"
@ -1875,6 +1903,13 @@ cbor@8.1.0:
dependencies:
nofilter "^3.1.0"
chainsaw@~0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/chainsaw/-/chainsaw-0.1.0.tgz#5eab50b28afe58074d0d58291388828b5e5fbc98"
integrity sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=
dependencies:
traverse ">=0.3.0 <0.4"
chalk@4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.0.0.tgz#6e98081ed2d17faab615eb52ac66ec1fe6209e72"
@ -2789,6 +2824,13 @@ dotenv@^8.2.0:
resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a"
integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==
duplexer2@~0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1"
integrity sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=
dependencies:
readable-stream "^2.0.2"
ecc-jsbn@~0.1.1:
version "0.1.2"
resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
@ -3480,6 +3522,16 @@ fsevents@~2.1.2:
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e"
integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==
fstream@^1.0.12:
version "1.0.12"
resolved "https://registry.yarnpkg.com/fstream/-/fstream-1.0.12.tgz#4e8ba8ee2d48be4f7d0de505455548eae5932045"
integrity sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==
dependencies:
graceful-fs "^4.1.2"
inherits "~2.0.0"
mkdirp ">=0.5 0"
rimraf "2"
function-bind@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
@ -3690,7 +3742,7 @@ graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.4:
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb"
integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==
graceful-fs@^4.2.0:
graceful-fs@^4.2.0, graceful-fs@^4.2.2:
version "4.2.8"
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a"
integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==
@ -4007,7 +4059,7 @@ inflight@^1.0.4:
once "^1.3.0"
wrappy "1"
inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3:
inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
version "2.0.4"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@ -4800,6 +4852,11 @@ lilconfig@^2.0.3:
resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-2.0.3.tgz#68f3005e921dafbd2a2afb48379986aa6d2579fd"
integrity sha512-EHKqr/+ZvdKCifpNrJCKxBTgk5XupZA3y/aCPY9mxfgBzmgh93Mt/WqjjQ38oMxXuvDokaKiM3lAgvSH2sjtHg==
listenercount@~1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/listenercount/-/listenercount-1.0.1.tgz#84c8a72ab59c4725321480c975e6508342e70937"
integrity sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=
loader-runner@^4.2.0:
version "4.2.0"
resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.2.0.tgz#d7022380d66d14c5fb1d496b89864ebcfd478384"
@ -5204,7 +5261,7 @@ mkdirp-classic@^0.5.3:
resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==
mkdirp@0.x, mkdirp@^0.5.4:
mkdirp@0.x, "mkdirp@>=0.5 0", mkdirp@^0.5.4:
version "0.5.5"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def"
integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==
@ -6598,7 +6655,7 @@ readable-stream@1.1.x:
isarray "0.0.1"
string_decoder "~0.10.x"
readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.2.2:
readable-stream@^2.0.0, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.2.2, readable-stream@~2.3.6:
version "2.3.7"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
@ -6780,6 +6837,13 @@ reusify@^1.0.4:
resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==
rimraf@2:
version "2.7.1"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec"
integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==
dependencies:
glob "^7.1.3"
rimraf@3.0.2, rimraf@^3.0.0, rimraf@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a"
@ -6914,7 +6978,7 @@ set-blocking@^2.0.0, set-blocking@~2.0.0:
resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc=
setimmediate@^1.0.5:
setimmediate@^1.0.5, setimmediate@~1.0.4:
version "1.0.5"
resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285"
integrity sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=
@ -7584,6 +7648,11 @@ trace-redirect@1.0.6:
resolved "https://registry.yarnpkg.com/trace-redirect/-/trace-redirect-1.0.6.tgz#ac629b5bf8247d30dde5a35fe9811b811075b504"
integrity sha512-UUfa1DjjU5flcjMdaFIiIEGDTyu2y/IiMjOX4uGXa7meKBS4vD4f2Uy/tken9Qkd4Jsm4sRsfZcIIPqrRVF3Mg==
"traverse@>=0.3.0 <0.4":
version "0.3.9"
resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9"
integrity sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=
ts-jest@^25.2.1:
version "25.5.1"
resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-25.5.1.tgz#2913afd08f28385d54f2f4e828be4d261f4337c7"
@ -7827,6 +7896,22 @@ unpipe@1.0.0:
resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=
unzipper@0.10.11:
version "0.10.11"
resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.10.11.tgz#0b4991446472cbdb92ee7403909f26c2419c782e"
integrity sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==
dependencies:
big-integer "^1.6.17"
binary "~0.3.0"
bluebird "~3.4.1"
buffer-indexof-polyfill "~1.0.0"
duplexer2 "~0.1.4"
fstream "^1.0.12"
graceful-fs "^4.2.2"
listenercount "~1.0.1"
readable-stream "~2.3.6"
setimmediate "~1.0.4"
uri-js@^4.2.2:
version "4.2.2"
resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0"

20
packages/client/src/pages/admin/emojis.vue
View File

@ -182,6 +182,26 @@ const menu = (ev: MouseEvent) => {
});
});
}
}, {
icon: 'fas fa-upload',
text: i18n.locale.import,
action: async () => {
const file = await selectFile(ev.currentTarget || ev.target);
os.api('admin/emoji/import-zip', {
fileId: file.id,
})
.then(() => {
os.alert({
type: 'info',
text: i18n.locale.importRequested,
});
}).catch((e) => {
os.alert({
type: 'error',
text: e.message,
});
});
}
}], ev.currentTarget || ev.target);
};