diff --git a/package.json b/package.json index 19e6184023..b6d4fccbca 100644 --- a/package.json +++ b/package.json @@ -169,6 +169,7 @@ "parse5": "5.1.0", "portscanner": "2.2.0", "progress-bar-webpack-plugin": "1.11.0", + "promise-limit": "2.7.0", "promise-sequential": "1.1.1", "pug": "2.0.3", "punycode": "2.1.1", diff --git a/src/tools/move-drive-files.ts b/src/tools/move-drive-files.ts index 0c201ff8d5..782ea664af 100644 --- a/src/tools/move-drive-files.ts +++ b/src/tools/move-drive-files.ts @@ -1,10 +1,12 @@ import * as Minio from 'minio'; import * as uuid from 'uuid'; -const sequential = require('promise-sequential'); -import DriveFile, { DriveFileChunk, getDriveFileBucket } from '../models/drive-file'; +import * as promiseLimit from 'promise-limit'; +import DriveFile, { DriveFileChunk, getDriveFileBucket, IDriveFile } from '../models/drive-file'; import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../models/drive-file-thumbnail'; import config from '../config'; +const limit = promiseLimit(16); + DriveFile.find({ $or: [{ withoutChunks: { $exists: false } @@ -19,59 +21,63 @@ DriveFile.find({ }).then(async files => { console.log(`there is ${files.length} files`); - await sequential(files.map(file => async () => { - file = await DriveFile.findOne({ _id: file._id }); + await Promise.all(files.map(file => limit(() => job(file)))); - const minio = new Minio.Client(config.drive.config); - - const name = file.filename; - const keyDir = `${config.drive.prefix}/${uuid.v4()}`; - const key = `${keyDir}/${name}`; - const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`; - const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`; - - const baseUrl = config.drive.baseUrl - || `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`; - - const bucket = await getDriveFileBucket(); - const readable = bucket.openDownloadStream(file._id); - - await minio.putObject(config.drive.bucket, key, readable, file.length, { - 'Content-Type': file.contentType, - 'Cache-Control': 'max-age=31536000, immutable' - }); - - await DriveFile.findOneAndUpdate({ _id: file._id }, { - $set: { - 'metadata.withoutChunks': true, - 'metadata.storage': 'minio', - 'metadata.storageProps': { - key: key, - thumbnailKey: thumbnailKey - }, - 'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`, - } - }); - - // チャンクをすべて削除 - await DriveFileChunk.remove({ - files_id: file._id - }); - - //#region サムネイルもあれば削除 - const thumbnail = await DriveFileThumbnail.findOne({ - 'metadata.originalId': file._id - }); - - if (thumbnail) { - await DriveFileThumbnailChunk.remove({ - files_id: thumbnail._id - }); - - await DriveFileThumbnail.remove({ _id: thumbnail._id }); - } - //#endregion - - console.log('done', file._id); - })); + console.log('ALL DONE'); }); + +async function job(file: IDriveFile): Promise { + file = await DriveFile.findOne({ _id: file._id }); + + const minio = new Minio.Client(config.drive.config); + + const name = file.filename; + const keyDir = `${config.drive.prefix}/${uuid.v4()}`; + const key = `${keyDir}/${name}`; + const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`; + const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`; + + const baseUrl = config.drive.baseUrl + || `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`; + + const bucket = await getDriveFileBucket(); + const readable = bucket.openDownloadStream(file._id); + + await minio.putObject(config.drive.bucket, key, readable, file.length, { + 'Content-Type': file.contentType, + 'Cache-Control': 'max-age=31536000, immutable' + }); + + await DriveFile.findOneAndUpdate({ _id: file._id }, { + $set: { + 'metadata.withoutChunks': true, + 'metadata.storage': 'minio', + 'metadata.storageProps': { + key: key, + thumbnailKey: thumbnailKey + }, + 'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`, + } + }); + + // チャンクをすべて削除 + await DriveFileChunk.remove({ + files_id: file._id + }); + + //#region サムネイルもあれば削除 + const thumbnail = await DriveFileThumbnail.findOne({ + 'metadata.originalId': file._id + }); + + if (thumbnail) { + await DriveFileThumbnailChunk.remove({ + files_id: thumbnail._id + }); + + await DriveFileThumbnail.remove({ _id: thumbnail._id }); + } + //#endregion + + console.log('done', file._id); +}