並列に処理するように
This commit is contained in:
syuilo
parent
07560a4fdd
commit
3a80b59986
|
|
@ -169,6 +169,7 @@
|
|||
"parse5": "5.1.0",
|
||||
"portscanner": "2.2.0",
|
||||
"progress-bar-webpack-plugin": "1.11.0",
|
||||
"promise-limit": "2.7.0",
|
||||
"promise-sequential": "1.1.1",
|
||||
"pug": "2.0.3",
|
||||
"punycode": "2.1.1",
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import * as Minio from 'minio';
|
||||
import * as uuid from 'uuid';
|
||||
const sequential = require('promise-sequential');
|
||||
import DriveFile, { DriveFileChunk, getDriveFileBucket } from '../models/drive-file';
|
||||
import * as promiseLimit from 'promise-limit';
|
||||
import DriveFile, { DriveFileChunk, getDriveFileBucket, IDriveFile } from '../models/drive-file';
|
||||
import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../models/drive-file-thumbnail';
|
||||
import config from '../config';
|
||||
|
||||
const limit = promiseLimit(16);
|
||||
|
||||
DriveFile.find({
|
||||
$or: [{
|
||||
withoutChunks: { $exists: false }
|
||||
|
|
@ -19,59 +21,63 @@ DriveFile.find({
|
|||
}).then(async files => {
|
||||
console.log(`there is ${files.length} files`);
|
||||
|
||||
await sequential(files.map(file => async () => {
|
||||
file = await DriveFile.findOne({ _id: file._id });
|
||||
await Promise.all(files.map(file => limit(() => job(file))));
|
||||
|
||||
const minio = new Minio.Client(config.drive.config);
|
||||
|
||||
const name = file.filename;
|
||||
const keyDir = `${config.drive.prefix}/${uuid.v4()}`;
|
||||
const key = `${keyDir}/${name}`;
|
||||
const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`;
|
||||
const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`;
|
||||
|
||||
const baseUrl = config.drive.baseUrl
|
||||
|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
|
||||
|
||||
const bucket = await getDriveFileBucket();
|
||||
const readable = bucket.openDownloadStream(file._id);
|
||||
|
||||
await minio.putObject(config.drive.bucket, key, readable, file.length, {
|
||||
'Content-Type': file.contentType,
|
||||
'Cache-Control': 'max-age=31536000, immutable'
|
||||
});
|
||||
|
||||
await DriveFile.findOneAndUpdate({ _id: file._id }, {
|
||||
$set: {
|
||||
'metadata.withoutChunks': true,
|
||||
'metadata.storage': 'minio',
|
||||
'metadata.storageProps': {
|
||||
key: key,
|
||||
thumbnailKey: thumbnailKey
|
||||
},
|
||||
'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`,
|
||||
}
|
||||
});
|
||||
|
||||
// チャンクをすべて削除
|
||||
await DriveFileChunk.remove({
|
||||
files_id: file._id
|
||||
});
|
||||
|
||||
//#region サムネイルもあれば削除
|
||||
const thumbnail = await DriveFileThumbnail.findOne({
|
||||
'metadata.originalId': file._id
|
||||
});
|
||||
|
||||
if (thumbnail) {
|
||||
await DriveFileThumbnailChunk.remove({
|
||||
files_id: thumbnail._id
|
||||
});
|
||||
|
||||
await DriveFileThumbnail.remove({ _id: thumbnail._id });
|
||||
}
|
||||
//#endregion
|
||||
|
||||
console.log('done', file._id);
|
||||
}));
|
||||
console.log('ALL DONE');
|
||||
});
|
||||
|
||||
async function job(file: IDriveFile): Promise<any> {
|
||||
file = await DriveFile.findOne({ _id: file._id });
|
||||
|
||||
const minio = new Minio.Client(config.drive.config);
|
||||
|
||||
const name = file.filename;
|
||||
const keyDir = `${config.drive.prefix}/${uuid.v4()}`;
|
||||
const key = `${keyDir}/${name}`;
|
||||
const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`;
|
||||
const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`;
|
||||
|
||||
const baseUrl = config.drive.baseUrl
|
||||
|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
|
||||
|
||||
const bucket = await getDriveFileBucket();
|
||||
const readable = bucket.openDownloadStream(file._id);
|
||||
|
||||
await minio.putObject(config.drive.bucket, key, readable, file.length, {
|
||||
'Content-Type': file.contentType,
|
||||
'Cache-Control': 'max-age=31536000, immutable'
|
||||
});
|
||||
|
||||
await DriveFile.findOneAndUpdate({ _id: file._id }, {
|
||||
$set: {
|
||||
'metadata.withoutChunks': true,
|
||||
'metadata.storage': 'minio',
|
||||
'metadata.storageProps': {
|
||||
key: key,
|
||||
thumbnailKey: thumbnailKey
|
||||
},
|
||||
'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`,
|
||||
}
|
||||
});
|
||||
|
||||
// チャンクをすべて削除
|
||||
await DriveFileChunk.remove({
|
||||
files_id: file._id
|
||||
});
|
||||
|
||||
//#region サムネイルもあれば削除
|
||||
const thumbnail = await DriveFileThumbnail.findOne({
|
||||
'metadata.originalId': file._id
|
||||
});
|
||||
|
||||
if (thumbnail) {
|
||||
await DriveFileThumbnailChunk.remove({
|
||||
files_id: thumbnail._id
|
||||
});
|
||||
|
||||
await DriveFileThumbnail.remove({ _id: thumbnail._id });
|
||||
}
|
||||
//#endregion
|
||||
|
||||
console.log('done', file._id);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue