From 64aedcaa6b3e9170e77c428ee306830464b42bcf Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 03:46:30 +0900 Subject: [PATCH 01/10] =?UTF-8?q?add-file-to-drive=20-=20Promise=E7=99=BE?= =?UTF-8?q?=E7=83=88=E6=8B=B3=E3=81=A8=E3=83=A1=E3=83=A2=E3=83=AA=E5=89=8A?= =?UTF-8?q?=E6=B8=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 9 +- src/api/common/add-file-to-drive.ts | 338 +++++++++++------- .../endpoints/drive/files/upload_from_url.ts | 74 +++- 3 files changed, 269 insertions(+), 152 deletions(-) diff --git a/package.json b/package.json index 5b26ee574..235e64f7c 100644 --- a/package.json +++ b/package.json @@ -54,13 +54,14 @@ "@types/node": "8.0.49", "@types/page": "1.5.32", "@types/proxy-addr": "2.0.0", - "@types/seedrandom": "2.4.27", "@types/ratelimiter": "2.1.28", "@types/redis": "2.8.1", - "@types/request": "2.0.7", + "@types/request": "^2.0.7", "@types/rimraf": "2.0.2", "@types/riot": "3.6.1", + "@types/seedrandom": "2.4.27", "@types/serve-favicon": "2.2.29", + "@types/tmp": "0.0.33", "@types/uuid": "3.4.3", "@types/webpack": "3.8.0", "@types/webpack-stream": "3.2.8", @@ -111,7 +112,6 @@ "deep-equal": "1.0.1", "deepcopy": "0.6.3", "diskusage": "0.2.2", - "download": "6.2.5", "elasticsearch": "13.3.1", "escape-regexp": "0.0.1", "express": "4.15.4", @@ -140,7 +140,7 @@ "recaptcha-promise": "0.1.3", "reconnecting-websocket": "3.2.2", "redis": "2.8.0", - "request": "2.83.0", + "request": "^2.83.0", "rimraf": "2.6.2", "riot": "3.7.4", "rndstr": "1.0.0", @@ -152,6 +152,7 @@ "syuilo-password-strength": "0.0.1", "tcp-port-used": "0.1.2", "textarea-caret": "3.0.2", + "tmp": "0.0.33", "ts-node": "3.3.0", "typescript": "2.6.1", "uuid": "3.1.0", diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index a96906d29..a7c7cb464 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -9,28 +9,34 @@ import DriveFolder from '../models/drive-folder'; import serialize from '../serializers/drive-file'; import event from '../event'; import config from '../../conf'; -import { Duplex } from 'stream'; +import { Buffer } from 'buffer'; +import * as fs from 'fs'; +import * as tmp from 'tmp'; +import * as stream from 'stream'; const log = debug('misskey:register-drive-file'); -const addToGridFS = (name, binary, type, metadata): Promise => new Promise(async (resolve, reject) => { - const dataStream = new Duplex(); - dataStream.push(binary); - dataStream.push(null); +const tmpFile = (): Promise => new Promise((resolve, reject) => { + tmp.file((e, path) => { + if (e) return reject(e) + resolve(path) + }) +}) - const bucket = await getGridFSBucket(); - const writeStream = bucket.openUploadStream(name, { contentType: type, metadata }); - writeStream.once('finish', (doc) => { resolve(doc); }); - writeStream.on('error', reject); - dataStream.pipe(writeStream); -}); +const addToGridFS = (name: string, readable: stream.Readable, type: string, metadata: any): Promise => + getGridFSBucket() + .then(bucket => new Promise((resolve, reject) => { + const writeStream = bucket.openUploadStream(name, { contentType: type, metadata }); + writeStream.once('finish', (doc) => { resolve(doc); }); + writeStream.on('error', reject); + readable.pipe(writeStream); + })) /** * Add file to drive * * @param user User who wish to add file - * @param fileName File name - * @param data Contents + * @param file File path, binary, or readableStream * @param comment Comment * @param type File type * @param folderId Folder ID @@ -39,139 +45,201 @@ const addToGridFS = (name, binary, type, metadata): Promise => new Promise( */ export default ( user: any, - data: Buffer, + file: string | Buffer | stream.Readable, name: string = null, comment: string = null, folderId: mongodb.ObjectID = null, force: boolean = false -) => new Promise(async (resolve, reject) => { +) => new Promise((resolve, reject) => { log(`registering ${name} (user: ${user.username})`); - // File size - const size = data.byteLength; - - log(`size is ${size}`); - - // File type - let mime = 'application/octet-stream'; - const type = fileType(data); - if (type !== null) { - mime = type.mime; - - if (name === null) { - name = `untitled.${type.ext}`; + // Get file path + new Promise((res: (v: string) => void, rej) => { + if (typeof file === 'string') { + res(file) + return } - } else { - if (name === null) { - name = 'untitled'; + if (file instanceof Buffer) { + tmpFile() + .then(path => { + fs.writeFile(path, file, (err) => { + if (err) rej(err) + res(path) + }) + }) + .catch(rej) + return } - } - - log(`type is ${mime}`); - - // Generate hash - const hash = crypto - .createHash('md5') - .update(data) - .digest('hex') as string; - - log(`hash is ${hash}`); - - if (!force) { - // Check if there is a file with the same hash - const much = await DriveFile.findOne({ - md5: hash, - 'metadata.user_id': user._id - }); - - if (much !== null) { - log('file with same hash is found'); - return resolve(much); - } else { - log('file with same hash is not found'); + if (typeof file === 'object' && typeof file.read === 'function') { + tmpFile() + .then(path => { + const readable: stream.Readable = file + const writable = fs.createWriteStream(path) + readable + .on('error', rej) + .on('end', () => { + res(path) + }) + .pipe(writable) + .on('error', rej) + }) + .catch(rej) } - } + rej(new Error('un-compatible file.')) + }) + // Calculate hash, get content type and get file size + .then(path => Promise.all([ + path, + // hash + ((): Promise => new Promise((res, rej) => { + const readable = fs.createReadStream(path) + const hash = crypto.createHash('md5') + readable + .on('error', rej) + .on('end', () => { + res(hash.digest('hex')) + }) + .pipe(hash) + .on('error', rej) + }))(), + // mime + ((): Promise<[string, string | null]> => new Promise((res, rej) => { + const readable = fs.createReadStream(path) + readable + .on('error', rej) + .once('data', (buffer: Buffer) => { + readable.destroy() + const type = fileType(buffer) + if (!type) { + return res(['application/octet-stream', null]) + } + return res([type.mime, type.ext]) + }) + }))(), + // size + ((): Promise => new Promise((res, rej) => { + fs.stat(path, (err, stats) => { + if (err) return rej(err) + res(stats.size) + }) + }))() + ])) + .then(async ([path, hash, [mime, ext], size]) => { + log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`) - // Calculate drive usage - const usage = ((await DriveFile - .aggregate([ - { $match: { 'metadata.user_id': user._id } }, - { $project: { - length: true - }}, - { $group: { - _id: null, - usage: { $sum: '$length' } - }} - ]))[0] || { - usage: 0 - }).usage; + // detect name + const detectedName: string = name || (ext ? `untitled.${ext}` : 'untitled'); - log(`drive usage is ${usage}`); + if (!force) { + // Check if there is a file with the same hash + const much = await DriveFile.findOne({ + md5: hash, + 'metadata.user_id': user._id + }); - // If usage limit exceeded - if (usage + size > user.drive_capacity) { - return reject('no-free-space'); - } - - // If the folder is specified - let folder: any = null; - if (folderId !== null) { - folder = await DriveFolder - .findOne({ - _id: folderId, - user_id: user._id - }); - - if (folder === null) { - return reject('folder-not-found'); - } - } - - let properties: any = null; - - // If the file is an image - if (/^image\/.*$/.test(mime)) { - // Calculate width and height to save in property - const g = gm(data, name); - const size = await prominence(g).size(); - properties = { - width: size.width, - height: size.height - }; - - log('image width and height is calculated'); - } - - // Create DriveFile document - const file = await addToGridFS(name, data, mime, { - user_id: user._id, - folder_id: folder !== null ? folder._id : null, - comment: comment, - properties: properties - }); - - log(`drive file has been created ${file._id}`); - - resolve(file); - - // Serialize - const fileObj = await serialize(file); - - // Publish drive_file_created event - event(user._id, 'drive_file_created', fileObj); - - // Register to search database - if (config.elasticsearch.enable) { - const es = require('../../db/elasticsearch'); - es.index({ - index: 'misskey', - type: 'drive_file', - id: file._id.toString(), - body: { - name: file.name, - user_id: user._id.toString() + if (much !== null) { + log('file with same hash is found'); + return resolve(much); + } else { + log('file with same hash is not found'); + } } - }); - } + + const [properties, folder] = await Promise.all([ + // properties + (async () => { + if (!/^image\/.*$/.test(mime)) { + return null + } + // If the file is an image, calculate width and height to save in property + const g = gm(data, name); + const size = await prominence(g).size(); + const properties = { + width: size.width, + height: size.height + }; + log('image width and height is calculated'); + return properties + })(), + // folder + (async () => { + if (!folderId) { + return null + } + const driveFolder = await DriveFolder.findOne({ + _id: folderId, + user_id: user._id + }) + if (!driveFolder) { + throw 'folder-not-found' + } + return driveFolder + })(), + // usage checker + (async () => { + // Calculate drive usage + const usage = await DriveFile + .aggregate([ + { $match: { 'metadata.user_id': user._id } }, + { + $project: { + length: true + } + }, + { + $group: { + _id: null, + usage: { $sum: '$length' } + } + } + ]) + .then((aggregates: any[]) => { + if (aggregates.length > 0) { + return aggregates[0].usage + } + return 0 + }); + + log(`drive usage is ${usage}`); + + // If usage limit exceeded + if (usage + size > user.drive_capacity) { + throw 'no-free-space'; + } + })() + ]) + + const readable = fs.createReadStream(path) + + return addToGridFS(name, readable, mime, { + user_id: user._id, + folder_id: folder !== null ? folder._id : null, + comment: comment, + properties: properties + }) + }) + .then(file => { + log(`drive file has been created ${file._id}`); + resolve(file) + return serialize(file) + }) + .then(serializedFile => { + // Publish drive_file_created event + event(user._id, 'drive_file_created', fileObj); + + // Register to search database + if (config.elasticsearch.enable) { + const es = require('../../db/elasticsearch'); + es.index({ + index: 'misskey', + type: 'drive_file', + id: file._id.toString(), + body: { + name: file.name, + user_id: user._id.toString() + } + }); + } + }) + .catch(reject) }); diff --git a/src/api/endpoints/drive/files/upload_from_url.ts b/src/api/endpoints/drive/files/upload_from_url.ts index 46cfffb69..9c759994e 100644 --- a/src/api/endpoints/drive/files/upload_from_url.ts +++ b/src/api/endpoints/drive/files/upload_from_url.ts @@ -2,11 +2,17 @@ * Module dependencies */ import * as URL from 'url'; -const download = require('download'); import $ from 'cafy'; import { validateFileName } from '../../../models/drive-file'; import serialize from '../../../serializers/drive-file'; import create from '../../../common/add-file-to-drive'; +import * as debug from 'debug'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +import * as request from 'request'; +import * as crypto from 'crypto'; + +const log = debug('misskey:endpoint:upload_from_url') /** * Create a file from a URL @@ -15,7 +21,7 @@ import create from '../../../common/add-file-to-drive'; * @param {any} user * @return {Promise} */ -module.exports = (params, user) => new Promise(async (res, rej) => { +module.exports = (params, user) => new Promise((res, rej) => { // Get 'url' parameter // TODO: Validate this url const [url, urlErr] = $(params.url).string().$; @@ -30,15 +36,57 @@ module.exports = (params, user) => new Promise(async (res, rej) => { const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$; if (folderIdErr) return rej('invalid folder_id param'); - // Download file - const data = await download(url); - - // Create file - const driveFile = await create(user, data, name, null, folderId); - - // Serialize - const fileObj = await serialize(driveFile); - - // Response - res(fileObj); + // Create temp file + new Promise((res, rej) => { + tmp.file((e, path) => { + if (e) return rej(e) + res(path) + }) + }) + // Download file + .then((path: string) => new Promise((res, rej) => { + const writable = fs.createWriteStream(path) + request(url) + .on('error', rej) + .on('end', () => { + writable.close() + res(path) + }) + .pipe(writable) + .on('error', rej) + })) + // Calculate hash & content-type + .then((path: string) => new Promise((res, rej) => { + const readable = fs.createReadStream(path) + const hash = crypto.createHash('md5') + readable + .on('error', rej) + .on('end', () => { + hash.end() + res([path, hash.digest('hex')]) + }) + .pipe(hash) + .on('error', rej) + })) + // Create file + .then((rv: string[]) => new Promise((res, rej) => { + const [path, hash] = rv + create(user, { + stream: fs.createReadStream(path), + name, + hash + }, null, folderId) + .then(driveFile => { + res(driveFile) + // crean-up + fs.unlink(path, (e) => { + if (e) log(e.stack) + }) + }) + .catch(rej) + })) + // Serialize + .then(serialize) + .then(res) + .catch(rej) }); From e56f716a89227c76dfc05e48b3ca438f766f85b4 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 03:47:42 +0900 Subject: [PATCH 02/10] format --- src/api/common/add-file-to-drive.ts | 92 +++++++++---------- .../endpoints/drive/files/upload_from_url.ts | 38 ++++---- 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index a7c7cb464..c6a4c4791 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -18,10 +18,10 @@ const log = debug('misskey:register-drive-file'); const tmpFile = (): Promise => new Promise((resolve, reject) => { tmp.file((e, path) => { - if (e) return reject(e) - resolve(path) - }) -}) + if (e) return reject(e); + resolve(path); + }); +}); const addToGridFS = (name: string, readable: stream.Readable, type: string, metadata: any): Promise => getGridFSBucket() @@ -30,7 +30,7 @@ const addToGridFS = (name: string, readable: stream.Readable, type: string, meta writeStream.once('finish', (doc) => { resolve(doc); }); writeStream.on('error', reject); readable.pipe(writeStream); - })) + })); /** * Add file to drive @@ -56,76 +56,76 @@ export default ( // Get file path new Promise((res: (v: string) => void, rej) => { if (typeof file === 'string') { - res(file) - return + res(file); + return; } if (file instanceof Buffer) { tmpFile() .then(path => { fs.writeFile(path, file, (err) => { - if (err) rej(err) - res(path) - }) + if (err) rej(err); + res(path); + }); }) - .catch(rej) - return + .catch(rej); + return; } if (typeof file === 'object' && typeof file.read === 'function') { tmpFile() .then(path => { - const readable: stream.Readable = file - const writable = fs.createWriteStream(path) + const readable: stream.Readable = file; + const writable = fs.createWriteStream(path); readable .on('error', rej) .on('end', () => { - res(path) + res(path); }) .pipe(writable) - .on('error', rej) + .on('error', rej); }) - .catch(rej) + .catch(rej); } - rej(new Error('un-compatible file.')) + rej(new Error('un-compatible file.')); }) // Calculate hash, get content type and get file size .then(path => Promise.all([ path, // hash ((): Promise => new Promise((res, rej) => { - const readable = fs.createReadStream(path) - const hash = crypto.createHash('md5') + const readable = fs.createReadStream(path); + const hash = crypto.createHash('md5'); readable .on('error', rej) .on('end', () => { - res(hash.digest('hex')) + res(hash.digest('hex')); }) .pipe(hash) - .on('error', rej) + .on('error', rej); }))(), // mime ((): Promise<[string, string | null]> => new Promise((res, rej) => { - const readable = fs.createReadStream(path) + const readable = fs.createReadStream(path); readable .on('error', rej) .once('data', (buffer: Buffer) => { - readable.destroy() - const type = fileType(buffer) + readable.destroy(); + const type = fileType(buffer); if (!type) { - return res(['application/octet-stream', null]) + return res(['application/octet-stream', null]); } - return res([type.mime, type.ext]) - }) + return res([type.mime, type.ext]); + }); }))(), // size ((): Promise => new Promise((res, rej) => { fs.stat(path, (err, stats) => { - if (err) return rej(err) - res(stats.size) - }) + if (err) return rej(err); + res(stats.size); + }); }))() ])) .then(async ([path, hash, [mime, ext], size]) => { - log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`) + log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`); // detect name const detectedName: string = name || (ext ? `untitled.${ext}` : 'untitled'); @@ -149,7 +149,7 @@ export default ( // properties (async () => { if (!/^image\/.*$/.test(mime)) { - return null + return null; } // If the file is an image, calculate width and height to save in property const g = gm(data, name); @@ -159,21 +159,21 @@ export default ( height: size.height }; log('image width and height is calculated'); - return properties + return properties; })(), // folder (async () => { if (!folderId) { - return null + return null; } const driveFolder = await DriveFolder.findOne({ _id: folderId, user_id: user._id - }) + }); if (!driveFolder) { - throw 'folder-not-found' + throw 'folder-not-found'; } - return driveFolder + return driveFolder; })(), // usage checker (async () => { @@ -195,9 +195,9 @@ export default ( ]) .then((aggregates: any[]) => { if (aggregates.length > 0) { - return aggregates[0].usage + return aggregates[0].usage; } - return 0 + return 0; }); log(`drive usage is ${usage}`); @@ -207,21 +207,21 @@ export default ( throw 'no-free-space'; } })() - ]) + ]); - const readable = fs.createReadStream(path) + const readable = fs.createReadStream(path); return addToGridFS(name, readable, mime, { user_id: user._id, folder_id: folder !== null ? folder._id : null, comment: comment, properties: properties - }) + }); }) .then(file => { log(`drive file has been created ${file._id}`); - resolve(file) - return serialize(file) + resolve(file); + return serialize(file); }) .then(serializedFile => { // Publish drive_file_created event @@ -241,5 +241,5 @@ export default ( }); } }) - .catch(reject) + .catch(reject); }); diff --git a/src/api/endpoints/drive/files/upload_from_url.ts b/src/api/endpoints/drive/files/upload_from_url.ts index 9c759994e..60332b4af 100644 --- a/src/api/endpoints/drive/files/upload_from_url.ts +++ b/src/api/endpoints/drive/files/upload_from_url.ts @@ -12,7 +12,7 @@ import * as fs from 'fs'; import * as request from 'request'; import * as crypto from 'crypto'; -const log = debug('misskey:endpoint:upload_from_url') +const log = debug('misskey:endpoint:upload_from_url'); /** * Create a file from a URL @@ -39,54 +39,54 @@ module.exports = (params, user) => new Promise((res, rej) => { // Create temp file new Promise((res, rej) => { tmp.file((e, path) => { - if (e) return rej(e) - res(path) - }) + if (e) return rej(e); + res(path); + }); }) // Download file .then((path: string) => new Promise((res, rej) => { - const writable = fs.createWriteStream(path) + const writable = fs.createWriteStream(path); request(url) .on('error', rej) .on('end', () => { - writable.close() - res(path) + writable.close(); + res(path); }) .pipe(writable) - .on('error', rej) + .on('error', rej); })) // Calculate hash & content-type .then((path: string) => new Promise((res, rej) => { - const readable = fs.createReadStream(path) - const hash = crypto.createHash('md5') + const readable = fs.createReadStream(path); + const hash = crypto.createHash('md5'); readable .on('error', rej) .on('end', () => { - hash.end() - res([path, hash.digest('hex')]) + hash.end(); + res([path, hash.digest('hex')]); }) .pipe(hash) - .on('error', rej) + .on('error', rej); })) // Create file .then((rv: string[]) => new Promise((res, rej) => { - const [path, hash] = rv + const [path, hash] = rv; create(user, { stream: fs.createReadStream(path), name, hash }, null, folderId) .then(driveFile => { - res(driveFile) + res(driveFile); // crean-up fs.unlink(path, (e) => { - if (e) log(e.stack) - }) + if (e) log(e.stack); + }); }) - .catch(rej) + .catch(rej); })) // Serialize .then(serialize) .then(res) - .catch(rej) + .catch(rej); }); From f6c63fbe14e47a01a9b09c3911a7c128375440c9 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 03:56:39 +0900 Subject: [PATCH 03/10] =?UTF-8?q?add-file-to-drive=20-=20gm=E3=81=AB?= =?UTF-8?q?=E6=B8=A1=E3=81=99=E5=BC=95=E6=95=B0=E3=82=92=E6=AD=A3=E3=81=97?= =?UTF-8?q?=E3=81=8F=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/common/add-file-to-drive.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index c6a4c4791..1aa21f71a 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -152,7 +152,7 @@ export default ( return null; } // If the file is an image, calculate width and height to save in property - const g = gm(data, name); + const g = gm(fs.createReadStream(path), name); const size = await prominence(g).size(); const properties = { width: size.width, From 54804f4a642176134fa835eca86a047a87a704d2 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 04:11:53 +0900 Subject: [PATCH 04/10] =?UTF-8?q?add-file-to-drive=20-=20hash=E3=81=8Cstre?= =?UTF-8?q?am=E3=82=92=E5=8F=97=E3=81=91=E3=82=8B=E6=99=82=E3=80=81hash?= =?UTF-8?q?=E3=82=82=E3=81=BE=E3=81=9Fstream=E3=81=AA=E3=81=AE=E3=81=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/common/add-file-to-drive.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index 1aa21f71a..5f7d255e4 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -94,13 +94,16 @@ export default ( ((): Promise => new Promise((res, rej) => { const readable = fs.createReadStream(path); const hash = crypto.createHash('md5'); + const chunks = []; readable .on('error', rej) - .on('end', () => { - res(hash.digest('hex')); - }) .pipe(hash) - .on('error', rej); + .on('error', rej) + .on('data', (chunk) => chunks.push(chunk)) + .on('end', () => { + const buffer = Buffer.concat(chunks); + res(buffer.toString('hex')); + }); }))(), // mime ((): Promise<[string, string | null]> => new Promise((res, rej) => { From 47f98fbab76e8680f5a7c99037b3b237c7256ca2 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 04:28:51 +0900 Subject: [PATCH 05/10] =?UTF-8?q?=E3=83=90=E3=82=B0=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/common/add-file-to-drive.ts | 35 ++++---- .../endpoints/drive/files/upload_from_url.ts | 81 +++++++------------ 2 files changed, 46 insertions(+), 70 deletions(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index 5f7d255e4..1c8965e31 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -214,7 +214,7 @@ export default ( const readable = fs.createReadStream(path); - return addToGridFS(name, readable, mime, { + return addToGridFS(detectedName, readable, mime, { user_id: user._id, folder_id: folder !== null ? folder._id : null, comment: comment, @@ -224,25 +224,26 @@ export default ( .then(file => { log(`drive file has been created ${file._id}`); resolve(file); - return serialize(file); - }) - .then(serializedFile => { - // Publish drive_file_created event - event(user._id, 'drive_file_created', fileObj); - // Register to search database - if (config.elasticsearch.enable) { - const es = require('../../db/elasticsearch'); - es.index({ - index: 'misskey', - type: 'drive_file', - id: file._id.toString(), - body: { - name: file.name, - user_id: user._id.toString() + serialize(file) + .then(serializedFile => { + // Publish drive_file_created event + event(user._id, 'drive_file_created', serializedFile); + + // Register to search database + if (config.elasticsearch.enable) { + const es = require('../../db/elasticsearch'); + es.index({ + index: 'misskey', + type: 'drive_file', + id: file._id.toString(), + body: { + name: file.name, + user_id: user._id.toString() + } + }); } }); - } }) .catch(reject); }); diff --git a/src/api/endpoints/drive/files/upload_from_url.ts b/src/api/endpoints/drive/files/upload_from_url.ts index 60332b4af..519e0bdf6 100644 --- a/src/api/endpoints/drive/files/upload_from_url.ts +++ b/src/api/endpoints/drive/files/upload_from_url.ts @@ -10,7 +10,6 @@ import * as debug from 'debug'; import * as tmp from 'tmp'; import * as fs from 'fs'; import * as request from 'request'; -import * as crypto from 'crypto'; const log = debug('misskey:endpoint:upload_from_url'); @@ -21,11 +20,11 @@ const log = debug('misskey:endpoint:upload_from_url'); * @param {any} user * @return {Promise} */ -module.exports = (params, user) => new Promise((res, rej) => { +module.exports = async (params, user): Promise => { // Get 'url' parameter // TODO: Validate this url const [url, urlErr] = $(params.url).string().$; - if (urlErr) return rej('invalid url param'); + if (urlErr) throw 'invalid url param'; let name = URL.parse(url).pathname.split('/').pop(); if (!validateFileName(name)) { @@ -34,59 +33,35 @@ module.exports = (params, user) => new Promise((res, rej) => { // Get 'folder_id' parameter const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$; - if (folderIdErr) return rej('invalid folder_id param'); + if (folderIdErr) throw 'invalid folder_id param'; // Create temp file - new Promise((res, rej) => { + const path = await new Promise((res: (string) => void, rej) => { tmp.file((e, path) => { if (e) return rej(e); res(path); }); - }) - // Download file - .then((path: string) => new Promise((res, rej) => { - const writable = fs.createWriteStream(path); - request(url) - .on('error', rej) - .on('end', () => { - writable.close(); - res(path); - }) - .pipe(writable) - .on('error', rej); - })) - // Calculate hash & content-type - .then((path: string) => new Promise((res, rej) => { - const readable = fs.createReadStream(path); - const hash = crypto.createHash('md5'); - readable - .on('error', rej) - .on('end', () => { - hash.end(); - res([path, hash.digest('hex')]); - }) - .pipe(hash) - .on('error', rej); - })) - // Create file - .then((rv: string[]) => new Promise((res, rej) => { - const [path, hash] = rv; - create(user, { - stream: fs.createReadStream(path), - name, - hash - }, null, folderId) - .then(driveFile => { - res(driveFile); - // crean-up - fs.unlink(path, (e) => { - if (e) log(e.stack); - }); - }) - .catch(rej); - })) - // Serialize - .then(serialize) - .then(res) - .catch(rej); -}); + }); + + // write content at URL to temp file + await new Promise((res, rej) => { + const writable = fs.createWriteStream(path); + request(url) + .on('error', rej) + .on('end', () => { + writable.close(); + res(path); + }) + .pipe(writable) + .on('error', rej); + }); + + const driveFile = await create(user, path, name, null, folderId); + + // clean-up + fs.unlink(path, (e) => { + if (e) log(e.stack); + }); + + return serialize(driveFile); +}; From 342345cc2fb95e86d65095e0c9996441950ad628 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 04:35:25 +0900 Subject: [PATCH 06/10] =?UTF-8?q?create=20-=20=E3=83=90=E3=83=83=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=92=E4=BD=BF=E7=94=A8=E3=81=97=E3=81=AA=E3=81=84?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/endpoints/drive/files/create.ts | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/api/endpoints/drive/files/create.ts b/src/api/endpoints/drive/files/create.ts index 7967c3118..7546eca30 100644 --- a/src/api/endpoints/drive/files/create.ts +++ b/src/api/endpoints/drive/files/create.ts @@ -1,7 +1,6 @@ /** * Module dependencies */ -import * as fs from 'fs'; import $ from 'cafy'; import { validateFileName } from '../../../models/drive-file'; import serialize from '../../../serializers/drive-file'; @@ -15,15 +14,11 @@ import create from '../../../common/add-file-to-drive'; * @param {any} user * @return {Promise} */ -module.exports = (file, params, user) => new Promise(async (res, rej) => { +module.exports = async (file, params, user): Promise => { if (file == null) { - return rej('file is required'); + throw 'file is required'; } - // TODO: 非同期にしたい。Promise対応してないんだろうか... - const buffer = fs.readFileSync(file.path); - fs.unlink(file.path, (err) => { if (err) console.log(err); }); - // Get 'name' parameter let name = file.originalname; if (name !== undefined && name !== null) { @@ -33,7 +28,7 @@ module.exports = (file, params, user) => new Promise(async (res, rej) => { } else if (name === 'blob') { name = null; } else if (!validateFileName(name)) { - return rej('invalid name'); + throw 'invalid name'; } } else { name = null; @@ -41,14 +36,11 @@ module.exports = (file, params, user) => new Promise(async (res, rej) => { // Get 'folder_id' parameter const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$; - if (folderIdErr) return rej('invalid folder_id param'); + if (folderIdErr) throw 'invalid folder_id param'; // Create file - const driveFile = await create(user, buffer, name, null, folderId); + const driveFile = await create(user, file.path, name, null, folderId); // Serialize - const fileObj = await serialize(driveFile); - - // Response - res(fileObj); -}); + return serialize(driveFile); +}; From 519bb82b039dd037667f106b29f74bc0dffb4b3a Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 04:39:21 +0900 Subject: [PATCH 07/10] =?UTF-8?q?add-file-to-drive=20-=20=E3=83=90?= =?UTF-8?q?=E3=83=83=E3=83=95=E3=82=A1=E5=8F=97=E3=81=91=E4=BB=98=E3=81=91?= =?UTF-8?q?=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/common/add-file-to-drive.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index 1c8965e31..e5d9dd7c1 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -36,7 +36,7 @@ const addToGridFS = (name: string, readable: stream.Readable, type: string, meta * Add file to drive * * @param user User who wish to add file - * @param file File path, binary, or readableStream + * @param file File path or readableStream * @param comment Comment * @param type File type * @param folderId Folder ID @@ -45,7 +45,7 @@ const addToGridFS = (name: string, readable: stream.Readable, type: string, meta */ export default ( user: any, - file: string | Buffer | stream.Readable, + file: string | stream.Readable, name: string = null, comment: string = null, folderId: mongodb.ObjectID = null, @@ -59,17 +59,6 @@ export default ( res(file); return; } - if (file instanceof Buffer) { - tmpFile() - .then(path => { - fs.writeFile(path, file, (err) => { - if (err) rej(err); - res(path); - }); - }) - .catch(rej); - return; - } if (typeof file === 'object' && typeof file.read === 'function') { tmpFile() .then(path => { From 51faa7a227ad8f8489d03da937c78667442beb0e Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 04:54:47 +0900 Subject: [PATCH 08/10] =?UTF-8?q?add-file-to-drive=20-=20=E8=A6=8B?= =?UTF-8?q?=E9=80=9A=E3=81=97=E3=82=92=E8=89=AF=E3=81=8F=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/common/add-file-to-drive.ts | 295 ++++++++++++++-------------- 1 file changed, 149 insertions(+), 146 deletions(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index e5d9dd7c1..eeb92005a 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -32,29 +32,18 @@ const addToGridFS = (name: string, readable: stream.Readable, type: string, meta readable.pipe(writeStream); })); -/** - * Add file to drive - * - * @param user User who wish to add file - * @param file File path or readableStream - * @param comment Comment - * @param type File type - * @param folderId Folder ID - * @param force If set to true, forcibly upload the file even if there is a file with the same hash. - * @return Object that represents added file - */ -export default ( +const addFile = async ( user: any, file: string | stream.Readable, name: string = null, comment: string = null, folderId: mongodb.ObjectID = null, force: boolean = false -) => new Promise((resolve, reject) => { +) => { log(`registering ${name} (user: ${user.username})`); // Get file path - new Promise((res: (v: string) => void, rej) => { + const path = await new Promise((res: (v: string) => void, rej) => { if (typeof file === 'string') { res(file); return; @@ -75,141 +64,155 @@ export default ( .catch(rej); } rej(new Error('un-compatible file.')); - }) - // Calculate hash, get content type and get file size - .then(path => Promise.all([ - path, - // hash - ((): Promise => new Promise((res, rej) => { - const readable = fs.createReadStream(path); - const hash = crypto.createHash('md5'); - const chunks = []; - readable - .on('error', rej) - .pipe(hash) - .on('error', rej) - .on('data', (chunk) => chunks.push(chunk)) - .on('end', () => { - const buffer = Buffer.concat(chunks); - res(buffer.toString('hex')); - }); - }))(), - // mime - ((): Promise<[string, string | null]> => new Promise((res, rej) => { - const readable = fs.createReadStream(path); - readable - .on('error', rej) - .once('data', (buffer: Buffer) => { - readable.destroy(); - const type = fileType(buffer); - if (!type) { - return res(['application/octet-stream', null]); - } - return res([type.mime, type.ext]); - }); - }))(), - // size - ((): Promise => new Promise((res, rej) => { - fs.stat(path, (err, stats) => { - if (err) return rej(err); - res(stats.size); - }); - }))() - ])) - .then(async ([path, hash, [mime, ext], size]) => { - log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`); - - // detect name - const detectedName: string = name || (ext ? `untitled.${ext}` : 'untitled'); - - if (!force) { - // Check if there is a file with the same hash - const much = await DriveFile.findOne({ - md5: hash, - 'metadata.user_id': user._id - }); - - if (much !== null) { - log('file with same hash is found'); - return resolve(much); - } else { - log('file with same hash is not found'); - } - } - - const [properties, folder] = await Promise.all([ - // properties - (async () => { - if (!/^image\/.*$/.test(mime)) { - return null; - } - // If the file is an image, calculate width and height to save in property - const g = gm(fs.createReadStream(path), name); - const size = await prominence(g).size(); - const properties = { - width: size.width, - height: size.height - }; - log('image width and height is calculated'); - return properties; - })(), - // folder - (async () => { - if (!folderId) { - return null; - } - const driveFolder = await DriveFolder.findOne({ - _id: folderId, - user_id: user._id - }); - if (!driveFolder) { - throw 'folder-not-found'; - } - return driveFolder; - })(), - // usage checker - (async () => { - // Calculate drive usage - const usage = await DriveFile - .aggregate([ - { $match: { 'metadata.user_id': user._id } }, - { - $project: { - length: true - } - }, - { - $group: { - _id: null, - usage: { $sum: '$length' } - } - } - ]) - .then((aggregates: any[]) => { - if (aggregates.length > 0) { - return aggregates[0].usage; - } - return 0; - }); - - log(`drive usage is ${usage}`); - - // If usage limit exceeded - if (usage + size > user.drive_capacity) { - throw 'no-free-space'; - } - })() - ]); + }); + // Calculate hash, get content type and get file size + const [hash, [mime, ext], size] = await Promise.all([ + // hash + ((): Promise => new Promise((res, rej) => { const readable = fs.createReadStream(path); - - return addToGridFS(detectedName, readable, mime, { - user_id: user._id, - folder_id: folder !== null ? folder._id : null, - comment: comment, - properties: properties + const hash = crypto.createHash('md5'); + const chunks = []; + readable + .on('error', rej) + .pipe(hash) + .on('error', rej) + .on('data', (chunk) => chunks.push(chunk)) + .on('end', () => { + const buffer = Buffer.concat(chunks); + res(buffer.toString('hex')); + }); + }))(), + // mime + ((): Promise<[string, string | null]> => new Promise((res, rej) => { + const readable = fs.createReadStream(path); + readable + .on('error', rej) + .once('data', (buffer: Buffer) => { + readable.destroy(); + const type = fileType(buffer); + if (!type) { + return res(['application/octet-stream', null]); + } + return res([type.mime, type.ext]); + }); + }))(), + // size + ((): Promise => new Promise((res, rej) => { + fs.stat(path, (err, stats) => { + if (err) return rej(err); + res(stats.size); }); - }) + }))() + ]); + + log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`); + + // detect name + const detectedName: string = name || (ext ? `untitled.${ext}` : 'untitled'); + + if (!force) { + // Check if there is a file with the same hash + const much = await DriveFile.findOne({ + md5: hash, + 'metadata.user_id': user._id + }); + + if (much !== null) { + log('file with same hash is found'); + return much; + } else { + log('file with same hash is not found'); + } + } + + const [properties, folder] = await Promise.all([ + // properties + (async () => { + if (!/^image\/.*$/.test(mime)) { + return null; + } + // If the file is an image, calculate width and height to save in property + const g = gm(fs.createReadStream(path), name); + const size = await prominence(g).size(); + const properties = { + width: size.width, + height: size.height + }; + log('image width and height is calculated'); + return properties; + })(), + // folder + (async () => { + if (!folderId) { + return null; + } + const driveFolder = await DriveFolder.findOne({ + _id: folderId, + user_id: user._id + }); + if (!driveFolder) { + throw 'folder-not-found'; + } + return driveFolder; + })(), + // usage checker + (async () => { + // Calculate drive usage + const usage = await DriveFile + .aggregate([ + { $match: { 'metadata.user_id': user._id } }, + { + $project: { + length: true + } + }, + { + $group: { + _id: null, + usage: { $sum: '$length' } + } + } + ]) + .then((aggregates: any[]) => { + if (aggregates.length > 0) { + return aggregates[0].usage; + } + return 0; + }); + + log(`drive usage is ${usage}`); + + // If usage limit exceeded + if (usage + size > user.drive_capacity) { + throw 'no-free-space'; + } + })() + ]); + + const readable = fs.createReadStream(path); + + return addToGridFS(detectedName, readable, mime, { + user_id: user._id, + folder_id: folder !== null ? folder._id : null, + comment: comment, + properties: properties + }); +}; + +/** + * Add file to drive + * + * @param user User who wish to add file + * @param file File path or readableStream + * @param comment Comment + * @param type File type + * @param folderId Folder ID + * @param force If set to true, forcibly upload the file even if there is a file with the same hash. + * @return Object that represents added file + */ +export default (user: any, file: string | stream.Readable, ...args) => new Promise((resolve, reject) => { + addFile(user, file, ...args) .then(file => { log(`drive file has been created ${file._id}`); resolve(file); From d8a3b4ff1d418eb2aab30237d797dc4844858abe Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 05:10:28 +0900 Subject: [PATCH 09/10] =?UTF-8?q?add-file-to-drive=20-=20=E8=B2=AC?= =?UTF-8?q?=E5=8B=99=E3=81=AE=E5=88=86=E5=89=B2=E3=81=A8=E3=83=86=E3=83=B3?= =?UTF-8?q?=E3=83=9D=E3=83=A9=E3=83=AA=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E3=82=92=E5=89=8A=E9=99=A4=E3=81=99=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/common/add-file-to-drive.ts | 61 +++++++++++++++++------------ 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index eeb92005a..6a728d0d1 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -34,7 +34,7 @@ const addToGridFS = (name: string, readable: stream.Readable, type: string, meta const addFile = async ( user: any, - file: string | stream.Readable, + path: string, name: string = null, comment: string = null, folderId: mongodb.ObjectID = null, @@ -42,30 +42,6 @@ const addFile = async ( ) => { log(`registering ${name} (user: ${user.username})`); - // Get file path - const path = await new Promise((res: (v: string) => void, rej) => { - if (typeof file === 'string') { - res(file); - return; - } - if (typeof file === 'object' && typeof file.read === 'function') { - tmpFile() - .then(path => { - const readable: stream.Readable = file; - const writable = fs.createWriteStream(path); - readable - .on('error', rej) - .on('end', () => { - res(path); - }) - .pipe(writable) - .on('error', rej); - }) - .catch(rej); - } - rej(new Error('un-compatible file.')); - }); - // Calculate hash, get content type and get file size const [hash, [mime, ext], size] = await Promise.all([ // hash @@ -212,7 +188,40 @@ const addFile = async ( * @return Object that represents added file */ export default (user: any, file: string | stream.Readable, ...args) => new Promise((resolve, reject) => { - addFile(user, file, ...args) + // Get file path + new Promise((res: (v: [string, boolean]) => void, rej) => { + if (typeof file === 'string') { + res([file, false]); + return; + } + if (typeof file === 'object' && typeof file.read === 'function') { + tmpFile() + .then(path => { + const readable: stream.Readable = file; + const writable = fs.createWriteStream(path); + readable + .on('error', rej) + .on('end', () => { + res([path, true]); + }) + .pipe(writable) + .on('error', rej); + }) + .catch(rej); + } + rej(new Error('un-compatible file.')); + }).then(([path, remove]): Promise => new Promise((res, rej) => { + addFile(user, path, ...args) + .then(file => { + res(file) + if (remove) { + fs.unlink(path, (e) => { + if (e) log(e.stack) + }) + } + }) + .catch(rej) + })) .then(file => { log(`drive file has been created ${file._id}`); resolve(file); From aabfe3c87365952bdffa757ea30d8631daf7de4f Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 05:12:48 +0900 Subject: [PATCH 10/10] format --- src/api/common/add-file-to-drive.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index 6a728d0d1..7defbc631 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -213,14 +213,14 @@ export default (user: any, file: string | stream.Readable, ...args) => new Promi }).then(([path, remove]): Promise => new Promise((res, rej) => { addFile(user, path, ...args) .then(file => { - res(file) + res(file); if (remove) { fs.unlink(path, (e) => { - if (e) log(e.stack) - }) + if (e) log(e.stack); + }); } }) - .catch(rej) + .catch(rej); })) .then(file => { log(`drive file has been created ${file._id}`);