working s3 integration, commented out load credentials from json

This commit is contained in:
Abhinav Adduri 2017-06-06 10:24:58 -07:00 committed by Danny Coates
parent f377aa4551
commit af4edfef16
No known key found for this signature in database
GPG Key ID: 4C442633C62E00CB
6 changed files with 116 additions and 86 deletions

View File

@ -5,7 +5,6 @@ class FileReceiver extends EventEmitter {
constructor() { constructor() {
super(); super();
this.salt = strToIv(location.pathname.slice(10, -1)); this.salt = strToIv(location.pathname.slice(10, -1));
window.salt = this.salt;
} }
download() { download() {
@ -31,8 +30,6 @@ class FileReceiver extends EventEmitter {
let blob = new Blob([this.response]); let blob = new Blob([this.response]);
let fileReader = new FileReader(); let fileReader = new FileReader();
fileReader.onload = function() { fileReader.onload = function() {
window.data = this.result;
console.log(this.result);
resolve({ resolve({
data: this.result, data: this.result,
fname: xhr fname: xhr

View File

@ -85,10 +85,13 @@ class FileSender extends EventEmitter {
xhr.onreadystatechange = () => { xhr.onreadystatechange = () => {
if (xhr.readyState == XMLHttpRequest.DONE) { if (xhr.readyState == XMLHttpRequest.DONE) {
// uuid field and url field
let responseObj = JSON.parse(xhr.responseText);
resolve({ resolve({
url: responseObj.url,
fileId: fileId, fileId: fileId,
secretKey: keydata.k, secretKey: keydata.k,
deleteToken: xhr.responseText deleteToken: responseObj.uuid
}); });
} }
}; };

View File

@ -58,8 +58,7 @@ $(document).ready(function() {
progress.innerText = `Progress: ${percentComplete}%`; progress.innerText = `Progress: ${percentComplete}%`;
}); });
fileSender.upload().then(info => { fileSender.upload().then(info => {
const url = `${window.location const url = info.url.trim() + `#${info.secretKey}`.trim();
.origin}/download/${info.fileId}/#${info.secretKey}`;
$('#link').attr('value', url); $('#link').attr('value', url);
link.innerHTML = url; link.innerHTML = url;
localStorage.setItem(info.fileId, info.deleteToken); localStorage.setItem(info.fileId, info.deleteToken);

View File

@ -10,11 +10,13 @@
"convict": "^3.0.0", "convict": "^3.0.0",
"express": "^4.15.3", "express": "^4.15.3",
"fs-extra": "^3.0.1", "fs-extra": "^3.0.1",
"node-fetch": "^1.7.1",
"path": "^0.12.7", "path": "^0.12.7",
"redis": "^2.7.1" "redis": "^2.7.1"
}, },
"devDependencies": { "devDependencies": {
"browserify": "^14.4.0", "browserify": "^14.4.0",
"cross-env": "^5.0.0",
"prettier": "^1.3.1", "prettier": "^1.3.1",
"watchify": "^3.9.0" "watchify": "^3.9.0"
}, },
@ -22,7 +24,8 @@
"repository": "mozilla/something-awesome", "repository": "mozilla/something-awesome",
"scripts": { "scripts": {
"format": "prettier --single-quote --write 'frontend/src/*.js' 'server/*.js'", "format": "prettier --single-quote --write 'frontend/src/*.js' 'server/*.js'",
"start": "watchify frontend/src/main.js -o public/bundle.js -d | node server/portal_server.js", "dev": "watchify frontend/src/main.js -o public/bundle.js -d | node server/portal_server.js",
"start": "watchify frontend/src/main.js -o public/bundle.js -d | cross-env NODE_ENV=production node server/portal_server.js",
"test": "echo \"Error: no test specified\" && exit 1" "test": "echo \"Error: no test specified\" && exit 1"
} }
} }

View File

@ -1,16 +1,22 @@
const convict = require('convict'); const convict = require('convict');
let api_key = 'INSERT API KEY HERE';
let conf = convict({ let conf = convict({
aws_credentials: { aws_credentials: {
region: 'us-west-2', region: 'us-west-2',
bucketName: 'testpilot-p2p' bucketName: 'testpilot-p2p'
},
bitly_credentials: {
api_key: api_key
},
env: {
format: ['production', 'development'],
default: 'development',
env: 'NODE_ENV'
} }
}) });
// var env = conf.get('env');
// conf.loadFile('./config/' + env + '.json');
// Perform validation // Perform validation
conf.validate({ allowed: 'strict' }); conf.validate({ allowed: 'strict' });
module.exports = conf; module.exports = conf.getProperties();

View File

@ -1,24 +1,25 @@
const express = require("express") const express = require('express');
const busboy = require("connect-busboy"); const busboy = require('connect-busboy');
const path = require("path"); const path = require('path');
const fs = require("fs-extra"); const fs = require('fs-extra');
const bodyParser = require("body-parser"); const bodyParser = require('body-parser');
const crypto = require("crypto"); const crypto = require('crypto');
const conf = require('./config.js'); const conf = require('./config.js');
const stream = require('stream'); const stream = require('stream');
const fetch = require('node-fetch');
let aws_credentials = conf.get('aws_credentials'); let aws_credentials = conf.aws_credentials;
let bitly_credentials = conf.bitly_credentials;
let isProduction = conf.env === 'production';
const AWS = require('aws-sdk'); const AWS = require('aws-sdk');
AWS.config.loadFromPath('../../.aws/credentials');
const s3 = new AWS.S3(); const s3 = new AWS.S3();
const app = express();
const redis = require('redis');
const redis_client = redis.createClient();
const app = express() redis_client.on('error', err => {
const redis = require("redis"),
redis_client = redis.createClient();
redis_client.on("error", (err) => {
console.log(err); console.log(err);
}); });
@ -33,24 +34,26 @@ app.get('/download/:id', (req, res) => {
app.get('/assets/download/:id', (req, res) => { app.get('/assets/download/:id', (req, res) => {
let id = req.params.id; let id = req.params.id;
if (!validateID(id)) { if (!validateID(id)) {
res.send(404); res.sendStatus(404);
return; return;
} }
redis_client.hget(id, "filename", (err, reply) => { // maybe some expiration logic too redis_client.hget(id, 'filename', (err, reply) => {
// maybe some expiration logic too
if (!reply) { if (!reply) {
res.sendStatus(404); res.sendStatus(404);
} else { } else {
let params = { let params = {
Bucket: aws_credentials.bucketName, Bucket: aws_credentials.bucketName,
Key: id Key: id
} };
s3.headObject(params, function(err, data) { s3.headObject(params, function(err, data) {
res.writeHead(200, {"Content-Disposition": "attachment; filename=" + reply, res.writeHead(200, {
"Content-Type": "application/octet-stream", 'Content-Disposition': 'attachment; filename=' + reply,
"Content-Length": data.ContentLength}); 'Content-Type': 'application/octet-stream',
'Content-Length': data.ContentLength
});
let file_stream = s3.getObject(params).createReadStream(); let file_stream = s3.getObject(params).createReadStream();
file_stream.on('finish', () => { file_stream.on('finish', () => {
@ -59,26 +62,13 @@ app.get('/assets/download/:id', (req, res) => {
if (!err) { if (!err) {
console.log('Deleted off s3.'); console.log('Deleted off s3.');
} }
}) });
}); });
file_stream.pipe(res); file_stream.pipe(res);
}); });
// s3.getObject(params, function(err, data) {
// if (err) {
// console.log(err, err.stack); // an error occurred
// res.sendStatus(404);
// }
// else {
// }
// })
} }
}) });
}); });
app.post('/delete/:id', (req, res) => { app.post('/delete/:id', (req, res) => {
@ -95,43 +85,42 @@ app.post('/delete/:id', (req, res) => {
res.sendStatus(404); res.sendStatus(404);
} }
redis_client.hget(id, "delete", (err, reply) => { redis_client.hget(id, 'delete', (err, reply) => {
if (!reply || (delete_token !== reply)) { if (!reply || delete_token !== reply) {
res.sendStatus(404); res.sendStatus(404);
} else { } else {
redis_client.del(id); redis_client.del(id);
let params = { let params = {
Bucket: aws_credentials.bucketName, Bucket: aws_credentials.bucketName,
Key: id Key: id
} };
s3.deleteObject(params, function(err, data) { s3.deleteObject(params, function(err, data) {
if (!err) { if (!err) {
console.log('Deleted off s3.'); console.log('Deleted off s3.');
} }
}) });
res.sendStatus(200); res.sendStatus(200);
} }
}); });
}); });
app.post("/upload/:id", (req, res, next) => { app.post('/upload/:id', (req, res, next) => {
if (!validateID(req.params.id)) { if (!validateID(req.params.id)) {
res.send(404); res.send(404);
return; return;
} }
req.pipe(req.busboy); req.pipe(req.busboy);
req.busboy.on("file", (fieldname, file, filename) => { req.busboy.on('file', (fieldname, file, filename) => {
console.log("Uploading: " + filename); console.log('Uploading: ' + filename);
let params = { let params = {
Bucket: aws_credentials.bucketName, Bucket: aws_credentials.bucketName,
Key: req.params.id, Key: req.params.id,
Body: file Body: file
} };
s3.upload(params, function(err, data) { s3.upload(params, function(err, data) {
if (err) { if (err) {
@ -140,21 +129,54 @@ app.post("/upload/:id", (req, res, next) => {
let id = req.params.id; let id = req.params.id;
let uuid = crypto.randomBytes(10).toString('hex'); let uuid = crypto.randomBytes(10).toString('hex');
redis_client.hmset([id, "filename", filename, "delete", uuid]); redis_client.hmset([id, 'filename', filename, 'delete', uuid]);
redis_client.expire(id, 86400000); redis_client.expire(id, 86400000);
console.log("Upload Finished of " + filename); console.log('Upload Finished of ' + filename);
res.send(uuid);
} if (isProduction) {
let url =
req.protocol +
`://` +
req.get('host') +
'/download/' +
req.params.id +
'/';
fetch(
'https://api-ssl.bitly.com/v3/shorten?access_token=' +
bitly_credentials.api_key +
'&longUrl=' +
encodeURIComponent(url) +
'&format=txt'
)
.then(res => {
return res.text();
}) })
.then(body => {
res.json({
uuid: uuid,
url: body
});
});
} else {
res.json({
uuid: uuid,
url: url
});
}
}
}); });
}); });
}); });
app.listen(3000, () => { let server = app.listen(3000, () => {
console.log('Portal app listening on port 3000!'); console.log('Portal app listening on port 3000!');
}); });
let validateID = route_id => { let validateID = route_id => {
return route_id.match(/^[0-9a-fA-F]{32}$/) !== null; return route_id.match(/^[0-9a-fA-F]{32}$/) !== null;
}; };
if (bitly_credentials.api_key === 'INSERT API KEY HERE') {
throw new Error('Copy paste a bitly API key into server/config.js');
}