Merge pull request #45 from mozilla/s3-integration

S3 integration
This commit is contained in:
Danny Coates 2017-06-07 16:34:52 -07:00 committed by GitHub
commit a4437a341e
11 changed files with 990 additions and 340 deletions

View File

@ -34,7 +34,7 @@ class FileReceiver extends EventEmitter {
data: this.result,
fname: xhr
.getResponseHeader('Content-Disposition')
.match(/filename="(.+)"/)[1]
.match(/=(.+)/)[1]
});
};

View File

@ -85,10 +85,13 @@ class FileSender extends EventEmitter {
xhr.onreadystatechange = () => {
if (xhr.readyState == XMLHttpRequest.DONE) {
// uuid field and url field
let responseObj = JSON.parse(xhr.responseText);
resolve({
url: responseObj.url,
fileId: fileId,
secretKey: keydata.k,
deleteToken: xhr.responseText
deleteToken: responseObj.uuid
});
}
};

View File

@ -58,8 +58,7 @@ $(document).ready(function() {
progress.innerText = `Progress: ${percentComplete}%`;
});
fileSender.upload().then(info => {
const url = `${window.location
.origin}/download/${info.fileId}/#${info.secretKey}`;
const url = info.url.trim() + `#${info.secretKey}`.trim();
$('#link').attr('value', url);
link.innerHTML = url;
localStorage.setItem(info.fileId, info.deleteToken);

919
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -4,15 +4,23 @@
"version": "1.0.0",
"author": "Mozilla (https://mozilla.org)",
"dependencies": {
"aws-sdk": "^2.62.0",
"body-parser": "^1.17.2",
"bytes": "^2.5.0",
"color-convert": "^1.9.0",
"connect-busboy": "0.0.2",
"convict": "^3.0.0",
"express": "^4.15.3",
"express-handlebars": "^3.0.0",
"fs-extra": "^3.0.1",
"node-fetch": "^1.7.1",
"path": "^0.12.7",
"redis": "^2.7.1"
},
"devDependencies": {
"browserify": "^14.4.0",
"buffer-shims": "^1.0.0",
"cross-env": "^5.0.0",
"prettier": "^1.3.1",
"watchify": "^3.9.0"
},
@ -20,7 +28,8 @@
"repository": "mozilla/something-awesome",
"scripts": {
"format": "prettier --single-quote --write 'frontend/src/*.js' 'server/*.js'",
"start": "watchify frontend/src/main.js -o public/bundle.js -d | node server/portal_server.js",
"dev": "watchify frontend/src/main.js -o public/bundle.js -d | node server/portal_server.js",
"start": "watchify frontend/src/main.js -o public/bundle.js -d | cross-env NODE_ENV=production node server/portal_server.js",
"test": "echo \"Error: no test specified\" && exit 1"
}
}

35
server/config.js Normal file
View File

@ -0,0 +1,35 @@
const convict = require('convict');
let conf = convict({
bitly_key: {
format: String,
default: 'localhost',
env: 'P2P_BITLY_KEY'
},
s3_bucket: {
format: String,
default: 'localhost',
env: 'P2P_S3_BUCKET'
},
redis_host: {
format: String,
default: 'localhost',
env: 'P2P_REDIS_HOST'
},
listen_port: {
format: 'port',
default: 1443,
arg: 'port',
env: 'P2P_LISTEN_PORT'
},
env: {
format: ['production', 'development'],
default: 'development',
env: 'NODE_ENV'
}
});
// Perform validation
conf.validate({ allowed: 'strict' });
module.exports = conf.getProperties();

View File

@ -1,49 +1,85 @@
const express = require('express');
const exphbs = require('express-handlebars');
const busboy = require('connect-busboy');
const path = require('path');
const fs = require('fs-extra');
const bodyParser = require('body-parser');
const crypto = require('crypto');
const stream = require('stream');
const fetch = require('node-fetch');
const bytes = require('bytes');
const conf = require('./config.js');
const storage = require('./storage.js');
let notLocalHost =
conf.env === 'production' &&
conf.s3_bucket !== 'localhost' &&
conf.bitly_key !== 'localhost';
const app = express();
const redis = require('redis'),
client = redis.createClient();
client.on('error', err => {
console.log(err);
});
app.engine('handlebars', exphbs({ defaultLayout: 'main' }));
app.set('view engine', 'handlebars');
app.use(busboy());
app.use(bodyParser.json());
app.use(express.static(path.join(__dirname, '../public')));
app.get('/', (req, res) => {
res.render('index');
});
app.get('/download/:id', (req, res) => {
res.sendFile(path.join(__dirname + '/../public/download.html'));
let id = req.params.id;
storage.filename(id).then(filename => {
storage
.length(id)
.then(contentLength => {
res.render('download', {
filename: filename,
filesize: bytes(contentLength)
});
})
.catch(() => {
res.render('download');
});
});
});
app.get('/assets/download/:id', (req, res) => {
let id = req.params.id;
if (!validateID(id)) {
res.send(404);
res.sendStatus(404);
return;
}
client.hget(id, 'filename', (err, reply) => {
// maybe some expiration logic too
if (!reply) {
res.sendStatus(404);
} else {
res.setHeader('Content-Disposition', 'attachment; filename=' + reply);
res.setHeader('Content-Type', 'application/octet-stream');
res.download(__dirname + '/../static/' + id, reply, err => {
if (!err) {
client.del(id);
fs.unlinkSync(__dirname + '/../static/' + id);
}
storage
.filename(id)
.then(reply => {
storage.length(id).then(contentLength => {
res.writeHead(200, {
'Content-Disposition': 'attachment; filename=' + reply,
'Content-Type': 'application/octet-stream',
'Content-Length': contentLength
});
});
}
});
let file_stream = storage.get(id);
file_stream.on(notLocalHost ? 'finish' : 'close', () => {
storage.forceDelete(id).then(err => {
if (!err) {
console.log('Deleted.');
}
});
});
file_stream.pipe(res);
})
.catch(err => {
res.sendStatus(404);
});
});
app.post('/delete/:id', (req, res) => {
@ -60,15 +96,14 @@ app.post('/delete/:id', (req, res) => {
res.sendStatus(404);
}
client.hget(id, 'delete', (err, reply) => {
if (!reply) {
res.sendStatus(404);
} else {
client.del(id);
fs.unlinkSync(__dirname + '/../static/' + id);
res.sendStatus(200);
}
});
storage
.delete(id, delete_token)
.then(err => {
if (!err) {
console.log('Deleted.');
}
})
.catch(err => res.sendStatus(404));
});
app.post('/upload/:id', (req, res, next) => {
@ -77,34 +112,19 @@ app.post('/upload/:id', (req, res, next) => {
return;
}
let fstream;
req.pipe(req.busboy);
req.busboy.on('file', (fieldname, file, filename) => {
console.log('Uploading: ' + filename);
let url = `${req.protocol}://${req.get('host')}/download/${req.params.id}/`;
//Path where image will be uploaded
fstream = fs.createWriteStream(__dirname + '/../static/' + req.params.id);
file.pipe(fstream);
fstream.on('close', () => {
let id = req.params.id;
let uuid = crypto.randomBytes(10).toString('hex');
client.hmset([id, 'filename', filename, 'delete', uuid]);
// delete the file off the server in 24 hours
// setTimeout(() => {
// fs.unlinkSync(__dirname + "/static/" + id);
// }, 86400000);
client.expire(id, 86400000);
console.log('Upload Finished of ' + filename);
res.send(uuid);
storage.set(req.params.id, file, filename, url).then(linkAndID => {
res.json(linkAndID);
});
});
});
app.listen(3000, () => {
console.log('Portal app listening on port 3000!');
let server = app.listen(conf.listen_port, () => {
console.log(`Portal app listening on port ${conf.listen_port}!`);
});
let validateID = route_id => {

212
server/storage.js Normal file
View File

@ -0,0 +1,212 @@
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const conf = require('./config.js');
const fs = require('fs');
const path = require('path');
const fetch = require('node-fetch');
const crypto = require('crypto');
const redis = require('redis');
const redis_client = redis.createClient();
redis_client.on('error', err => {
console.log(err);
});
let notLocalhost =
conf.env === 'production' &&
conf.s3_bucket !== 'localhost' &&
conf.bitly_key !== 'localhost';
if (notLocalhost) {
module.exports = {
filename: filename,
length: awsLength,
get: awsGet,
set: awsSet,
delete: awsDelete,
forceDelete: awsForceDelete
};
} else {
module.exports = {
filename: filename,
length: localLength,
get: localGet,
set: localSet,
delete: localDelete,
forceDelete: localForceDelete
};
}
function filename(id) {
return new Promise((resolve, reject) => {
redis_client.hget(id, 'filename', (err, reply) => {
if (!err) {
resolve(reply);
} else {
reject();
}
});
});
}
function localLength(id) {
return new Promise((resolve, reject) => {
try {
resolve(fs.statSync(__dirname + '/../static/' + id).size);
} catch (err) {
reject();
}
});
}
function localGet(id) {
return fs.createReadStream(__dirname + '/../static/' + id);
}
function localSet(id, file, filename, url) {
return new Promise((resolve, reject) => {
fstream = fs.createWriteStream(__dirname + '/../static/' + id);
file.pipe(fstream);
fstream.on('close', () => {
let uuid = crypto.randomBytes(10).toString('hex');
redis_client.hmset([id, 'filename', filename, 'delete', uuid]);
redis_client.expire(id, 86400000);
console.log('Upload Finished of ' + filename);
resolve({
uuid: uuid,
url: url
});
});
fstream.on('error', () => reject());
});
}
function localDelete(id, delete_token) {
return new Promise((resolve, reject) => {
redis_client.hget(id, 'delete', (err, reply) => {
if (!reply || delete_token !== reply) {
reject();
} else {
redis_client.del(id);
resolve(fs.unlinkSync(__dirname + '/../static/' + id));
}
});
});
}
function localForceDelete(id) {
return new Promise((resolve, reject) => {
redis_client.del(id);
resolve(fs.unlinkSync(__dirname + '/../static/' + id));
});
}
function awsLength(id) {
let params = {
Bucket: conf.s3_bucket,
Key: id
};
return new Promise((resolve, reject) => {
s3.headObject(params, function(err, data) {
if (!err) {
resolve(data.ContentLength);
} else {
reject();
}
});
});
}
function awsGet(id) {
let params = {
Bucket: conf.s3_bucket,
Key: id
};
return s3.getObject(params).createReadStream();
}
function awsSet(id, file, filename, url) {
let params = {
Bucket: conf.s3_bucket,
Key: id,
Body: file
};
return new Promise((resolve, reject) => {
s3.upload(params, function(err, data) {
if (err) {
console.log(err, err.stack); // an error occurred
reject();
} else {
let uuid = crypto.randomBytes(10).toString('hex');
redis_client.hmset([id, 'filename', filename, 'delete', uuid]);
redis_client.expire(id, 86400000);
console.log('Upload Finished of ' + filename);
if (conf.bitly_key) {
fetch(
'https://api-ssl.bitly.com/v3/shorten?access_token=' +
conf.bitly_key +
'&longUrl=' +
encodeURIComponent(url) +
'&format=txt'
)
.then(res => {
return res.text();
})
.then(body => {
resolve({
uuid: uuid,
url: body
});
});
} else {
resolve({
uuid: uuid,
url: url
});
}
}
});
});
}
function awsDelete(id, delete_token) {
return new Promise((resolve, reject) => {
redis_client.hget(id, 'delete', (err, reply) => {
if (!reply || delete_token !== reply) {
reject();
} else {
redis_client.del(id);
let params = {
Bucket: conf.s3_bucket,
Key: id
};
s3.deleteObject(params, function(err, data) {
resolve(err);
});
}
});
});
}
function awsForceDelete(id) {
return new Promise((resolve, reject) => {
redis_client.del(id);
let params = {
Bucket: conf.s3_bucket,
Key: id
};
s3.deleteObject(params, function(err, data) {
resolve(err);
});
});
}

View File

@ -11,9 +11,13 @@
<div class="main-window">
<div id="download">
{{#if filename}}
<div class="title">
Your friend is sending you a file:
</div>
<span> {{{filename}}} ({{{filesize}}})
<div class="share-window">
<button id="download-btn" onclick="download()">Download File</button>
<img id="expired-img" src="/resources/link_expired.png"/>
@ -21,6 +25,18 @@
<div class="send-new" id="send-file">
Send your own files
</div>
{{else}}
<div class="title">
This link has expired or never existed in the first place.
</div>
<div class="share-window">
<img src="/resources/link_expired.png"/>
</div>
<div class="send-new" id="send-file">
Send your own files
</div>
{{/if}}
</div>
</div>

View File

@ -3,9 +3,9 @@
<head>
<title>Firefox Fileshare</title>
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
<script src="/bundle.js"></script>
<script src="./bundle.js"></script>
<link rel="stylesheet" href="https://code.cdn.mozilla.net/fonts/fira.css">
<link rel="stylesheet" type="text/css" href="/main.css" />
<link rel="stylesheet" type="text/css" href="./main.css" />
</head>
<body>

View File

@ -0,0 +1 @@
{{{body}}}