Merge pull request #45 from mozilla/s3-integration

S3 integration

frontend/src/fileReceiver.js

@@ -34,7 +34,7 @@ class FileReceiver extends EventEmitter {
               data: this.result,
               fname: xhr
                 .getResponseHeader('Content-Disposition')
-                .match(/filename="(.+)"/)[1]
+                .match(/=(.+)/)[1]
             });
           };
 

frontend/src/fileSender.js

@@ -85,10 +85,13 @@ class FileSender extends EventEmitter {
 
           xhr.onreadystatechange = () => {
             if (xhr.readyState == XMLHttpRequest.DONE) {
+              // uuid field and url field
+              let responseObj = JSON.parse(xhr.responseText);
               resolve({
+                url: responseObj.url,
                 fileId: fileId,
                 secretKey: keydata.k,
-                deleteToken: xhr.responseText
+                deleteToken: responseObj.uuid
               });
             }
           };

frontend/src/upload.js

@@ -58,8 +58,7 @@ $(document).ready(function() {
       progress.innerText = `Progress: ${percentComplete}%`;
     });
     fileSender.upload().then(info => {
-      const url = `${window.location
+      const url = info.url.trim() + `#${info.secretKey}`.trim();
-        .origin}/download/${info.fileId}/#${info.secretKey}`;
       $('#link').attr('value', url);
       link.innerHTML = url;
       localStorage.setItem(info.fileId, info.deleteToken);

package.json

@@ -4,15 +4,23 @@
   "version": "1.0.0",
   "author": "Mozilla (https://mozilla.org)",
   "dependencies": {
+    "aws-sdk": "^2.62.0",
     "body-parser": "^1.17.2",
+    "bytes": "^2.5.0",
+    "color-convert": "^1.9.0",
     "connect-busboy": "0.0.2",
+    "convict": "^3.0.0",
     "express": "^4.15.3",
+    "express-handlebars": "^3.0.0",
     "fs-extra": "^3.0.1",
+    "node-fetch": "^1.7.1",
     "path": "^0.12.7",
     "redis": "^2.7.1"
   },
   "devDependencies": {
     "browserify": "^14.4.0",
+    "buffer-shims": "^1.0.0",
+    "cross-env": "^5.0.0",
     "prettier": "^1.3.1",
     "watchify": "^3.9.0"
   },
@@ -20,7 +28,8 @@
   "repository": "mozilla/something-awesome",
   "scripts": {
     "format": "prettier --single-quote --write 'frontend/src/*.js' 'server/*.js'",
-    "start": "watchify frontend/src/main.js -o public/bundle.js -d | node server/portal_server.js",
+    "dev": "watchify frontend/src/main.js -o public/bundle.js -d | node server/portal_server.js",
+    "start": "watchify frontend/src/main.js -o public/bundle.js -d | cross-env NODE_ENV=production node server/portal_server.js",
     "test": "echo \"Error: no test specified\" && exit 1"
   }
 }

server/config.js

@@ -0,0 +1,35 @@
+const convict = require('convict');
+
+let conf = convict({
+  bitly_key: {
+    format: String,
+    default: 'localhost',
+    env: 'P2P_BITLY_KEY'
+  },
+  s3_bucket: {
+    format: String,
+    default: 'localhost',
+    env: 'P2P_S3_BUCKET'
+  },
+  redis_host: {
+    format: String,
+    default: 'localhost',
+    env: 'P2P_REDIS_HOST'
+  },
+  listen_port: {
+    format: 'port',
+    default: 1443,
+    arg: 'port',
+    env: 'P2P_LISTEN_PORT'
+  },
+  env: {
+    format: ['production', 'development'],
+    default: 'development',
+    env: 'NODE_ENV'
+  }
+});
+
+// Perform validation
+conf.validate({ allowed: 'strict' });
+
+module.exports = conf.getProperties();

server/portal_server.js

@@ -1,49 +1,85 @@
 const express = require('express');
+const exphbs = require('express-handlebars');
 const busboy = require('connect-busboy');
 const path = require('path');
 const fs = require('fs-extra');
 const bodyParser = require('body-parser');
 const crypto = require('crypto');
+const stream = require('stream');
+const fetch = require('node-fetch');
+const bytes = require('bytes');
+const conf = require('./config.js');
+const storage = require('./storage.js');
+
+let notLocalHost =
+  conf.env === 'production' &&
+  conf.s3_bucket !== 'localhost' &&
+  conf.bitly_key !== 'localhost';
 
 const app = express();
-const redis = require('redis'),
-  client = redis.createClient();
 
-client.on('error', err => {
+app.engine('handlebars', exphbs({ defaultLayout: 'main' }));
-  console.log(err);
+app.set('view engine', 'handlebars');
-});
 
 app.use(busboy());
 app.use(bodyParser.json());
+
 app.use(express.static(path.join(__dirname, '../public')));
 
+app.get('/', (req, res) => {
+  res.render('index');
+});
+
 app.get('/download/:id', (req, res) => {
-  res.sendFile(path.join(__dirname + '/../public/download.html'));
+  let id = req.params.id;
+  storage.filename(id).then(filename => {
+    storage
+      .length(id)
+      .then(contentLength => {
+        res.render('download', {
+          filename: filename,
+          filesize: bytes(contentLength)
+        });
+      })
+      .catch(() => {
+        res.render('download');
+      });
+  });
 });
 
 app.get('/assets/download/:id', (req, res) => {
   let id = req.params.id;
   if (!validateID(id)) {
-    res.send(404);
+    res.sendStatus(404);
     return;
   }
 
-  client.hget(id, 'filename', (err, reply) => {
+  storage
-    // maybe some expiration logic too
+    .filename(id)
-    if (!reply) {
+    .then(reply => {
-      res.sendStatus(404);
+      storage.length(id).then(contentLength => {
-    } else {
+        res.writeHead(200, {
-      res.setHeader('Content-Disposition', 'attachment; filename=' + reply);
+          'Content-Disposition': 'attachment; filename=' + reply,
-      res.setHeader('Content-Type', 'application/octet-stream');
+          'Content-Type': 'application/octet-stream',
-
+          'Content-Length': contentLength
-      res.download(__dirname + '/../static/' + id, reply, err => {
+        });
-        if (!err) {
-          client.del(id);
-          fs.unlinkSync(__dirname + '/../static/' + id);
-        }
       });
-    }
+
-  });
+      let file_stream = storage.get(id);
+
+      file_stream.on(notLocalHost ? 'finish' : 'close', () => {
+        storage.forceDelete(id).then(err => {
+          if (!err) {
+            console.log('Deleted.');
+          }
+        });
+      });
+
+      file_stream.pipe(res);
+    })
+    .catch(err => {
+      res.sendStatus(404);
+    });
 });
 
 app.post('/delete/:id', (req, res) => {
@@ -60,15 +96,14 @@ app.post('/delete/:id', (req, res) => {
     res.sendStatus(404);
   }
 
-  client.hget(id, 'delete', (err, reply) => {
+  storage
-    if (!reply) {
+    .delete(id, delete_token)
-      res.sendStatus(404);
+    .then(err => {
-    } else {
+      if (!err) {
-      client.del(id);
+        console.log('Deleted.');
-      fs.unlinkSync(__dirname + '/../static/' + id);
+      }
-      res.sendStatus(200);
+    })
-    }
+    .catch(err => res.sendStatus(404));
-  });
 });
 
 app.post('/upload/:id', (req, res, next) => {
@@ -77,34 +112,19 @@ app.post('/upload/:id', (req, res, next) => {
     return;
   }
 
-  let fstream;
   req.pipe(req.busboy);
   req.busboy.on('file', (fieldname, file, filename) => {
     console.log('Uploading: ' + filename);
+    let url = `${req.protocol}://${req.get('host')}/download/${req.params.id}/`;
 
-    //Path where image will be uploaded
+    storage.set(req.params.id, file, filename, url).then(linkAndID => {
-    fstream = fs.createWriteStream(__dirname + '/../static/' + req.params.id);
+      res.json(linkAndID);
-    file.pipe(fstream);
-    fstream.on('close', () => {
-      let id = req.params.id;
-      let uuid = crypto.randomBytes(10).toString('hex');
-
-      client.hmset([id, 'filename', filename, 'delete', uuid]);
-
-      // delete the file off the server in 24 hours
-      // setTimeout(() => {
-      //   fs.unlinkSync(__dirname + "/static/" + id);
-      // }, 86400000);
-
-      client.expire(id, 86400000);
-      console.log('Upload Finished of ' + filename);
-      res.send(uuid);
     });
   });
 });
 
-app.listen(3000, () => {
+let server = app.listen(conf.listen_port, () => {
-  console.log('Portal app listening on port 3000!');
+  console.log(`Portal app listening on port ${conf.listen_port}!`);
 });
 
 let validateID = route_id => {

server/storage.js

@@ -0,0 +1,212 @@
+const AWS = require('aws-sdk');
+const s3 = new AWS.S3();
+
+const conf = require('./config.js');
+const fs = require('fs');
+const path = require('path');
+const fetch = require('node-fetch');
+const crypto = require('crypto');
+
+const redis = require('redis');
+const redis_client = redis.createClient();
+
+redis_client.on('error', err => {
+  console.log(err);
+});
+
+let notLocalhost =
+  conf.env === 'production' &&
+  conf.s3_bucket !== 'localhost' &&
+  conf.bitly_key !== 'localhost';
+
+if (notLocalhost) {
+  module.exports = {
+    filename: filename,
+    length: awsLength,
+    get: awsGet,
+    set: awsSet,
+    delete: awsDelete,
+    forceDelete: awsForceDelete
+  };
+} else {
+  module.exports = {
+    filename: filename,
+    length: localLength,
+    get: localGet,
+    set: localSet,
+    delete: localDelete,
+    forceDelete: localForceDelete
+  };
+}
+
+function filename(id) {
+  return new Promise((resolve, reject) => {
+    redis_client.hget(id, 'filename', (err, reply) => {
+      if (!err) {
+        resolve(reply);
+      } else {
+        reject();
+      }
+    });
+  });
+}
+
+function localLength(id) {
+  return new Promise((resolve, reject) => {
+    try {
+      resolve(fs.statSync(__dirname + '/../static/' + id).size);
+    } catch (err) {
+      reject();
+    }
+  });
+}
+
+function localGet(id) {
+  return fs.createReadStream(__dirname + '/../static/' + id);
+}
+
+function localSet(id, file, filename, url) {
+  return new Promise((resolve, reject) => {
+    fstream = fs.createWriteStream(__dirname + '/../static/' + id);
+    file.pipe(fstream);
+    fstream.on('close', () => {
+      let uuid = crypto.randomBytes(10).toString('hex');
+
+      redis_client.hmset([id, 'filename', filename, 'delete', uuid]);
+      redis_client.expire(id, 86400000);
+      console.log('Upload Finished of ' + filename);
+      resolve({
+        uuid: uuid,
+        url: url
+      });
+    });
+
+    fstream.on('error', () => reject());
+  });
+}
+
+function localDelete(id, delete_token) {
+  return new Promise((resolve, reject) => {
+    redis_client.hget(id, 'delete', (err, reply) => {
+      if (!reply || delete_token !== reply) {
+        reject();
+      } else {
+        redis_client.del(id);
+        resolve(fs.unlinkSync(__dirname + '/../static/' + id));
+      }
+    });
+  });
+}
+
+function localForceDelete(id) {
+  return new Promise((resolve, reject) => {
+    redis_client.del(id);
+    resolve(fs.unlinkSync(__dirname + '/../static/' + id));
+  });
+}
+
+function awsLength(id) {
+  let params = {
+    Bucket: conf.s3_bucket,
+    Key: id
+  };
+  return new Promise((resolve, reject) => {
+    s3.headObject(params, function(err, data) {
+      if (!err) {
+        resolve(data.ContentLength);
+      } else {
+        reject();
+      }
+    });
+  });
+}
+
+function awsGet(id) {
+  let params = {
+    Bucket: conf.s3_bucket,
+    Key: id
+  };
+
+  return s3.getObject(params).createReadStream();
+}
+
+function awsSet(id, file, filename, url) {
+  let params = {
+    Bucket: conf.s3_bucket,
+    Key: id,
+    Body: file
+  };
+
+  return new Promise((resolve, reject) => {
+    s3.upload(params, function(err, data) {
+      if (err) {
+        console.log(err, err.stack); // an error occurred
+        reject();
+      } else {
+        let uuid = crypto.randomBytes(10).toString('hex');
+
+        redis_client.hmset([id, 'filename', filename, 'delete', uuid]);
+
+        redis_client.expire(id, 86400000);
+        console.log('Upload Finished of ' + filename);
+        if (conf.bitly_key) {
+          fetch(
+            'https://api-ssl.bitly.com/v3/shorten?access_token=' +
+              conf.bitly_key +
+              '&longUrl=' +
+              encodeURIComponent(url) +
+              '&format=txt'
+          )
+            .then(res => {
+              return res.text();
+            })
+            .then(body => {
+              resolve({
+                uuid: uuid,
+                url: body
+              });
+            });
+        } else {
+          resolve({
+            uuid: uuid,
+            url: url
+          });
+        }
+      }
+    });
+  });
+}
+
+function awsDelete(id, delete_token) {
+  return new Promise((resolve, reject) => {
+    redis_client.hget(id, 'delete', (err, reply) => {
+      if (!reply || delete_token !== reply) {
+        reject();
+      } else {
+        redis_client.del(id);
+        let params = {
+          Bucket: conf.s3_bucket,
+          Key: id
+        };
+
+        s3.deleteObject(params, function(err, data) {
+          resolve(err);
+        });
+      }
+    });
+  });
+}
+
+function awsForceDelete(id) {
+  return new Promise((resolve, reject) => {
+    redis_client.del(id);
+    let params = {
+      Bucket: conf.s3_bucket,
+      Key: id
+    };
+
+    s3.deleteObject(params, function(err, data) {
+      resolve(err);
+    });
+  });
+}

views/download.handlebars

@@ -11,9 +11,13 @@
 
   <div class="main-window">
     <div id="download">
+      {{#if filename}}
       <div class="title">
         Your friend is sending you a file:
       </div>
+      
+      <span> {{{filename}}} ({{{filesize}}})
+
       <div class="share-window">
         <button id="download-btn" onclick="download()">Download File</button>
         <img id="expired-img" src="/resources/link_expired.png"/>
@@ -21,6 +25,18 @@
       <div class="send-new" id="send-file">
         Send your own files
       </div>
+      {{else}}
+      <div class="title">
+        This link has expired or never existed in the first place.
+      </div>
+
+      <div class="share-window">
+        <img src="/resources/link_expired.png"/>
+      </div>
+      <div class="send-new" id="send-file">
+        Send your own files
+      </div>
+      {{/if}}
     </div>
   </div>
 

views/index.handlebars

@@ -3,9 +3,9 @@
 <head>
   <title>Firefox Fileshare</title>
   <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
-  <script src="/bundle.js"></script>
+  <script src="./bundle.js"></script>
   <link rel="stylesheet" href="https://code.cdn.mozilla.net/fonts/fira.css">
-  <link rel="stylesheet" type="text/css" href="/main.css" />
+  <link rel="stylesheet" type="text/css" href="./main.css" />
 </head>
 <body>
 

views/layouts/main.handlebars

@@ -0,0 +1 @@
+{{{body}}}