Merge branch 'vnext' of https://github.com/mozilla/send into ios-prototype-3

app/api.js

@@ -135,7 +135,7 @@ async function upload(stream, metadata, verifierB64, onprogress, canceller) {
   const endpoint =
     window.location.protocol === 'file:'
       ? 'wss://send2.dev.lcip.org/api/ws'
-      : `${protocol}//${host}:${port}/api/ws`;
+      : `${protocol}//${host}${port ? ':' : ''}${port}/api/ws`;
 
   const ws = await asyncInitWebSocket(endpoint);
 
@@ -270,7 +270,7 @@ function download(id, keychain, onprogress, canceller) {
       }
     });
     const auth = await keychain.authHeader();
-    xhr.open('get', `/api/download/${id}`);
+    xhr.open('get', `/api/download/blob/${id}`);
     xhr.setRequestHeader('Authorization', auth);
     xhr.responseType = 'blob';
     xhr.send();

app/capabilities.js

@@ -0,0 +1,75 @@
+async function checkCrypto() {
+  try {
+    const key = await crypto.subtle.generateKey(
+      {
+        name: 'AES-GCM',
+        length: 128
+      },
+      true,
+      ['encrypt', 'decrypt']
+    );
+    await crypto.subtle.exportKey('raw', key);
+    await crypto.subtle.encrypt(
+      {
+        name: 'AES-GCM',
+        iv: crypto.getRandomValues(new Uint8Array(12)),
+        tagLength: 128
+      },
+      key,
+      new ArrayBuffer(8)
+    );
+    await crypto.subtle.importKey(
+      'raw',
+      crypto.getRandomValues(new Uint8Array(16)),
+      'PBKDF2',
+      false,
+      ['deriveKey']
+    );
+    await crypto.subtle.importKey(
+      'raw',
+      crypto.getRandomValues(new Uint8Array(16)),
+      'HKDF',
+      false,
+      ['deriveKey']
+    );
+    return true;
+  } catch (err) {
+    return false;
+  }
+}
+
+function checkStreams() {
+  try {
+    new ReadableStream({
+      pull() {}
+    });
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+function polyfillStreams() {
+  try {
+    require('@mattiasbuelens/web-streams-polyfill');
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+export default async function capabilities() {
+  const crypto = await checkCrypto();
+  const nativeStreams = checkStreams();
+  const polyStreams = nativeStreams ? false : polyfillStreams();
+
+  return {
+    crypto,
+    streamUpload: nativeStreams || polyStreams,
+    streamDownload:
+      nativeStreams &&
+      'serviceWorker' in navigator &&
+      !/safari/i.test(navigator.userAgent),
+    multifile: nativeStreams || polyStreams
+  };
+}

app/ece.js

@@ -48,7 +48,7 @@ class ECETransformer {
         name: 'AES-GCM',
         length: 128
       },
-      false,
+      true, // Edge polyfill requires key to be extractable to encrypt :/
       ['encrypt', 'decrypt']
     );
   }

app/fileManager.js

@@ -177,7 +177,9 @@ export default function(state, emitter) {
     try {
       const start = Date.now();
       metrics.startedDownload({ size: file.size, ttl: file.ttl });
-      const dl = state.transfer.download();
+      const dl = state.transfer.download({
+        stream: state.capabilities.streamDownload
+      });
       render();
       await dl;
       const time = Date.now() - start;

app/fileReceiver.js

@@ -1,7 +1,9 @@
 import Nanobus from 'nanobus';
 import Keychain from './keychain';
 import { delay, bytes } from './utils';
-import { metadata } from './api';
+import { downloadFile, metadata } from './api';
+import { blobStream } from './streams';
+import Zip from './zip';
 
 export default class FileReceiver extends Nanobus {
   constructor(fileInfo) {
@@ -52,22 +54,6 @@ export default class FileReceiver extends Nanobus {
     this.state = 'ready';
   }
 
-  async streamToArrayBuffer(stream, streamSize, onprogress) {
-    const result = new Uint8Array(streamSize);
-    let offset = 0;
-    const reader = stream.getReader();
-    let state = await reader.read();
-    while (!state.done) {
-      result.set(state.value, offset);
-      offset += state.value.length;
-      state = await reader.read();
-      onprogress([offset, streamSize]);
-    }
-
-    onprogress([streamSize, streamSize]);
-    return result.slice(0, offset).buffer;
-  }
-
   sendMessageToSw(msg) {
     return new Promise((resolve, reject) => {
       const channel = new MessageChannel();
@@ -86,7 +72,46 @@ export default class FileReceiver extends Nanobus {
     });
   }
 
-  async download(noSave = false) {
+  async downloadBlob(noSave = false) {
+    this.state = 'downloading';
+    this.downloadRequest = await downloadFile(
+      this.fileInfo.id,
+      this.keychain,
+      p => {
+        this.progress = p;
+        this.emit('progress');
+      }
+    );
+    try {
+      const ciphertext = await this.downloadRequest.result;
+      this.downloadRequest = null;
+      this.msg = 'decryptingFile';
+      this.state = 'decrypting';
+      this.emit('decrypting');
+      let size = this.fileInfo.size;
+      let plainStream = this.keychain.decryptStream(blobStream(ciphertext));
+      if (this.fileInfo.type === 'send-archive') {
+        const zip = new Zip(this.fileInfo.manifest, plainStream);
+        plainStream = zip.stream;
+        size = zip.size;
+      }
+      const plaintext = await streamToArrayBuffer(plainStream, size);
+      if (!noSave) {
+        await saveFile({
+          plaintext,
+          name: decodeURIComponent(this.fileInfo.name),
+          type: this.fileInfo.type
+        });
+      }
+      this.msg = 'downloadFinish';
+      this.state = 'complete';
+    } catch (e) {
+      this.downloadRequest = null;
+      throw e;
+    }
+  }
+
+  async downloadStream(noSave = false) {
     const onprogress = p => {
       this.progress = p;
       this.emit('progress');
@@ -156,4 +181,64 @@ export default class FileReceiver extends Nanobus {
       throw e;
     }
   }
+
+  download(options) {
+    if (options.stream) {
+      return this.downloadStream(options.noSave);
+    }
+    return this.downloadBlob(options.noSave);
+  }
+}
+
+async function streamToArrayBuffer(stream, size) {
+  const result = new Uint8Array(size);
+  let offset = 0;
+  const reader = stream.getReader();
+  let state = await reader.read();
+  while (!state.done) {
+    result.set(state.value, offset);
+    offset += state.value.length;
+    state = await reader.read();
+  }
+
+  return result.buffer;
+}
+
+async function saveFile(file) {
+  return new Promise(function(resolve, reject) {
+    const dataView = new DataView(file.plaintext);
+    const blob = new Blob([dataView], { type: file.type });
+
+    if (navigator.msSaveBlob) {
+      navigator.msSaveBlob(blob, file.name);
+      return resolve();
+    } else if (/iPhone|fxios/i.test(navigator.userAgent)) {
+      // This method is much slower but createObjectURL
+      // is buggy on iOS
+      const reader = new FileReader();
+      reader.addEventListener('loadend', function() {
+        if (reader.error) {
+          return reject(reader.error);
+        }
+        if (reader.result) {
+          const a = document.createElement('a');
+          a.href = reader.result;
+          a.download = file.name;
+          document.body.appendChild(a);
+          a.click();
+        }
+        resolve();
+      });
+      reader.readAsDataURL(blob);
+    } else {
+      const downloadUrl = URL.createObjectURL(blob);
+      const a = document.createElement('a');
+      a.href = downloadUrl;
+      a.download = file.name;
+      document.body.appendChild(a);
+      a.click();
+      URL.revokeObjectURL(downloadUrl);
+      setTimeout(resolve, 100);
+    }
+  });
 }

app/main.js

@@ -1,29 +1,33 @@
 import 'fast-text-encoding'; // MS Edge support
 import 'fluent-intl-polyfill';
 import app from './routes';
+import capabilities from './capabilities';
 import locale from '../common/locales';
 import fileManager from './fileManager';
 import dragManager from './dragManager';
 import pasteManager from './pasteManager';
-import { canHasSend } from './utils';
 import storage from './storage';
 import metrics from './metrics';
 import experiments from './experiments';
 import Raven from 'raven-js';
 import './main.css';
 
+(async function start() {
   if (navigator.doNotTrack !== '1' && window.RAVEN_CONFIG) {
     Raven.config(window.SENTRY_ID, window.RAVEN_CONFIG).install();
   }
-
+  const capa = await capabilities();
+  if (capa.streamDownload) {
+    navigator.serviceWorker.register('/serviceWorker.js');
+  }
   app.use((state, emitter) => {
+    state.capabilities = capa;
     state.transfer = null;
     state.fileInfo = null;
     state.translate = locale.getTranslator();
     state.storage = storage;
     state.raven = Raven;
     window.appState = state;
-  emitter.on('DOMContentLoaded', async function checkSupport() {
     let unsupportedReason = null;
     if (
       // Firefox < 50
@@ -32,8 +36,7 @@ app.use((state, emitter) => {
     ) {
       unsupportedReason = 'outdated';
     }
-    const ok = await canHasSend();
-    if (!ok) {
+    if (!state.capabilities.crypto) {
       unsupportedReason = /firefox/i.test(navigator.userAgent)
         ? 'outdated'
         : 'gcm';
@@ -44,15 +47,10 @@ app.use((state, emitter) => {
       );
     }
   });
-});
-
-app.use(() => {
-  navigator.serviceWorker.register('/serviceWorker.js');
-});
   app.use(metrics);
   app.use(fileManager);
   app.use(dragManager);
   app.use(experiments);
   app.use(pasteManager);
-
   app.mount('body');
+})();

app/serviceWorker.js

@@ -75,7 +75,7 @@ async function decryptStream(id) {
 
 self.onfetch = event => {
   const req = event.request;
-  if (req.url.includes('/api/download')) {
+  if (/\/api\/download\/[A-Fa-f0-9]{4,}/.test(req.url)) {
     const id = req.url.split('/')[5];
     event.respondWith(decryptStream(id));
   }

app/utils.js

@@ -22,46 +22,6 @@ function loadShim(polyfill) {
   });
 }
 
-async function canHasSend() {
-  try {
-    const key = await crypto.subtle.generateKey(
-      {
-        name: 'AES-GCM',
-        length: 128
-      },
-      true,
-      ['encrypt', 'decrypt']
-    );
-    await crypto.subtle.encrypt(
-      {
-        name: 'AES-GCM',
-        iv: crypto.getRandomValues(new Uint8Array(12)),
-        tagLength: 128
-      },
-      key,
-      new ArrayBuffer(8)
-    );
-    await crypto.subtle.importKey(
-      'raw',
-      crypto.getRandomValues(new Uint8Array(16)),
-      'PBKDF2',
-      false,
-      ['deriveKey']
-    );
-    await crypto.subtle.importKey(
-      'raw',
-      crypto.getRandomValues(new Uint8Array(16)),
-      'HKDF',
-      false,
-      ['deriveKey']
-    );
-    return true;
-  } catch (err) {
-    console.error(err);
-    return false;
-  }
-}
-
 function isFile(id) {
   return /^[0-9a-fA-F]{10}$/.test(id);
 }
@@ -179,7 +139,6 @@ module.exports = {
   arrayToB64,
   b64ToArray,
   loadShim,
-  canHasSend,
   isFile,
   openLinksInNewTab
 };

package-lock.json

@@ -88,6 +88,15 @@
       "integrity": "sha1-9vGlzl05caSt6RoR0i1MRZrNN18=",
       "dev": true
     },
+    "@mattiasbuelens/web-streams-polyfill": {
+      "version": "0.1.0-alpha.5",
+      "resolved": "https://registry.npmjs.org/@mattiasbuelens/web-streams-polyfill/-/web-streams-polyfill-0.1.0-alpha.5.tgz",
+      "integrity": "sha512-KduiboN8xXQT+XAe76935VriUy62Yv/1EyTE43xmP+GEtPKpyUklKvYaXPq1fZpibMgXIIi/B2Qx0dReam5RKQ==",
+      "dev": true,
+      "requires": {
+        "@types/whatwg-streams": "0.0.6"
+      }
+    },
     "@mrmlnc/readdir-enhanced": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz",
@@ -122,6 +131,12 @@
         "samsam": "1.3.0"
       }
     },
+    "@types/whatwg-streams": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/@types/whatwg-streams/-/whatwg-streams-0.0.6.tgz",
+      "integrity": "sha512-O4Hat94N1RUCObqAbVUtd6EcucseqBcpfbFXzy12CYF6BQVHWR+ztDA3YPjewCmdKHYZ5VA7TZ5hq2bMyqxiBw==",
+      "dev": true
+    },
     "@webassemblyjs/ast": {
       "version": "1.5.13",
       "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.5.13.tgz",

package.json

@@ -56,6 +56,7 @@
   },
   "devDependencies": {
     "@dannycoates/webpack-dev-server": "^3.1.4",
+    "@mattiasbuelens/web-streams-polyfill": "0.1.0-alpha.5",
     "asmcrypto.js": "^0.22.0",
     "babel-core": "^6.26.3",
     "babel-loader": "^7.1.4",

server/routes/index.js

@@ -57,6 +57,7 @@ module.exports = function(app) {
   app.get('/completed', language, pages.blank);
   app.get('/unsupported/:reason', language, pages.unsupported);
   app.get(`/api/download/:id${ID_REGEX}`, auth, require('./download'));
+  app.get(`/api/download/blob/:id${ID_REGEX}`, auth, require('./download'));
   app.get(`/api/exists/:id${ID_REGEX}`, require('./exists'));
   app.get(`/api/metadata/:id${ID_REGEX}`, auth, require('./metadata'));
   app.post('/api/upload', require('./upload'));

test/frontend/tests/workflow-tests.js

@@ -7,7 +7,7 @@ const headless = /Headless/.test(navigator.userAgent);
 // TODO: save on headless doesn't work as it used to since it now
 // follows a link instead of fetch. Maybe there's a way to make it
 // work? For now always set noSave.
-const noSave = true || !headless; // only run the saveFile code if headless
+const options = { noSave: true || !headless, stream: true }; // only run the saveFile code if headless
 
 // FileSender uses a File in real life but a Blob works for testing
 const blob = new Blob([new ArrayBuffer(1024 * 128)], { type: 'text/plain' });
@@ -27,10 +27,10 @@ describe('Upload / Download flow', function() {
       requiresPassword: false
     });
     await fr.getMetadata();
-    await fr.download(noSave);
+    await fr.download(options);
 
     try {
-      await fr.download(noSave);
+      await fr.download(options);
       assert.fail('downloaded again');
     } catch (e) {
       assert.equal(e.message, '404');
@@ -50,7 +50,7 @@ describe('Upload / Download flow', function() {
       password: 'magic'
     });
     await fr.getMetadata();
-    await fr.download(noSave);
+    await fr.download(options);
     assert.equal(fr.state, 'complete');
   });
 
@@ -75,7 +75,7 @@ describe('Upload / Download flow', function() {
     try {
       // We can't decrypt without IV from metadata
       // but let's try to download anyway
-      await fr.download(noSave);
+      await fr.download(options);
       assert.fail('downloaded file with bad password');
     } catch (e) {
       assert.equal(e.message, '401');
@@ -135,7 +135,7 @@ describe('Upload / Download flow', function() {
     await fr.getMetadata();
     fr.once('progress', () => fr.cancel());
     try {
-      await fr.download(noSave);
+      await fr.download(options);
       assert.fail('not cancelled');
     } catch (e) {
       assert.equal(e.message, '0');
@@ -153,7 +153,7 @@ describe('Upload / Download flow', function() {
       requiresPassword: false
     });
     await fr.getMetadata();
-    await fr.download(noSave);
+    await fr.download(options);
     await file.updateDownloadCount();
     assert.equal(file.dtotal, 1);
   });
@@ -171,7 +171,7 @@ describe('Upload / Download flow', function() {
     fr.once('progress', () => fr.cancel());
 
     try {
-      await fr.download(noSave);
+      await fr.download(options);
       assert.fail('not cancelled');
     } catch (e) {
       await file.updateDownloadCount();
@@ -190,15 +190,15 @@ describe('Upload / Download flow', function() {
     });
     await file.changeLimit(2);
     await fr.getMetadata();
-    await fr.download(noSave);
+    await fr.download(options);
     await file.updateDownloadCount();
     assert.equal(file.dtotal, 1);
 
-    await fr.download(noSave);
+    await fr.download(options);
     await file.updateDownloadCount();
     assert.equal(file.dtotal, 2);
     try {
-      await fr.download(noSave);
+      await fr.download(options);
       assert.fail('downloaded too many times');
     } catch (e) {
       assert.equal(e.message, '404');

webpack.config.js

@@ -89,6 +89,7 @@ const web = {
           {
             // Strip asserts from our deps, mainly choojs family
             include: [path.resolve(__dirname, 'node_modules')],
+            exclude: [path.resolve(__dirname, 'node_modules/crc')],
             loader: 'webpack-unassert-loader'
           }
         ]