saves stream to file

2018-07-06 15:49:50 -07:00 · 2018-07-06 15:49:50 -07:00 · f98bc0878c
parent 62ed0a411f
commit f98bc0878c
14 changed files with 136 additions and 201 deletions
--- a/app/api.js
+++ b/app/api.js
@ -202,9 +202,10 @@ export function uploadWs(encrypted, info, metadata, verifierB64, onprogress) {

 ////////////////////////

-async function downloadS(id, keychain, onprogress, signal) {
+async function downloadS(id, keychain, signal) {
  const auth = await keychain.authHeader();

+  //this will be already funneled through serviceworker
  const response = await fetch(`/api/download/${id}`, {
    signal: signal,
    method: 'GET',
@ -225,20 +226,18 @@ async function downloadS(id, keychain, onprogress, signal) {
  //right now only chrome allows obtaining a stream from fetch
  //for other browsers we fetch as a blob and convert to polyfill stream later
  if (response.body) {
-    console.log("STREAM")
    return RS(response.body);
  }
  return response.blob();
-
 }

-async function tryDownloadStream(id, keychain, onprogress, signal, tries = 1) {
+async function tryDownloadStream(id, keychain, signal, tries = 1) {
  try {
-    const result = await downloadS(id, keychain, onprogress, signal);
+    const result = await downloadS(id, keychain, signal);
    return result;
  } catch (e) {
    if (e.message === '401' && --tries > 0) {
-      return tryDownloadStream(id, keychain, onprogress, signal, tries);
+      return tryDownloadStream(id, keychain, signal, tries);
    }
    if (e.name === 'AbortError') {
      throw new Error('0');
@ -247,14 +246,14 @@ async function tryDownloadStream(id, keychain, onprogress, signal, tries = 1) {
  }
 }

-export function downloadStream(id, keychain, onprogress) {
+export function downloadStream(id, keychain) {
  const controller = new AbortController();
  function cancel() {
    controller.abort();
  }
  return {
    cancel,
-    result: tryDownloadStream(id, keychain, onprogress, controller.signal, 2)
+    result: tryDownloadStream(id, keychain, controller.signal, 2)
  };
 }

--- a/app/ece.js
+++ b/app/ece.js
@ -1,8 +1,16 @@
 require('buffer');
-import { TransformStream as PolyTS, ReadableStream as PolyRS } from 'web-streams-polyfill';
-import { createReadableStreamWrapper, createTransformStreamWrapper } from '@mattiasbuelens/web-streams-adapter';
+/*
+import {
+  TransformStream as PolyTS,
+  ReadableStream as PolyRS
+} from 'web-streams-polyfill';
+import {
+  createReadableStreamWrapper,
+  createTransformStreamWrapper
+} from '@mattiasbuelens/web-streams-adapter';
 const toTS = createTransformStreamWrapper(PolyTS);
 const toRS = createReadableStreamWrapper(PolyRS);
+*/

 const NONCE_LENGTH = 12;
 const TAG_LENGTH = 16;
@ -15,7 +23,7 @@ const encoder = new TextEncoder();

 function generateSalt(len) {
  const randSalt = new Uint8Array(len);
-  window.crypto.getRandomValues(randSalt);
+  crypto.getRandomValues(randSalt);
  return randSalt.buffer;
 }

@ -31,7 +39,7 @@ class ECETransformer {
  }

  async generateKey() {
-    const inputKey = await window.crypto.subtle.importKey(
+    const inputKey = await crypto.subtle.importKey(
      'raw',
      this.ikm,
      'HKDF',
@ -39,7 +47,7 @@ class ECETransformer {
      ['deriveKey']
    );

-    return window.crypto.subtle.deriveKey(
+    return crypto.subtle.deriveKey(
      {
        name: 'HKDF',
        salt: this.salt,
@ -57,7 +65,7 @@ class ECETransformer {
  }

  async generateNonceBase() {
-    const inputKey = await window.crypto.subtle.importKey(
+    const inputKey = await crypto.subtle.importKey(
      'raw',
      this.ikm,
      'HKDF',
@ -65,9 +73,9 @@ class ECETransformer {
      ['deriveKey']
    );

-    const base = await window.crypto.subtle.exportKey(
+    const base = await crypto.subtle.exportKey(
      'raw',
-      await window.crypto.subtle.deriveKey(
+      await crypto.subtle.deriveKey(
        {
          name: 'HKDF',
          salt: this.salt,
@ -156,7 +164,7 @@ class ECETransformer {

  async encryptRecord(buffer, seq, isLast) {
    const nonce = this.generateNonce(seq);
-    const encrypted = await window.crypto.subtle.encrypt(
+    const encrypted = await crypto.subtle.encrypt(
      { name: 'AES-GCM', iv: nonce },
      this.key,
      this.pad(buffer, isLast)
@ -166,7 +174,7 @@ class ECETransformer {

  async decryptRecord(buffer, seq, isLast) {
    const nonce = this.generateNonce(seq);
-    const data = await window.crypto.subtle.decrypt(
+    const data = await crypto.subtle.decrypt(
      {
        name: 'AES-GCM',
        iv: nonce,
@ -266,7 +274,7 @@ class StreamSlicer {
  constructor(rs, mode) {
    this.mode = mode;
    this.rs = rs;
-    this.chunkSize = (mode === MODE_ENCRYPT) ? (rs - 17) : 21;
+    this.chunkSize = mode === MODE_ENCRYPT ? rs - 17 : 21;
    this.partialChunk = new Uint8Array(this.chunkSize); //where partial chunks are saved
    this.offset = 0;
  }
@ -285,7 +293,7 @@ class StreamSlicer {
    let i = 0;

    if (this.offset > 0) {
-      const len = Math.min(chunk.byteLength, (this.chunkSize - this.offset));
+      const len = Math.min(chunk.byteLength, this.chunkSize - this.offset);
      this.partialChunk.set(chunk.slice(0, len), this.offset);
      this.offset += len;
      i += len;
@ -297,7 +305,7 @@ class StreamSlicer {
    }

    while (i < chunk.byteLength) {
-      if ((chunk.byteLength - i) >= this.chunkSize) {
+      if (chunk.byteLength - i >= this.chunkSize) {
        const record = chunk.slice(i, i + this.chunkSize);
        i += this.chunkSize;
        this.send(record, controller);
@ -318,17 +326,6 @@ class StreamSlicer {
  }
 }

-async function stream2blob(stream) {
-  const chunks = [];
-  const reader = stream.getReader();
-  let state = await reader.read();
-  while (!state.done) {
-    chunks.push(state.value);
-    state = await reader.read();
-  }
-  return new Blob(chunks);
-}
-
 /*
 input: a blob or a ReadableStream containing data to be transformed
 key:  Uint8Array containing key of size KEY_LENGTH 
@ -354,7 +351,8 @@ export default class ECE {
  info() {
    return {
      recordSize: this.rs,
-      fileSize: 21 + this.input.size + 16 * Math.floor(this.input.size / (this.rs - 17))
+      fileSize:
+        21 + this.input.size + 16 * Math.floor(this.input.size / (this.rs - 17))
    };
  }

@ -362,13 +360,19 @@ export default class ECE {
    let inputStream;

    if (this.input instanceof Blob) {
-      inputStream = toRS(new ReadableStream(new BlobSlicer(this.input, this.rs, this.mode)));
+      inputStream = new ReadableStream(
+        new BlobSlicer(this.input, this.rs, this.mode)
+      ); //inputStream = toRS(new ReadableStream(new BlobSlicer(this.input, this.rs, this.mode)));
    } else {
-      const sliceStream = toTS(new TransformStream(new StreamSlicer(this.rs, this.mode)));
+      const sliceStream = new TransformStream(
+        new StreamSlicer(this.rs, this.mode)
+      ); //const sliceStream = toTS(new TransformStream(new StreamSlicer(this.rs, this.mode)));
      inputStream = this.input.pipeThrough(sliceStream);
    }

-    const cryptoStream = toTS(new TransformStream(new ECETransformer(this.mode, this.key, this.rs, this.salt)));
-    return inputStream.pipeThrough(cryptoStream);
+    const cryptoStream = new TransformStream(
+      new ECETransformer(this.mode, this.key, this.rs, this.salt)
+    ); //const cryptoStream = toTS(new TransformStream(new ECETransformer(this.mode, this.key, this.rs, this.salt)));
+    return inputStream.pipeThrough(cryptoStream); //return toRS(inputStream.pipeThrough(cryptoStream));
  }
 }
--- a/app/fileManager.js
+++ b/app/fileManager.js
@ -36,12 +36,6 @@ export default function(state, emitter) {
    }
  }

-  function register() {
-    navigator.serviceWorker.register('/serviceWorker.js')
-    .then( reg => console.log("registration successful or already installed"))
-    .catch( e => console.log(e) );
-  }
-
  function updateProgress() {
    if (updateTitle) {
      emitter.emit('DOMTitleChange', percent(state.transfer.progressRatio));
@ -156,6 +150,7 @@ export default function(state, emitter) {

  emitter.on('getMetadata', async () => {
    const file = state.fileInfo;
+
    const receiver = new FileReceiver(file);
    try {
      await receiver.getMetadata();
@ -169,12 +164,6 @@ export default function(state, emitter) {
      }
    }

-    const info = {
-      key: file.secretKey,
-      nonce: file.nonce
-    }
-    navigator.serviceWorker.controller.postMessage(info);
-
    render();
  });

--- a/app/fileReceiver.js
+++ b/app/fileReceiver.js
@ -1,7 +1,7 @@
 import Nanobus from 'nanobus';
 import Keychain from './keychain';
 import { bytes } from './utils';
-import { metadata, downloadFile, downloadStream } from './api';
+import { metadata } from './api';

 export default class FileReceiver extends Nanobus {
  constructor(fileInfo) {
@ -52,7 +52,6 @@ export default class FileReceiver extends Nanobus {
  }

  async streamToArrayBuffer(stream, streamSize, onprogress) {
-    try {
    const result = new Uint8Array(streamSize);
    let offset = 0;
    const reader = stream.getReader();
@ -66,93 +65,44 @@ export default class FileReceiver extends Nanobus {

    onprogress([streamSize, streamSize]);
    return result.slice(0, offset).buffer;
-    } catch (e) {
-      console.log(e);
-      throw (e);
-    }
  }

  async download(noSave = false) {
    const onprogress = p => {
      this.progress = p;
      this.emit('progress');
-    }
+    };

    try {
      this.state = 'downloading';
-      this.downloadRequest = downloadStream(
-        this.fileInfo.id,
-        this.keychain
-      );
+
+      const auth = await this.keychain.authHeader();
+      const info = {
+        key: this.fileInfo.secretKey,
+        nonce: this.fileInfo.nonce,
+        filename: this.fileInfo.name,
+        auth: auth
+      };
+      navigator.serviceWorker.controller.postMessage(info);

      onprogress([0, this.fileInfo.size]);
-      const download = await this.downloadRequest.result;
-      const plainstream = this.keychain.decryptStream(download);
-
-      //temporary
-      const plaintext = await this.streamToArrayBuffer(
-        plainstream,
-        this.fileInfo.size,
-        onprogress
-      );
-      this.downloadRequest = null;
-
-      this.msg = 'decryptingFile';
-      this.state = 'decrypting';
-      this.emit('decrypting');

      if (!noSave) {
-        await saveFile({
-          plaintext,
-          name: decodeURIComponent(this.fileInfo.name),
-          type: this.fileInfo.type
-        });
+        const downloadUrl = `${location.protocol}//${
+          location.host
+        }/api/download/${this.fileInfo.id}`;
+        const a = document.createElement('a');
+        a.href = downloadUrl;
+        document.body.appendChild(a);
+        a.click();
+        URL.revokeObjectURL(downloadUrl);
      }

-      this.msg = 'downloadFinish';
-      this.state = 'complete';
+      //this.msg = 'downloadFinish';
+      //this.state = 'complete';
    } catch (e) {
      this.downloadRequest = null;
      throw e;
    }
  }
 }
-
-async function saveFile(file) {
-  return new Promise(function(resolve, reject) {
-    const dataView = new DataView(file.plaintext);
-    const blob = new Blob([dataView], { type: file.type });
-
-    if (navigator.msSaveBlob) {
-      navigator.msSaveBlob(blob, file.name);
-      return resolve();
-    } else if (/iPhone|fxios/i.test(navigator.userAgent)) {
-      // This method is much slower but createObjectURL
-      // is buggy on iOS
-      const reader = new FileReader();
-      reader.addEventListener('loadend', function() {
-        if (reader.error) {
-          return reject(reader.error);
-        }
-        if (reader.result) {
-          const a = document.createElement('a');
-          a.href = reader.result;
-          a.download = file.name;
-          document.body.appendChild(a);
-          a.click();
-        }
-        resolve();
-      });
-      reader.readAsDataURL(blob);
-    } else {
-      const downloadUrl = URL.createObjectURL(blob);
-      const a = document.createElement('a');
-      a.href = downloadUrl;
-      a.download = file.name;
-      document.body.appendChild(a);
-      a.click();
-      URL.revokeObjectURL(downloadUrl);
-      setTimeout(resolve, 100);
-    }
-  });
-}
--- a/app/keychain.js
+++ b/app/keychain.js
@ -9,14 +9,14 @@ export default class Keychain {
    if (ivB64) {
      this.iv = b64ToArray(ivB64);
    } else {
-      this.iv = window.crypto.getRandomValues(new Uint8Array(12));
+      this.iv = crypto.getRandomValues(new Uint8Array(12));
    }
    if (secretKeyB64) {
      this.rawSecret = b64ToArray(secretKeyB64);
    } else {
-      this.rawSecret = window.crypto.getRandomValues(new Uint8Array(16));
+      this.rawSecret = crypto.getRandomValues(new Uint8Array(16));
    }
-    this.secretKeyPromise = window.crypto.subtle.importKey(
+    this.secretKeyPromise = crypto.subtle.importKey(
      'raw',
      this.rawSecret,
      'HKDF',
@ -24,7 +24,7 @@ export default class Keychain {
      ['deriveKey']
    );
    this.encryptKeyPromise = this.secretKeyPromise.then(function(secretKey) {
-      return window.crypto.subtle.deriveKey(
+      return crypto.subtle.deriveKey(
        {
          name: 'HKDF',
          salt: new Uint8Array(),
@ -41,7 +41,7 @@ export default class Keychain {
      );
    });
    this.metaKeyPromise = this.secretKeyPromise.then(function(secretKey) {
-      return window.crypto.subtle.deriveKey(
+      return crypto.subtle.deriveKey(
        {
          name: 'HKDF',
          salt: new Uint8Array(),
@ -58,7 +58,7 @@ export default class Keychain {
      );
    });
    this.authKeyPromise = this.secretKeyPromise.then(function(secretKey) {
-      return window.crypto.subtle.deriveKey(
+      return crypto.subtle.deriveKey(
        {
          name: 'HKDF',
          salt: new Uint8Array(),
@ -91,12 +91,12 @@ export default class Keychain {
  }

  setPassword(password, shareUrl) {
-    this.authKeyPromise = window.crypto.subtle
+    this.authKeyPromise = crypto.subtle
      .importKey('raw', encoder.encode(password), { name: 'PBKDF2' }, false, [
        'deriveKey'
      ])
      .then(passwordKey =>
-        window.crypto.subtle.deriveKey(
+        crypto.subtle.deriveKey(
          {
            name: 'PBKDF2',
            salt: encoder.encode(shareUrl),
@ -115,7 +115,7 @@ export default class Keychain {
  }

  setAuthKey(authKeyB64) {
-    this.authKeyPromise = window.crypto.subtle.importKey(
+    this.authKeyPromise = crypto.subtle.importKey(
      'raw',
      b64ToArray(authKeyB64),
      {
@ -129,13 +129,13 @@ export default class Keychain {

  async authKeyB64() {
    const authKey = await this.authKeyPromise;
-    const rawAuth = await window.crypto.subtle.exportKey('raw', authKey);
+    const rawAuth = await crypto.subtle.exportKey('raw', authKey);
    return arrayToB64(new Uint8Array(rawAuth));
  }

  async authHeader() {
    const authKey = await this.authKeyPromise;
-    const sig = await window.crypto.subtle.sign(
+    const sig = await crypto.subtle.sign(
      {
        name: 'HMAC'
      },
@ -147,7 +147,7 @@ export default class Keychain {

  async encryptFile(plaintext) {
    const encryptKey = await this.encryptKeyPromise;
-    const ciphertext = await window.crypto.subtle.encrypt(
+    const ciphertext = await crypto.subtle.encrypt(
      {
        name: 'AES-GCM',
        iv: this.iv,
@ -161,7 +161,7 @@ export default class Keychain {

  async encryptMetadata(metadata) {
    const metaKey = await this.metaKeyPromise;
-    const ciphertext = await window.crypto.subtle.encrypt(
+    const ciphertext = await crypto.subtle.encrypt(
      {
        name: 'AES-GCM',
        iv: new Uint8Array(12),
@ -194,7 +194,7 @@ export default class Keychain {

  async decryptFile(ciphertext) {
    const encryptKey = await this.encryptKeyPromise;
-    const plaintext = await window.crypto.subtle.decrypt(
+    const plaintext = await crypto.subtle.decrypt(
      {
        name: 'AES-GCM',
        iv: this.iv,
@ -208,7 +208,7 @@ export default class Keychain {

  async decryptMetadata(ciphertext) {
    const metaKey = await this.metaKeyPromise;
-    const plaintext = await window.crypto.subtle.decrypt(
+    const plaintext = await crypto.subtle.decrypt(
      {
        name: 'AES-GCM',
        iv: new Uint8Array(12),
--- a/app/main.js
+++ b/app/main.js
@ -9,18 +9,11 @@ import storage from './storage';
 import metrics from './metrics';
 import experiments from './experiments';
 import Raven from 'raven-js';
-import assets from '../common/assets';

 if (navigator.doNotTrack !== '1' && window.RAVEN_CONFIG) {
  Raven.config(window.SENTRY_ID, window.RAVEN_CONFIG).install();
 }

-function register(state, emitter) {
-  navigator.serviceWorker.register('serviceWorker.js')
-  .then( reg => console.log("registration successful or already installed"))
-  .catch( e => console.log(e) );
-}
-
 app.use((state, emitter) => {
  state.transfer = null;
  state.fileInfo = null;
@ -51,7 +44,9 @@ app.use((state, emitter) => {
  });
 });

-app.use(register);
+app.use(() => {
+  navigator.serviceWorker.register('/serviceWorker.js');
+});
 app.use(metrics);
 app.use(fileManager);
 app.use(dragManager);
--- a/app/serviceWorker.js
+++ b/app/serviceWorker.js
@ -1,38 +1,41 @@
 import Keychain from './keychain';

-self.addEventListener('install', (event) => {
-  console.log("install event on sw")
+self.addEventListener('install', event => {
  self.skipWaiting();
 });

 async function decryptStream(request) {
-  console.log("DOWNLOAD FETCH")
-  //make actual request to server, get response back, decrypt it, send it
-  const response = await fetch(req, 
-    {
+  const response = await fetch(request.url, {
    method: 'GET',
-      headers: { Authorization: auth }
-    }
-  );
+    headers: { Authorization: self.auth }
+  });

  if (response.status !== 200) {
-    console.log(response.status)
-    throw new Error(response.status);
-  }
-
-  const body = response.body;
-  console.log(body);
-
    return response;
  }

-self.onfetch = (event) => {
+  const body = response.body; //stream
+  const decrypted = self.keychain.decryptStream(body);
+
+  const headers = {
+    headers: {
+      'Content-Disposition': 'attachment; filename=' + self.filename
+    }
+  };
+
+  const newRes = new Response(decrypted, headers);
+  return newRes;
+}
+
+self.onfetch = event => {
  const req = event.request.clone();
  if (req.url.includes('/api/download')) {
    event.respondWith(decryptStream(req));
  }
 };

-self.onmessage = (event) => {
+self.onmessage = event => {
  self.keychain = new Keychain(event.data.key, event.data.nonce);
+  self.filename = event.data.filename;
+  self.auth = event.data.auth;
 };
--- a/app/utils.js
+++ b/app/utils.js
@ -24,7 +24,7 @@ function loadShim(polyfill) {

 async function canHasSend() {
  try {
-    const key = await window.crypto.subtle.generateKey(
+    const key = await crypto.subtle.generateKey(
      {
        name: 'AES-GCM',
        length: 128
@ -32,25 +32,25 @@ async function canHasSend() {
      true,
      ['encrypt', 'decrypt']
    );
-    await window.crypto.subtle.encrypt(
+    await crypto.subtle.encrypt(
      {
        name: 'AES-GCM',
-        iv: window.crypto.getRandomValues(new Uint8Array(12)),
+        iv: crypto.getRandomValues(new Uint8Array(12)),
        tagLength: 128
      },
      key,
      new ArrayBuffer(8)
    );
-    await window.crypto.subtle.importKey(
+    await crypto.subtle.importKey(
      'raw',
-      window.crypto.getRandomValues(new Uint8Array(16)),
+      crypto.getRandomValues(new Uint8Array(16)),
      'PBKDF2',
      false,
      ['deriveKey']
    );
-    await window.crypto.subtle.importKey(
+    await crypto.subtle.importKey(
      'raw',
-      window.crypto.getRandomValues(new Uint8Array(16)),
+      crypto.getRandomValues(new Uint8Array(16)),
      'HKDF',
      false,
      ['deriveKey']
@ -75,7 +75,7 @@ function copyToClipboard(str) {
  if (navigator.userAgent.match(/iphone|ipad|ipod/i)) {
    const range = document.createRange();
    range.selectNodeContents(aux);
-    const sel = window.getSelection();
+    const sel = getSelection();
    sel.removeAllRanges();
    sel.addRange(range);
    aux.setSelectionRange(0, str.length);
--- a/package-lock.json
+++ b/package-lock.json
@ -17701,6 +17701,11 @@
        "any-observable": "0.2.0"
      }
    },
+    "streamsaver": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/streamsaver/-/streamsaver-1.0.1.tgz",
+      "integrity": "sha1-R11ASXO15pJqVX8OTNhVijUg4Hw="
+    },
    "strftime": {
      "version": "0.10.0",
      "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.0.tgz",
--- a/package.json
+++ b/package.json
@ -122,7 +122,6 @@
    "aws-sdk": "^2.206.0",
    "babel-plugin-transform-runtime": "^6.23.0",
    "babel-polyfill": "^6.26.0",
-    "babel-runtime": "^6.26.0",
    "choo": "^6.10.0",
    "cldr-core": "^32.0.0",
    "convict": "^4.0.1",
--- a/server/bin/prod.js
+++ b/server/bin/prod.js
@ -11,6 +11,7 @@ if (config.sentry_dsn) {
 }

 const app = express();
+
 expressWs(app, null, { perMessageDeflate: false });
 app.ws('/api/ws', require('../routes/ws'));
 routes(app);
--- a/test/frontend/tests/streaming-tests.js
+++ b/test/frontend/tests/streaming-tests.js
@ -49,7 +49,7 @@ describe('Streaming', function() {
    it('can decrypt', async function() {
      const encBlob = new Blob([encrypted]);
      const ece = new ECE(encBlob, key, 'decrypt', rs);
-      const decStream = await ece.transform()
+      const decStream = await ece.transform();

      const reader = decStream.getReader();
      let result = Buffer.from([]);
--- a/webpackSw.config.js
+++ b/webpackSw.config.js
@ -1,16 +1,9 @@
 const path = require('path');
-const webpack = require('webpack');

 const regularJSOptions = {
  babelrc: false,
-  presets: [['env', { modules: false  }], 'stage-2'],
-  // yo-yoify converts html template strings to direct dom api calls
-  plugins: [
-    "transform-runtime", {
-      //"polyfill": false,
-      //"regenerator": true
-    }
-  ]
+  presets: [['env'], 'stage-2'],
+  plugins: ['transform-runtime']
 };

 const entry = {
@ -24,15 +17,12 @@ module.exports = {
    path: path.resolve(__dirname, 'dist'),
    publicPath: '/'
  },
+
  module: {
    rules: [
      {
        loader: 'babel-loader',
-        // exclude: /node_modules/,
-        include: [
-          path.resolve(__dirname, 'app'),
-          path.resolve(__dirname, 'node_modules/buffer')
-        ],
+        exclude: /node_modules/,
        options: regularJSOptions
      }
    ]