From 62ed0a411fa79d4e7f9505abfab529ca9a9a3b32 Mon Sep 17 00:00:00 2001 From: Emily Date: Thu, 5 Jul 2018 12:40:49 -0700 Subject: [PATCH] sw --- app/api.js | 58 ++++++++--------- app/ece.js | 85 +++++++++++++++---------- app/fileManager.js | 13 ++++ app/fileReceiver.js | 86 ++++++++------------------ app/fileSender.js | 2 +- app/keychain.js | 13 ++-- app/main.js | 8 +++ app/serviceWorker.js | 38 ++++++++++++ build/generate_asset_map.js | 1 + package-lock.json | 13 ++++ package.json | 5 +- server/routes/jsconfig.js | 2 + test/frontend/tests/api-tests.js | 6 +- test/frontend/tests/streaming-tests.js | 6 +- webpack.config.js | 2 +- webpackSw.config.js | 40 ++++++++++++ 16 files changed, 245 insertions(+), 133 deletions(-) create mode 100644 app/serviceWorker.js create mode 100644 webpackSw.config.js diff --git a/app/api.js b/app/api.js index 0219d006..168b3abc 100644 --- a/app/api.js +++ b/app/api.js @@ -1,4 +1,7 @@ import { arrayToB64, b64ToArray, delay } from './utils'; +import { ReadableStream as PolyRS} from 'web-streams-polyfill'; +import { createReadableStreamWrapper } from '@mattiasbuelens/web-streams-adapter'; +const RS = createReadableStreamWrapper(PolyRS); function post(obj) { return { @@ -201,38 +204,32 @@ export function uploadWs(encrypted, info, metadata, verifierB64, onprogress) { async function downloadS(id, keychain, onprogress, signal) { const auth = await keychain.authHeader(); - try { - const response = await fetch(`/api/download/${id}`, { - signal: signal , - method: 'GET', - headers: {'Authorization': auth} - }); - if (response.status !== 200) { - throw new Error(response.status); - } + const response = await fetch(`/api/download/${id}`, { + signal: signal, + method: 'GET', + headers: { Authorization: auth } + }); - const authHeader = response.headers.get('WWW-Authenticate'); - if (authHeader) { - keychain.nonce = parseNonce(authHeader); - } - - const fileSize = response.headers.get('Content-Length'); - onprogress([0, fileSize]); - - console.log(response.body); - if (response.body) { - return response.body; - } - return response.blob(); - - } catch (err) { - if (err.name === 'AbortError') { - throw new Error('0'); - } else { - throw err; - } + if (response.status !== 200) { + throw new Error(response.status); } + + const authHeader = response.headers.get('WWW-Authenticate'); + if (authHeader) { + keychain.nonce = parseNonce(authHeader); + } + + const fileSize = response.headers.get('Content-Length'); + + //right now only chrome allows obtaining a stream from fetch + //for other browsers we fetch as a blob and convert to polyfill stream later + if (response.body) { + console.log("STREAM") + return RS(response.body); + } + return response.blob(); + } async function tryDownloadStream(id, keychain, onprogress, signal, tries = 1) { @@ -243,6 +240,9 @@ async function tryDownloadStream(id, keychain, onprogress, signal, tries = 1) { if (e.message === '401' && --tries > 0) { return tryDownloadStream(id, keychain, onprogress, signal, tries); } + if (e.name === 'AbortError') { + throw new Error('0'); + } throw e; } } diff --git a/app/ece.js b/app/ece.js index 12605811..e80b2e65 100644 --- a/app/ece.js +++ b/app/ece.js @@ -1,12 +1,15 @@ require('buffer'); -import { TransformStream } from 'web-streams-polyfill'; +import { TransformStream as PolyTS, ReadableStream as PolyRS } from 'web-streams-polyfill'; +import { createReadableStreamWrapper, createTransformStreamWrapper } from '@mattiasbuelens/web-streams-adapter'; +const toTS = createTransformStreamWrapper(PolyTS); +const toRS = createReadableStreamWrapper(PolyRS); const NONCE_LENGTH = 12; const TAG_LENGTH = 16; const KEY_LENGTH = 16; const MODE_ENCRYPT = 'encrypt'; const MODE_DECRYPT = 'decrypt'; -const RS = 1048576; +const RS = 1024 * 1024; const encoder = new TextEncoder(); @@ -218,13 +221,14 @@ class ECETransformer { } async flush(controller) { + //console.log('ece stream ends') if (this.prevChunk) { await this.transformPrevChunk(true, controller); } } } -class BlobSlicer { +export class BlobSlicer { constructor(blob, rs, mode) { this.blob = blob; this.index = 0; @@ -262,28 +266,27 @@ class StreamSlicer { constructor(rs, mode) { this.mode = mode; this.rs = rs; - this.chunkSize = mode === MODE_ENCRYPT ? rs - 17 : 21; + this.chunkSize = (mode === MODE_ENCRYPT) ? (rs - 17) : 21; this.partialChunk = new Uint8Array(this.chunkSize); //where partial chunks are saved - this.offset = 0; + this.offset = 0; } send(buf, controller) { - //console.log("sent a record") controller.enqueue(buf); - if (this.chunkSize === 21) { + if (this.chunkSize === 21 && this.mode === MODE_DECRYPT) { this.chunkSize = this.rs; - this.partialChunk = new Uint8Array(this.chunkSize); } + this.partialChunk = new Uint8Array(this.chunkSize); } - //reslice input uint8arrays into record sized chunks + //reslice input into record sized chunks transform(chunk, controller) { - //console.log('Received chunk') // with %d bytes.', chunk.byteLength) + //console.log('Received chunk with %d bytes.', chunk.byteLength) let i = 0; - if (this.offset > 0) { //send off the partial chunk + if (this.offset > 0) { const len = Math.min(chunk.byteLength, (this.chunkSize - this.offset)); - this.partialChunk.set((chunk.slice(0, len)), this.offset); + this.partialChunk.set(chunk.slice(0, len), this.offset); this.offset += len; i += len; @@ -293,32 +296,41 @@ class StreamSlicer { } } - while (i < chunk.byteLength) { //send off whole records and stick last bit in partialChunk - if ((chunk.byteLength - i) > this.chunkSize) { + while (i < chunk.byteLength) { + if ((chunk.byteLength - i) >= this.chunkSize) { const record = chunk.slice(i, i + this.chunkSize); i += this.chunkSize; this.send(record, controller); } else { - const end = chunk.slice(i, end); + const end = chunk.slice(i, this.chunkSize); + i += end.length; this.partialChunk.set(end); this.offset = end.length; - i += end.length; } } } flush(controller) { + //console.log('slice stream ends') if (this.offset > 0) { - console.log("sent a partial record") controller.enqueue(this.partialChunk.slice(0, this.offset)); } } } - +async function stream2blob(stream) { + const chunks = []; + const reader = stream.getReader(); + let state = await reader.read(); + while (!state.done) { + chunks.push(state.value); + state = await reader.read(); + } + return new Blob(chunks); +} /* -input: a blob or a readable stream containing data to be transformed +input: a blob or a ReadableStream containing data to be transformed key: Uint8Array containing key of size KEY_LENGTH mode: string, either 'encrypt' or 'decrypt' rs: int containing record size, optional @@ -326,26 +338,37 @@ salt: ArrayBuffer containing salt of KEY_LENGTH length, optional */ export default class ECE { constructor(input, key, mode, rs, salt) { + this.input = input; + this.key = key; + this.mode = mode; + this.rs = rs; + this.salt = salt; if (rs === undefined) { - rs = RS; + this.rs = RS; } if (salt === undefined) { - salt = generateSalt(KEY_LENGTH); + this.salt = generateSalt(KEY_LENGTH); } + } + info() { + return { + recordSize: this.rs, + fileSize: 21 + this.input.size + 16 * Math.floor(this.input.size / (this.rs - 17)) + }; + } + + transform() { let inputStream; - if (input instanceof Blob) { - this.streamInfo = { - recordSize: rs, - fileSize: 21 + input.size + 16 * Math.floor(input.size / (rs - 17)) - }; - inputStream = new ReadableStream(new BlobSlicer(input, rs, mode)); + + if (this.input instanceof Blob) { + inputStream = toRS(new ReadableStream(new BlobSlicer(this.input, this.rs, this.mode))); } else { - const sliceStream = new TransformStream(new StreamSlicer(rs, mode)); - inputStream = input.pipeThrough(sliceStream); + const sliceStream = toTS(new TransformStream(new StreamSlicer(this.rs, this.mode))); + inputStream = this.input.pipeThrough(sliceStream); } - const ts = new TransformStream(new ECETransformer(mode, key, rs, salt)); - this.stream = inputStream.pipeThrough(ts); + const cryptoStream = toTS(new TransformStream(new ECETransformer(this.mode, this.key, this.rs, this.salt))); + return inputStream.pipeThrough(cryptoStream); } } diff --git a/app/fileManager.js b/app/fileManager.js index 786f99f8..88f024f9 100644 --- a/app/fileManager.js +++ b/app/fileManager.js @@ -36,6 +36,12 @@ export default function(state, emitter) { } } + function register() { + navigator.serviceWorker.register('/serviceWorker.js') + .then( reg => console.log("registration successful or already installed")) + .catch( e => console.log(e) ); + } + function updateProgress() { if (updateTitle) { emitter.emit('DOMTitleChange', percent(state.transfer.progressRatio)); @@ -162,6 +168,13 @@ export default function(state, emitter) { } } } + + const info = { + key: file.secretKey, + nonce: file.nonce + } + navigator.serviceWorker.controller.postMessage(info); + render(); }); diff --git a/app/fileReceiver.js b/app/fileReceiver.js index 5ca9f62b..cf23dcb2 100644 --- a/app/fileReceiver.js +++ b/app/fileReceiver.js @@ -1,7 +1,7 @@ import Nanobus from 'nanobus'; import Keychain from './keychain'; import { bytes } from './utils'; -import { metadata, downloadFile, downloadStream} from './api'; +import { metadata, downloadFile, downloadStream } from './api'; export default class FileReceiver extends Nanobus { constructor(fileInfo) { @@ -51,89 +51,56 @@ export default class FileReceiver extends Nanobus { this.state = 'ready'; } - /* - async streamToArrayBuffer(stream, streamSize) { - try { - var finish; - const promise = new Promise((resolve) => { - finish = resolve; - }); - const result = new Uint8Array(streamSize); - let offset = 0; - - - const writer = new WritableStream( - { - write(chunk) { - result.set(state.value, offset); - offset += state.value.length; - }, - close() { - //resolve a promise or something - finish.resolve(); - } - } - ); - - stream.pipeTo(writer); - - await promise; - return result.slice(0, offset).buffer; - - } catch (e) { - console.log(e) - } - } - */ - - async streamToArrayBuffer(stream, streamSize) { + async streamToArrayBuffer(stream, streamSize, onprogress) { try { const result = new Uint8Array(streamSize); let offset = 0; - console.log("reading...") const reader = stream.getReader(); let state = await reader.read(); - console.log("read done") while (!state.done) { result.set(state.value, offset); offset += state.value.length; state = await reader.read(); + onprogress([offset, streamSize]); } + onprogress([streamSize, streamSize]); return result.slice(0, offset).buffer; } catch (e) { - console.log(e) + console.log(e); + throw (e); } } async download(noSave = false) { - this.state = 'downloading'; - this.downloadRequest = await downloadStream( - this.fileInfo.id, - this.keychain, - p => { - this.progress = p; - this.emit('progress'); - } - ); + const onprogress = p => { + this.progress = p; + this.emit('progress'); + } try { + this.state = 'downloading'; + this.downloadRequest = downloadStream( + this.fileInfo.id, + this.keychain + ); - const ciphertext = await this.downloadRequest.result; + onprogress([0, this.fileInfo.size]); + const download = await this.downloadRequest.result; + const plainstream = this.keychain.decryptStream(download); + + //temporary + const plaintext = await this.streamToArrayBuffer( + plainstream, + this.fileInfo.size, + onprogress + ); this.downloadRequest = null; + this.msg = 'decryptingFile'; this.state = 'decrypting'; this.emit('decrypting'); - const dec = this.keychain.decryptStream(ciphertext); - - let plaintext = await this.streamToArrayBuffer( - dec.stream, - this.fileInfo.size - ); - - if (plaintext === undefined) { plaintext = (new Uint8Array(1)).buffer; } - if (!noSave) { await saveFile({ plaintext, @@ -144,7 +111,6 @@ export default class FileReceiver extends Nanobus { this.msg = 'downloadFinish'; this.state = 'complete'; - } catch (e) { this.downloadRequest = null; throw e; diff --git a/app/fileSender.js b/app/fileSender.js index 7c3c4de1..08cf307e 100644 --- a/app/fileSender.js +++ b/app/fileSender.js @@ -65,7 +65,7 @@ export default class FileSender extends Nanobus { this.msg = 'encryptingFile'; this.emit('encrypting'); - const enc = this.keychain.encryptStream(this.file); + const enc = await this.keychain.encryptStream(this.file); const metadata = await this.keychain.encryptMetadata(this.file); const authKeyB64 = await this.keychain.authKeyB64(); diff --git a/app/keychain.js b/app/keychain.js index 75778990..d02ff83b 100644 --- a/app/keychain.js +++ b/app/keychain.js @@ -180,13 +180,16 @@ export default class Keychain { } encryptStream(plaintext) { - const enc = new ECE(plaintext, this.rawSecret, 'encrypt'); - return enc; + const ece = new ECE(plaintext, this.rawSecret, 'encrypt'); + return { + stream: ece.transform(), + streamInfo: ece.info() + }; } - decryptStream(encstream) { - const dec = new ECE(encstream, this.rawSecret, 'decrypt'); - return dec; + decryptStream(cryptotext) { + const ece = new ECE(cryptotext, this.rawSecret, 'decrypt'); + return ece.transform(); } async decryptFile(ciphertext) { diff --git a/app/main.js b/app/main.js index 353afa1a..fb13587a 100644 --- a/app/main.js +++ b/app/main.js @@ -9,11 +9,18 @@ import storage from './storage'; import metrics from './metrics'; import experiments from './experiments'; import Raven from 'raven-js'; +import assets from '../common/assets'; if (navigator.doNotTrack !== '1' && window.RAVEN_CONFIG) { Raven.config(window.SENTRY_ID, window.RAVEN_CONFIG).install(); } +function register(state, emitter) { + navigator.serviceWorker.register('serviceWorker.js') + .then( reg => console.log("registration successful or already installed")) + .catch( e => console.log(e) ); +} + app.use((state, emitter) => { state.transfer = null; state.fileInfo = null; @@ -44,6 +51,7 @@ app.use((state, emitter) => { }); }); +app.use(register); app.use(metrics); app.use(fileManager); app.use(dragManager); diff --git a/app/serviceWorker.js b/app/serviceWorker.js new file mode 100644 index 00000000..624a6028 --- /dev/null +++ b/app/serviceWorker.js @@ -0,0 +1,38 @@ +import Keychain from './keychain'; + +self.addEventListener('install', (event) => { + console.log("install event on sw") + self.skipWaiting(); +}); + +async function decryptStream(request) { + console.log("DOWNLOAD FETCH") + //make actual request to server, get response back, decrypt it, send it + const response = await fetch(req, + { + method: 'GET', + headers: { Authorization: auth } + } + ); + + if (response.status !== 200) { + console.log(response.status) + throw new Error(response.status); + } + + const body = response.body; + console.log(body); + + return response; +} + +self.onfetch = (event) => { + const req = event.request.clone(); + if (req.url.includes('/api/download')) { + event.respondWith(decryptStream(req)); + } +}; + +self.onmessage = (event) => { + self.keychain = new Keychain(event.data.key, event.data.nonce); +}; \ No newline at end of file diff --git a/build/generate_asset_map.js b/build/generate_asset_map.js index de0999c9..97a93432 100644 --- a/build/generate_asset_map.js +++ b/build/generate_asset_map.js @@ -20,6 +20,7 @@ module.exports = function() { const files = fs.readdirSync(path.join(__dirname, '..', 'assets')); const code = `module.exports = { "package.json": require('../package.json'), + "serviceWorker.js" : require('../app/serviceWorker.js'), ${files.map(kv).join(',\n')} };`; return { diff --git a/package-lock.json b/package-lock.json index 6247b8a6..b8c6358e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -89,6 +89,11 @@ "integrity": "sha1-9vGlzl05caSt6RoR0i1MRZrNN18=", "dev": true }, + "@mattiasbuelens/web-streams-adapter": { + "version": "0.1.0-alpha.1", + "resolved": "https://registry.npmjs.org/@mattiasbuelens/web-streams-adapter/-/web-streams-adapter-0.1.0-alpha.1.tgz", + "integrity": "sha512-8YK2ZY6CAgrzFGfW2uPyNDMYvh7OmWjrlbdP+GeHiMJhzPF3XwrQaHyLQ4IZqGTj8NW879ttfbcqbLqQxWvtsw==" + }, "@sinonjs/formatio": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-2.0.0.tgz", @@ -1172,6 +1177,14 @@ "regenerator-transform": "0.10.1" } }, + "babel-plugin-transform-runtime": { + "version": "6.23.0", + "resolved": "https://registry.npmjs.org/babel-plugin-transform-runtime/-/babel-plugin-transform-runtime-6.23.0.tgz", + "integrity": "sha1-iEkNRGUC6puOfvsP4J7E2ZR5se4=", + "requires": { + "babel-runtime": "6.26.0" + } + }, "babel-plugin-transform-strict-mode": { "version": "6.24.1", "resolved": "https://registry.npmjs.org/babel-plugin-transform-strict-mode/-/babel-plugin-transform-strict-mode-6.24.1.tgz", diff --git a/package.json b/package.json index b927d113..a0685123 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "prepush": "npm test", "check": "nsp check", "clean": "rimraf dist", - "build": "npm run clean && webpack -p", + "build": "npm run clean && webpack -p && webpack --config webpackSw.config.js -p", "lint": "npm-run-all lint:*", "lint:css": "stylelint app/*.css app/**/*.css", "lint:js": "eslint .", @@ -118,8 +118,11 @@ "webpack-unassert-loader": "^1.2.0" }, "dependencies": { + "@mattiasbuelens/web-streams-adapter": "0.1.0-alpha.1", "aws-sdk": "^2.206.0", + "babel-plugin-transform-runtime": "^6.23.0", "babel-polyfill": "^6.26.0", + "babel-runtime": "^6.26.0", "choo": "^6.10.0", "cldr-core": "^32.0.0", "convict": "^4.0.1", diff --git a/server/routes/jsconfig.js b/server/routes/jsconfig.js index 23e07374..20854998 100644 --- a/server/routes/jsconfig.js +++ b/server/routes/jsconfig.js @@ -1,4 +1,5 @@ const config = require('../config'); +const assets = require('../../common/assets'); let sentry = ''; if (config.sentry_id) { @@ -36,6 +37,7 @@ if (isIE && !isUnsupportedPage) { } var MAXFILESIZE = ${config.max_file_size}; var EXPIRE_SECONDS = ${config.expire_seconds}; +var SERVICEWORKER = '${assets.get('serviceWorker.js')}'; ${ga} ${sentry} `; diff --git a/test/frontend/tests/api-tests.js b/test/frontend/tests/api-tests.js index 22e221e4..301a09de 100644 --- a/test/frontend/tests/api-tests.js +++ b/test/frontend/tests/api-tests.js @@ -13,7 +13,7 @@ describe('API', function() { describe('websocket upload', function() { it('returns file info on success', async function() { const keychain = new Keychain(); - const enc = keychain.encryptStream(plaintext); + const enc = await keychain.encryptStream(plaintext); const meta = await keychain.encryptMetadata(metadata); const verifierB64 = await keychain.authKeyB64(); const p = function() {}; @@ -22,12 +22,12 @@ describe('API', function() { const result = await up.result; assert.ok(result.url); assert.ok(result.id); - assert.ok(result.ownerToken); + assert.ok(result.ownerToken); }); it('can be cancelled', async function() { const keychain = new Keychain(); - const enc = keychain.encryptStream(plaintext); + const enc = await keychain.encryptStream(plaintext); const meta = await keychain.encryptMetadata(metadata); const verifierB64 = await keychain.authKeyB64(); const p = function() {}; diff --git a/test/frontend/tests/streaming-tests.js b/test/frontend/tests/streaming-tests.js index 355fcaa0..77fdec5a 100644 --- a/test/frontend/tests/streaming-tests.js +++ b/test/frontend/tests/streaming-tests.js @@ -31,7 +31,8 @@ describe('Streaming', function() { const blob = new Blob([str], { type: 'text/plain' }); it('can encrypt', async function() { - const encStream = new ECE(blob, key, 'encrypt', rs, salt).stream; + const ece = new ECE(blob, key, 'encrypt', rs, salt); + const encStream = await ece.transform(); const reader = encStream.getReader(); let result = Buffer.from([]); @@ -47,7 +48,8 @@ describe('Streaming', function() { it('can decrypt', async function() { const encBlob = new Blob([encrypted]); - const decStream = await new ECE(encBlob, key, 'decrypt', rs).stream; + const ece = new ECE(encBlob, key, 'decrypt', rs); + const decStream = await ece.transform() const reader = decStream.getReader(); let result = Buffer.from([]); diff --git a/webpack.config.js b/webpack.config.js index 6619f697..dcf4ffb8 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -40,7 +40,7 @@ module.exports = { test: /\.js$/, oneOf: [ { - include: require.resolve('./assets/cryptofill'), + include: [require.resolve('./assets/cryptofill')], use: [ { loader: 'file-loader', diff --git a/webpackSw.config.js b/webpackSw.config.js new file mode 100644 index 00000000..ded7bc29 --- /dev/null +++ b/webpackSw.config.js @@ -0,0 +1,40 @@ +const path = require('path'); +const webpack = require('webpack'); + +const regularJSOptions = { + babelrc: false, + presets: [['env', { modules: false }], 'stage-2'], + // yo-yoify converts html template strings to direct dom api calls + plugins: [ + "transform-runtime", { + //"polyfill": false, + //"regenerator": true + } + ] +}; + +const entry = { + serviceWorker: ['./app/serviceWorker.js'] +}; + +module.exports = { + entry, + output: { + filename: '[name].js', + path: path.resolve(__dirname, 'dist'), + publicPath: '/' + }, + module: { + rules: [ + { + loader: 'babel-loader', + // exclude: /node_modules/, + include: [ + path.resolve(__dirname, 'app'), + path.resolve(__dirname, 'node_modules/buffer') + ], + options: regularJSOptions + } + ] + } +}; \ No newline at end of file