From d14aeb29e96cedae824cf61e13dcdc90f518c472 Mon Sep 17 00:00:00 2001 From: Danny Coates Date: Tue, 31 Jul 2018 11:29:26 -0700 Subject: [PATCH 1/4] began adding capability flags --- app/api.js | 2 +- app/capabilities.js | 74 ++++++++++++++++ app/fileManager.js | 4 +- app/fileReceiver.js | 121 ++++++++++++++++++++++---- app/main.js | 52 ++++++----- app/serviceWorker.js | 2 +- app/utils.js | 41 --------- package-lock.json | 15 ++++ package.json | 1 + server/routes/index.js | 1 + test/frontend/tests/workflow-tests.js | 22 ++--- webpack.config.js | 1 + 12 files changed, 236 insertions(+), 100 deletions(-) create mode 100644 app/capabilities.js diff --git a/app/api.js b/app/api.js index 28aa271c..b47ee84e 100644 --- a/app/api.js +++ b/app/api.js @@ -270,7 +270,7 @@ function download(id, keychain, onprogress, canceller) { } }); const auth = await keychain.authHeader(); - xhr.open('get', `/api/download/${id}`); + xhr.open('get', `/api/download/blob/${id}`); xhr.setRequestHeader('Authorization', auth); xhr.responseType = 'blob'; xhr.send(); diff --git a/app/capabilities.js b/app/capabilities.js new file mode 100644 index 00000000..1603a002 --- /dev/null +++ b/app/capabilities.js @@ -0,0 +1,74 @@ +async function checkCrypto() { + try { + const key = await crypto.subtle.generateKey( + { + name: 'AES-GCM', + length: 128 + }, + true, + ['encrypt', 'decrypt'] + ); + await crypto.subtle.encrypt( + { + name: 'AES-GCM', + iv: crypto.getRandomValues(new Uint8Array(12)), + tagLength: 128 + }, + key, + new ArrayBuffer(8) + ); + await crypto.subtle.importKey( + 'raw', + crypto.getRandomValues(new Uint8Array(16)), + 'PBKDF2', + false, + ['deriveKey'] + ); + await crypto.subtle.importKey( + 'raw', + crypto.getRandomValues(new Uint8Array(16)), + 'HKDF', + false, + ['deriveKey'] + ); + return true; + } catch (err) { + return false; + } +} + +function checkStreams() { + try { + new ReadableStream({ + pull() {} + }); + return true; + } catch (e) { + return false; + } +} + +function polyfillStreams() { + try { + require('@mattiasbuelens/web-streams-polyfill'); + return true; + } catch (e) { + return false; + } +} + +export default async function capabilities() { + const crypto = await checkCrypto(); + const nativeStreams = checkStreams(); + const polyStreams = nativeStreams ? false : polyfillStreams(); + + return { + crypto, + streamUpload: nativeStreams || polyStreams, + streamDownload: + nativeStreams && + 'serviceWorker' in navigator && + !/safari/i.test(navigator.userAgent), + multifile: nativeStreams || polyStreams + }; +} diff --git a/app/fileManager.js b/app/fileManager.js index bc89ff75..087cddc2 100644 --- a/app/fileManager.js +++ b/app/fileManager.js @@ -177,7 +177,9 @@ export default function(state, emitter) { try { const start = Date.now(); metrics.startedDownload({ size: file.size, ttl: file.ttl }); - const dl = state.transfer.download(); + const dl = state.transfer.download({ + stream: state.capabilities.streamDownload + }); render(); await dl; const time = Date.now() - start; diff --git a/app/fileReceiver.js b/app/fileReceiver.js index b22097ad..8f50dd2b 100644 --- a/app/fileReceiver.js +++ b/app/fileReceiver.js @@ -1,7 +1,9 @@ import Nanobus from 'nanobus'; import Keychain from './keychain'; import { delay, bytes } from './utils'; -import { metadata } from './api'; +import { downloadFile, metadata } from './api'; +import { blobStream } from './streams'; +import Zip from './zip'; export default class FileReceiver extends Nanobus { constructor(fileInfo) { @@ -52,22 +54,6 @@ export default class FileReceiver extends Nanobus { this.state = 'ready'; } - async streamToArrayBuffer(stream, streamSize, onprogress) { - const result = new Uint8Array(streamSize); - let offset = 0; - const reader = stream.getReader(); - let state = await reader.read(); - while (!state.done) { - result.set(state.value, offset); - offset += state.value.length; - state = await reader.read(); - onprogress([offset, streamSize]); - } - - onprogress([streamSize, streamSize]); - return result.slice(0, offset).buffer; - } - sendMessageToSw(msg) { return new Promise((resolve, reject) => { const channel = new MessageChannel(); @@ -86,7 +72,46 @@ export default class FileReceiver extends Nanobus { }); } - async download(noSave = false) { + async downloadBlob(noSave = false) { + this.state = 'downloading'; + this.downloadRequest = await downloadFile( + this.fileInfo.id, + this.keychain, + p => { + this.progress = p; + this.emit('progress'); + } + ); + try { + const ciphertext = await this.downloadRequest.result; + this.downloadRequest = null; + this.msg = 'decryptingFile'; + this.state = 'decrypting'; + this.emit('decrypting'); + let size = this.fileInfo.size; + let plainStream = this.keychain.decryptStream(blobStream(ciphertext)); + if (this.fileInfo.type === 'send-archive') { + const zip = new Zip(this.fileInfo.manifest, plainStream); + plainStream = zip.stream; + size = zip.size; + } + const plaintext = await streamToArrayBuffer(plainStream, size); + if (!noSave) { + await saveFile({ + plaintext, + name: decodeURIComponent(this.fileInfo.name), + type: this.fileInfo.type + }); + } + this.msg = 'downloadFinish'; + this.state = 'complete'; + } catch (e) { + this.downloadRequest = null; + throw e; + } + } + + async downloadStream(noSave = false) { const onprogress = p => { this.progress = p; this.emit('progress'); @@ -156,4 +181,64 @@ export default class FileReceiver extends Nanobus { throw e; } } + + download(options) { + if (options.stream) { + return this.downloadStream(options.noSave); + } + return this.downloadBlob(options.noSave); + } +} + +async function streamToArrayBuffer(stream, size) { + const result = new Uint8Array(size); + let offset = 0; + const reader = stream.getReader(); + let state = await reader.read(); + while (!state.done) { + result.set(state.value, offset); + offset += state.value.length; + state = await reader.read(); + } + + return result.buffer; +} + +async function saveFile(file) { + return new Promise(function(resolve, reject) { + const dataView = new DataView(file.plaintext); + const blob = new Blob([dataView], { type: file.type }); + + if (navigator.msSaveBlob) { + navigator.msSaveBlob(blob, file.name); + return resolve(); + } else if (/iPhone|fxios/i.test(navigator.userAgent)) { + // This method is much slower but createObjectURL + // is buggy on iOS + const reader = new FileReader(); + reader.addEventListener('loadend', function() { + if (reader.error) { + return reject(reader.error); + } + if (reader.result) { + const a = document.createElement('a'); + a.href = reader.result; + a.download = file.name; + document.body.appendChild(a); + a.click(); + } + resolve(); + }); + reader.readAsDataURL(blob); + } else { + const downloadUrl = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = downloadUrl; + a.download = file.name; + document.body.appendChild(a); + a.click(); + URL.revokeObjectURL(downloadUrl); + setTimeout(resolve, 100); + } + }); } diff --git a/app/main.js b/app/main.js index e1f5dbda..ce4140a2 100644 --- a/app/main.js +++ b/app/main.js @@ -1,29 +1,33 @@ import 'fast-text-encoding'; // MS Edge support import 'fluent-intl-polyfill'; import app from './routes'; +import capabilities from './capabilities'; import locale from '../common/locales'; import fileManager from './fileManager'; import dragManager from './dragManager'; import pasteManager from './pasteManager'; -import { canHasSend } from './utils'; import storage from './storage'; import metrics from './metrics'; import experiments from './experiments'; import Raven from 'raven-js'; import './main.css'; -if (navigator.doNotTrack !== '1' && window.RAVEN_CONFIG) { - Raven.config(window.SENTRY_ID, window.RAVEN_CONFIG).install(); -} - -app.use((state, emitter) => { - state.transfer = null; - state.fileInfo = null; - state.translate = locale.getTranslator(); - state.storage = storage; - state.raven = Raven; - window.appState = state; - emitter.on('DOMContentLoaded', async function checkSupport() { +(async function start() { + if (navigator.doNotTrack !== '1' && window.RAVEN_CONFIG) { + Raven.config(window.SENTRY_ID, window.RAVEN_CONFIG).install(); + } + const capa = await capabilities(); + if (capa.streamDownload) { + navigator.serviceWorker.register('/serviceWorker.js'); + } + app.use((state, emitter) => { + state.capabilities = capa; + state.transfer = null; + state.fileInfo = null; + state.translate = locale.getTranslator(); + state.storage = storage; + state.raven = Raven; + window.appState = state; let unsupportedReason = null; if ( // Firefox < 50 @@ -32,8 +36,7 @@ app.use((state, emitter) => { ) { unsupportedReason = 'outdated'; } - const ok = await canHasSend(); - if (!ok) { + if (!state.capabilities.crypto) { unsupportedReason = /firefox/i.test(navigator.userAgent) ? 'outdated' : 'gcm'; @@ -44,15 +47,10 @@ app.use((state, emitter) => { ); } }); -}); - -app.use(() => { - navigator.serviceWorker.register('/serviceWorker.js'); -}); -app.use(metrics); -app.use(fileManager); -app.use(dragManager); -app.use(experiments); -app.use(pasteManager); - -app.mount('body'); + app.use(metrics); + app.use(fileManager); + app.use(dragManager); + app.use(experiments); + app.use(pasteManager); + app.mount('body'); +})(); diff --git a/app/serviceWorker.js b/app/serviceWorker.js index 5dc70110..bcf1aac4 100644 --- a/app/serviceWorker.js +++ b/app/serviceWorker.js @@ -75,7 +75,7 @@ async function decryptStream(id) { self.onfetch = event => { const req = event.request; - if (req.url.includes('/api/download')) { + if (/\/api\/download\/[A-Fa-f0-9]{4,}/.test(req.url)) { const id = req.url.split('/')[5]; event.respondWith(decryptStream(id)); } diff --git a/app/utils.js b/app/utils.js index e3c900a6..8baa6617 100644 --- a/app/utils.js +++ b/app/utils.js @@ -22,46 +22,6 @@ function loadShim(polyfill) { }); } -async function canHasSend() { - try { - const key = await crypto.subtle.generateKey( - { - name: 'AES-GCM', - length: 128 - }, - true, - ['encrypt', 'decrypt'] - ); - await crypto.subtle.encrypt( - { - name: 'AES-GCM', - iv: crypto.getRandomValues(new Uint8Array(12)), - tagLength: 128 - }, - key, - new ArrayBuffer(8) - ); - await crypto.subtle.importKey( - 'raw', - crypto.getRandomValues(new Uint8Array(16)), - 'PBKDF2', - false, - ['deriveKey'] - ); - await crypto.subtle.importKey( - 'raw', - crypto.getRandomValues(new Uint8Array(16)), - 'HKDF', - false, - ['deriveKey'] - ); - return true; - } catch (err) { - console.error(err); - return false; - } -} - function isFile(id) { return /^[0-9a-fA-F]{10}$/.test(id); } @@ -179,7 +139,6 @@ module.exports = { arrayToB64, b64ToArray, loadShim, - canHasSend, isFile, openLinksInNewTab }; diff --git a/package-lock.json b/package-lock.json index c1c04328..53d3f9b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -88,6 +88,15 @@ "integrity": "sha1-9vGlzl05caSt6RoR0i1MRZrNN18=", "dev": true }, + "@mattiasbuelens/web-streams-polyfill": { + "version": "0.1.0-alpha.5", + "resolved": "https://registry.npmjs.org/@mattiasbuelens/web-streams-polyfill/-/web-streams-polyfill-0.1.0-alpha.5.tgz", + "integrity": "sha512-KduiboN8xXQT+XAe76935VriUy62Yv/1EyTE43xmP+GEtPKpyUklKvYaXPq1fZpibMgXIIi/B2Qx0dReam5RKQ==", + "dev": true, + "requires": { + "@types/whatwg-streams": "0.0.6" + } + }, "@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", @@ -122,6 +131,12 @@ "samsam": "1.3.0" } }, + "@types/whatwg-streams": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/@types/whatwg-streams/-/whatwg-streams-0.0.6.tgz", + "integrity": "sha512-O4Hat94N1RUCObqAbVUtd6EcucseqBcpfbFXzy12CYF6BQVHWR+ztDA3YPjewCmdKHYZ5VA7TZ5hq2bMyqxiBw==", + "dev": true + }, "@webassemblyjs/ast": { "version": "1.5.13", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.5.13.tgz", diff --git a/package.json b/package.json index 2be8bff1..daf0f25a 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ }, "devDependencies": { "@dannycoates/webpack-dev-server": "^3.1.4", + "@mattiasbuelens/web-streams-polyfill": "0.1.0-alpha.5", "asmcrypto.js": "^0.22.0", "babel-core": "^6.26.3", "babel-loader": "^7.1.4", diff --git a/server/routes/index.js b/server/routes/index.js index 3cf1600a..d258c662 100644 --- a/server/routes/index.js +++ b/server/routes/index.js @@ -57,6 +57,7 @@ module.exports = function(app) { app.get('/completed', language, pages.blank); app.get('/unsupported/:reason', language, pages.unsupported); app.get(`/api/download/:id${ID_REGEX}`, auth, require('./download')); + app.get(`/api/download/blob/:id${ID_REGEX}`, auth, require('./download')); app.get(`/api/exists/:id${ID_REGEX}`, require('./exists')); app.get(`/api/metadata/:id${ID_REGEX}`, auth, require('./metadata')); app.post('/api/upload', require('./upload')); diff --git a/test/frontend/tests/workflow-tests.js b/test/frontend/tests/workflow-tests.js index b440596d..a54907f7 100644 --- a/test/frontend/tests/workflow-tests.js +++ b/test/frontend/tests/workflow-tests.js @@ -7,7 +7,7 @@ const headless = /Headless/.test(navigator.userAgent); // TODO: save on headless doesn't work as it used to since it now // follows a link instead of fetch. Maybe there's a way to make it // work? For now always set noSave. -const noSave = true || !headless; // only run the saveFile code if headless +const options = { noSave: true || !headless, stream: true }; // only run the saveFile code if headless // FileSender uses a File in real life but a Blob works for testing const blob = new Blob([new ArrayBuffer(1024 * 128)], { type: 'text/plain' }); @@ -27,10 +27,10 @@ describe('Upload / Download flow', function() { requiresPassword: false }); await fr.getMetadata(); - await fr.download(noSave); + await fr.download(options); try { - await fr.download(noSave); + await fr.download(options); assert.fail('downloaded again'); } catch (e) { assert.equal(e.message, '404'); @@ -50,7 +50,7 @@ describe('Upload / Download flow', function() { password: 'magic' }); await fr.getMetadata(); - await fr.download(noSave); + await fr.download(options); assert.equal(fr.state, 'complete'); }); @@ -75,7 +75,7 @@ describe('Upload / Download flow', function() { try { // We can't decrypt without IV from metadata // but let's try to download anyway - await fr.download(noSave); + await fr.download(options); assert.fail('downloaded file with bad password'); } catch (e) { assert.equal(e.message, '401'); @@ -135,7 +135,7 @@ describe('Upload / Download flow', function() { await fr.getMetadata(); fr.once('progress', () => fr.cancel()); try { - await fr.download(noSave); + await fr.download(options); assert.fail('not cancelled'); } catch (e) { assert.equal(e.message, '0'); @@ -153,7 +153,7 @@ describe('Upload / Download flow', function() { requiresPassword: false }); await fr.getMetadata(); - await fr.download(noSave); + await fr.download(options); await file.updateDownloadCount(); assert.equal(file.dtotal, 1); }); @@ -171,7 +171,7 @@ describe('Upload / Download flow', function() { fr.once('progress', () => fr.cancel()); try { - await fr.download(noSave); + await fr.download(options); assert.fail('not cancelled'); } catch (e) { await file.updateDownloadCount(); @@ -190,15 +190,15 @@ describe('Upload / Download flow', function() { }); await file.changeLimit(2); await fr.getMetadata(); - await fr.download(noSave); + await fr.download(options); await file.updateDownloadCount(); assert.equal(file.dtotal, 1); - await fr.download(noSave); + await fr.download(options); await file.updateDownloadCount(); assert.equal(file.dtotal, 2); try { - await fr.download(noSave); + await fr.download(options); assert.fail('downloaded too many times'); } catch (e) { assert.equal(e.message, '404'); diff --git a/webpack.config.js b/webpack.config.js index 24b5f291..ad0922ec 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -88,6 +88,7 @@ const web = { { // Strip asserts from our deps, mainly choojs family include: [path.resolve(__dirname, 'node_modules')], + exclude: [path.resolve(__dirname, 'node_modules/crc')], loader: 'webpack-unassert-loader' } ] From 9501bf7270e1190b856f98db295e3ac25ec51e16 Mon Sep 17 00:00:00 2001 From: Danny Coates Date: Tue, 31 Jul 2018 12:19:27 -0700 Subject: [PATCH 2/4] colon in wss url breaks edge when port is not explicit --- app/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api.js b/app/api.js index b47ee84e..3d394c46 100644 --- a/app/api.js +++ b/app/api.js @@ -135,7 +135,7 @@ async function upload(stream, metadata, verifierB64, onprogress, canceller) { const endpoint = window.location.protocol === 'file:' ? 'wss://send2.dev.lcip.org/api/ws' - : `${protocol}//${host}:${port}/api/ws`; + : `${protocol}//${host}${port ? ':' : ''}${port}/api/ws`; const ws = await asyncInitWebSocket(endpoint); From 4fabd539d10333a52901994cac7ed6c1184ebd88 Mon Sep 17 00:00:00 2001 From: Danny Coates Date: Tue, 31 Jul 2018 15:10:12 -0700 Subject: [PATCH 3/4] added exportKey to capabilities check --- app/capabilities.js | 1 + 1 file changed, 1 insertion(+) diff --git a/app/capabilities.js b/app/capabilities.js index 1603a002..1d6441d8 100644 --- a/app/capabilities.js +++ b/app/capabilities.js @@ -8,6 +8,7 @@ async function checkCrypto() { true, ['encrypt', 'decrypt'] ); + await crypto.subtle.exportKey('raw', key); await crypto.subtle.encrypt( { name: 'AES-GCM', From a0edff1ead295395120e6e13c9546b4d96e0976f Mon Sep 17 00:00:00 2001 From: Danny Coates Date: Tue, 31 Jul 2018 15:37:04 -0700 Subject: [PATCH 4/4] make ece encrypt key extractable for Edge --- app/ece.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ece.js b/app/ece.js index f46c9f61..3622dbd7 100644 --- a/app/ece.js +++ b/app/ece.js @@ -48,7 +48,7 @@ class ECETransformer { name: 'AES-GCM', length: 128 }, - false, + true, // Edge polyfill requires key to be extractable to encrypt :/ ['encrypt', 'decrypt'] ); }