diff --git a/app/api.js b/app/api.js index 42d7d512..7ec1341e 100644 --- a/app/api.js +++ b/app/api.js @@ -91,47 +91,107 @@ export async function setPassword(id, owner_token, keychain) { return response.ok; } -export function uploadFile( - encrypted, - metadata, - verifierB64, - keychain, - onprogress -) { - const xhr = new XMLHttpRequest(); - const upload = { - cancel: function() { - xhr.abort(); - }, - result: new Promise(function(resolve, reject) { - xhr.addEventListener('loadend', function() { - const authHeader = xhr.getResponseHeader('WWW-Authenticate'); - if (authHeader) { - keychain.nonce = parseNonce(authHeader); - } - if (xhr.status === 200) { - const responseObj = JSON.parse(xhr.responseText); - return resolve({ - url: responseObj.url, - id: responseObj.id, - ownerToken: responseObj.owner +function asyncInitWebSocket(server) { + return new Promise(resolve => { + const ws = new WebSocket(server); + ws.onopen = () => { + resolve(ws); + }; + }); +} + +function listenForResponse(ws, canceller) { + return new Promise((resolve, reject) => { + ws.addEventListener('message', function(msg) { + try { + const response = JSON.parse(msg.data); + if (response.error) { + throw new Error(response.error); + } else { + resolve({ + url: response.url, + id: response.id, + ownerToken: response.owner }); } - reject(new Error(xhr.status)); - }); - }) - }; - const blob = new Blob([encrypted], { type: 'application/octet-stream' }); - xhr.upload.addEventListener('progress', function(event) { - if (event.lengthComputable) { - onprogress([event.loaded, event.total]); - } + } catch (e) { + ws.close(); + canceller.cancelled = true; + canceller.error = e; + reject(e); + } + }); }); - xhr.open('post', '/api/upload', true); - xhr.setRequestHeader('X-File-Metadata', arrayToB64(new Uint8Array(metadata))); - xhr.setRequestHeader('Authorization', `send-v1 ${verifierB64}`); - xhr.send(blob); - return upload; +} + +async function upload( + stream, + streamInfo, + metadata, + verifierB64, + onprogress, + canceller +) { + const host = window.location.hostname; + const port = window.location.port; + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + const ws = await asyncInitWebSocket(`${protocol}//${host}:${port}/api/ws`); + + try { + const metadataHeader = arrayToB64(new Uint8Array(metadata)); + const fileMeta = { + fileMetadata: metadataHeader, + authorization: `send-v1 ${verifierB64}` + }; + + const responsePromise = listenForResponse(ws, canceller); + + ws.send(JSON.stringify(fileMeta)); + + const reader = stream.getReader(); + let state = await reader.read(); + let size = 0; + while (!state.done) { + const buf = state.value; + if (canceller.cancelled) { + throw canceller.error; + } + + ws.send(buf); + + onprogress([Math.min(streamInfo.fileSize, size), streamInfo.fileSize]); + size += streamInfo.recordSize; + state = await reader.read(); + } + const footer = new Uint8Array([0]); + ws.send(footer); + + const response = await responsePromise; //promise only fufills if response is good + ws.close(); + return response; + } catch (e) { + ws.close(4000); + throw e; + } +} + +export function uploadWs(encrypted, info, metadata, verifierB64, onprogress) { + const canceller = { cancelled: false }; + + return { + cancel: function() { + canceller.error = new Error(0); + canceller.cancelled = true; + }, + result: upload( + encrypted, + info, + metadata, + verifierB64, + onprogress, + canceller + ) + }; } function download(id, keychain, onprogress, canceller) { @@ -151,11 +211,7 @@ function download(id, keychain, onprogress, canceller) { } const blob = new Blob([xhr.response]); - const fileReader = new FileReader(); - fileReader.readAsArrayBuffer(blob); - fileReader.onload = function() { - resolve(this.result); - }; + resolve(blob); }); xhr.addEventListener('progress', function(event) { if (event.lengthComputable && event.target.status === 200) { diff --git a/app/ece.js b/app/ece.js new file mode 100644 index 00000000..eb86c6c0 --- /dev/null +++ b/app/ece.js @@ -0,0 +1,292 @@ +require('buffer'); +import { ReadableStream, TransformStream } from 'web-streams-polyfill'; + +const NONCE_LENGTH = 12; +const TAG_LENGTH = 16; +const KEY_LENGTH = 16; +const MODE_ENCRYPT = 'encrypt'; +const MODE_DECRYPT = 'decrypt'; +const RS = 1048576; + +const encoder = new TextEncoder(); + +function generateSalt(len) { + const randSalt = new Uint8Array(len); + window.crypto.getRandomValues(randSalt); + return randSalt.buffer; +} + +class ECETransformer { + constructor(mode, ikm, rs, salt) { + this.mode = mode; + this.prevChunk; + this.seq = 0; + this.firstchunk = true; + this.rs = rs; + this.ikm = ikm.buffer; + this.salt = salt; + } + + async generateKey() { + const inputKey = await window.crypto.subtle.importKey( + 'raw', + this.ikm, + 'HKDF', + false, + ['deriveKey'] + ); + + return window.crypto.subtle.deriveKey( + { + name: 'HKDF', + salt: this.salt, + info: encoder.encode('Content-Encoding: aes128gcm\0'), + hash: 'SHA-256' + }, + inputKey, + { + name: 'AES-GCM', + length: 128 + }, + false, + ['encrypt', 'decrypt'] + ); + } + + async generateNonceBase() { + const inputKey = await window.crypto.subtle.importKey( + 'raw', + this.ikm, + 'HKDF', + false, + ['deriveKey'] + ); + + const base = await window.crypto.subtle.exportKey( + 'raw', + await window.crypto.subtle.deriveKey( + { + name: 'HKDF', + salt: this.salt, + info: encoder.encode('Content-Encoding: nonce\0'), + hash: 'SHA-256' + }, + inputKey, + { + name: 'AES-GCM', + length: 128 + }, + true, + ['encrypt', 'decrypt'] + ) + ); + + return Buffer.from(base.slice(0, NONCE_LENGTH)); + } + + generateNonce(seq) { + if (seq > 0xffffffff) { + throw new Error('record sequence number exceeds limit'); + } + const nonce = Buffer.from(this.nonceBase); + const m = nonce.readUIntBE(nonce.length - 4, 4); + const xor = (m ^ seq) >>> 0; //forces unsigned int xor + nonce.writeUIntBE(xor, nonce.length - 4, 4); + + return nonce; + } + + pad(data, isLast) { + const len = data.length; + if (len + TAG_LENGTH >= this.rs) { + throw new Error('data too large for record size'); + } + + if (isLast) { + const padding = Buffer.alloc(1); + padding.writeUInt8(2, 0); + return Buffer.concat([data, padding]); + } else { + const padding = Buffer.alloc(this.rs - len - TAG_LENGTH); + padding.fill(0); + padding.writeUInt8(1, 0); + return Buffer.concat([data, padding]); + } + } + + unpad(data, isLast) { + for (let i = data.length - 1; i >= 0; i--) { + if (data[i]) { + if (isLast) { + if (data[i] !== 2) { + throw new Error('delimiter of final record is not 2'); + } + } else { + if (data[i] !== 1) { + throw new Error('delimiter of not final record is not 1'); + } + } + return data.slice(0, i); + } + } + throw new Error('no delimiter found'); + } + + createHeader() { + const nums = Buffer.alloc(5); + nums.writeUIntBE(this.rs, 0, 4); + nums.writeUIntBE(0, 4, 1); + return Buffer.concat([Buffer.from(this.salt), nums]); + } + + readHeader(buffer) { + if (buffer.length < 21) { + throw new Error('chunk too small for reading header'); + } + const header = {}; + header.salt = buffer.buffer.slice(0, KEY_LENGTH); + header.rs = buffer.readUIntBE(KEY_LENGTH, 4); + const idlen = buffer.readUInt8(KEY_LENGTH + 4); + header.length = idlen + KEY_LENGTH + 5; + return header; + } + + async encryptRecord(buffer, seq, isLast) { + const nonce = this.generateNonce(seq); + const encrypted = await window.crypto.subtle.encrypt( + { name: 'AES-GCM', iv: nonce }, + this.key, + this.pad(buffer, isLast) + ); + return Buffer.from(encrypted); + } + + async decryptRecord(buffer, seq, isLast) { + const nonce = this.generateNonce(seq); + const data = await window.crypto.subtle.decrypt( + { + name: 'AES-GCM', + iv: nonce, + tagLength: 128 + }, + this.key, + buffer + ); + + return this.unpad(Buffer.from(data), isLast); + } + + async start(controller) { + if (this.mode === MODE_ENCRYPT) { + this.key = await this.generateKey(); + this.nonceBase = await this.generateNonceBase(); + controller.enqueue(this.createHeader()); + } else if (this.mode !== MODE_DECRYPT) { + throw new Error('mode must be either encrypt or decrypt'); + } + } + + async transformPrevChunk(isLast, controller) { + if (this.mode === MODE_ENCRYPT) { + controller.enqueue( + await this.encryptRecord(this.prevChunk, this.seq, isLast) + ); + this.seq++; + } else { + if (this.seq === 0) { + //the first chunk during decryption contains only the header + const header = this.readHeader(this.prevChunk); + this.salt = header.salt; + this.rs = header.rs; + this.key = await this.generateKey(); + this.nonceBase = await this.generateNonceBase(); + } else { + controller.enqueue( + await this.decryptRecord(this.prevChunk, this.seq - 1, isLast) + ); + } + this.seq++; + } + } + + async transform(chunk, controller) { + if (!this.firstchunk) { + await this.transformPrevChunk(false, controller); + } + this.firstchunk = false; + this.prevChunk = Buffer.from(chunk.buffer); + } + + async flush(controller) { + if (this.prevChunk) { + await this.transformPrevChunk(true, controller); + } + } +} + +class BlobSlicer { + constructor(blob, rs, mode) { + this.blob = blob; + this.index = 0; + this.mode = mode; + this.chunkSize = mode === MODE_ENCRYPT ? rs - 17 : rs; + } + + pull(controller) { + return new Promise((resolve, reject) => { + const bytesLeft = this.blob.size - this.index; + if (bytesLeft <= 0) { + controller.close(); + return resolve(); + } + let size = 1; + if (this.mode === MODE_DECRYPT && this.index === 0) { + size = Math.min(21, bytesLeft); + } else { + size = Math.min(this.chunkSize, bytesLeft); + } + const blob = this.blob.slice(this.index, this.index + size); + const reader = new FileReader(); + reader.onload = () => { + controller.enqueue(new Uint8Array(reader.result)); + resolve(); + }; + reader.onerror = reject; + reader.readAsArrayBuffer(blob); + this.index += size; + }); + } +} + +class BlobSliceStream extends ReadableStream { + constructor(blob, size, mode) { + super(new BlobSlicer(blob, size, mode)); + } +} + +/* +input: a blob containing data to be transformed +key: Uint8Array containing key of size KEY_LENGTH +mode: string, either 'encrypt' or 'decrypt' +rs: int containing record size, optional +salt: ArrayBuffer containing salt of KEY_LENGTH length, optional +*/ +export default class ECE { + constructor(input, key, mode, rs, salt) { + if (rs === undefined) { + rs = RS; + } + if (salt === undefined) { + salt = generateSalt(KEY_LENGTH); + } + + this.streamInfo = { + recordSize: rs, + fileSize: 21 + input.size + 16 * Math.floor(input.size / (rs - 17)) + }; + const inputStream = new BlobSliceStream(input, rs, mode); + + const ts = new TransformStream(new ECETransformer(mode, key, rs, salt)); + this.stream = inputStream.pipeThrough(ts); + } +} diff --git a/app/fileManager.js b/app/fileManager.js index 786f99f8..e29a05e2 100644 --- a/app/fileManager.js +++ b/app/fileManager.js @@ -52,7 +52,7 @@ export default function(state, emitter) { checkFiles(); }); - emitter.on('navigate', checkFiles); + //emitter.on('navigate', checkFiles); emitter.on('render', () => { lastRender = Date.now(); diff --git a/app/fileReceiver.js b/app/fileReceiver.js index 27dccf6d..2a22fbca 100644 --- a/app/fileReceiver.js +++ b/app/fileReceiver.js @@ -51,6 +51,21 @@ export default class FileReceiver extends Nanobus { this.state = 'ready'; } + async streamToArrayBuffer(stream, streamSize) { + const reader = stream.getReader(); + const result = new Uint8Array(streamSize); + let offset = 0; + + let state = await reader.read(); + while (!state.done) { + result.set(state.value, offset); + offset += state.value.length; + state = await reader.read(); + } + + return result.buffer; + } + async download(noSave = false) { this.state = 'downloading'; this.downloadRequest = await downloadFile( @@ -61,13 +76,21 @@ export default class FileReceiver extends Nanobus { this.emit('progress'); } ); + try { const ciphertext = await this.downloadRequest.result; this.downloadRequest = null; this.msg = 'decryptingFile'; this.state = 'decrypting'; this.emit('decrypting'); - const plaintext = await this.keychain.decryptFile(ciphertext); + + const dec = await this.keychain.decryptStream(ciphertext); + const plainstream = dec.stream; + const plaintext = await this.streamToArrayBuffer( + plainstream, + dec.streamInfo.fileSize + ); + if (!noSave) { await saveFile({ plaintext, diff --git a/app/fileSender.js b/app/fileSender.js index 99f2ad12..7c3c4de1 100644 --- a/app/fileSender.js +++ b/app/fileSender.js @@ -3,7 +3,7 @@ import Nanobus from 'nanobus'; import OwnedFile from './ownedFile'; import Keychain from './keychain'; import { arrayToB64, bytes } from './utils'; -import { uploadFile } from './api'; +import { uploadWs } from './api'; export default class FileSender extends Nanobus { constructor(file) { @@ -59,28 +59,31 @@ export default class FileSender extends Nanobus { async upload() { const start = Date.now(); - const plaintext = await this.readFile(); if (this.cancelled) { throw new Error(0); } this.msg = 'encryptingFile'; this.emit('encrypting'); - const encrypted = await this.keychain.encryptFile(plaintext); + + const enc = this.keychain.encryptStream(this.file); const metadata = await this.keychain.encryptMetadata(this.file); const authKeyB64 = await this.keychain.authKeyB64(); - if (this.cancelled) { - throw new Error(0); - } - this.uploadRequest = uploadFile( - encrypted, + + this.uploadRequest = uploadWs( + enc.stream, + enc.streamInfo, metadata, authKeyB64, - this.keychain, p => { this.progress = p; this.emit('progress'); } ); + + if (this.cancelled) { + throw new Error(0); + } + this.msg = 'fileSizeProgress'; this.emit('progress'); // HACK to kick MS Edge try { diff --git a/app/keychain.js b/app/keychain.js index eacffec3..75778990 100644 --- a/app/keychain.js +++ b/app/keychain.js @@ -1,5 +1,5 @@ import { arrayToB64, b64ToArray } from './utils'; - +import ECE from './ece.js'; const encoder = new TextEncoder(); const decoder = new TextDecoder(); @@ -179,6 +179,16 @@ export default class Keychain { return ciphertext; } + encryptStream(plaintext) { + const enc = new ECE(plaintext, this.rawSecret, 'encrypt'); + return enc; + } + + decryptStream(encstream) { + const dec = new ECE(encstream, this.rawSecret, 'decrypt'); + return dec; + } + async decryptFile(ciphertext) { const encryptKey = await this.encryptKeyPromise; const plaintext = await window.crypto.subtle.decrypt( diff --git a/docs/build.md b/docs/build.md index 78ebd66f..f440f410 100644 --- a/docs/build.md +++ b/docs/build.md @@ -2,11 +2,11 @@ Send has two build configurations, development and production. Both can be run v # Development -`npm start` launches a `webpack-dev-server` on port 8080 that compiles the assets and watches files for changes. It also serves the backend API and frontend unit tests via the `server/dev.js` entrypoint. The frontend tests can be run in the browser by navigating to http://localhost:8080/test and will rerun automatically as the watched files are saved with changes. +`npm start` launches a `webpack-dev-server` on port 8080 that compiles the assets and watches files for changes. It also serves the backend API and frontend unit tests via the `server/bin/dev.js` entrypoint. The frontend tests can be run in the browser by navigating to http://localhost:8080/test and will rerun automatically as the watched files are saved with changes. # Production -`npm run build` compiles the assets and writes the files to the `dist/` directory. `npm run prod` launches an Express server on port 1443 that serves the backend API and frontend static assets from `dist/` via the `server/prod.js` entrypoint. +`npm run build` compiles the assets and writes the files to the `dist/` directory. `npm run prod` launches an Express server on port 1443 that serves the backend API and frontend static assets from `dist/` via the `server/bin/prod.js` entrypoint. # Notable differences diff --git a/package-lock.json b/package-lock.json index 509909c1..6247b8a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -485,8 +485,7 @@ "async-limiter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", - "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==", - "dev": true + "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" }, "asynckit": { "version": "0.4.0", @@ -2613,8 +2612,7 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "cosmiconfig": { "version": "4.0.0", @@ -3628,7 +3626,6 @@ "version": "3.5.4", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.5.4.tgz", "integrity": "sha512-JzYSLYMhoVVBe8+mbHQ4KgpvHpm0DZpJuL8PY93Vyv1fW7jYJ90LoXa1di/CVbJM+TgMs91rbDapE/RNIfnJsA==", - "dev": true, "requires": { "end-of-stream": "1.4.1", "inherits": "2.0.3", @@ -3640,7 +3637,6 @@ "version": "2.3.5", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.5.tgz", "integrity": "sha512-tK0yDhrkygt/knjowCUiWP9YdV7c5R+8cR0r/kt9ZhBU906Fs6RpQJCEilamRJj1Nx2rWI6LkW9gKqjTkshhEw==", - "dev": true, "requires": { "core-util-is": "1.0.2", "inherits": "2.0.3", @@ -3655,7 +3651,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", - "dev": true, "requires": { "safe-buffer": "5.1.1" } @@ -3719,7 +3714,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", - "dev": true, "requires": { "once": "1.4.0" } @@ -4373,6 +4367,14 @@ } } }, + "express-ws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/express-ws/-/express-ws-4.0.0.tgz", + "integrity": "sha512-KEyUw8AwRET2iFjFsI1EJQrJ/fHeGiJtgpYgEWG3yDv4l/To/m3a2GaYfeGyB3lsWdvbesjF5XCMx+SVBgAAYw==", + "requires": { + "ws": "5.2.0" + } + }, "extend": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", @@ -6496,6 +6498,15 @@ "ctype": "0.5.3" } }, + "http_ece": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/http_ece/-/http_ece-1.0.5.tgz", + "integrity": "sha1-tgZg+q8UIVEC0Uk+pyDc2StTNy8=", + "dev": true, + "requires": { + "urlsafe-base64": "1.0.0" + } + }, "https-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", @@ -12113,7 +12124,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dev": true, "requires": { "wrappy": "1.0.2" } @@ -15771,8 +15781,7 @@ "process-nextick-args": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", - "dev": true + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" }, "progress": { "version": "2.0.0", @@ -15894,6 +15903,19 @@ "proxy-from-env": "1.0.0", "rimraf": "2.6.2", "ws": "3.3.3" + }, + "dependencies": { + "ws": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz", + "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==", + "dev": true, + "requires": { + "async-limiter": "1.0.0", + "safe-buffer": "5.1.1", + "ultron": "1.1.1" + } + } } }, "q": { @@ -17655,8 +17677,7 @@ "stream-shift": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz", - "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=", - "dev": true + "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=" }, "stream-to-observable": { "version": "0.2.0", @@ -18713,8 +18734,7 @@ "ultron": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz", - "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==", - "dev": true + "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==" }, "unassert": { "version": "1.5.1", @@ -19004,6 +19024,12 @@ } } }, + "urlsafe-base64": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/urlsafe-base64/-/urlsafe-base64-1.0.0.tgz", + "integrity": "sha1-I/iQaabGL0bPOh07ABac77kL4MY=", + "dev": true + }, "use": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/use/-/use-2.0.2.tgz", @@ -19123,8 +19149,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "dev": true + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "util.promisify": { "version": "1.0.0", @@ -19295,6 +19320,12 @@ "minimalistic-assert": "1.0.0" } }, + "web-streams-polyfill": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-1.3.2.tgz", + "integrity": "sha1-NxkkXpCSgtk5Z4JfRLzVUOnAOZU=", + "dev": true + }, "webidl-conversions": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", @@ -19794,6 +19825,53 @@ "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==", "dev": true }, + "websocket-stream": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/websocket-stream/-/websocket-stream-5.1.2.tgz", + "integrity": "sha512-lchLOk435iDWs0jNuL+hiU14i3ERSrMA0IKSiJh7z6X/i4XNsutBZrtqu2CPOZuA4G/zabiqVAos0vW+S7GEVw==", + "requires": { + "duplexify": "3.5.4", + "inherits": "2.0.3", + "readable-stream": "2.3.6", + "safe-buffer": "5.1.1", + "ws": "3.3.3", + "xtend": "4.0.1" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "5.1.1" + } + }, + "ws": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz", + "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==", + "requires": { + "async-limiter": "1.0.0", + "safe-buffer": "5.1.1", + "ultron": "1.1.1" + } + } + } + }, "whatwg-encoding": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.3.tgz", @@ -19882,8 +19960,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "wreck": { "version": "12.5.1", @@ -19922,14 +19999,11 @@ } }, "ws": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz", - "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==", - "dev": true, + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.0.tgz", + "integrity": "sha512-c18dMeW+PEQdDFzkhDsnBAlS4Z8KGStBQQUcQ5mf7Nf689jyGk0594L+i9RaQuf4gog6SvWLJorz2NfSaqxZ7w==", "requires": { - "async-limiter": "1.0.0", - "safe-buffer": "5.1.1", - "ultron": "1.1.1" + "async-limiter": "1.0.0" } }, "x-is-function": { diff --git a/package.json b/package.json index 90dca69e..b927d113 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "test-integration": "docker-compose up --abort-on-container-exit --exit-code-from integration-tests --build --remove-orphans --quiet-pull && docker-compose down", "test-integration-stage": "cross-env BASE_URL=https://send.stage.mozaws.net npm run test-integration", "start": "npm run clean && cross-env NODE_ENV=development webpack-dev-server", - "prod": "node server/prod.js" + "prod": "node server/bin/prod.js" }, "lint-staged": { "*.js": [ @@ -82,6 +82,7 @@ "git-rev-sync": "^1.10.0", "github-changes": "^1.1.2", "html-loader": "^0.5.5", + "http_ece": "^1.0.5", "husky": "^0.14.3", "lint-staged": "^7.0.0", "mocha": "^5.0.4", @@ -109,6 +110,7 @@ "svgo-loader": "^2.1.0", "testpilot-ga": "^0.3.0", "val-loader": "^1.1.0", + "web-streams-polyfill": "^1.3.2", "webpack": "^3.11.0", "webpack-dev-middleware": "^2.0.6", "webpack-dev-server": "2.9.1", @@ -122,13 +124,15 @@ "cldr-core": "^32.0.0", "convict": "^4.0.1", "express": "^4.16.2", + "express-ws": "^4.0.0", "fluent": "^0.6.3", "fluent-langneg": "^0.1.0", "helmet": "^3.12.0", "mkdirp": "^0.5.1", "mozlog": "^2.2.0", "raven": "^2.4.2", - "redis": "^2.8.0" + "redis": "^2.8.0", + "websocket-stream": "^5.1.2" }, "availableLanguages": [ "en-US", diff --git a/server/bin/dev.js b/server/bin/dev.js new file mode 100644 index 00000000..d94eb8c4 --- /dev/null +++ b/server/bin/dev.js @@ -0,0 +1,23 @@ +const assets = require('../../common/assets'); +const locales = require('../../common/locales'); +const routes = require('../routes'); +const pages = require('../routes/pages'); +const tests = require('../../test/frontend/routes'); +const express = require('express'); +const expressWs = require('express-ws'); +const config = require('../config'); + +const wsapp = express(); +expressWs(wsapp, null, { perMessageDeflate: false }); +wsapp.ws('/api/ws', require('../routes/ws')); +wsapp.listen(8081, config.listen_address); + +module.exports = function(app, devServer) { + assets.setMiddleware(devServer.middleware); + locales.setMiddleware(devServer.middleware); + routes(app); + tests(app); + // webpack-dev-server routes haven't been added yet + // so wait for next tick to add 404 handler + process.nextTick(() => app.use(pages.notfound)); +}; diff --git a/server/prod.js b/server/bin/prod.js similarity index 59% rename from server/prod.js rename to server/bin/prod.js index 4177b160..d52393ed 100644 --- a/server/prod.js +++ b/server/bin/prod.js @@ -1,20 +1,22 @@ const express = require('express'); const path = require('path'); const Raven = require('raven'); -const config = require('./config'); -const routes = require('./routes'); -const pages = require('./routes/pages'); +const config = require('../config'); +const routes = require('../routes'); +const pages = require('../routes/pages'); +const expressWs = require('express-ws'); if (config.sentry_dsn) { Raven.config(config.sentry_dsn).install(); } const app = express(); - +expressWs(app, null, { perMessageDeflate: false }); +app.ws('/api/ws', require('../routes/ws')); routes(app); app.use( - express.static(path.resolve(__dirname, '../dist/'), { + express.static(path.resolve(__dirname, '../../dist/'), { setHeaders: function(res) { res.set('Cache-Control', 'public, max-age=31536000, immutable'); res.removeHeader('Pragma'); diff --git a/server/bin/test.js b/server/bin/test.js new file mode 100644 index 00000000..b5f349a5 --- /dev/null +++ b/server/bin/test.js @@ -0,0 +1,18 @@ +const assets = require('../../common/assets'); +const locales = require('../../common/locales'); +const routes = require('../routes'); +const pages = require('../routes/pages'); +const tests = require('../../test/frontend/routes'); +const expressWs = require('express-ws'); + +module.exports = function(app, devServer) { + assets.setMiddleware(devServer.middleware); + locales.setMiddleware(devServer.middleware); + expressWs(app, null, { perMessageDeflate: false }); + app.ws('/api/ws', require('../routes/ws')); + routes(app); + tests(app); + // webpack-dev-server routes haven't been added yet + // so wait for next tick to add 404 handler + process.nextTick(() => app.use(pages.notfound)); +}; diff --git a/server/dev.js b/server/dev.js deleted file mode 100644 index babb9b12..00000000 --- a/server/dev.js +++ /dev/null @@ -1,15 +0,0 @@ -const assets = require('../common/assets'); -const locales = require('../common/locales'); -const routes = require('./routes'); -const pages = require('./routes/pages'); -const tests = require('../test/frontend/routes'); - -module.exports = function(app, devServer) { - assets.setMiddleware(devServer.middleware); - locales.setMiddleware(devServer.middleware); - routes(app); - tests(app); - // webpack-dev-server routes haven't been added yet - // so wait for next tick to add 404 handler - process.nextTick(() => app.use(pages.notfound)); -}; diff --git a/server/routes/ws.js b/server/routes/ws.js new file mode 100644 index 00000000..f2f6250d --- /dev/null +++ b/server/routes/ws.js @@ -0,0 +1,73 @@ +const crypto = require('crypto'); +const storage = require('../storage'); +const config = require('../config'); +const mozlog = require('../log'); +const Limiter = require('../limiter'); +const Parser = require('../streamparser'); +const wsStream = require('websocket-stream/stream'); + +const log = mozlog('send.upload'); + +module.exports = async function(ws, req) { + let fileStream; + + ws.on('close', e => { + if (e !== 1000 && fileStream !== undefined) { + fileStream.destroy(); + } + }); + + ws.once('message', async function(message) { + try { + const newId = crypto.randomBytes(5).toString('hex'); + const owner = crypto.randomBytes(10).toString('hex'); + + const fileInfo = JSON.parse(message); + const metadata = fileInfo.fileMetadata; + const auth = fileInfo.authorization; + + if (!metadata || !auth) { + ws.send( + JSON.stringify({ + error: 400 + }) + ); + return ws.close(); + } + + const meta = { + owner, + metadata, + auth: auth.split(' ')[1], + nonce: crypto.randomBytes(16).toString('base64') + }; + + const protocol = config.env === 'production' ? 'https' : req.protocol; + const url = `${protocol}://${req.get('host')}/download/${newId}/`; + + const limiter = new Limiter(config.max_file_size); + const parser = new Parser(); + fileStream = wsStream(ws, { binary: true }) + .pipe(limiter) + .pipe(parser); + storage.set(newId, fileStream, meta); + + ws.send( + JSON.stringify({ + url, + owner: meta.owner, + id: newId, + authentication: `send-v1 ${meta.nonce}` + }) + ); + } catch (e) { + log.error('upload', e); + ws.send( + JSON.stringify({ + error: e === 'limit' ? 413 : 500 + }) + ); + ws.close(); + } + }); +}; diff --git a/server/streamparser.js b/server/streamparser.js new file mode 100644 index 00000000..d7f7b110 --- /dev/null +++ b/server/streamparser.js @@ -0,0 +1,23 @@ +const { Transform } = require('stream'); + +class StreamParser extends Transform { + constructor() { + super(); + let res; + this.promise = new Promise(resolve => { + res = resolve; + }); + this.res = res; + } + + _transform(chunk, encoding, callback) { + if (chunk.byteLength === 1 && chunk[0] === 0) { + this.res(); + } else { + this.push(chunk); + } + callback(); + } +} + +module.exports = StreamParser; diff --git a/test/frontend/runner.js b/test/frontend/runner.js index d474b2d8..3ae95f2f 100644 --- a/test/frontend/runner.js +++ b/test/frontend/runner.js @@ -7,7 +7,7 @@ const webpack = require('webpack'); const config = require('../../webpack.config'); const middleware = require('webpack-dev-middleware'); const express = require('express'); -const devRoutes = require('../../server/dev'); +const devRoutes = require('../../server/bin/test'); const app = express(); const wpm = middleware(webpack(config), { logLevel: 'silent' }); diff --git a/test/frontend/tests/api-tests.js b/test/frontend/tests/api-tests.js index fb470add..22e221e4 100644 --- a/test/frontend/tests/api-tests.js +++ b/test/frontend/tests/api-tests.js @@ -3,21 +3,22 @@ import * as api from '../../../app/api'; import Keychain from '../../../app/keychain'; const encoder = new TextEncoder(); -const plaintext = encoder.encode('hello world!'); +const plaintext = new Blob([encoder.encode('hello world!')]); const metadata = { name: 'test.txt', type: 'text/plain' }; describe('API', function() { - describe('uploadFile', function() { + describe('websocket upload', function() { it('returns file info on success', async function() { const keychain = new Keychain(); - const encrypted = await keychain.encryptFile(plaintext); + const enc = keychain.encryptStream(plaintext); const meta = await keychain.encryptMetadata(metadata); const verifierB64 = await keychain.authKeyB64(); const p = function() {}; - const up = api.uploadFile(encrypted, meta, verifierB64, keychain, p); + const up = api.uploadWs(enc.stream, enc.streamInfo, meta, verifierB64, p); + const result = await up.result; assert.ok(result.url); assert.ok(result.id); @@ -26,11 +27,11 @@ describe('API', function() { it('can be cancelled', async function() { const keychain = new Keychain(); - const encrypted = await keychain.encryptFile(plaintext); + const enc = keychain.encryptStream(plaintext); const meta = await keychain.encryptMetadata(metadata); const verifierB64 = await keychain.authKeyB64(); const p = function() {}; - const up = api.uploadFile(encrypted, meta, verifierB64, keychain, p); + const up = api.uploadWs(enc.stream, enc.streamInfo, meta, verifierB64, p); up.cancel(); try { await up.result; diff --git a/test/frontend/tests/streaming-tests.js b/test/frontend/tests/streaming-tests.js new file mode 100644 index 00000000..355fcaa0 --- /dev/null +++ b/test/frontend/tests/streaming-tests.js @@ -0,0 +1,64 @@ +const ece = require('http_ece'); +require('buffer'); + +import assert from 'assert'; +import { b64ToArray } from '../../../app/utils'; +import ECE from '../../../app/ece.js'; + +const rs = 36; + +const str = 'You are the dancing queen, young and sweet, only seventeen.'; +const testSalt = 'I1BsxtFttlv3u_Oo94xnmw'; +const keystr = 'yqdlZ-tYemfogSmv7Ws5PQ'; + +const buffer = Buffer.from(str); +const params = { + version: 'aes128gcm', + rs: rs, + salt: testSalt, + keyid: '', + key: keystr +}; + +const encrypted = ece.encrypt(buffer, params); +const decrypted = ece.decrypt(encrypted, params); + +describe('Streaming', function() { + //testing against http_ece's implementation + describe('ECE', function() { + const key = b64ToArray(keystr); + const salt = b64ToArray(testSalt).buffer; + const blob = new Blob([str], { type: 'text/plain' }); + + it('can encrypt', async function() { + const encStream = new ECE(blob, key, 'encrypt', rs, salt).stream; + const reader = encStream.getReader(); + + let result = Buffer.from([]); + + let state = await reader.read(); + while (!state.done) { + result = Buffer.concat([result, state.value]); + state = await reader.read(); + } + + assert.deepEqual(result, encrypted); + }); + + it('can decrypt', async function() { + const encBlob = new Blob([encrypted]); + const decStream = await new ECE(encBlob, key, 'decrypt', rs).stream; + + const reader = decStream.getReader(); + let result = Buffer.from([]); + + let state = await reader.read(); + while (!state.done) { + result = Buffer.concat([result, state.value]); + state = await reader.read(); + } + + assert.deepEqual(result, decrypted); + }); + }); +}); diff --git a/test/frontend/tests/workflow-tests.js b/test/frontend/tests/workflow-tests.js index 23bc8efc..eed956da 100644 --- a/test/frontend/tests/workflow-tests.js +++ b/test/frontend/tests/workflow-tests.js @@ -93,7 +93,7 @@ describe('Upload / Download flow', function() { fs.cancel(); // before encrypting try { await up; - assert.fail('not cancelled'); + assert.fail('not cancelled 1'); } catch (e) { assert.equal(e.message, '0'); } @@ -101,7 +101,7 @@ describe('Upload / Download flow', function() { fs.once('encrypting', () => fs.cancel()); try { await fs.upload(); - assert.fail('not cancelled'); + assert.fail('not cancelled 2'); } catch (e) { assert.equal(e.message, '0'); } @@ -109,7 +109,7 @@ describe('Upload / Download flow', function() { fs.once('progress', () => fs.cancel()); try { await fs.upload(); - assert.fail('not cancelled'); + assert.fail('not cancelled 3'); } catch (e) { assert.equal(e.message, '0'); } diff --git a/webpack.config.js b/webpack.config.js index 65bc21e0..6619f697 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -209,6 +209,13 @@ module.exports = { devServer: { compress: true, host: '0.0.0.0', - before: IS_DEV ? require('./server/dev') : undefined + before: IS_DEV ? require('./server/bin/dev') : undefined, + proxy: { + '/api/ws': { + target: 'ws://localhost:8081', + ws: true, + secure: false + } + } } };