natty
/
fox-send
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

saves stream to file

This commit is contained in:
Emily 2018-07-06 15:49:50 -07:00
parent 62ed0a411f
commit f98bc0878c
14 changed files with 136 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
app/api.js
View File

@ -1,5 +1,5 @@
import { arrayToB64, b64ToArray, delay } from './utils';
import { ReadableStream as PolyRS} from 'web-streams-polyfill';
import { ReadableStream as PolyRS } from 'web-streams-polyfill';
import { createReadableStreamWrapper } from '@mattiasbuelens/web-streams-adapter';
const RS = createReadableStreamWrapper(PolyRS);
@ -202,9 +202,10 @@ export function uploadWs(encrypted, info, metadata, verifierB64, onprogress) {
////////////////////////
async function downloadS(id, keychain, onprogress, signal) {
async function downloadS(id, keychain, signal) {
const auth = await keychain.authHeader();
//this will be already funneled through serviceworker
const response = await fetch(`/api/download/${id}`, {
signal: signal,
method: 'GET',
@ -223,22 +224,20 @@ async function downloadS(id, keychain, onprogress, signal) {
const fileSize = response.headers.get('Content-Length');
//right now only chrome allows obtaining a stream from fetch
//for other browsers we fetch as a blob and convert to polyfill stream later
//for other browsers we fetch as a blob and convert to polyfill stream later
if (response.body) {
console.log("STREAM")
return RS(response.body);
}
return response.blob();
}
async function tryDownloadStream(id, keychain, onprogress, signal, tries = 1) {
async function tryDownloadStream(id, keychain, signal, tries = 1) {
try {
const result = await downloadS(id, keychain, onprogress, signal);
const result = await downloadS(id, keychain, signal);
return result;
} catch (e) {
if (e.message === '401' && --tries > 0) {
return tryDownloadStream(id, keychain, onprogress, signal, tries);
return tryDownloadStream(id, keychain, signal, tries);
}
if (e.name === 'AbortError') {
throw new Error('0');
@ -247,14 +246,14 @@ async function tryDownloadStream(id, keychain, onprogress, signal, tries = 1) {
}
}
export function downloadStream(id, keychain, onprogress) {
export function downloadStream(id, keychain) {
const controller = new AbortController();
function cancel() {
controller.abort();
}
return {
cancel,
result: tryDownloadStream(id, keychain, onprogress, controller.signal, 2)
result: tryDownloadStream(id, keychain, controller.signal, 2)
};
}

62
app/ece.js
View File

@ -1,8 +1,16 @@
require('buffer');
import { TransformStream as PolyTS, ReadableStream as PolyRS } from 'web-streams-polyfill';
import { createReadableStreamWrapper, createTransformStreamWrapper } from '@mattiasbuelens/web-streams-adapter';
/*
import {
TransformStream as PolyTS,
ReadableStream as PolyRS
} from 'web-streams-polyfill';
import {
createReadableStreamWrapper,
createTransformStreamWrapper
} from '@mattiasbuelens/web-streams-adapter';
const toTS = createTransformStreamWrapper(PolyTS);
const toRS = createReadableStreamWrapper(PolyRS);
*/
const NONCE_LENGTH = 12;
const TAG_LENGTH = 16;
@ -15,7 +23,7 @@ const encoder = new TextEncoder();
function generateSalt(len) {
const randSalt = new Uint8Array(len);
window.crypto.getRandomValues(randSalt);
crypto.getRandomValues(randSalt);
return randSalt.buffer;
}
@ -31,7 +39,7 @@ class ECETransformer {
}
async generateKey() {
const inputKey = await window.crypto.subtle.importKey(
const inputKey = await crypto.subtle.importKey(
'raw',
this.ikm,
'HKDF',
@ -39,7 +47,7 @@ class ECETransformer {
['deriveKey']
);
return window.crypto.subtle.deriveKey(
return crypto.subtle.deriveKey(
{
name: 'HKDF',
salt: this.salt,
@ -57,7 +65,7 @@ class ECETransformer {
}
async generateNonceBase() {
const inputKey = await window.crypto.subtle.importKey(
const inputKey = await crypto.subtle.importKey(
'raw',
this.ikm,
'HKDF',
@ -65,9 +73,9 @@ class ECETransformer {
['deriveKey']
);
const base = await window.crypto.subtle.exportKey(
const base = await crypto.subtle.exportKey(
'raw',
await window.crypto.subtle.deriveKey(
await crypto.subtle.deriveKey(
{
name: 'HKDF',
salt: this.salt,
@ -156,7 +164,7 @@ class ECETransformer {
async encryptRecord(buffer, seq, isLast) {
const nonce = this.generateNonce(seq);
const encrypted = await window.crypto.subtle.encrypt(
const encrypted = await crypto.subtle.encrypt(
{ name: 'AES-GCM', iv: nonce },
this.key,
this.pad(buffer, isLast)
@ -166,7 +174,7 @@ class ECETransformer {
async decryptRecord(buffer, seq, isLast) {
const nonce = this.generateNonce(seq);
const data = await window.crypto.subtle.decrypt(
const data = await crypto.subtle.decrypt(
{
name: 'AES-GCM',
iv: nonce,
@ -266,7 +274,7 @@ class StreamSlicer {
constructor(rs, mode) {
this.mode = mode;
this.rs = rs;
this.chunkSize = (mode === MODE_ENCRYPT) ? (rs - 17) : 21;
this.chunkSize = mode === MODE_ENCRYPT ? rs - 17 : 21;
this.partialChunk = new Uint8Array(this.chunkSize); //where partial chunks are saved
this.offset = 0;
}
@ -285,7 +293,7 @@ class StreamSlicer {
let i = 0;
if (this.offset > 0) {
const len = Math.min(chunk.byteLength, (this.chunkSize - this.offset));
const len = Math.min(chunk.byteLength, this.chunkSize - this.offset);
this.partialChunk.set(chunk.slice(0, len), this.offset);
this.offset += len;
i += len;
@ -297,7 +305,7 @@ class StreamSlicer {
}
while (i < chunk.byteLength) {
if ((chunk.byteLength - i) >= this.chunkSize) {
if (chunk.byteLength - i >= this.chunkSize) {
const record = chunk.slice(i, i + this.chunkSize);
i += this.chunkSize;
this.send(record, controller);
@ -318,17 +326,6 @@ class StreamSlicer {
}
}
async function stream2blob(stream) {
const chunks = [];
const reader = stream.getReader();
let state = await reader.read();
while (!state.done) {
chunks.push(state.value);
state = await reader.read();
}
return new Blob(chunks);
}
/*
input: a blob or a ReadableStream containing data to be transformed
key: Uint8Array containing key of size KEY_LENGTH
@ -354,7 +351,8 @@ export default class ECE {
info() {
return {
recordSize: this.rs,
fileSize: 21 + this.input.size + 16 * Math.floor(this.input.size / (this.rs - 17))
fileSize:
21 + this.input.size + 16 * Math.floor(this.input.size / (this.rs - 17))
};
}
@ -362,13 +360,19 @@ export default class ECE {
let inputStream;
if (this.input instanceof Blob) {
inputStream = toRS(new ReadableStream(new BlobSlicer(this.input, this.rs, this.mode)));
inputStream = new ReadableStream(
new BlobSlicer(this.input, this.rs, this.mode)
); //inputStream = toRS(new ReadableStream(new BlobSlicer(this.input, this.rs, this.mode)));
} else {
const sliceStream = toTS(new TransformStream(new StreamSlicer(this.rs, this.mode)));
const sliceStream = new TransformStream(
new StreamSlicer(this.rs, this.mode)
); //const sliceStream = toTS(new TransformStream(new StreamSlicer(this.rs, this.mode)));
inputStream = this.input.pipeThrough(sliceStream);
}
const cryptoStream = toTS(new TransformStream(new ECETransformer(this.mode, this.key, this.rs, this.salt)));
return inputStream.pipeThrough(cryptoStream);
const cryptoStream = new TransformStream(
new ECETransformer(this.mode, this.key, this.rs, this.salt)
); //const cryptoStream = toTS(new TransformStream(new ECETransformer(this.mode, this.key, this.rs, this.salt)));
return inputStream.pipeThrough(cryptoStream); //return toRS(inputStream.pipeThrough(cryptoStream));
}
}

13
app/fileManager.js
View File

@ -36,12 +36,6 @@ export default function(state, emitter) {
}
}
function register() {
navigator.serviceWorker.register('/serviceWorker.js')
.then( reg => console.log("registration successful or already installed"))
.catch( e => console.log(e) );
}
function updateProgress() {
if (updateTitle) {
emitter.emit('DOMTitleChange', percent(state.transfer.progressRatio));
@ -156,6 +150,7 @@ export default function(state, emitter) {
emitter.on('getMetadata', async () => {
const file = state.fileInfo;
const receiver = new FileReceiver(file);
try {
await receiver.getMetadata();
@ -169,12 +164,6 @@ export default function(state, emitter) {
}
}
const info = {
key: file.secretKey,
nonce: file.nonce
}
navigator.serviceWorker.controller.postMessage(info);
render();
});

116
app/fileReceiver.js
View File

@ -1,7 +1,7 @@
import Nanobus from 'nanobus';
import Keychain from './keychain';
import { bytes } from './utils';
import { metadata, downloadFile, downloadStream } from './api';
import { metadata } from './api';
export default class FileReceiver extends Nanobus {
constructor(fileInfo) {
@ -52,107 +52,57 @@ export default class FileReceiver extends Nanobus {
}
async streamToArrayBuffer(stream, streamSize, onprogress) {
try {
const result = new Uint8Array(streamSize);
let offset = 0;
const reader = stream.getReader();
let state = await reader.read();
while (!state.done) {
result.set(state.value, offset);
offset += state.value.length;
state = await reader.read();
onprogress([offset, streamSize]);
}
onprogress([streamSize, streamSize]);
return result.slice(0, offset).buffer;
} catch (e) {
console.log(e);
throw (e);
const result = new Uint8Array(streamSize);
let offset = 0;
const reader = stream.getReader();
let state = await reader.read();
while (!state.done) {
result.set(state.value, offset);
offset += state.value.length;
state = await reader.read();
onprogress([offset, streamSize]);
}
onprogress([streamSize, streamSize]);
return result.slice(0, offset).buffer;
}
async download(noSave = false) {
const onprogress = p => {
this.progress = p;
this.emit('progress');
}
};
try {
this.state = 'downloading';
this.downloadRequest = downloadStream(
this.fileInfo.id,
this.keychain
);
const auth = await this.keychain.authHeader();
const info = {
key: this.fileInfo.secretKey,
nonce: this.fileInfo.nonce,
filename: this.fileInfo.name,
auth: auth
};
navigator.serviceWorker.controller.postMessage(info);
onprogress([0, this.fileInfo.size]);
const download = await this.downloadRequest.result;
const plainstream = this.keychain.decryptStream(download);
//temporary
const plaintext = await this.streamToArrayBuffer(
plainstream,
this.fileInfo.size,
onprogress
);
this.downloadRequest = null;
this.msg = 'decryptingFile';
this.state = 'decrypting';
this.emit('decrypting');
if (!noSave) {
await saveFile({
plaintext,
name: decodeURIComponent(this.fileInfo.name),
type: this.fileInfo.type
});
const downloadUrl = `${location.protocol}//${
location.host
}/api/download/${this.fileInfo.id}`;
const a = document.createElement('a');
a.href = downloadUrl;
document.body.appendChild(a);
a.click();
URL.revokeObjectURL(downloadUrl);
}
this.msg = 'downloadFinish';
this.state = 'complete';
//this.msg = 'downloadFinish';
//this.state = 'complete';
} catch (e) {
this.downloadRequest = null;
throw e;
}
}
}
async function saveFile(file) {
return new Promise(function(resolve, reject) {
const dataView = new DataView(file.plaintext);
const blob = new Blob([dataView], { type: file.type });
if (navigator.msSaveBlob) {
navigator.msSaveBlob(blob, file.name);
return resolve();
} else if (/iPhone|fxios/i.test(navigator.userAgent)) {
// This method is much slower but createObjectURL
// is buggy on iOS
const reader = new FileReader();
reader.addEventListener('loadend', function() {
if (reader.error) {
return reject(reader.error);
}
if (reader.result) {
const a = document.createElement('a');
a.href = reader.result;
a.download = file.name;
document.body.appendChild(a);
a.click();
}
resolve();
});
reader.readAsDataURL(blob);
} else {
const downloadUrl = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = downloadUrl;
a.download = file.name;
document.body.appendChild(a);
a.click();
URL.revokeObjectURL(downloadUrl);
setTimeout(resolve, 100);
}
});
}

30
app/keychain.js
View File

@ -9,14 +9,14 @@ export default class Keychain {
if (ivB64) {
this.iv = b64ToArray(ivB64);
} else {
this.iv = window.crypto.getRandomValues(new Uint8Array(12));
this.iv = crypto.getRandomValues(new Uint8Array(12));
}
if (secretKeyB64) {
this.rawSecret = b64ToArray(secretKeyB64);
} else {
this.rawSecret = window.crypto.getRandomValues(new Uint8Array(16));
this.rawSecret = crypto.getRandomValues(new Uint8Array(16));
}
this.secretKeyPromise = window.crypto.subtle.importKey(
this.secretKeyPromise = crypto.subtle.importKey(
'raw',
this.rawSecret,
'HKDF',
@ -24,7 +24,7 @@ export default class Keychain {
['deriveKey']
);
this.encryptKeyPromise = this.secretKeyPromise.then(function(secretKey) {
return window.crypto.subtle.deriveKey(
return crypto.subtle.deriveKey(
{
name: 'HKDF',
salt: new Uint8Array(),
@ -41,7 +41,7 @@ export default class Keychain {
);
});
this.metaKeyPromise = this.secretKeyPromise.then(function(secretKey) {
return window.crypto.subtle.deriveKey(
return crypto.subtle.deriveKey(
{
name: 'HKDF',
salt: new Uint8Array(),
@ -58,7 +58,7 @@ export default class Keychain {
);
});
this.authKeyPromise = this.secretKeyPromise.then(function(secretKey) {
return window.crypto.subtle.deriveKey(
return crypto.subtle.deriveKey(
{
name: 'HKDF',
salt: new Uint8Array(),
@ -91,12 +91,12 @@ export default class Keychain {
}
setPassword(password, shareUrl) {
this.authKeyPromise = window.crypto.subtle
this.authKeyPromise = crypto.subtle
.importKey('raw', encoder.encode(password), { name: 'PBKDF2' }, false, [
'deriveKey'
])
.then(passwordKey =>
window.crypto.subtle.deriveKey(
crypto.subtle.deriveKey(
{
name: 'PBKDF2',
salt: encoder.encode(shareUrl),
@ -115,7 +115,7 @@ export default class Keychain {
}
setAuthKey(authKeyB64) {
this.authKeyPromise = window.crypto.subtle.importKey(
this.authKeyPromise = crypto.subtle.importKey(
'raw',
b64ToArray(authKeyB64),
{
@ -129,13 +129,13 @@ export default class Keychain {
async authKeyB64() {
const authKey = await this.authKeyPromise;
const rawAuth = await window.crypto.subtle.exportKey('raw', authKey);
const rawAuth = await crypto.subtle.exportKey('raw', authKey);
return arrayToB64(new Uint8Array(rawAuth));
}
async authHeader() {
const authKey = await this.authKeyPromise;
const sig = await window.crypto.subtle.sign(
const sig = await crypto.subtle.sign(
{
name: 'HMAC'
},
@ -147,7 +147,7 @@ export default class Keychain {
async encryptFile(plaintext) {
const encryptKey = await this.encryptKeyPromise;
const ciphertext = await window.crypto.subtle.encrypt(
const ciphertext = await crypto.subtle.encrypt(
{
name: 'AES-GCM',
iv: this.iv,
@ -161,7 +161,7 @@ export default class Keychain {
async encryptMetadata(metadata) {
const metaKey = await this.metaKeyPromise;
const ciphertext = await window.crypto.subtle.encrypt(
const ciphertext = await crypto.subtle.encrypt(
{
name: 'AES-GCM',
iv: new Uint8Array(12),
@ -194,7 +194,7 @@ export default class Keychain {
async decryptFile(ciphertext) {
const encryptKey = await this.encryptKeyPromise;
const plaintext = await window.crypto.subtle.decrypt(
const plaintext = await crypto.subtle.decrypt(
{
name: 'AES-GCM',
iv: this.iv,
@ -208,7 +208,7 @@ export default class Keychain {
async decryptMetadata(ciphertext) {
const metaKey = await this.metaKeyPromise;
const plaintext = await window.crypto.subtle.decrypt(
const plaintext = await crypto.subtle.decrypt(
{
name: 'AES-GCM',
iv: new Uint8Array(12),

11
app/main.js
View File

@ -9,18 +9,11 @@ import storage from './storage';
import metrics from './metrics';
import experiments from './experiments';
import Raven from 'raven-js';
import assets from '../common/assets';
if (navigator.doNotTrack !== '1' && window.RAVEN_CONFIG) {
Raven.config(window.SENTRY_ID, window.RAVEN_CONFIG).install();
}
function register(state, emitter) {
navigator.serviceWorker.register('serviceWorker.js')
.then( reg => console.log("registration successful or already installed"))
.catch( e => console.log(e) );
}
app.use((state, emitter) => {
state.transfer = null;
state.fileInfo = null;
@ -51,7 +44,9 @@ app.use((state, emitter) => {
});
});
app.use(register);
app.use(() => {
navigator.serviceWorker.register('/serviceWorker.js');
});
app.use(metrics);
app.use(fileManager);
app.use(dragManager);

39
app/serviceWorker.js
View File

@ -1,38 +1,41 @@
import Keychain from './keychain';
self.addEventListener('install', (event) => {
console.log("install event on sw")
self.addEventListener('install', event => {
self.skipWaiting();
});
async function decryptStream(request) {
console.log("DOWNLOAD FETCH")
//make actual request to server, get response back, decrypt it, send it
const response = await fetch(req,
{
method: 'GET',
headers: { Authorization: auth }
}
);
const response = await fetch(request.url, {
method: 'GET',
headers: { Authorization: self.auth }
});
if (response.status !== 200) {
console.log(response.status)
throw new Error(response.status);
return response;
}
const body = response.body;
console.log(body);
const body = response.body; //stream
const decrypted = self.keychain.decryptStream(body);
return response;
const headers = {
headers: {
'Content-Disposition': 'attachment; filename=' + self.filename
}
};
const newRes = new Response(decrypted, headers);
return newRes;
}
self.onfetch = (event) => {
self.onfetch = event => {
const req = event.request.clone();
if (req.url.includes('/api/download')) {
event.respondWith(decryptStream(req));
}
};
self.onmessage = (event) => {
self.onmessage = event => {
self.keychain = new Keychain(event.data.key, event.data.nonce);
};
self.filename = event.data.filename;
self.auth = event.data.auth;
};

16
app/utils.js
View File

@ -24,7 +24,7 @@ function loadShim(polyfill) {
async function canHasSend() {
try {
const key = await window.crypto.subtle.generateKey(
const key = await crypto.subtle.generateKey(
{
name: 'AES-GCM',
length: 128
@ -32,25 +32,25 @@ async function canHasSend() {
true,
['encrypt', 'decrypt']
);
await window.crypto.subtle.encrypt(
await crypto.subtle.encrypt(
{
name: 'AES-GCM',
iv: window.crypto.getRandomValues(new Uint8Array(12)),
iv: crypto.getRandomValues(new Uint8Array(12)),
tagLength: 128
},
key,
new ArrayBuffer(8)
);
await window.crypto.subtle.importKey(
await crypto.subtle.importKey(
'raw',
window.crypto.getRandomValues(new Uint8Array(16)),
crypto.getRandomValues(new Uint8Array(16)),
'PBKDF2',
false,
['deriveKey']
);
await window.crypto.subtle.importKey(
await crypto.subtle.importKey(
'raw',
window.crypto.getRandomValues(new Uint8Array(16)),
crypto.getRandomValues(new Uint8Array(16)),
'HKDF',
false,
['deriveKey']
@ -75,7 +75,7 @@ function copyToClipboard(str) {
if (navigator.userAgent.match(/iphone|ipad|ipod/i)) {
const range = document.createRange();
range.selectNodeContents(aux);
const sel = window.getSelection();
const sel = getSelection();
sel.removeAllRanges();
sel.addRange(range);
aux.setSelectionRange(0, str.length);

5
package-lock.json generated
View File

@ -17701,6 +17701,11 @@
"any-observable": "0.2.0"
}
},
"streamsaver": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/streamsaver/-/streamsaver-1.0.1.tgz",
"integrity": "sha1-R11ASXO15pJqVX8OTNhVijUg4Hw="
},
"strftime": {
"version": "0.10.0",
"resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.0.tgz",

1
package.json
View File

@ -122,7 +122,6 @@
"aws-sdk": "^2.206.0",
"babel-plugin-transform-runtime": "^6.23.0",
"babel-polyfill": "^6.26.0",
"babel-runtime": "^6.26.0",
"choo": "^6.10.0",
"cldr-core": "^32.0.0",
"convict": "^4.0.1",

1
server/bin/prod.js
View File

@ -11,6 +11,7 @@ if (config.sentry_dsn) {
}
const app = express();
expressWs(app, null, { perMessageDeflate: false });
app.ws('/api/ws', require('../routes/ws'));
routes(app);

2
test/frontend/tests/api-tests.js
View File

@ -22,7 +22,7 @@ describe('API', function() {
const result = await up.result;
assert.ok(result.url);
assert.ok(result.id);
assert.ok(result.ownerToken);
assert.ok(result.ownerToken);
});
it('can be cancelled', async function() {

2
test/frontend/tests/streaming-tests.js
View File

@ -49,7 +49,7 @@ describe('Streaming', function() {
it('can decrypt', async function() {
const encBlob = new Blob([encrypted]);
const ece = new ECE(encBlob, key, 'decrypt', rs);
const decStream = await ece.transform()
const decStream = await ece.transform();
const reader = decStream.getReader();
let result = Buffer.from([]);

20
webpackSw.config.js
View File

@ -1,16 +1,9 @@
const path = require('path');
const webpack = require('webpack');
const regularJSOptions = {
babelrc: false,
presets: [['env', { modules: false }], 'stage-2'],
// yo-yoify converts html template strings to direct dom api calls
plugins: [
"transform-runtime", {
//"polyfill": false,
//"regenerator": true
}
]
presets: [['env'], 'stage-2'],
plugins: ['transform-runtime']
};
const entry = {
@ -24,17 +17,14 @@ module.exports = {
path: path.resolve(__dirname, 'dist'),
publicPath: '/'
},
module: {
rules: [
{
loader: 'babel-loader',
// exclude: /node_modules/,
include: [
path.resolve(__dirname, 'app'),
path.resolve(__dirname, 'node_modules/buffer')
],
exclude: /node_modules/,
options: regularJSOptions
}
]
}
};
};