refactored stream transforms

This commit is contained in:
Danny Coates 2018-07-18 16:39:14 -07:00
parent f58b6194ce
commit f12d3abe79
No known key found for this signature in database
GPG Key ID: 4C442633C62E00CB
5 changed files with 65 additions and 80 deletions

View File

@ -1,15 +1,12 @@
import 'buffer'; import 'buffer';
import { import { transform } from './streams';
TStream as TransformStream,
RStream as ReadableStream
} from './streams';
const NONCE_LENGTH = 12; const NONCE_LENGTH = 12;
const TAG_LENGTH = 16; const TAG_LENGTH = 16;
const KEY_LENGTH = 16; const KEY_LENGTH = 16;
const MODE_ENCRYPT = 'encrypt'; const MODE_ENCRYPT = 'encrypt';
const MODE_DECRYPT = 'decrypt'; const MODE_DECRYPT = 'decrypt';
const RS = 1024 * 1024; const RS = 1024 * 64;
const encoder = new TextEncoder(); const encoder = new TextEncoder();
@ -277,6 +274,7 @@ class StreamSlicer {
this.chunkSize = this.rs; this.chunkSize = this.rs;
} }
this.partialChunk = new Uint8Array(this.chunkSize); this.partialChunk = new Uint8Array(this.chunkSize);
this.offset = 0;
} }
//reslice input into record sized chunks //reslice input into record sized chunks
@ -292,26 +290,25 @@ class StreamSlicer {
if (this.offset === this.chunkSize) { if (this.offset === this.chunkSize) {
this.send(this.partialChunk, controller); this.send(this.partialChunk, controller);
this.offset = 0;
} }
} }
while (i < chunk.byteLength) { while (i < chunk.byteLength) {
if (chunk.byteLength - i >= this.chunkSize) { const remainingBytes = chunk.byteLength - i;
if (remainingBytes >= this.chunkSize) {
const record = chunk.slice(i, i + this.chunkSize); const record = chunk.slice(i, i + this.chunkSize);
i += this.chunkSize; i += this.chunkSize;
this.send(record, controller); this.send(record, controller);
} else { } else {
const end = chunk.slice(i, this.chunkSize); const end = chunk.slice(i, i + remainingBytes);
i += end.length; i += end.byteLength;
this.partialChunk.set(end); this.partialChunk.set(end);
this.offset = end.length; this.offset = end.byteLength;
} }
} }
} }
flush(controller) { flush(controller) {
//console.log('slice stream ends')
if (this.offset > 0) { if (this.offset > 0) {
controller.enqueue(this.partialChunk.slice(0, this.offset)); controller.enqueue(this.partialChunk.slice(0, this.offset));
} }
@ -356,15 +353,11 @@ export default class ECE {
new BlobSlicer(this.input, this.rs, this.mode) new BlobSlicer(this.input, this.rs, this.mode)
); );
} else { } else {
const sliceStream = new TransformStream( inputStream = transform(this.input, new StreamSlicer(this.rs, this.mode));
new StreamSlicer(this.rs, this.mode)
);
inputStream = this.input.pipeThrough(sliceStream);
} }
return transform(
const cryptoStream = new TransformStream( inputStream,
new ECETransformer(this.mode, this.key, this.rs, this.salt) new ECETransformer(this.mode, this.key, this.rs, this.salt)
); );
return inputStream.pipeThrough(cryptoStream);
} }
} }

View File

@ -1,6 +1,6 @@
import Keychain from './keychain'; import Keychain from './keychain';
import { downloadStream } from './api'; import { downloadStream } from './api';
import { TStream as TransformStream, wrapReadable } from './streams'; import { transform } from './streams';
let noSave = false; let noSave = false;
const map = new Map(); const map = new Map();
@ -17,29 +17,26 @@ async function decryptStream(request) {
const id = request.url.split('/')[5]; const id = request.url.split('/')[5];
try { try {
const file = map.get(id); const file = map.get(id);
const keychain = new Keychain(file.key);
file.download = downloadStream(id, file.keychain); file.download = downloadStream(id, keychain);
const stream = await file.download.result; const body = await file.download.result;
// eslint-disable-next-line no-undef const readStream = transform(body, {
const progStream = new TransformStream({
transform: (chunk, controller) => { transform: (chunk, controller) => {
file.progress += chunk.length; file.progress += chunk.length;
controller.enqueue(chunk); controller.enqueue(chunk);
} }
}); });
const decrypted = keychain.decryptStream(readStream);
const readStream = wrapReadable(stream).pipeThrough(progStream);
const decrypted = file.keychain.decryptStream(readStream);
const headers = { const headers = {
'Content-Disposition': 'attachment; filename=' + file.filename, 'Content-Disposition': 'attachment; filename=' + file.filename,
'Content-Type': file.type, 'Content-Type': file.type,
'Content-Length': file.size 'Content-Length': file.size
}; };
const body = decrypted.isPony ? decrypted.toNative() : decrypted; return new Response(decrypted, { headers });
return new Response(body, { headers });
} catch (e) { } catch (e) {
if (noSave) { if (noSave) {
return new Response(null, { status: e.message }); return new Response(null, { status: e.message });
@ -47,6 +44,9 @@ async function decryptStream(request) {
const redirectRes = await fetch(`/download/${id}`); const redirectRes = await fetch(`/download/${id}`);
return new Response(redirectRes.body, { status: 302 }); return new Response(redirectRes.body, { status: 302 });
} finally {
// TODO: need to clean up, but not break progress
// map.delete(id)
} }
} }
@ -61,7 +61,7 @@ self.onmessage = event => {
if (event.data.request === 'init') { if (event.data.request === 'init') {
noSave = event.data.noSave; noSave = event.data.noSave;
const info = { const info = {
keychain: new Keychain(event.data.key), key: event.data.key,
filename: event.data.filename, filename: event.data.filename,
type: event.data.type, type: event.data.type,
size: event.data.size, size: event.data.size,
@ -69,24 +69,29 @@ self.onmessage = event => {
cancelled: false cancelled: false
}; };
if (event.data.requiresPassword) { if (event.data.requiresPassword) {
info.keychain.setPassword(event.data.password, event.data.url); info.password = event.data.password;
info.url = event.data.url;
} }
map.set(event.data.id, info); map.set(event.data.id, info);
event.ports[0].postMessage('file info received'); event.ports[0].postMessage('file info received');
} else if (event.data.request === 'progress') { } else if (event.data.request === 'progress') {
const file = map.get(event.data.id); const file = map.get(event.data.id);
if (file.cancelled) { if (!file) {
event.ports[0].postMessage({ progress: 0 });
} else if (file.cancelled) {
event.ports[0].postMessage({ error: 'cancelled' }); event.ports[0].postMessage({ error: 'cancelled' });
} else { } else {
event.ports[0].postMessage({ progress: file.progress }); event.ports[0].postMessage({ progress: file.progress });
} }
} else if (event.data.request === 'cancel') { } else if (event.data.request === 'cancel') {
const file = map.get(event.data.id); const file = map.get(event.data.id);
if (file) {
file.cancelled = true; file.cancelled = true;
if (file.download) { if (file.download) {
file.download.cancel(); file.download.cancel();
} }
}
event.ports[0].postMessage('download cancelled'); event.ports[0].postMessage('download cancelled');
} }
}; };

View File

@ -1,35 +1,36 @@
/* global TransformStream ReadableStream */ /* global ReadableStream */
import { createReadableStreamWrapper } from '@mattiasbuelens/web-streams-adapter';
import {
TransformStream as TransformStreamPony,
ReadableStream as ReadableStreamPony
} from 'web-streams-ponyfill';
const toNativeReadable = createReadableStreamWrapper(ReadableStream); export function transform(readable, transformer) {
const toPonyReadable = createReadableStreamWrapper(ReadableStreamPony); const reader = readable.getReader();
const tstream = new ReadableStream({
export let TStream; start(controller) {
if (typeof TransformStream === 'function') { if (transformer.start) {
TStream = TransformStream; return transformer.start(controller);
} else {
TStream = TransformStreamPony;
TStream.prototype.isPony = true;
}
export let RStream = ReadableStream;
try {
new ReadableStream().pipeThrough(new TransformStream());
} catch (e) {
RStream = ReadableStreamPony;
RStream.prototype.isPony = true;
RStream.prototype.toNative = function() {
return toNativeReadable(this);
};
}
export function wrapReadable(stream) {
if (RStream === ReadableStream) {
return stream;
} }
return toPonyReadable(stream); },
async pull(controller) {
let enqueued = false;
const c = {
enqueue(d) {
enqueued = true;
controller.enqueue(d);
}
};
while (!enqueued) {
const x = await reader.read();
if (x.done) {
if (transformer.flush) {
await transformer.flush(controller);
}
return controller.close();
}
await transformer.transform(x.value, c);
}
},
cancel() {
readable.cancel();
}
});
return tstream;
} }

12
package-lock.json generated
View File

@ -88,12 +88,6 @@
"integrity": "sha1-9vGlzl05caSt6RoR0i1MRZrNN18=", "integrity": "sha1-9vGlzl05caSt6RoR0i1MRZrNN18=",
"dev": true "dev": true
}, },
"@mattiasbuelens/web-streams-adapter": {
"version": "0.1.0-alpha.1",
"resolved": "https://registry.npmjs.org/@mattiasbuelens/web-streams-adapter/-/web-streams-adapter-0.1.0-alpha.1.tgz",
"integrity": "sha512-8YK2ZY6CAgrzFGfW2uPyNDMYvh7OmWjrlbdP+GeHiMJhzPF3XwrQaHyLQ4IZqGTj8NW879ttfbcqbLqQxWvtsw==",
"dev": true
},
"@mrmlnc/readdir-enhanced": { "@mrmlnc/readdir-enhanced": {
"version": "2.2.1", "version": "2.2.1",
"resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz",
@ -16277,12 +16271,6 @@
"minimalistic-assert": "1.0.1" "minimalistic-assert": "1.0.1"
} }
}, },
"web-streams-ponyfill": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/web-streams-ponyfill/-/web-streams-ponyfill-1.4.2.tgz",
"integrity": "sha512-LCHW+fE2UBJ2vjhqJujqmoxh1ytEDEr0dPO3CabMdMDJPKmsaxzS90V1Ar6LtNE5VHLqxR4YMEj1i4lzMAccIA==",
"dev": true
},
"webpack": { "webpack": {
"version": "4.16.0", "version": "4.16.0",
"resolved": "https://registry.npmjs.org/webpack/-/webpack-4.16.0.tgz", "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.16.0.tgz",

View File

@ -57,7 +57,6 @@
}, },
"devDependencies": { "devDependencies": {
"@dannycoates/webpack-dev-server": "^3.1.4", "@dannycoates/webpack-dev-server": "^3.1.4",
"@mattiasbuelens/web-streams-adapter": "0.1.0-alpha.1",
"asmcrypto.js": "^0.22.0", "asmcrypto.js": "^0.22.0",
"babel-core": "^6.26.3", "babel-core": "^6.26.3",
"babel-loader": "^7.1.4", "babel-loader": "^7.1.4",
@ -113,7 +112,6 @@
"svgo-loader": "^2.1.0", "svgo-loader": "^2.1.0",
"testpilot-ga": "^0.3.0", "testpilot-ga": "^0.3.0",
"val-loader": "^1.1.1", "val-loader": "^1.1.1",
"web-streams-ponyfill": "^1.4.2",
"webpack": "^4.15.1", "webpack": "^4.15.1",
"webpack-cli": "^3.0.8", "webpack-cli": "^3.0.8",
"webpack-dev-middleware": "^3.1.3", "webpack-dev-middleware": "^3.1.3",