refactored stream transforms

This commit is contained in:
Danny Coates 2018-07-18 16:39:14 -07:00
parent f58b6194ce
commit f12d3abe79
No known key found for this signature in database
GPG Key ID: 4C442633C62E00CB
5 changed files with 65 additions and 80 deletions

View File

@ -1,15 +1,12 @@
import 'buffer';
import {
TStream as TransformStream,
RStream as ReadableStream
} from './streams';
import { transform } from './streams';
const NONCE_LENGTH = 12;
const TAG_LENGTH = 16;
const KEY_LENGTH = 16;
const MODE_ENCRYPT = 'encrypt';
const MODE_DECRYPT = 'decrypt';
const RS = 1024 * 1024;
const RS = 1024 * 64;
const encoder = new TextEncoder();
@ -277,6 +274,7 @@ class StreamSlicer {
this.chunkSize = this.rs;
}
this.partialChunk = new Uint8Array(this.chunkSize);
this.offset = 0;
}
//reslice input into record sized chunks
@ -292,26 +290,25 @@ class StreamSlicer {
if (this.offset === this.chunkSize) {
this.send(this.partialChunk, controller);
this.offset = 0;
}
}
while (i < chunk.byteLength) {
if (chunk.byteLength - i >= this.chunkSize) {
const remainingBytes = chunk.byteLength - i;
if (remainingBytes >= this.chunkSize) {
const record = chunk.slice(i, i + this.chunkSize);
i += this.chunkSize;
this.send(record, controller);
} else {
const end = chunk.slice(i, this.chunkSize);
i += end.length;
const end = chunk.slice(i, i + remainingBytes);
i += end.byteLength;
this.partialChunk.set(end);
this.offset = end.length;
this.offset = end.byteLength;
}
}
}
flush(controller) {
//console.log('slice stream ends')
if (this.offset > 0) {
controller.enqueue(this.partialChunk.slice(0, this.offset));
}
@ -356,15 +353,11 @@ export default class ECE {
new BlobSlicer(this.input, this.rs, this.mode)
);
} else {
const sliceStream = new TransformStream(
new StreamSlicer(this.rs, this.mode)
);
inputStream = this.input.pipeThrough(sliceStream);
inputStream = transform(this.input, new StreamSlicer(this.rs, this.mode));
}
const cryptoStream = new TransformStream(
return transform(
inputStream,
new ECETransformer(this.mode, this.key, this.rs, this.salt)
);
return inputStream.pipeThrough(cryptoStream);
}
}

View File

@ -1,6 +1,6 @@
import Keychain from './keychain';
import { downloadStream } from './api';
import { TStream as TransformStream, wrapReadable } from './streams';
import { transform } from './streams';
let noSave = false;
const map = new Map();
@ -17,29 +17,26 @@ async function decryptStream(request) {
const id = request.url.split('/')[5];
try {
const file = map.get(id);
const keychain = new Keychain(file.key);
file.download = downloadStream(id, file.keychain);
file.download = downloadStream(id, keychain);
const stream = await file.download.result;
const body = await file.download.result;
// eslint-disable-next-line no-undef
const progStream = new TransformStream({
const readStream = transform(body, {
transform: (chunk, controller) => {
file.progress += chunk.length;
controller.enqueue(chunk);
}
});
const readStream = wrapReadable(stream).pipeThrough(progStream);
const decrypted = file.keychain.decryptStream(readStream);
const decrypted = keychain.decryptStream(readStream);
const headers = {
'Content-Disposition': 'attachment; filename=' + file.filename,
'Content-Type': file.type,
'Content-Length': file.size
};
const body = decrypted.isPony ? decrypted.toNative() : decrypted;
return new Response(body, { headers });
return new Response(decrypted, { headers });
} catch (e) {
if (noSave) {
return new Response(null, { status: e.message });
@ -47,6 +44,9 @@ async function decryptStream(request) {
const redirectRes = await fetch(`/download/${id}`);
return new Response(redirectRes.body, { status: 302 });
} finally {
// TODO: need to clean up, but not break progress
// map.delete(id)
}
}
@ -61,7 +61,7 @@ self.onmessage = event => {
if (event.data.request === 'init') {
noSave = event.data.noSave;
const info = {
keychain: new Keychain(event.data.key),
key: event.data.key,
filename: event.data.filename,
type: event.data.type,
size: event.data.size,
@ -69,24 +69,29 @@ self.onmessage = event => {
cancelled: false
};
if (event.data.requiresPassword) {
info.keychain.setPassword(event.data.password, event.data.url);
info.password = event.data.password;
info.url = event.data.url;
}
map.set(event.data.id, info);
event.ports[0].postMessage('file info received');
} else if (event.data.request === 'progress') {
const file = map.get(event.data.id);
if (file.cancelled) {
if (!file) {
event.ports[0].postMessage({ progress: 0 });
} else if (file.cancelled) {
event.ports[0].postMessage({ error: 'cancelled' });
} else {
event.ports[0].postMessage({ progress: file.progress });
}
} else if (event.data.request === 'cancel') {
const file = map.get(event.data.id);
if (file) {
file.cancelled = true;
if (file.download) {
file.download.cancel();
}
}
event.ports[0].postMessage('download cancelled');
}
};

View File

@ -1,35 +1,36 @@
/* global TransformStream ReadableStream */
import { createReadableStreamWrapper } from '@mattiasbuelens/web-streams-adapter';
import {
TransformStream as TransformStreamPony,
ReadableStream as ReadableStreamPony
} from 'web-streams-ponyfill';
/* global ReadableStream */
const toNativeReadable = createReadableStreamWrapper(ReadableStream);
const toPonyReadable = createReadableStreamWrapper(ReadableStreamPony);
export let TStream;
if (typeof TransformStream === 'function') {
TStream = TransformStream;
} else {
TStream = TransformStreamPony;
TStream.prototype.isPony = true;
}
export let RStream = ReadableStream;
try {
new ReadableStream().pipeThrough(new TransformStream());
} catch (e) {
RStream = ReadableStreamPony;
RStream.prototype.isPony = true;
RStream.prototype.toNative = function() {
return toNativeReadable(this);
};
}
export function wrapReadable(stream) {
if (RStream === ReadableStream) {
return stream;
export function transform(readable, transformer) {
const reader = readable.getReader();
const tstream = new ReadableStream({
start(controller) {
if (transformer.start) {
return transformer.start(controller);
}
return toPonyReadable(stream);
},
async pull(controller) {
let enqueued = false;
const c = {
enqueue(d) {
enqueued = true;
controller.enqueue(d);
}
};
while (!enqueued) {
const x = await reader.read();
if (x.done) {
if (transformer.flush) {
await transformer.flush(controller);
}
return controller.close();
}
await transformer.transform(x.value, c);
}
},
cancel() {
readable.cancel();
}
});
return tstream;
}

12
package-lock.json generated
View File

@ -88,12 +88,6 @@
"integrity": "sha1-9vGlzl05caSt6RoR0i1MRZrNN18=",
"dev": true
},
"@mattiasbuelens/web-streams-adapter": {
"version": "0.1.0-alpha.1",
"resolved": "https://registry.npmjs.org/@mattiasbuelens/web-streams-adapter/-/web-streams-adapter-0.1.0-alpha.1.tgz",
"integrity": "sha512-8YK2ZY6CAgrzFGfW2uPyNDMYvh7OmWjrlbdP+GeHiMJhzPF3XwrQaHyLQ4IZqGTj8NW879ttfbcqbLqQxWvtsw==",
"dev": true
},
"@mrmlnc/readdir-enhanced": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz",
@ -16277,12 +16271,6 @@
"minimalistic-assert": "1.0.1"
}
},
"web-streams-ponyfill": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/web-streams-ponyfill/-/web-streams-ponyfill-1.4.2.tgz",
"integrity": "sha512-LCHW+fE2UBJ2vjhqJujqmoxh1ytEDEr0dPO3CabMdMDJPKmsaxzS90V1Ar6LtNE5VHLqxR4YMEj1i4lzMAccIA==",
"dev": true
},
"webpack": {
"version": "4.16.0",
"resolved": "https://registry.npmjs.org/webpack/-/webpack-4.16.0.tgz",

View File

@ -57,7 +57,6 @@
},
"devDependencies": {
"@dannycoates/webpack-dev-server": "^3.1.4",
"@mattiasbuelens/web-streams-adapter": "0.1.0-alpha.1",
"asmcrypto.js": "^0.22.0",
"babel-core": "^6.26.3",
"babel-loader": "^7.1.4",
@ -113,7 +112,6 @@
"svgo-loader": "^2.1.0",
"testpilot-ga": "^0.3.0",
"val-loader": "^1.1.1",
"web-streams-ponyfill": "^1.4.2",
"webpack": "^4.15.1",
"webpack-cli": "^3.0.8",
"webpack-dev-middleware": "^3.1.3",