fix download workflow tests

This commit is contained in:
Emily 2018-07-11 16:52:46 -07:00
parent 6db3009e5f
commit ff7969a7ef
8 changed files with 327 additions and 99 deletions

View File

@ -1,7 +1,4 @@
import { arrayToB64, b64ToArray, delay } from './utils'; import { arrayToB64, b64ToArray, delay } from './utils';
import { ReadableStream as PolyRS } from 'web-streams-polyfill';
import { createReadableStreamWrapper } from '@mattiasbuelens/web-streams-adapter';
const RS = createReadableStreamWrapper(PolyRS);
function post(obj) { function post(obj) {
return { return {
@ -58,10 +55,12 @@ export async function setParams(id, owner_token, params) {
export async function fileInfo(id, owner_token) { export async function fileInfo(id, owner_token) {
const response = await fetch(`/api/info/${id}`, post({ owner_token })); const response = await fetch(`/api/info/${id}`, post({ owner_token }));
if (response.ok) { if (response.ok) {
const obj = await response.json(); const obj = await response.json();
return obj; return obj;
} }
throw new Error(response.status); throw new Error(response.status);
} }
@ -211,23 +210,17 @@ async function downloadS(id, keychain, signal) {
headers: { Authorization: auth } headers: { Authorization: auth }
}); });
if (response.status !== 200) {
throw new Error(response.status);
}
const authHeader = response.headers.get('WWW-Authenticate'); const authHeader = response.headers.get('WWW-Authenticate');
if (authHeader) { if (authHeader) {
keychain.nonce = parseNonce(authHeader); keychain.nonce = parseNonce(authHeader);
} }
const fileSize = response.headers.get('Content-Length'); if (response.status !== 200) {
throw new Error(response.status);
//right now only chrome allows obtaining a stream from fetch
//for other browsers we fetch as a blob and convert to polyfill stream later
if (response.body) {
return RS(response.body);
} }
return response.blob(); //const fileSize = response.headers.get('Content-Length');
return response.body;
} }
async function tryDownloadStream(id, keychain, signal, tries = 1) { async function tryDownloadStream(id, keychain, signal, tries = 1) {

View File

@ -1,16 +1,4 @@
require('buffer'); require('buffer');
/*
import {
TransformStream as PolyTS,
ReadableStream as PolyRS
} from 'web-streams-polyfill';
import {
createReadableStreamWrapper,
createTransformStreamWrapper
} from '@mattiasbuelens/web-streams-adapter';
const toTS = createTransformStreamWrapper(PolyTS);
const toRS = createReadableStreamWrapper(PolyRS);
*/
const NONCE_LENGTH = 12; const NONCE_LENGTH = 12;
const TAG_LENGTH = 16; const TAG_LENGTH = 16;
@ -362,17 +350,19 @@ export default class ECE {
if (this.input instanceof Blob) { if (this.input instanceof Blob) {
inputStream = new ReadableStream( inputStream = new ReadableStream(
new BlobSlicer(this.input, this.rs, this.mode) new BlobSlicer(this.input, this.rs, this.mode)
); //inputStream = toRS(new ReadableStream(new BlobSlicer(this.input, this.rs, this.mode))); );
} else { } else {
// eslint-disable-next-line no-undef
const sliceStream = new TransformStream( const sliceStream = new TransformStream(
new StreamSlicer(this.rs, this.mode) new StreamSlicer(this.rs, this.mode)
); //const sliceStream = toTS(new TransformStream(new StreamSlicer(this.rs, this.mode))); );
inputStream = this.input.pipeThrough(sliceStream); inputStream = this.input.pipeThrough(sliceStream);
} }
// eslint-disable-next-line no-undef
const cryptoStream = new TransformStream( const cryptoStream = new TransformStream(
new ECETransformer(this.mode, this.key, this.rs, this.salt) new ECETransformer(this.mode, this.key, this.rs, this.salt)
); //const cryptoStream = toTS(new TransformStream(new ECETransformer(this.mode, this.key, this.rs, this.salt))); );
return inputStream.pipeThrough(cryptoStream); //return toRS(inputStream.pipeThrough(cryptoStream)); return inputStream.pipeThrough(cryptoStream);
} }
} }

View File

@ -1,7 +1,7 @@
import Nanobus from 'nanobus'; import Nanobus from 'nanobus';
import Keychain from './keychain'; import Keychain from './keychain';
import { delay, bytes } from './utils'; import { delay, bytes } from './utils';
import { parseNonce, metadata } from './api'; import { metadata } from './api';
export default class FileReceiver extends Nanobus { export default class FileReceiver extends Nanobus {
constructor(fileInfo) { constructor(fileInfo) {
@ -72,12 +72,15 @@ export default class FileReceiver extends Nanobus {
const channel = new MessageChannel(); const channel = new MessageChannel();
channel.port1.onmessage = function(event) { channel.port1.onmessage = function(event) {
if (event.data.error !== undefined) { if (event.data === undefined) {
reject('bad response from serviceWorker');
} else if (event.data.error !== undefined) {
reject(event.data.error); reject(event.data.error);
} else { } else {
resolve(event.data); resolve(event.data);
} }
}; };
navigator.serviceWorker.controller.postMessage(msg, [channel.port2]); navigator.serviceWorker.controller.postMessage(msg, [channel.port2]);
}); });
} }
@ -90,26 +93,34 @@ export default class FileReceiver extends Nanobus {
this.downloadRequest = { this.downloadRequest = {
cancel: () => { cancel: () => {
this.sendMessageToSw('cancel'); this.sendMessageToSw({ request: 'cancel', id: this.fileInfo.id });
//throw new Error(0); throw new Error(0);
} }
}; };
try { try {
this.state = 'downloading'; this.state = 'downloading';
const auth = await this.keychain.authHeader();
const info = { const info = {
key: this.fileInfo.secretKey, request: 'init',
nonce: this.fileInfo.nonce, id: this.fileInfo.id,
filename: this.fileInfo.name, filename: this.fileInfo.name,
auth: auth key: this.fileInfo.secretKey,
requiresPassword: this.fileInfo.requiresPassword,
password: this.fileInfo.password,
url: this.fileInfo.url,
noSave
}; };
await this.sendMessageToSw(info); await this.sendMessageToSw(info);
console.log('SENDING REQUEST FROM PAGE ONCE'); onprogress([0, this.fileInfo.size]);
if (!noSave) { if (noSave) {
const res = await fetch(`/api/download/${this.fileInfo.id}`);
if (res.status !== 200) {
throw new Error(res.status);
}
} else {
const downloadUrl = `${location.protocol}//${ const downloadUrl = `${location.protocol}//${
location.host location.host
}/api/download/${this.fileInfo.id}`; }/api/download/${this.fileInfo.id}`;
@ -119,14 +130,13 @@ export default class FileReceiver extends Nanobus {
a.click(); a.click();
URL.revokeObjectURL(downloadUrl); URL.revokeObjectURL(downloadUrl);
const auth = await this.sendMessageToSw('authHeader');
if (auth) {
this.keychain.nonce = parseNonce(auth);
}
let prog = 0; let prog = 0;
while (prog < this.fileInfo.size) { while (prog < this.fileInfo.size) {
prog = await this.sendMessageToSw('progress'); const msg = await this.sendMessageToSw({
request: 'progress',
id: this.fileInfo.id
});
prog = msg.progress;
onprogress([prog, this.fileInfo.size]); onprogress([prog, this.fileInfo.size]);
await delay(); await delay();
} }
@ -137,9 +147,6 @@ export default class FileReceiver extends Nanobus {
this.state = 'complete'; this.state = 'complete';
} catch (e) { } catch (e) {
this.downloadRequest = null; this.downloadRequest = null;
if (e === 'cancelled') {
throw new Error(0);
}
throw e; throw e;
} }
} }

View File

@ -1,44 +1,50 @@
import Keychain from './keychain'; import Keychain from './keychain';
import { downloadStream } from './api';
let noSave = false;
const map = new Map();
self.addEventListener('install', event => { self.addEventListener('install', event => {
self.skipWaiting(); self.skipWaiting();
}); });
self.addEventListener('activate', event => {
self.clients.claim();
});
async function decryptStream(request) { async function decryptStream(request) {
self.controller = new AbortController(); const id = request.url.split('/')[5];
//console.log('SW INTERCEPTED DOWNLOAD'); try {
const file = map.get(id);
const response = await fetch(request.url, { file.download = downloadStream(id, file.keychain);
method: 'GET',
headers: { Authorization: self.auth },
signal: controller.signal
});
if (response.status !== 200) { const stream = await file.download.result;
return response;
}
self.authHeader = response.headers.get('WWW-Authenticate');
const body = response.body; //stream
// eslint-disable-next-line no-undef
const progStream = new TransformStream({ const progStream = new TransformStream({
transform: (chunk, controller) => { transform: (chunk, controller) => {
self.progress += chunk.length; file.progress += chunk.length;
controller.enqueue(chunk); controller.enqueue(chunk);
} }
}); });
const decrypted = self.keychain.decryptStream(body.pipeThrough(progStream)); const readStream = stream.pipeThrough(progStream);
const decrypted = file.keychain.decryptStream(readStream);
const headers = { const headers = {
headers: { 'Content-Disposition': 'attachment; filename=' + file.filename
'Content-Disposition': 'attachment; filename=' + self.filename
}
}; };
const newRes = new Response(decrypted, headers); return new Response(decrypted, { headers });
return newRes; } catch (e) {
if (noSave) {
return new Response(null, { status: e.message });
}
const redirectRes = await fetch(`/download/${id}`);
return new Response(redirectRes.body, { status: 302 });
}
} }
self.onfetch = event => { self.onfetch = event => {
@ -49,25 +55,32 @@ self.onfetch = event => {
}; };
self.onmessage = event => { self.onmessage = event => {
if (event.data.key) { if (event.data.request === 'init') {
self.keychain = new Keychain(event.data.key, event.data.nonce); noSave = event.data.noSave;
self.filename = event.data.filename; const info = {
self.auth = event.data.auth; keychain: new Keychain(event.data.key),
self.progress = 0; filename: event.data.filename,
self.cancelled = false; progress: 0,
cancelled: false
};
if (event.data.requiresPassword) {
info.keychain.setPassword(event.data.password, event.data.url);
}
map.set(event.data.id, info);
event.ports[0].postMessage('file info received'); event.ports[0].postMessage('file info received');
} else if (event.data === 'progress') { } else if (event.data.request === 'progress') {
if (self.cancelled) { const file = map.get(event.data.id);
if (file.cancelled) {
event.ports[0].postMessage({ error: 'cancelled' }); event.ports[0].postMessage({ error: 'cancelled' });
} else { } else {
event.ports[0].postMessage(self.progress); event.ports[0].postMessage({ progress: file.progress });
} }
} else if (event.data === 'authHeader') { } else if (event.data.request === 'cancel') {
event.ports[0].postMessage(self.authHeader); const file = map.get(event.data.id);
} else if (event.data === 'cancel') { file.cancelled = true;
self.cancelled = true; if (file.download) {
if (self.controller) { file.download.cancel();
self.controller.abort();
} }
event.ports[0].postMessage('download cancelled'); event.ports[0].postMessage('download cancelled');
} }

View File

@ -30,7 +30,7 @@
"test:frontend": "cross-env NODE_ENV=development node test/frontend/runner.js && nyc report --reporter=html", "test:frontend": "cross-env NODE_ENV=development node test/frontend/runner.js && nyc report --reporter=html",
"test-integration": "docker-compose up --abort-on-container-exit --exit-code-from integration-tests --build --remove-orphans --quiet-pull && docker-compose down", "test-integration": "docker-compose up --abort-on-container-exit --exit-code-from integration-tests --build --remove-orphans --quiet-pull && docker-compose down",
"test-integration-stage": "cross-env BASE_URL=https://send.stage.mozaws.net npm run test-integration", "test-integration-stage": "cross-env BASE_URL=https://send.stage.mozaws.net npm run test-integration",
"start": "npm run clean && cross-env NODE_ENV=development webpack-dev-server", "start": "npm run clean && cross-env NODE_ENV=development webpack-dev-server --config webpack.dev.config.js",
"prod": "node server/bin/prod.js" "prod": "node server/bin/prod.js"
}, },
"lint-staged": { "lint-staged": {

View File

@ -1,5 +1,4 @@
const config = require('../config'); const config = require('../config');
const assets = require('../../common/assets');
let sentry = ''; let sentry = '';
if (config.sentry_id) { if (config.sentry_id) {
@ -37,7 +36,6 @@ if (isIE && !isUnsupportedPage) {
} }
var MAXFILESIZE = ${config.max_file_size}; var MAXFILESIZE = ${config.max_file_size};
var EXPIRE_SECONDS = ${config.expire_seconds}; var EXPIRE_SECONDS = ${config.expire_seconds};
var SERVICEWORKER = '${assets.get('serviceWorker.js')}';
${ga} ${ga}
${sentry} ${sentry}
`; `;

View File

@ -8,6 +8,7 @@ const noSave = !headless; // only run the saveFile code if headless
// FileSender uses a File in real life but a Blob works for testing // FileSender uses a File in real life but a Blob works for testing
const blob = new Blob(['hello world!'], { type: 'text/plain' }); const blob = new Blob(['hello world!'], { type: 'text/plain' });
blob.name = 'test.txt'; blob.name = 'test.txt';
navigator.serviceWorker.register('/serviceWorker.js');
describe('Upload / Download flow', function() { describe('Upload / Download flow', function() {
it('can only download once by default', async function() { it('can only download once by default', async function() {
@ -67,7 +68,7 @@ describe('Upload / Download flow', function() {
try { try {
// We can't decrypt without IV from metadata // We can't decrypt without IV from metadata
// but let's try to download anyway // but let's try to download anyway
await fr.download(); await fr.download(noSave);
assert.fail('downloaded file with bad password'); assert.fail('downloaded file with bad password');
} catch (e) { } catch (e) {
assert.equal(e.message, '401'); assert.equal(e.message, '401');

226
webpack.dev.config.js Normal file
View File

@ -0,0 +1,226 @@
const path = require('path');
const webpack = require('webpack');
const CopyPlugin = require('copy-webpack-plugin');
const ManifestPlugin = require('webpack-manifest-plugin');
const ExtractTextPlugin = require('extract-text-webpack-plugin');
const IS_DEV = process.env.NODE_ENV === 'development';
const regularJSOptions = {
babelrc: false,
presets: [['env', { modules: false }], 'stage-2'],
// yo-yoify converts html template strings to direct dom api calls
plugins: ['yo-yoify']
};
const entry = {
// babel-polyfill and fluent are directly included in vendor
// because they are not explicitly referenced by app
vendor: ['babel-polyfill', 'fluent'],
app: ['./app/main.js'],
style: ['./app/main.css'],
serviceWorker: ['./app/serviceWorker.js']
};
if (IS_DEV) {
entry.tests = ['./test/frontend/index.js'];
// istanbul instruments the source for code coverage
regularJSOptions.plugins.push('istanbul');
}
module.exports = {
entry,
output: {
filename: '[name].js',
path: path.resolve(__dirname, 'dist'),
publicPath: '/'
},
devtool: IS_DEV && 'inline-source-map',
module: {
rules: [
{
test: /\.js$/,
oneOf: [
{
include: [require.resolve('./assets/cryptofill')],
use: [
{
loader: 'file-loader',
options: {
name: '[name].[hash:8].[ext]'
}
}
]
},
{
// inlines version from package.json into header/index.js
include: require.resolve('./app/templates/header'),
use: [
{
loader: 'babel-loader',
options: regularJSOptions
},
'./build/version_loader'
]
},
{
// fluent gets exposed as a global so that each language script
// can load independently and share it.
include: [path.dirname(require.resolve('fluent'))],
use: [
{
loader: 'expose-loader',
options: 'fluent'
},
{
loader: 'babel-loader',
options: {
presets: [['env', { modules: false }], 'stage-3']
}
}
]
},
{
include: [
path.resolve(__dirname, 'app'),
path.resolve(__dirname, 'common')
]
},
{
loader: 'babel-loader',
include: [
// some dependencies need to get re-babeled because we
// have different targets than their default configs
path.resolve(__dirname, 'node_modules/testpilot-ga/src'),
path.resolve(__dirname, 'node_modules/fluent-intl-polyfill'),
path.resolve(__dirname, 'node_modules/intl-pluralrules')
],
options: regularJSOptions
},
{
// Strip asserts from our deps, mainly choojs family
include: [path.resolve(__dirname, 'node_modules')],
loader: 'webpack-unassert-loader'
}
]
},
{
test: /\.(png|jpg)$/,
loader: 'file-loader',
options: {
name: '[name].[hash:8].[ext]'
}
},
{
test: /\.svg$/,
use: [
{
loader: 'file-loader',
options: {
name: '[name].[hash:8].[ext]'
}
},
{
loader: 'svgo-loader',
options: {
plugins: [
{ removeViewBox: false }, // true causes stretched images
{ convertStyleToAttrs: true }, // for CSP, no unsafe-eval
{ removeTitle: true } // for smallness
]
}
}
]
},
{
// creates style.css with all styles
test: /\.css$/,
use: ExtractTextPlugin.extract({
use: [
{
loader: 'css-loader',
options: { modules: false, importLoaders: 1 }
},
'postcss-loader'
]
})
},
{
// creates version.json for /__version__ from package.json
test: require.resolve('./package.json'),
use: [
{
loader: 'file-loader',
options: {
name: 'version.json'
}
},
'extract-loader',
'./build/package_json_loader'
]
},
{
// creates a js script for each ftl
test: /\.ftl$/,
use: [
{
loader: 'file-loader',
options: {
name: '[path][name].[hash:8].js'
}
},
'extract-loader',
'./build/fluent_loader'
]
},
{
// creates test.js for /test
test: require.resolve('./test/frontend/index.js'),
use: ['babel-loader', 'val-loader']
},
{
// loads all assets from assets/ for use by common/assets.js
test: require.resolve('./build/generate_asset_map.js'),
use: ['babel-loader', 'val-loader']
},
{
// loads all the ftl from public/locales for use by common/locales.js
test: require.resolve('./build/generate_l10n_map.js'),
use: ['babel-loader', 'val-loader']
}
]
},
plugins: [
new CopyPlugin([
{
context: 'public',
from: '*.*'
}
]),
new webpack.IgnorePlugin(/dist/), // used in common/*.js
new webpack.IgnorePlugin(/require-from-string/), // used in common/locales.js
new webpack.HashedModuleIdsPlugin(),
// new webpack.optimize.CommonsChunkPlugin({
// name: 'vendor',
// minChunks: ({ resource }) => /node_modules/.test(resource)
// }),
// new webpack.optimize.CommonsChunkPlugin({
// name: 'runtime'
// }),
new ExtractTextPlugin({
filename: 'style.[contenthash:8].css'
}),
new ManifestPlugin() // used by server side to resolve hashed assets
],
devServer: {
compress: true,
host: '0.0.0.0',
before: IS_DEV ? require('./server/bin/dev') : undefined,
proxy: {
'/api/ws': {
target: 'ws://localhost:8081',
ws: true,
secure: false
}
}
}
};