natty
/
fox-send
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1378 from mozilla/new-ws 

Use resumable uploads to GCS
This commit is contained in:
Danny Coates 2019-08-08 08:54:00 -07:00 committed by GitHub
parent 259a5a5f24 27be72e0cd
commit 0729064753
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 173 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
app/api.js
View File

@ -147,7 +147,16 @@ function asyncInitWebSocket(server) {
function listenForResponse(ws, canceller) { function listenForResponse(ws, canceller) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
function handleClose(event) {
// a 'close' event before a 'message' event means the request failed
ws.removeEventListener('message', handleMessage);
const error = canceller.cancelled
? canceller.error
: new Error('connection closed');
reject(error);
}
function handleMessage(msg) { function handleMessage(msg) {
ws.removeEventListener('close', handleClose);
try { try {
const response = JSON.parse(msg.data); const response = JSON.parse(msg.data);
if (response.error) { if (response.error) {
@ -156,13 +165,11 @@ function listenForResponse(ws, canceller) {
resolve(response); resolve(response);
} }
} catch (e) { } catch (e) {
ws.close();
canceller.cancelled = true;
canceller.error = e;
reject(e); reject(e);
} }
} }
ws.addEventListener('message', handleMessage, { once: true }); ws.addEventListener('message', handleMessage, { once: true });
ws.addEventListener('close', handleClose, { once: true });
}); });
} }
@ -205,29 +212,35 @@ async function upload(
let state = await reader.read(); let state = await reader.read();
let size = 0; let size = 0;
while (!state.done) { while (!state.done) {
const buf = state.value;
if (canceller.cancelled) { if (canceller.cancelled) {
throw canceller.error; ws.close();
} }
if (ws.readyState !== WebSocket.OPEN) {
break;
}
const buf = state.value;
ws.send(buf); ws.send(buf);
onprogress(size); onprogress(size);
size += buf.length; size += buf.length;
state = await reader.read(); state = await reader.read();
while (ws.bufferedAmount > ECE_RECORD_SIZE * 2) { while (
ws.bufferedAmount > ECE_RECORD_SIZE * 2 &&
ws.readyState === WebSocket.OPEN &&
!canceller.cancelled
) {
await delay(); await delay();
} }
} }
const footer = new Uint8Array([0]); if (ws.readyState === WebSocket.OPEN) {
ws.send(footer); ws.send(new Uint8Array([0])); //EOF
}
await completedResponse; await completedResponse;
ws.close();
return uploadInfo; return uploadInfo;
} catch (e) { } finally {
ws.close(4000); if (![WebSocket.CLOSED, WebSocket.CLOSING].includes(ws.readyState)) {
throw e; ws.close();
}
} }
} }

320
package-lock.json generated
View File

@ -1243,6 +1243,25 @@
"minimalistic-crypto-utils": "^1.0.0" "minimalistic-crypto-utils": "^1.0.0"
} }
}, },
"@dannycoates/express-ws": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/@dannycoates/express-ws/-/express-ws-5.0.3.tgz",
"integrity": "sha512-EvHGP+NU2GVeQQDzlJRclN86UxxGJP5er4v8ojVn55cu7zhLLXZ5/QQJu0RcIzh4WHMjgYEE7/bKCohVK946og==",
"requires": {
"esm": "^3.0.84",
"ws": "^7.1.1"
},
"dependencies": {
"ws": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/ws/-/ws-7.1.1.tgz",
"integrity": "sha512-o41D/WmDeca0BqYhsr3nJzQyg9NF5X8l/UdnFNux9cS3lwB+swm8qGWX5rn+aD6xfBU3rGmtHij7g7x6LxFU3A==",
"requires": {
"async-limiter": "^1.0.0"
}
}
}
},
"@dannycoates/webcrypto-liner": { "@dannycoates/webcrypto-liner": {
"version": "0.1.37", "version": "0.1.37",
"resolved": "https://registry.npmjs.org/@dannycoates/webcrypto-liner/-/webcrypto-liner-0.1.37.tgz", "resolved": "https://registry.npmjs.org/@dannycoates/webcrypto-liner/-/webcrypto-liner-0.1.37.tgz",
@ -1509,9 +1528,9 @@
} }
}, },
"@sinonjs/samsam": { "@sinonjs/samsam": {
"version": "3.3.1", "version": "3.3.2",
"resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-3.3.1.tgz", "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-3.3.2.tgz",
"integrity": "sha512-wRSfmyd81swH0hA1bxJZJ57xr22kC07a1N4zuIL47yTS04bDk6AoCkczcqHEjcRPmJ+FruGJ9WBQiJwMtIElFw==", "integrity": "sha512-ILO/rR8LfAb60Y1Yfp9vxfYAASK43NFC2mLzpvLUbCQY/Qu8YwReboseu8aheCEkyElZF2L2T9mHcR2bgdvZyA==",
"dev": true, "dev": true,
"requires": { "requires": {
"@sinonjs/commons": "^1.0.2", "@sinonjs/commons": "^1.0.2",
@ -2340,9 +2359,9 @@
} }
}, },
"aws-sdk": { "aws-sdk": {
"version": "2.503.0", "version": "2.505.0",
"resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.503.0.tgz", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.505.0.tgz",
"integrity": "sha512-DPwRxhPYCGNvKL9rUhpAoOVpMRyISbVMlXykLQknYo7wyOI+jYcyA7t8H7IFPAqj4ZG+R+Au9tO/KT4im+2gbg==", "integrity": "sha512-VSPSUl1xt0K+YAjNGbrNOOJ/q6X9GBIeg+vzxBBy4oX9SkZitKVuvIB6uc4fio9XQkYKvCdl8V/vMsQWyIvVgw==",
"requires": { "requires": {
"buffer": "4.9.1", "buffer": "4.9.1",
"events": "1.1.1", "events": "1.1.1",
@ -2484,9 +2503,9 @@
} }
}, },
"base64-js": { "base64-js": {
"version": "1.3.0", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
"integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==" "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
}, },
"basic-auth": { "basic-auth": {
"version": "2.0.1", "version": "2.0.1",
@ -4094,9 +4113,9 @@
} }
}, },
"css-loader": { "css-loader": {
"version": "3.1.0", "version": "3.2.0",
"resolved": "https://registry.npmjs.org/css-loader/-/css-loader-3.1.0.tgz", "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-3.2.0.tgz",
"integrity": "sha512-MuL8WsF/KSrHCBCYaozBKlx+r7vIfUaDTEreo7wR7Vv3J6N0z6fqWjRk3e/6wjneitXN1r/Y9FTK1psYNOBdJQ==", "integrity": "sha512-QTF3Ud5H7DaZotgdcJjGMvyDj5F3Pn1j/sC6VBEOVp94cbwqyIBdcs/quzj4MC1BKQSrTpQznegH/5giYbhnCQ==",
"dev": true, "dev": true,
"requires": { "requires": {
"camelcase": "^5.3.1", "camelcase": "^5.3.1",
@ -4174,15 +4193,15 @@
} }
}, },
"postcss-value-parser": { "postcss-value-parser": {
"version": "4.0.0", "version": "4.0.2",
"resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.0.0.tgz", "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.0.2.tgz",
"integrity": "sha512-ESPktioptiSUchCKgggAkzdmkgzKfmp0EU8jXH+5kbIUB+unr0Y4CY9SRMvibuvYUBjNh1ACLbxqYNpdTQOteQ==", "integrity": "sha512-LmeoohTpp/K4UiyQCwuGWlONxXamGzCMtFxLq4W1nZVGIQLYvMCJx3yAF9qyyuFpflABI9yVdtJAqbihOsCsJQ==",
"dev": true "dev": true
}, },
"schema-utils": { "schema-utils": {
"version": "2.0.1", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.0.1.tgz", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.1.0.tgz",
"integrity": "sha512-HJFKJ4JixDpRur06QHwi8uu2kZbng318ahWEKgBjc0ZklcE4FDvmm2wghb448q0IRaABxIESt8vqPFvwgMB80A==", "integrity": "sha512-g6SViEZAfGNrToD82ZPUjq52KUPDYc+fN5+g6Euo5mLokl/9Yx14z0Cu4RR1m55HtBXejO0sBt+qw79axN+Fiw==",
"dev": true, "dev": true,
"requires": { "requires": {
"ajv": "^6.1.0", "ajv": "^6.1.0",
@ -5986,14 +6005,6 @@
} }
} }
}, },
"express-ws": {
"version": "github:dannycoates/express-ws#d0910a43b1802b22476362113557e20b18e185ba",
"from": "github:dannycoates/express-ws",
"requires": {
"esm": "^3.0.84",
"ws": "github:dannycoates/ws"
}
},
"extend": { "extend": {
"version": "3.0.2", "version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@ -6278,9 +6289,9 @@
} }
}, },
"file-loader": { "file-loader": {
"version": "4.1.0", "version": "4.2.0",
"resolved": "https://registry.npmjs.org/file-loader/-/file-loader-4.1.0.tgz", "resolved": "https://registry.npmjs.org/file-loader/-/file-loader-4.2.0.tgz",
"integrity": "sha512-ajDk1nlByoalZAGR4b0H6oD+EGlWnyW1qbSxzaUc7RFiqmn+RbXQQRbTc72jsiUIlVusJ4Et58ltds8ZwTfnAw==", "integrity": "sha512-+xZnaK5R8kBJrHK0/6HRlrKNamvVS5rjyuju+rnyxRGuwUJwpAMsVzUl5dz6rK8brkzjV6JpcFNjp6NqV0g1OQ==",
"dev": true, "dev": true,
"requires": { "requires": {
"loader-utils": "^1.2.3", "loader-utils": "^1.2.3",
@ -6288,9 +6299,9 @@
}, },
"dependencies": { "dependencies": {
"schema-utils": { "schema-utils": {
"version": "2.0.1", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.0.1.tgz", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.1.0.tgz",
"integrity": "sha512-HJFKJ4JixDpRur06QHwi8uu2kZbng318ahWEKgBjc0ZklcE4FDvmm2wghb448q0IRaABxIESt8vqPFvwgMB80A==", "integrity": "sha512-g6SViEZAfGNrToD82ZPUjq52KUPDYc+fN5+g6Euo5mLokl/9Yx14z0Cu4RR1m55HtBXejO0sBt+qw79axN+Fiw==",
"dev": true, "dev": true,
"requires": { "requires": {
"ajv": "^6.1.0", "ajv": "^6.1.0",
@ -6635,9 +6646,9 @@
}, },
"dependencies": { "dependencies": {
"graceful-fs": { "graceful-fs": {
"version": "4.2.0", "version": "4.2.1",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.1.tgz",
"integrity": "sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg==", "integrity": "sha512-b9usnbDGnD928gJB3LrCmxoibr3VE4U2SMo5PBuBnokWyDADTqDPXg4YpwKF1trpH+UbGp7QLicO3+aWEy0+mw==",
"dev": true "dev": true
} }
} }
@ -9649,9 +9660,10 @@
} }
}, },
"lodash": { "lodash": {
"version": "4.17.11", "version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
"dev": true
}, },
"lodash._reinterpolate": { "lodash._reinterpolate": {
"version": "3.0.0", "version": "3.0.0",
@ -9792,9 +9804,9 @@
"dev": true "dev": true
}, },
"lolex": { "lolex": {
"version": "4.0.1", "version": "4.2.0",
"resolved": "https://registry.npmjs.org/lolex/-/lolex-4.0.1.tgz", "resolved": "https://registry.npmjs.org/lolex/-/lolex-4.2.0.tgz",
"integrity": "sha512-UHuOBZ5jjsKuzbB/gRNNW8Vg8f00Emgskdq2kvZxgBJCS0aqquAuXai/SkWORlKeZEiNQWZjFZOqIUcH9LqKCw==", "integrity": "sha512-gKO5uExCXvSm6zbF562EvM+rd1kQDnB9AZBbiQVzf1ZmdDpxUSvpnAaVOP83N/31mRK8Ml8/VE8DMvsAZQ+7wg==",
"dev": true "dev": true
}, },
"longest-streak": { "longest-streak": {
@ -10274,9 +10286,9 @@
} }
}, },
"mixin-deep": { "mixin-deep": {
"version": "1.3.1", "version": "1.3.2",
"resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz", "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz",
"integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==", "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==",
"dev": true, "dev": true,
"requires": { "requires": {
"for-in": "^1.0.2", "for-in": "^1.0.2",
@ -10667,15 +10679,15 @@
"integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ=="
}, },
"nise": { "nise": {
"version": "1.4.10", "version": "1.5.1",
"resolved": "https://registry.npmjs.org/nise/-/nise-1.4.10.tgz", "resolved": "https://registry.npmjs.org/nise/-/nise-1.5.1.tgz",
"integrity": "sha512-sa0RRbj53dovjc7wombHmVli9ZihXbXCQ2uH3TNm03DyvOSIQbxg+pbqDKrk2oxMK1rtLGVlKxcB9rrc6X5YjA==", "integrity": "sha512-edFWm0fsFG2n318rfEnKlTZTkjlbVOFF9XIA+fj+Ed+Qz1laYW2lobwavWoMzGrYDHH1EpiNJgDfvGnkZztR/g==",
"dev": true, "dev": true,
"requires": { "requires": {
"@sinonjs/formatio": "^3.1.0", "@sinonjs/formatio": "^3.2.1",
"@sinonjs/text-encoding": "^0.7.1", "@sinonjs/text-encoding": "^0.7.1",
"just-extend": "^4.0.2", "just-extend": "^4.0.2",
"lolex": "^2.3.2", "lolex": "^4.1.0",
"path-to-regexp": "^1.7.0" "path-to-regexp": "^1.7.0"
}, },
"dependencies": { "dependencies": {
@ -10685,12 +10697,6 @@
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=",
"dev": true "dev": true
}, },
"lolex": {
"version": "2.7.5",
"resolved": "https://registry.npmjs.org/lolex/-/lolex-2.7.5.tgz",
"integrity": "sha512-l9x0+1offnKKIzYVjyXU2SiwhXDLekRzKyhnbyldPHvC7BvLPVpdNUNR2KeMAiCN2D/kLNttZgQD5WjSxuBx3Q==",
"dev": true
},
"path-to-regexp": { "path-to-regexp": {
"version": "1.7.0", "version": "1.7.0",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz",
@ -12567,9 +12573,9 @@
} }
}, },
"postcss-value-parser": { "postcss-value-parser": {
"version": "4.0.0", "version": "4.0.2",
"resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.0.0.tgz", "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.0.2.tgz",
"integrity": "sha512-ESPktioptiSUchCKgggAkzdmkgzKfmp0EU8jXH+5kbIUB+unr0Y4CY9SRMvibuvYUBjNh1ACLbxqYNpdTQOteQ==", "integrity": "sha512-LmeoohTpp/K4UiyQCwuGWlONxXamGzCMtFxLq4W1nZVGIQLYvMCJx3yAF9qyyuFpflABI9yVdtJAqbihOsCsJQ==",
"dev": true "dev": true
}, },
"source-map": { "source-map": {
@ -13649,6 +13655,16 @@
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz",
"integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=" "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs="
}, },
"reduce-css-calc": {
"version": "2.1.6",
"resolved": "https://registry.npmjs.org/reduce-css-calc/-/reduce-css-calc-2.1.6.tgz",
"integrity": "sha512-+l5/qlQmdsbM9h6JerJ/y5vR5Ci0k93aszLNpCmbadC3nBcbRGmIBm0s9Nj59i22LvCjTGftWzdQRwdknayxhw==",
"dev": true,
"requires": {
"css-unit-converter": "^1.1.1",
"postcss-value-parser": "^3.3.0"
}
},
"referrer-policy": { "referrer-policy": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.2.0.tgz", "resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.2.0.tgz",
@ -14265,6 +14281,11 @@
"pend": "~1.2.0" "pend": "~1.2.0"
} }
}, },
"lodash": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
},
"ms": { "ms": {
"version": "2.1.1", "version": "2.1.1",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
@ -14446,9 +14467,9 @@
"dev": true "dev": true
}, },
"set-value": { "set-value": {
"version": "2.0.0", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz",
"integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==", "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==",
"dev": true, "dev": true,
"requires": { "requires": {
"extend-shallow": "^2.0.1", "extend-shallow": "^2.0.1",
@ -14548,17 +14569,17 @@
} }
}, },
"sinon": { "sinon": {
"version": "7.3.2", "version": "7.4.1",
"resolved": "https://registry.npmjs.org/sinon/-/sinon-7.3.2.tgz", "resolved": "https://registry.npmjs.org/sinon/-/sinon-7.4.1.tgz",
"integrity": "sha512-thErC1z64BeyGiPvF8aoSg0LEnptSaWE7YhdWWbWXgelOyThent7uKOnnEh9zBxDbKixtr5dEko+ws1sZMuFMA==", "integrity": "sha512-7s9buHGHN/jqoy/v4bJgmt0m1XEkCEd/tqdHXumpBp0JSujaT4Ng84JU5wDdK4E85ZMq78NuDe0I3NAqXY8TFg==",
"dev": true, "dev": true,
"requires": { "requires": {
"@sinonjs/commons": "^1.4.0", "@sinonjs/commons": "^1.4.0",
"@sinonjs/formatio": "^3.2.1", "@sinonjs/formatio": "^3.2.1",
"@sinonjs/samsam": "^3.3.1", "@sinonjs/samsam": "^3.3.2",
"diff": "^3.5.0", "diff": "^3.5.0",
"lolex": "^4.0.1", "lolex": "^4.2.0",
"nise": "^1.4.10", "nise": "^1.5.1",
"supports-color": "^5.5.0" "supports-color": "^5.5.0"
}, },
"dependencies": { "dependencies": {
@ -15745,9 +15766,9 @@
} }
}, },
"tailwindcss": { "tailwindcss": {
"version": "1.0.6", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-1.0.6.tgz", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-1.1.0.tgz",
"integrity": "sha512-hhdvXpnlYNnxfZCpldMypSWRzgmoQbKBy3namGlsP0Gs+qM8EwF3DBVUPoq8wJYbBFjzrgatE4czWJ6f12Y9+Q==", "integrity": "sha512-hcxY5MUrY/LcdsX4Xr2aOF24eF4HaxDbHCbDu3W2+bCJRsXRhJmZfAgjipjsyhZult0YAa9HFp14xRXMqwUYTw==",
"dev": true, "dev": true,
"requires": { "requires": {
"autoprefixer": "^9.4.5", "autoprefixer": "^9.4.5",
@ -15762,7 +15783,8 @@
"postcss-js": "^2.0.0", "postcss-js": "^2.0.0",
"postcss-nested": "^4.1.1", "postcss-nested": "^4.1.1",
"postcss-selector-parser": "^6.0.0", "postcss-selector-parser": "^6.0.0",
"pretty-hrtime": "^1.0.3" "pretty-hrtime": "^1.0.3",
"reduce-css-calc": "^2.1.6"
}, },
"dependencies": { "dependencies": {
"ansi-styles": { "ansi-styles": {
@ -15845,9 +15867,9 @@
} }
}, },
"terser": { "terser": {
"version": "4.1.2", "version": "4.1.3",
"resolved": "https://registry.npmjs.org/terser/-/terser-4.1.2.tgz", "resolved": "https://registry.npmjs.org/terser/-/terser-4.1.3.tgz",
"integrity": "sha512-jvNoEQSPXJdssFwqPSgWjsOrb+ELoE+ILpHPKXC83tIxOlh2U75F1KuB2luLD/3a6/7K3Vw5pDn+hvu0C4AzSw==", "integrity": "sha512-on13d+cnpn5bMouZu+J8tPYQecsdRJCJuxFJ+FVoPBoLJgk5bCBkp+Uen2hWyi0KIUm6eDarnlAlH+KgIx/PuQ==",
"dev": true, "dev": true,
"requires": { "requires": {
"commander": "^2.20.0", "commander": "^2.20.0",
@ -15936,9 +15958,9 @@
"dev": true "dev": true
}, },
"webpack-sources": { "webpack-sources": {
"version": "1.4.1", "version": "1.4.3",
"resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.1.tgz", "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz",
"integrity": "sha512-XSz38193PTo/1csJabKaV4b53uRVotlMgqJXm3s3eje0Bu6gQTxYDqpD38CmQfDBA+gN+QqaGjasuC8I/7eW3Q==", "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==",
"dev": true, "dev": true,
"requires": { "requires": {
"source-list-map": "^2.0.0", "source-list-map": "^2.0.0",
@ -16271,11 +16293,6 @@
} }
} }
}, },
"ultron": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
"integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og=="
},
"unassert": { "unassert": {
"version": "1.5.1", "version": "1.5.1",
"resolved": "https://registry.npmjs.org/unassert/-/unassert-1.5.1.tgz", "resolved": "https://registry.npmjs.org/unassert/-/unassert-1.5.1.tgz",
@ -16354,38 +16371,15 @@
} }
}, },
"union-value": { "union-value": {
"version": "1.0.0", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
"integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=", "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==",
"dev": true, "dev": true,
"requires": { "requires": {
"arr-union": "^3.1.0", "arr-union": "^3.1.0",
"get-value": "^2.0.6", "get-value": "^2.0.6",
"is-extendable": "^0.1.1", "is-extendable": "^0.1.1",
"set-value": "^0.4.3" "set-value": "^2.0.1"
},
"dependencies": {
"extend-shallow": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
"integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
"dev": true,
"requires": {
"is-extendable": "^0.1.0"
}
},
"set-value": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz",
"integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=",
"dev": true,
"requires": {
"extend-shallow": "^2.0.1",
"is-extendable": "^0.1.1",
"is-plain-object": "^2.0.1",
"to-object-path": "^0.3.0"
}
}
} }
}, },
"uniq": { "uniq": {
@ -17185,34 +17179,34 @@
} }
}, },
"webpack": { "webpack": {
"version": "4.39.1", "version": "4.38.0",
"resolved": "https://registry.npmjs.org/webpack/-/webpack-4.39.1.tgz", "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.38.0.tgz",
"integrity": "sha512-/LAb2TJ2z+eVwisldp3dqTEoNhzp/TLCZlmZm3GGGAlnfIWDgOEE758j/9atklNLfRyhKbZTCOIoPqLJXeBLbQ==", "integrity": "sha512-lbuFsVOq8PZY+1Ytz/mYOvYOo+d4IJ31hHk/7iyoeWtwN33V+5HYotSH+UIb9tq914ey0Hot7z6HugD+je3sWw==",
"dev": true, "dev": true,
"requires": { "requires": {
"@webassemblyjs/ast": "1.8.5", "@webassemblyjs/ast": "1.8.5",
"@webassemblyjs/helper-module-context": "1.8.5", "@webassemblyjs/helper-module-context": "1.8.5",
"@webassemblyjs/wasm-edit": "1.8.5", "@webassemblyjs/wasm-edit": "1.8.5",
"@webassemblyjs/wasm-parser": "1.8.5", "@webassemblyjs/wasm-parser": "1.8.5",
"acorn": "^6.2.1", "acorn": "^6.2.0",
"ajv": "^6.10.2", "ajv": "^6.1.0",
"ajv-keywords": "^3.4.1", "ajv-keywords": "^3.1.0",
"chrome-trace-event": "^1.0.2", "chrome-trace-event": "^1.0.0",
"enhanced-resolve": "^4.1.0", "enhanced-resolve": "^4.1.0",
"eslint-scope": "^4.0.3", "eslint-scope": "^4.0.0",
"json-parse-better-errors": "^1.0.2", "json-parse-better-errors": "^1.0.2",
"loader-runner": "^2.4.0", "loader-runner": "^2.3.0",
"loader-utils": "^1.2.3", "loader-utils": "^1.1.0",
"memory-fs": "^0.4.1", "memory-fs": "~0.4.1",
"micromatch": "^3.1.10", "micromatch": "^3.1.8",
"mkdirp": "^0.5.1", "mkdirp": "~0.5.0",
"neo-async": "^2.6.1", "neo-async": "^2.5.0",
"node-libs-browser": "^2.2.1", "node-libs-browser": "^2.0.0",
"schema-utils": "^1.0.0", "schema-utils": "^1.0.0",
"tapable": "^1.1.3", "tapable": "^1.1.0",
"terser-webpack-plugin": "^1.4.1", "terser-webpack-plugin": "^1.1.0",
"watchpack": "^1.6.0", "watchpack": "^1.5.0",
"webpack-sources": "^1.4.1" "webpack-sources": "^1.3.0"
}, },
"dependencies": { "dependencies": {
"acorn": { "acorn": {
@ -17221,24 +17215,6 @@
"integrity": "sha512-JD0xT5FCRDNyjDda3Lrg/IxFscp9q4tiYtxE1/nOzlKCk7hIRuYjhq1kCNkbPjMRMZuFq20HNQn1I9k8Oj0E+Q==", "integrity": "sha512-JD0xT5FCRDNyjDda3Lrg/IxFscp9q4tiYtxE1/nOzlKCk7hIRuYjhq1kCNkbPjMRMZuFq20HNQn1I9k8Oj0E+Q==",
"dev": true "dev": true
}, },
"ajv": {
"version": "6.10.2",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz",
"integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==",
"dev": true,
"requires": {
"fast-deep-equal": "^2.0.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
}
},
"ajv-keywords": {
"version": "3.4.1",
"resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.4.1.tgz",
"integrity": "sha512-RO1ibKvd27e6FEShVFfPALuHI3WjSVNeK5FIsmme/LYRNxjKuNj+Dt7bucLa6NdSv3JcVTyMlm9kGR84z1XpaQ==",
"dev": true
},
"eslint-scope": { "eslint-scope": {
"version": "4.0.3", "version": "4.0.3",
"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz",
@ -17248,28 +17224,6 @@
"esrecurse": "^4.1.0", "esrecurse": "^4.1.0",
"estraverse": "^4.1.1" "estraverse": "^4.1.1"
} }
},
"source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
"dev": true
},
"tapable": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz",
"integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==",
"dev": true
},
"webpack-sources": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.1.tgz",
"integrity": "sha512-XSz38193PTo/1csJabKaV4b53uRVotlMgqJXm3s3eje0Bu6gQTxYDqpD38CmQfDBA+gN+QqaGjasuC8I/7eW3Q==",
"dev": true,
"requires": {
"source-list-map": "^2.0.0",
"source-map": "~0.6.1"
}
} }
} }
}, },
@ -17692,31 +17646,6 @@
"integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==", "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==",
"dev": true "dev": true
}, },
"websocket-stream": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/websocket-stream/-/websocket-stream-5.5.0.tgz",
"integrity": "sha512-EXy/zXb9kNHI07TIMz1oIUIrPZxQRA8aeJ5XYg5ihV8K4kD1DuA+FY6R96HfdIHzlSzS8HiISAfrm+vVQkZBug==",
"requires": {
"duplexify": "^3.5.1",
"inherits": "^2.0.1",
"readable-stream": "^2.3.3",
"safe-buffer": "^5.1.2",
"ws": "^3.2.0",
"xtend": "^4.0.0"
},
"dependencies": {
"ws": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
"integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
"requires": {
"async-limiter": "~1.0.0",
"safe-buffer": "~5.1.0",
"ultron": "~1.1.0"
}
}
}
},
"wgxpath": { "wgxpath": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/wgxpath/-/wgxpath-1.0.0.tgz", "resolved": "https://registry.npmjs.org/wgxpath/-/wgxpath-1.0.0.tgz",
@ -17819,8 +17748,9 @@
} }
}, },
"ws": { "ws": {
"version": "github:dannycoates/ws#c83cbb3bce478122cedcb8c475d9e86e1112824a", "version": "6.1.2",
"from": "github:dannycoates/ws", "resolved": "github:dannycoates/ws#c83cbb3bce478122cedcb8c475d9e86e1112824a",
"dev": true,
"requires": { "requires": {
"async-limiter": "~1.0.0" "async-limiter": "~1.0.0"
} }

19
package.json
View File

@ -70,13 +70,13 @@
"asmcrypto.js": "^0.22.0", "asmcrypto.js": "^0.22.0",
"babel-loader": "^8.0.6", "babel-loader": "^8.0.6",
"babel-plugin-istanbul": "^5.2.0", "babel-plugin-istanbul": "^5.2.0",
"base64-js": "^1.3.0", "base64-js": "^1.3.1",
"content-disposition": "^0.5.3", "content-disposition": "^0.5.3",
"copy-webpack-plugin": "^5.0.4", "copy-webpack-plugin": "^5.0.4",
"core-js": "^3.1.4", "core-js": "^3.1.4",
"crc": "^3.8.0", "crc": "^3.8.0",
"cross-env": "^5.2.0", "cross-env": "^5.2.0",
"css-loader": "^3.1.0", "css-loader": "^3.2.0",
"css-mqpacker": "^7.0.0", "css-mqpacker": "^7.0.0",
"cssnano": "^4.1.10", "cssnano": "^4.1.10",
"eslint": "^6.1.0", "eslint": "^6.1.0",
@ -88,7 +88,7 @@
"extract-loader": "^3.1.0", "extract-loader": "^3.1.0",
"extract-text-webpack-plugin": "^4.0.0-beta.0", "extract-text-webpack-plugin": "^4.0.0-beta.0",
"fast-text-encoding": "^1.0.0", "fast-text-encoding": "^1.0.0",
"file-loader": "^4.1.0", "file-loader": "^4.2.0",
"git-rev-sync": "^1.12.0", "git-rev-sync": "^1.12.0",
"html-loader": "^0.5.5", "html-loader": "^0.5.5",
"http_ece": "^1.1.0", "http_ece": "^1.1.0",
@ -112,14 +112,14 @@
"redis-mock": "^0.45.0", "redis-mock": "^0.45.0",
"rimraf": "^2.6.3", "rimraf": "^2.6.3",
"script-loader": "^0.7.2", "script-loader": "^0.7.2",
"sinon": "^7.3.2", "sinon": "^7.4.1",
"string-hash": "^1.1.3", "string-hash": "^1.1.3",
"stylelint": "^10.1.0", "stylelint": "^10.1.0",
"stylelint-config-standard": "^18.3.0", "stylelint-config-standard": "^18.3.0",
"stylelint-no-unsupported-browser-features": "^3.0.2", "stylelint-no-unsupported-browser-features": "^3.0.2",
"svgo": "^1.3.0", "svgo": "^1.3.0",
"svgo-loader": "^2.2.1", "svgo-loader": "^2.2.1",
"tailwindcss": "^1.0.6", "tailwindcss": "^1.1.0",
"val-loader": "^1.1.1", "val-loader": "^1.1.1",
"wdio-docker-service": "^1.4.2", "wdio-docker-service": "^1.4.2",
"wdio-dot-reporter": "0.0.10", "wdio-dot-reporter": "0.0.10",
@ -129,7 +129,7 @@
"wdio-selenium-standalone-service": "0.0.12", "wdio-selenium-standalone-service": "0.0.12",
"wdio-spec-reporter": "^0.1.5", "wdio-spec-reporter": "^0.1.5",
"webdriverio": "^4.14.4", "webdriverio": "^4.14.4",
"webpack": "^4.39.1", "webpack": "4.38.0",
"webpack-cli": "^3.3.6", "webpack-cli": "^3.3.6",
"webpack-dev-middleware": "^3.7.0", "webpack-dev-middleware": "^3.7.0",
"webpack-dev-server": "^3.7.2", "webpack-dev-server": "^3.7.2",
@ -137,16 +137,16 @@
"webpack-unassert-loader": "^1.2.0" "webpack-unassert-loader": "^1.2.0"
}, },
"dependencies": { "dependencies": {
"@dannycoates/express-ws": "^5.0.3",
"@fluent/bundle": "^0.13.0", "@fluent/bundle": "^0.13.0",
"@fluent/langneg": "^0.3.0", "@fluent/langneg": "^0.3.0",
"@google-cloud/storage": "^3.0.4", "@google-cloud/storage": "^3.0.4",
"aws-sdk": "^2.503.0", "aws-sdk": "^2.505.0",
"body-parser": "^1.19.0", "body-parser": "^1.19.0",
"choo": "^6.13.3", "choo": "^6.13.3",
"cldr-core": "^35.1.0", "cldr-core": "^35.1.0",
"convict": "^5.1.0", "convict": "^5.1.0",
"express": "^4.17.1", "express": "^4.17.1",
"express-ws": "github:dannycoates/express-ws",
"fxa-geodb": "^1.0.4", "fxa-geodb": "^1.0.4",
"helmet": "^3.20.0", "helmet": "^3.20.0",
"mkdirp": "^0.5.1", "mkdirp": "^0.5.1",
@ -155,8 +155,7 @@
"raven": "^2.6.4", "raven": "^2.6.4",
"redis": "^2.8.0", "redis": "^2.8.0",
"selenium-standalone": "^6.15.6", "selenium-standalone": "^6.15.6",
"ua-parser-js": "^0.7.20", "ua-parser-js": "^0.7.20"
"websocket-stream": "^5.5.0"
}, },
"availableLanguages": [ "availableLanguages": [
"en-US", "en-US",

2
server/bin/dev.js
View File

@ -3,7 +3,7 @@ const routes = require('../routes');
const pages = require('../routes/pages'); const pages = require('../routes/pages');
const tests = require('../../test/frontend/routes'); const tests = require('../../test/frontend/routes');
const express = require('express'); const express = require('express');
const expressWs = require('express-ws'); const expressWs = require('@dannycoates/express-ws');
const morgan = require('morgan'); const morgan = require('morgan');
const config = require('../config'); const config = require('../config');

2
server/bin/prod.js
View File

@ -4,7 +4,7 @@ const Raven = require('raven');
const config = require('../config'); const config = require('../config');
const routes = require('../routes'); const routes = require('../routes');
const pages = require('../routes/pages'); const pages = require('../routes/pages');
const expressWs = require('express-ws'); const expressWs = require('@dannycoates/express-ws');
if (config.sentry_dsn) { if (config.sentry_dsn) {
Raven.config(config.sentry_dsn).install(); Raven.config(config.sentry_dsn).install();

2
server/bin/test.js
View File

@ -2,7 +2,7 @@ const assets = require('../../common/assets');
const routes = require('../routes'); const routes = require('../routes');
const pages = require('../routes/pages'); const pages = require('../routes/pages');
const tests = require('../../test/frontend/routes'); const tests = require('../../test/frontend/routes');
const expressWs = require('express-ws'); const expressWs = require('@dannycoates/express-ws');
module.exports = function(app, devServer) { module.exports = function(app, devServer) {
assets.setMiddleware(devServer.middleware); assets.setMiddleware(devServer.middleware);

23
server/routes/ws.js
View File

@ -3,12 +3,11 @@ const storage = require('../storage');
const config = require('../config'); const config = require('../config');
const mozlog = require('../log'); const mozlog = require('../log');
const Limiter = require('../limiter'); const Limiter = require('../limiter');
const wsStream = require('websocket-stream/stream');
const fxa = require('../fxa'); const fxa = require('../fxa');
const { statUploadEvent } = require('../amplitude'); const { statUploadEvent } = require('../amplitude');
const { encryptedSize } = require('../../app/utils'); const { encryptedSize } = require('../../app/utils');
const { Duplex } = require('stream'); const { Transform } = require('stream');
const log = mozlog('send.upload'); const log = mozlog('send.upload');
@ -76,25 +75,19 @@ module.exports = function(ws, req) {
}) })
); );
const limiter = new Limiter(encryptedSize(maxFileSize)); const limiter = new Limiter(encryptedSize(maxFileSize));
const flowControl = new Duplex({ const eof = new Transform({
read() { transform: function(chunk, encoding, callback) {
ws.resume();
},
write(chunk, encoding, callback) {
if (chunk.length === 1 && chunk[0] === 0) { if (chunk.length === 1 && chunk[0] === 0) {
this.push(null); this.push(null);
} else { } else {
if (!this.push(chunk)) { this.push(chunk);
ws.pause();
}
} }
callback(); callback();
} }
}); });
const wsStream = ws.constructor.createWebSocketStream(ws);
fileStream = wsStream(ws, { binary: true }) fileStream = wsStream.pipe(eof).pipe(limiter); // limiter needs to be the last in the chain
.pipe(flowControl)
.pipe(limiter); // limiter needs to be the last in the chain
await storage.set(newId, fileStream, meta, timeLimit); await storage.set(newId, fileStream, meta, timeLimit);
@ -126,8 +119,8 @@ module.exports = function(ws, req) {
error: e === 'limit' ? 413 : 500 error: e === 'limit' ? 413 : 500
}) })
); );
}
}
ws.close(); ws.close();
}
}
}); });
}; };

2
server/storage/gcs.js
View File

@ -22,7 +22,7 @@ class GCSStorage {
.pipe( .pipe(
this.bucket.file(id).createWriteStream({ this.bucket.file(id).createWriteStream({
validation: false, validation: false,
resumable: false resumable: true
}) })
) )
.on('error', reject) .on('error', reject)

2
test/testServer.js
View File

@ -6,7 +6,7 @@ module.exports = {
const webpack = require('webpack'); const webpack = require('webpack');
const middleware = require('webpack-dev-middleware'); const middleware = require('webpack-dev-middleware');
const express = require('express'); const express = require('express');
const expressWs = require('express-ws'); const expressWs = require('@dannycoates/express-ws');
const assets = require('../common/assets'); const assets = require('../common/assets');
const routes = require('../server/routes'); const routes = require('../server/routes');
const tests = require('./frontend/routes'); const tests = require('./frontend/routes');