From 18e1609cb35d8d3da28d2d9686acbea34211c8c5 Mon Sep 17 00:00:00 2001 From: Danny Coates Date: Mon, 5 Mar 2018 16:29:09 -0800 Subject: [PATCH] updated docs --- README.md | 4 +- app/fileManager.js | 6 +- app/readme.md | 9 +++ build/generate_asset_map.js | 11 ++++ build/generate_l10n_map.js | 11 ++++ build/readme.md | 26 ++++++++ common/readme.md | 3 + docs/build.md | 22 +++++++ docs/encryption.md | 46 +++++++++++++ docs/experiments.md | 84 ++++++++++++++++++++++++ docs/localization.md | 29 ++++++++ package.json | 7 +- server/readme.md | 19 ++++++ test/{unit => backend}/auth-tests.js | 0 test/{unit => backend}/delete-tests.js | 0 test/{unit => backend}/info-tests.js | 0 test/{unit => backend}/language-tests.js | 0 test/{unit => backend}/metadata-tests.js | 0 test/{unit => backend}/owner-tests.js | 0 test/{unit => backend}/params-tests.js | 0 test/{unit => backend}/password-tests.js | 0 test/{unit => backend}/s3-tests.js | 0 test/{unit => backend}/storage-tests.js | 0 test/readme.md | 17 +++++ test/test_upload.txt | 1 - webpack.config.js | 28 ++++++-- 26 files changed, 309 insertions(+), 14 deletions(-) create mode 100644 app/readme.md create mode 100644 build/readme.md create mode 100644 common/readme.md create mode 100644 docs/build.md create mode 100644 docs/encryption.md create mode 100644 docs/experiments.md create mode 100644 docs/localization.md create mode 100644 server/readme.md rename test/{unit => backend}/auth-tests.js (100%) rename test/{unit => backend}/delete-tests.js (100%) rename test/{unit => backend}/info-tests.js (100%) rename test/{unit => backend}/language-tests.js (100%) rename test/{unit => backend}/metadata-tests.js (100%) rename test/{unit => backend}/owner-tests.js (100%) rename test/{unit => backend}/params-tests.js (100%) rename test/{unit => backend}/password-tests.js (100%) rename test/{unit => backend}/s3-tests.js (100%) rename test/{unit => backend}/storage-tests.js (100%) create mode 100644 test/readme.md delete mode 100644 test/test_upload.txt diff --git a/README.md b/README.md index 0e6e2d88..ccbb01cd 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![CircleCI](https://img.shields.io/circleci/project/github/mozilla/send.svg)](https://circleci.com/gh/mozilla/send) [![Available on Test Pilot](https://img.shields.io/badge/available_on-Test_Pilot-0996F8.svg)](https://testpilot.firefox.com/experiments/send) -**Docs:** [Docker](docs/docker.md), [Metrics](docs/metrics.md) +**Docs:** [FAQ](docs/faq.md), [Encryption](docs/encryption.md), [Build](docs/build.md), [Docker](docs/docker.md), [Metrics](docs/metrics.md), [More](docs/) --- @@ -71,6 +71,8 @@ The server is configured with environment variables. See [server/config.js](serv Firefox Send localization is managed via [Pontoon](https://pontoon.mozilla.org/projects/test-pilot-firefox-send/), not direct pull requests to the repository. If you want to fix a typo, add a new language, or simply know more about localization, please get in touch with the [existing localization team](https://pontoon.mozilla.org/teams/) for your language or Mozilla’s [l10n-drivers](https://wiki.mozilla.org/L10n:Mozilla_Team#Mozilla_Corporation) for guidance. +see also [docs/localization.md](docs/localization.md) + --- ## Contributing diff --git a/app/fileManager.js b/app/fileManager.js index 0dab3cba..897dda4f 100644 --- a/app/fileManager.js +++ b/app/fileManager.js @@ -104,8 +104,10 @@ export default function(state, emitter) { metrics.completedUpload(ownedFile); state.storage.addFile(ownedFile); - - document.getElementById('cancel-upload').hidden = 'hidden'; + const cancelBtn = document.getElementById('cancel-upload'); + if (cancelBtn) { + cancelBtn.hidden = 'hidden'; + } await delay(1000); await fadeOut('.page'); openLinksInNewTab(links, false); diff --git a/app/readme.md b/app/readme.md new file mode 100644 index 00000000..7708988a --- /dev/null +++ b/app/readme.md @@ -0,0 +1,9 @@ +# Application Code + +`app/` contains the browser code that gets bundled into `app.[hash].js`. It's got all the logic, crypto, and UI. All of it gets used in the browser, and some of it by the server for server side rendering. + +The main entrypoint for the browser is [main.js](./main.js) and on the server [routes/index.js](./routes/index.js) gets imported by [/server/routes/pages.js](../server/routes/pages.js) + +- `pages` contains display logic an markup for pages +- `routes` contains route definitions and logic +- `templates` contains ui elements smaller than pages diff --git a/build/generate_asset_map.js b/build/generate_asset_map.js index 1047e974..de0999c9 100644 --- a/build/generate_asset_map.js +++ b/build/generate_asset_map.js @@ -1,3 +1,14 @@ +/* + This code is included by both the server and frontend via + common/assets.js + + When included from the server the export will be the function. + + When included from the frontend (via webpack) the export will + be an object mapping file names to hashed file names. Example: + "send_logo.svg": "send_logo.5fcfdf0e.svg" +*/ + const fs = require('fs'); const path = require('path'); diff --git a/build/generate_l10n_map.js b/build/generate_l10n_map.js index 15c898dd..ed33c0f8 100644 --- a/build/generate_l10n_map.js +++ b/build/generate_l10n_map.js @@ -1,3 +1,14 @@ +/* + This code is included by both the server and frontend via + common/locales.js + + When included from the server the export will be the function. + + When included from the frontend (via webpack) the export will + be an object mapping ftl files to js files. Example: + "public/locales/en-US/send.ftl":"public/locales/en-US/send.6b4f8354.js" +*/ + const fs = require('fs'); const path = require('path'); diff --git a/build/readme.md b/build/readme.md new file mode 100644 index 00000000..ff0f9c5d --- /dev/null +++ b/build/readme.md @@ -0,0 +1,26 @@ +# Custom Loaders + +## Fluent Loader + +The fluent loader "compiles" `.ftl` files into `.js` files directly usable by both the frontend and server for localization. + +## Generate Asset Map + +This loader enumerates all the files in `assets/` so that `common/assets.js` can provide mappings from the source filename to the hashed filename used on the site. + +## Generate L10N Map + +This loader enumerates all the ftl files in `public/locales` so that the fluent loader can create it's js files. + +## Package.json Loader + +This loader creates a `version.json` file that gets exposed by the `/__version__` route from the `package.json` file and current git commit hash. + +## Version Loader + +This loader substitutes the string "VERSION" for the version string specified in `package.json`. This is a workaround because `package.json` already uses the `package_json_loader`. See [app/templates/header/index.js](../app/templates/header/index.js) for more info. + +# See Also + +- [docs/build.md](../docs/build.md) +- [webpack.config.js](../webpack.config.js) \ No newline at end of file diff --git a/common/readme.md b/common/readme.md new file mode 100644 index 00000000..9e351eee --- /dev/null +++ b/common/readme.md @@ -0,0 +1,3 @@ +# Common Code + +This directory contains code loaded by both the frontend `app` and backend `server`. The code here can be challenging to understand at first because the contexts for the two (three counting the dev server) environments that include them are quite different, but the purpose of these modules are quite simple, to provide mappings from the source assets (`copy-16.png`) to the concrete production assets (`copy-16.db66e0bf.svg`), similarly for localizations. \ No newline at end of file diff --git a/docs/build.md b/docs/build.md new file mode 100644 index 00000000..78ebd66f --- /dev/null +++ b/docs/build.md @@ -0,0 +1,22 @@ +Send has two build configurations, development and production. Both can be run via `npm` scripts, `npm start` for development and `npm run build` for production. Webpack is our only build tool and all configuration lives in [webpack.config.js](../webpack.config.js). + +# Development + +`npm start` launches a `webpack-dev-server` on port 8080 that compiles the assets and watches files for changes. It also serves the backend API and frontend unit tests via the `server/dev.js` entrypoint. The frontend tests can be run in the browser by navigating to http://localhost:8080/test and will rerun automatically as the watched files are saved with changes. + +# Production + +`npm run build` compiles the assets and writes the files to the `dist/` directory. `npm run prod` launches an Express server on port 1443 that serves the backend API and frontend static assets from `dist/` via the `server/prod.js` entrypoint. + +# Notable differences + +- Development compiles assets in memory, so no `dist/` directory is generated +- Development does not enable CSP headers +- Development frontend source is instrumented for code coverage +- Only development includes sourcemaps +- Only development exposes the `/test` route +- Production sets Cache-Control immutable headers on the hashed static assets + +# Custom Loaders + +The `build/` directory contains custom webpack loaders specific to Send. See [build/readme.md](../build/readme.md) for details on each loader. \ No newline at end of file diff --git a/docs/encryption.md b/docs/encryption.md new file mode 100644 index 00000000..d9481eeb --- /dev/null +++ b/docs/encryption.md @@ -0,0 +1,46 @@ +# File Encryption + +Send use 128-bit AES-GCM encryption via the [Web Crypto API](https://developer.mozilla.org/en-US/docs/Web/API/Web_Crypto_API) to encrypt files in the browser before uploading them to the server. The code is in [app/keychain.js](../app/keychain.js). + +## Steps + +### Uploading + +1. A new secret key is generated with `crypto.getRandomValues` +2. The secret key is used to derive 3 more keys via HKDF SHA-256 + - an encryption key for the file (AES-GCM) + - an encryption key for the file metadata (AES-GCM) + - a signing key for request authentication (HMAC SHA-256) +3. The file and metadata are encrypted with their corresponding keys +4. The encrypted data and signing key are uploaded to the server +5. An owner token and the share url are returned by the server and stored in local storage +6. The secret key is appended to the share url as a [#fragment](https://en.wikipedia.org/wiki/Fragment_identifier) and presented to the UI + +### Downloading + +1. The browser loads the share url page, which includes an authentication nonce +2. The browser imports the secret key from the url fragment +3. The same 3 keys as above are derived +4. The browser signs the nonce with it's signing key and requests the metadata +5. The encrypted metadata is decrypted and presented on the page +6. The browser makes another authenticated request to download the encrypted file +7. The browser downloads and decrypts the file +8. The file prompts the save dialog or automatically saves depending on the browser settings + +### Passwords + +A password may optionally be set to authenticate the download request. When a password is set the following steps occur. + +#### Sender + +1. The original signing key derived from the secret key is discarded +2. A new signing key is generated via PBKDF2 from the user entered password and the full share url (including secret key fragment) +3. The new key is sent to the server, authenticated by the owner token +4. The server stores the new key and marks the record as needing a password + +#### Downloader + +1. The browser loads the share url page, which includes an authentication nonce and indicator that the file requires a password +2. The user is prompted for the password and the signing key is derived +3. The browser requests the metadata using the key to sign the nonce +4. If the password was correct the metadata is returned, otherwise a 401 diff --git a/docs/experiments.md b/docs/experiments.md new file mode 100644 index 00000000..f627204b --- /dev/null +++ b/docs/experiments.md @@ -0,0 +1,84 @@ +# A/B experiment testing + +We're using Google Analytics Experiments for A/B testing. + +## Creating an experiment + +Navigate to the Behavior > Experiments section of Google Analytics and click the "Create experiment" button. + +The "Objective for this experiment" is the most complicated part. See the "Promo click (Goal ID 4 / Goal Set 1)" for an example. + +In step 2 add as many variants as you plan to test. The urls are not important since we aren't using their js library to choose the variants. The name will show up in the report so choose good ones. "Original page" becomes variant 0 and each variant increments by one. We'll use the numbers in our `app/experiments.js` code. + +Step 3 contains some script that we'll ignore. The important thing here is the **Experiment ID**. This is the value we need to name our experiment in `app/experiments.js`. Save the changes so far and wait until the code containing the experiment has been deployed to production **before** starting the experiment. + +## Experiment code + +Code for experiments live in [app/experiments.js](../app/experiments.js). There's an `experiments` object that contains the logic for deciding whether an experiment should run, which variant to use, and what to do. Each object needs to have these functions: + +### `eligible` function + +This function returns a boolean of whether this experiment should be active for this session. Any data available to the page can be used determine the result. + +### `variant` function + +This function returns which experimental group this session is placed in. The variant values need to match the values set up in Google Analytics, usually 0 thru N-1. This value is usually picked at random based on what percentage of each variant is desired. + +### `run` function + +This function gets the `variant` value chosen by the variant function and the `state` and `emitter` objects from the app. This function can do anything needed to change the app based on the experiment. A common pattern is to set or change a value on `state` that will be picked up by other parts of the app, like ui templates, to change how it looks or behaves. + +### Example + +Here's a full example of the experiment object: + +```js +const experiments = { + S9wqVl2SQ4ab2yZtqDI3Dw: { // The Experiment ID from Google Analytics + id: 'S9wqVl2SQ4ab2yZtqDI3Dw', + run: function(variant, state, emitter) { + switch (variant) { + case 1: + state.promo = 'blue'; + break; + case 2: + state.promo = 'pink'; + break; + default: + state.promo = 'grey'; + } + emitter.emit('render'); + }, + eligible: function() { + return ( + !/firefox|fxios/i.test(navigator.userAgent) && + document.querySelector('html').lang === 'en-US' + ); + }, + variant: function(state) { + const n = this.luckyNumber(state); + if (n < 0.33) { + return 0; + } + return n < 0.66 ? 1 : 2; + }, + luckyNumber: function(state) { + return luckyNumber( + `${this.id}:${state.storage.get('testpilot_ga__cid')}` + ); + } + } +}; +``` + +## Reporting results + +All metrics pings will include the variant and experiment id, but it's usually important to trigger a specific event to be counted as the experiment goal (the "Objective for this experiment" part from setup). Use an 'experiment' event to do this. For example: + +```js +emit('experiment', { cd3: 'promo' }); +``` + +where `emit` is the app emitter function passed to the [route handler](https://github.com/choojs/choo#approuteroutename-handlerstate-emit) + +The second argument can be an object with any additional parameters. It usually includes a custom dimension that we chose to filter on while creating the experiment in Google Analytics. \ No newline at end of file diff --git a/docs/localization.md b/docs/localization.md new file mode 100644 index 00000000..ad9072e1 --- /dev/null +++ b/docs/localization.md @@ -0,0 +1,29 @@ +# Localization + +Send is localized in over 50 languages. We use the [fluent](http://projectfluent.org/) library and store our translations in [FTL](http://projectfluent.org/fluent/guide/) files in `public/locales/`. `en-US` is our base language, and other languages are managed by [pontoon](https://pontoon.mozilla.org/projects/test-pilot-firefox-send/). + +## Process + +Strings are added or removed from [public/locales/en-US/send.ftl] as needed. Strings **MUST NOT** be *changed* after they've been commited and pushed to master. Changing a string requires creating a new ID with a new name (preferably descriptive instead of incremented) and deletion of the obsolete ID. It's often useful to add a comment above the string with info about how and where the string is used. + +Once new strings are commited to master they are available for translators in Pontoon. All languages other than `en-US` should be edited via Pontoon. Translations get automatically commited to the github master branch. + +### Activation + +The development environment includes all locales in `public/locales` via the `L10N_DEV` environment variable. Production uses `package.json` as the list of locales to use. Once a locale has enough string coverage it should be added to `package.json`. + +## Code + +In `app/` we use the `state.translate()` function to translate strings to the best matching language base on the user's `Accept-Language` header. It's a wrapper around fluent's [MessageContext.format](http://projectfluent.org/fluent.js/fluent/MessageContext.html). It works the same for both server and client side rendering. + +### Examples + +```js +// simple string +const finishedString = state.translate('downloadFinish') +// with parameters +const progressString = state.translate('downloadingPageProgress', { + filename: state.fileInfo.name, + size: bytes(state.fileInfo.size) +}) +``` diff --git a/package.json b/package.json index 6f5f1b62..4654ec3a 100644 --- a/package.json +++ b/package.json @@ -26,11 +26,10 @@ "contributors": "git shortlog -s | awk -F\\t '{print $2}' > CONTRIBUTORS", "release": "npm-run-all contributors changelog", "test": "npm-run-all test:*", - "test:backend": "nyc mocha --reporter=min test/unit", + "test:backend": "nyc mocha --reporter=min test/backend", "test:frontend": "cross-env NODE_ENV=development node test/frontend/runner.js && nyc report --reporter=html", - "start": "cross-env NODE_ENV=development webpack-dev-server", - "prod": "node server/prod.js", - "cover": "nyc --reporter=html mocha test/unit" + "start": "npm run clean && cross-env NODE_ENV=development webpack-dev-server", + "prod": "node server/prod.js" }, "lint-staged": { "*.js": [ diff --git a/server/readme.md b/server/readme.md new file mode 100644 index 00000000..84b7b283 --- /dev/null +++ b/server/readme.md @@ -0,0 +1,19 @@ +# Server Code + +The server provides the API, serves static assets, and renders the pages for Send. The production entrypoint is [prod.js](./prod.js) and the development entrypoint is [dev.js](./dev.js) via `webpack-dev-server`. + +## Server configuration + +[config.js](./config.js) contains the schema for our configuration options. Environment variables are the preferred method for setting configuration. + +## Middleware + +Contains authentication and localization middleware. + +## Routes + +Contains all the server routes and handlers for the API and pages + +## Storage + +Contains implementations of possible storage engines for the files and metadata \ No newline at end of file diff --git a/test/unit/auth-tests.js b/test/backend/auth-tests.js similarity index 100% rename from test/unit/auth-tests.js rename to test/backend/auth-tests.js diff --git a/test/unit/delete-tests.js b/test/backend/delete-tests.js similarity index 100% rename from test/unit/delete-tests.js rename to test/backend/delete-tests.js diff --git a/test/unit/info-tests.js b/test/backend/info-tests.js similarity index 100% rename from test/unit/info-tests.js rename to test/backend/info-tests.js diff --git a/test/unit/language-tests.js b/test/backend/language-tests.js similarity index 100% rename from test/unit/language-tests.js rename to test/backend/language-tests.js diff --git a/test/unit/metadata-tests.js b/test/backend/metadata-tests.js similarity index 100% rename from test/unit/metadata-tests.js rename to test/backend/metadata-tests.js diff --git a/test/unit/owner-tests.js b/test/backend/owner-tests.js similarity index 100% rename from test/unit/owner-tests.js rename to test/backend/owner-tests.js diff --git a/test/unit/params-tests.js b/test/backend/params-tests.js similarity index 100% rename from test/unit/params-tests.js rename to test/backend/params-tests.js diff --git a/test/unit/password-tests.js b/test/backend/password-tests.js similarity index 100% rename from test/unit/password-tests.js rename to test/backend/password-tests.js diff --git a/test/unit/s3-tests.js b/test/backend/s3-tests.js similarity index 100% rename from test/unit/s3-tests.js rename to test/backend/s3-tests.js diff --git a/test/unit/storage-tests.js b/test/backend/storage-tests.js similarity index 100% rename from test/unit/storage-tests.js rename to test/backend/storage-tests.js diff --git a/test/readme.md b/test/readme.md new file mode 100644 index 00000000..977bdf67 --- /dev/null +++ b/test/readme.md @@ -0,0 +1,17 @@ +# Tests + +To run all the tests use `npm test`. This will run the tests and produce a code coverage report at [coverage/index.html](../coverage/index.html). The full test suite is run as a hook on each `git push`. [Mocha](https://mochajs.org) is our preferred test runner. + +## Frontend + +Unit tests reside in `test/frontend/tests`. + +Frontend tests can be ran in the browser by running `npm start` and then browsing to http://localhost:8080/test. Doing it this way will watch for changes and rerun the suite automatically. + +You can also run them in headless Chrome by using `npm run test:frontend`. The results will be printed to the console. + +## Backend + +Unit tests reside in `test/backend` + +Backend test can be run with `npm run test:backend`. [Sinon](http://sinonjs.org/) and [proxyquire](https://github.com/thlorenz/proxyquire) are used for mocking. \ No newline at end of file diff --git a/test/test_upload.txt b/test/test_upload.txt deleted file mode 100644 index 273c1a9f..00000000 --- a/test/test_upload.txt +++ /dev/null @@ -1 +0,0 @@ -This is a test. \ No newline at end of file diff --git a/webpack.config.js b/webpack.config.js index 07e0112c..65bc21e0 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -8,10 +8,13 @@ const IS_DEV = process.env.NODE_ENV === 'development'; const regularJSOptions = { babelrc: false, presets: [['env', { modules: false }], 'stage-2'], + // yo-yoify converts html template strings to direct dom api calls plugins: ['yo-yoify'] }; const entry = { + // babel-polyfill and fluent are directly included in vendor + // because they are not explicitly referenced by app vendor: ['babel-polyfill', 'fluent'], app: ['./app/main.js'], style: ['./app/main.css'] @@ -19,6 +22,7 @@ const entry = { if (IS_DEV) { entry.tests = ['./test/frontend/index.js']; + // istanbul instruments the source for code coverage regularJSOptions.plugins.push('istanbul'); } @@ -47,6 +51,7 @@ module.exports = { ] }, { + // inlines version from package.json into header/index.js include: require.resolve('./app/templates/header'), use: [ { @@ -57,6 +62,8 @@ module.exports = { ] }, { + // fluent gets exposed as a global so that each language script + // can load independently and share it. include: [path.dirname(require.resolve('fluent'))], use: [ { @@ -76,6 +83,8 @@ module.exports = { include: [ path.resolve(__dirname, 'app'), path.resolve(__dirname, 'common'), + // some dependencies need to get re-babeled because we + // have different targets than their default configs path.resolve(__dirname, 'node_modules/testpilot-ga/src'), path.resolve(__dirname, 'node_modules/fluent-intl-polyfill'), path.resolve(__dirname, 'node_modules/intl-pluralrules') @@ -83,6 +92,7 @@ module.exports = { options: regularJSOptions }, { + // Strip asserts from our deps, mainly choojs family include: [path.resolve(__dirname, 'node_modules')], loader: 'webpack-unassert-loader' } @@ -108,15 +118,16 @@ module.exports = { loader: 'svgo-loader', options: { plugins: [ - { removeViewBox: false }, - { convertStyleToAttrs: true }, - { removeTitle: true } + { removeViewBox: false }, // true causes stretched images + { convertStyleToAttrs: true }, // for CSP, no unsafe-eval + { removeTitle: true } // for smallness ] } } ] }, { + // creates style.css with all styles test: /\.css$/, use: ExtractTextPlugin.extract({ use: [ @@ -129,6 +140,7 @@ module.exports = { }) }, { + // creates version.json for /__version__ from package.json test: require.resolve('./package.json'), use: [ { @@ -142,6 +154,7 @@ module.exports = { ] }, { + // creates a js script for each ftl test: /\.ftl$/, use: [ { @@ -155,14 +168,17 @@ module.exports = { ] }, { + // creates test.js for /test test: require.resolve('./test/frontend/index.js'), use: ['babel-loader', 'val-loader'] }, { + // loads all assets from assets/ for use by common/assets.js test: require.resolve('./build/generate_asset_map.js'), use: ['babel-loader', 'val-loader'] }, { + // loads all the ftl from public/locales for use by common/locales.js test: require.resolve('./build/generate_l10n_map.js'), use: ['babel-loader', 'val-loader'] } @@ -175,8 +191,8 @@ module.exports = { from: '*.*' } ]), - new webpack.IgnorePlugin(/dist/), - new webpack.IgnorePlugin(/require-from-string/), + new webpack.IgnorePlugin(/dist/), // used in common/*.js + new webpack.IgnorePlugin(/require-from-string/), // used in common/locales.js new webpack.HashedModuleIdsPlugin(), new webpack.optimize.CommonsChunkPlugin({ name: 'vendor', @@ -188,7 +204,7 @@ module.exports = { new ExtractTextPlugin({ filename: 'style.[contenthash:8].css' }), - new ManifestPlugin() + new ManifestPlugin() // used by server side to resolve hashed assets ], devServer: { compress: true,