From b5ef1785ab2b94e89fe99b2d51712e8cb66ad9d6 Mon Sep 17 00:00:00 2001 From: Danny Coates Date: Tue, 28 Jul 2020 09:31:09 -0700 Subject: [PATCH] replaced fxa-geodb with load balancer header Co-authored-by: timvisee --- package-lock.json | 68 ------------------------- package.json | 1 - server/amplitude.js | 43 +++++++--------- server/routes/delete.js | 2 + server/routes/download.js | 2 + server/routes/index.js | 104 +++++++++++++++++++++----------------- server/routes/metrics.js | 3 +- server/routes/ws.js | 2 + 8 files changed, 84 insertions(+), 141 deletions(-) diff --git a/package-lock.json b/package-lock.json index e9887ae6..80ad111d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5052,11 +5052,6 @@ "tweetnacl": "^0.14.3" } }, - "big-integer": { - "version": "1.6.48", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz", - "integrity": "sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==" - }, "big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -5105,11 +5100,6 @@ } } }, - "bluebird": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.2.tgz", - "integrity": "sha512-dhHTWMI7kMx5whMQntl7Vr9C6BvV10lFXDAasnqnrMYhXVCzzk6IO9Fo2L75jXHT07WrOngL1WDXOp+yYS91Yg==" - }, "bn.js": { "version": "4.11.8", "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz", @@ -6320,14 +6310,6 @@ "sha.js": "^2.4.8" } }, - "cron": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/cron/-/cron-1.5.0.tgz", - "integrity": "sha512-j7zMFLrcSta53xqOvETUt8ge+PM14GtF47gEGJJeVlM6qP24/eWHSgtiWiEiKBR2sHS8xZaBQZq4D7vFXg8dcQ==", - "requires": { - "moment-timezone": "^0.5.x" - } - }, "cross-env": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-6.0.3.tgz", @@ -9618,34 +9600,6 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, - "fxa-geodb": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/fxa-geodb/-/fxa-geodb-1.0.4.tgz", - "integrity": "sha512-f+uNgA+6OxmLAHhZvMztwPrByhkaVmSrKcb5Q1TI7Zz/onSQPYCJs388are7nWQdXI94pncqmSPxmT9kOUllEA==", - "requires": { - "bluebird": "3.5.2", - "cron": "1.5.0", - "maxmind": "2.8.0", - "mkdirp": "0.5.1", - "mozlog": "2.2.0", - "request": "2.88.0" - }, - "dependencies": { - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" - }, - "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "requires": { - "minimist": "0.0.8" - } - } - } - }, "gaxios": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-3.0.3.tgz", @@ -12455,15 +12409,6 @@ "integrity": "sha512-APMBEanjybaPzUrfqU0IMU5I0AswKMH7k8OTLs0vvV4KZpExkTkY87nR/zpbuTPj+gARop7aGUbl11pnDfW6xg==", "dev": true }, - "maxmind": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-2.8.0.tgz", - "integrity": "sha512-U3/jQRUoMf4pQ/Tm7JNtGRaM9z82fATB2TiGgs0kEKMPZn/UbOnlyGMRItJ2+KWrwjz9a7PqRzy3/haq9XfUOQ==", - "requires": { - "big-integer": "^1.6.31", - "tiny-lru": "^1.6.1" - } - }, "md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -13021,14 +12966,6 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" }, - "moment-timezone": { - "version": "0.5.28", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.28.tgz", - "integrity": "sha512-TDJkZvAyKIVWg5EtVqRzU97w0Rb0YVbfpqyjgu6GwXCAohVRqwZjf4fOzDE6p1Ch98Sro/8hQQi65WDXW5STPw==", - "requires": { - "moment": ">= 2.9.0" - } - }, "morgan": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.0.tgz", @@ -18835,11 +18772,6 @@ "integrity": "sha1-QFQRqOfmM5/mTbmiNN4R3DHgK9Q=", "dev": true }, - "tiny-lru": { - "version": "1.6.4", - "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-1.6.4.tgz", - "integrity": "sha512-Et+J3Css66XPSLWjLF9wmgbECsGiExlEL+jxsFerTQF6N6dpxswDTPAfIrAbQKO5c1uhgq2xvo5zMk1W+kBDNA==" - }, "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", diff --git a/package.json b/package.json index 66f38dfe..be0b583c 100644 --- a/package.json +++ b/package.json @@ -145,7 +145,6 @@ "configstore": "github:dannycoates/configstore#master", "convict": "^5.2.0", "express": "^4.17.1", - "fxa-geodb": "^1.0.4", "helmet": "^3.23.3", "mkdirp": "^0.5.1", "mozlog": "^2.2.0", diff --git a/server/amplitude.js b/server/amplitude.js index 787026f5..a69717fd 100644 --- a/server/amplitude.js +++ b/server/amplitude.js @@ -3,10 +3,6 @@ const fetch = require('node-fetch'); const config = require('./config'); const pkg = require('../package.json'); -const geoip = config.ip_db - ? require('fxa-geodb')({ dbPath: config.ip_db }) - : () => ({}); - const HOUR = 1000 * 60 * 60; function truncateToHour(timestamp) { @@ -24,20 +20,11 @@ function userId(fileId, ownerId) { return hash.digest('hex').substring(32); } -function location(ip) { - try { - return geoip(ip); - } catch (e) { - return {}; - } -} - function statUploadEvent(data) { - const loc = location(data.ip); const event = { session_id: -1, - country: loc.country, - region: loc.state, + country: data.country, + region: data.state, user_id: userId(data.id, data.owner), app_version: pkg.version, time: truncateToHour(Date.now()), @@ -57,11 +44,10 @@ function statUploadEvent(data) { } function statDownloadEvent(data) { - const loc = location(data.ip); const event = { session_id: -1, - country: loc.country, - region: loc.state, + country: data.country, + region: data.state, user_id: userId(data.id, data.owner), app_version: pkg.version, time: truncateToHour(Date.now()), @@ -77,11 +63,10 @@ function statDownloadEvent(data) { } function statDeleteEvent(data) { - const loc = location(data.ip); const event = { session_id: -1, - country: loc.country, - region: loc.state, + country: data.country, + region: data.state, user_id: userId(data.id, data.owner), app_version: pkg.version, time: truncateToHour(Date.now()), @@ -96,8 +81,16 @@ function statDeleteEvent(data) { return sendBatch([event]); } -function clientEvent(event, ua, language, session_id, deltaT, platform, ip) { - const loc = location(ip); +function clientEvent( + event, + ua, + language, + session_id, + deltaT, + platform, + country, + state +) { const ep = event.event_properties || {}; const up = event.user_properties || {}; const event_properties = { @@ -133,7 +126,7 @@ function clientEvent(event, ua, language, session_id, deltaT, platform, ip) { }; return { app_version: pkg.version, - country: loc.country, + country: country, device_id: event.device_id, event_properties, event_type: event.event_type, @@ -141,7 +134,7 @@ function clientEvent(event, ua, language, session_id, deltaT, platform, ip) { os_name: ua.os.name, os_version: ua.os.version, platform, - region: loc.state, + region: state, session_id, time: event.time + deltaT, user_id: event.user_id, diff --git a/server/routes/delete.js b/server/routes/delete.js index c0b70bd1..e909e90a 100644 --- a/server/routes/delete.js +++ b/server/routes/delete.js @@ -11,6 +11,8 @@ module.exports = async function(req, res) { statDeleteEvent({ id, ip: req.ip, + country: req.geo.country, + state: req.geo.state, owner: meta.owner, download_count: meta.dl, ttl, diff --git a/server/routes/download.js b/server/routes/download.js index 7fe0035b..63de2146 100644 --- a/server/routes/download.js +++ b/server/routes/download.js @@ -31,6 +31,8 @@ module.exports = async function(req, res) { statDownloadEvent({ id, ip: req.ip, + country: req.geo.country, + state: req.geo.state, owner: meta.owner, download_count: dl, ttl, diff --git a/server/routes/index.js b/server/routes/index.js index 7cb64e75..9b9ef5a3 100644 --- a/server/routes/index.js +++ b/server/routes/index.js @@ -32,55 +32,54 @@ module.exports = function(app) { }); if (!IS_DEV) { let csp = { - directives: { - defaultSrc: ["'self'"], - connectSrc: [ - "'self'", - 'wss://*.dev.lcip.org', - 'wss://*.send.nonprod.cloudops.mozgcp.net', - config.base_url.replace(/^https:\/\//, 'wss://'), - 'https://*.dev.lcip.org', - 'https://accounts.firefox.com', - 'https://*.accounts.firefox.com', - 'https://sentry.prod.mozaws.net' - ], - imgSrc: [ - "'self'", - 'https://*.dev.lcip.org', - 'https://firefoxusercontent.com', - 'https://secure.gravatar.com' - ], - scriptSrc: [ - "'self'", - function(req) { - return `'nonce-${req.cspNonce}'`; - } - ], - formAction: ["'none'"], - frameAncestors: ["'none'"], - objectSrc: ["'none'"], - reportUri: '/__cspreport__' - } + directives: { + defaultSrc: ["'self'"], + connectSrc: [ + "'self'", + 'wss://*.dev.lcip.org', + 'wss://*.send.nonprod.cloudops.mozgcp.net', + config.base_url.replace(/^https:\/\//, 'wss://'), + 'https://*.dev.lcip.org', + 'https://accounts.firefox.com', + 'https://*.accounts.firefox.com', + 'https://sentry.prod.mozaws.net' + ], + imgSrc: [ + "'self'", + 'https://*.dev.lcip.org', + 'https://firefoxusercontent.com', + 'https://secure.gravatar.com' + ], + scriptSrc: [ + "'self'", + function(req) { + return `'nonce-${req.cspNonce}'`; + } + ], + formAction: ["'none'"], + frameAncestors: ["'none'"], + objectSrc: ["'none'"], + reportUri: '/__cspreport__' } + }; - csp.directives.connectSrc.push(config.base_url.replace(/^https:\/\//,'wss://')) - if(config.fxa_csp_oauth_url != ""){ - csp.directives.connectSrc.push(config.fxa_csp_oauth_url) - } - if(config.fxa_csp_content_url != "" ){ - csp.directives.connectSrc.push(config.fxa_csp_content_url) - } - if(config.fxa_csp_profile_url != "" ){ - csp.directives.connectSrc.push(config.fxa_csp_profile_url) - } - if(config.fxa_csp_profileimage_url != ""){ - csp.directives.imgSrc.push(config.fxa_csp_profileimage_url) - } - - - app.use( - helmet.contentSecurityPolicy(csp) + csp.directives.connectSrc.push( + config.base_url.replace(/^https:\/\//, 'wss://') ); + if (config.fxa_csp_oauth_url != '') { + csp.directives.connectSrc.push(config.fxa_csp_oauth_url); + } + if (config.fxa_csp_content_url != '') { + csp.directives.connectSrc.push(config.fxa_csp_content_url); + } + if (config.fxa_csp_profile_url != '') { + csp.directives.connectSrc.push(config.fxa_csp_profile_url); + } + if (config.fxa_csp_profileimage_url != '') { + csp.directives.imgSrc.push(config.fxa_csp_profileimage_url); + } + + app.use(helmet.contentSecurityPolicy(csp)); } app.use(function(req, res, next) { @@ -91,6 +90,19 @@ module.exports = function(app) { ); next(); }); + app.use(function(req, res, next) { + try { + // set by the load balancer + const [country, state] = req.header('X-Client-Geo-Location').split(','); + req.geo = { + country, + state + }; + } catch (e) { + req.geo = {}; + } + next(); + }); app.use(bodyParser.json()); app.use(bodyParser.text()); app.get('/', language, pages.index); diff --git a/server/routes/metrics.js b/server/routes/metrics.js index 059e330e..0f6f64aa 100644 --- a/server/routes/metrics.js +++ b/server/routes/metrics.js @@ -12,7 +12,8 @@ module.exports = async function(req, res) { data.session_id + deltaT, deltaT, data.platform, - req.ip + req.geo.country, + req.geo.state ) ); const status = await sendBatch(events); diff --git a/server/routes/ws.js b/server/routes/ws.js index f56fad1d..c32b63a9 100644 --- a/server/routes/ws.js +++ b/server/routes/ws.js @@ -111,6 +111,8 @@ module.exports = function(ws, req) { statUploadEvent({ id: newId, ip: req.ip, + country: req.geo.country, + state: req.geo.state, owner, dlimit, timeLimit,