diff --git a/db/migrations/initial_schema.js b/db/migrations/initial_schema.js index 9658738912c..a76c7d5f1bd 100644 --- a/db/migrations/initial_schema.js +++ b/db/migrations/initial_schema.js @@ -8,23 +8,10 @@ exports.up = knex => { table.string("email"); table.string("verification_token").unique(); table.boolean("verified").defaultTo(false); - }) - .createTable("breaches", table => { - table.increments("id").primary(); - table.string("name").unique(); - table.json("meta"); - }) - .createTable("breached_hashes", table => { - table.increments("id").primary(); - table.integer("sha1_id").unsigned().references("id").inTable("email_hashes"); - table.integer("breach_id").unsigned().references("id").inTable("breaches"); - table.boolean("notified").defaultTo(false); }); }; exports.down = knex => { return knex.schema - .dropTableIfExists("breached_hashes") - .dropTableIfExists("email_hashes") - .dropTableIfExists("breaches"); + .dropTableIfExists("email_hashes"); }; diff --git a/db/models/breach.js b/db/models/breach.js deleted file mode 100644 index 13ab8e5eadf..00000000000 --- a/db/models/breach.js +++ /dev/null @@ -1,31 +0,0 @@ -"use strict"; - -const Model = require("objection").Model; -const path = require("path"); - -class Breach extends Model { - // Table name is the only required property. - static get tableName() { - return "breaches"; - } - - static get relationMappings() { - return { - email_hashes: { - relation: Model.ManyToManyRelation, - modelClass: path.join(__dirname, "emailhash"), - join: { - from: "breaches.id", - through: { - from: "breached_hashes.breach_id", - to: "breached_hashes.sha1_id", - extra: ["notified"], - }, - to: "email_hashes.id", - }, - }, - }; - } -} - -module.exports = Breach; diff --git a/db/models/emailhash.js b/db/models/emailhash.js index c4d16c5efe6..8ee2f07c32b 100644 --- a/db/models/emailhash.js +++ b/db/models/emailhash.js @@ -1,31 +1,12 @@ "use strict"; const Model = require("objection").Model; -const path = require("path"); class EmailHash extends Model { // Table name is the only required property. static get tableName() { return "email_hashes"; } - - static get relationMappings() { - return { - breaches: { - relation: Model.ManyToManyRelation, - modelClass: path.join(__dirname, "breach"), - join: { - from: "email_hashes.id", - through: { - from: "breached_hashes.sha1_id", - to: "breached_hashes.breach_id", - extra: ["notified"], - }, - to: "breaches.id", - }, - }, - }; - } } module.exports = EmailHash; diff --git a/db/utils.js b/db/utils.js index 9bf9a74f1c1..f63a99728fd 100644 --- a/db/utils.js +++ b/db/utils.js @@ -6,11 +6,9 @@ const Knex = require("knex"); const knexConfig = require("./knexfile"); const { Model } = require("objection"); -const Breach = require("./models/breach"); const EmailHash = require("./models/emailhash"); -// FIXME: TODO: resolve circular depenency b/w db/utils and hibp -// const HIBP = require("../hibp"); +const HIBP = require("../hibp"); const getSha1 = require("../sha1-utils"); const knex = Knex(knexConfig); @@ -18,27 +16,6 @@ Model.knex(knex); const DBUtils = { - async createBreach(name, meta) { - try { - return await Breach - .query() - .insert({ name, meta }); - } catch(e) { - console.error(e); - if (e.code && e.code === "23505") { - // Duplicate error, silently log. - console.error(`Duplicate breach: ${name}`); - return; - } - - throw e; - } - }, - - async deleteBreach(id) { - await Breach.query().deleteById(id); - }, - async addUnverifiedEmailHash(email) { return await EmailHash.query().insert( {email: email, sha1: getSha1(email), verification_token: uuidv4(), verified: false} @@ -110,8 +87,7 @@ const DBUtils = { }, async verifySubscriber(emailHash) { - // FIXME: TODO: resolve circular depenency b/w db/utils and hibp - // await HIBP.subscribeHash(emailHash.sha1); + await HIBP.subscribeHash(emailHash.sha1); return await emailHash.$query().patch({ verified: true }).returning("*"); }, @@ -127,85 +103,10 @@ const DBUtils = { }); }, - async getSubscribersForBreach(breach) { - return await breach - .$relatedQuery("email_hashes") - .whereNotNull("email") - .where("notified", false); - }, - async getSubscribersByHashes(hashes) { return await EmailHash.query().whereIn("sha1", hashes).andWhere("verified", "=", true); }, - async addBreachedHash(breachName, sha1) { - const addedEmailHash = await this._addEmailHash(sha1); - - const breachesByName = await Breach - .query() - .where("name", breachName); - - if (!breachesByName.length) { - return; - } - - const breach = breachesByName[0]; - - const relatedSha1 = await breach - .$relatedQuery("email_hashes") - .where("sha1", sha1); - - if (relatedSha1.length) { - // Already associated, nothing to do. - return; - } - - return await breach - .$relatedQuery("email_hashes") - .relate(addedEmailHash.id); - }, - - async addBreachedEmail(breachName, email) { - return await this.addBreachedHash(breachName, getSha1(email)); - }, - - async setBreachedHashNotified(breach, email) { - return await breach - .$relatedQuery("email_hashes") - .where("sha1", getSha1(email)) - .patch({ notified: true }); - }, - - async getBreachesForHash(sha1) { - return await this._getSha1EntryAndDo(sha1, async aEntry => { - return await aEntry - .$relatedQuery("breaches") - .orderBy("name"); - }, async () => { - return []; - }); - }, - - async getBreachesForHashPrefix(sha1Prefix) { - return await this._getSha1EntriesFromPrefixAndDo(sha1Prefix, async aEntries => { - return aEntries; - }, async () => { - return []; - }); - }, - - getBreachesForEmail(email) { - return this.getBreachesForHash(getSha1(email)); - }, - - async getBreachByName(breachName) { - return (await Breach.query().where("name", breachName))[0]; - }, - - async getBreachesByNames(breachNames) { - return await Breach.query().where("name", "in", breachNames); - }, - }; module.exports = DBUtils; diff --git a/hibp.js b/hibp.js index 562b11d1567..e15e2e46023 100644 --- a/hibp.js +++ b/hibp.js @@ -1,36 +1,67 @@ "use strict"; const got = require("got"); +const createDOMPurify = require("dompurify"); +const { JSDOM } = require("jsdom"); const AppConstants = require("./app-constants"); -const DBUtils = require("./db/utils"); const pkg = require("./package.json"); +const DOMPurify = createDOMPurify((new JSDOM("")).window); const HIBP_USER_AGENT = `${pkg.name}/${pkg.version}`; const HIBP = { - async req(path, options = {}) { - // Construct HIBP url and standard headers - const url = `${AppConstants.HIBP_KANON_API_ROOT}${path}?code=${encodeURIComponent(AppConstants.HIBP_KANON_API_TOKEN)}`; + _addStandardOptions (options = {}) { const hibpOptions = { headers: { "User-Agent": HIBP_USER_AGENT, }, json: true, }; - const reqOptions = Object.assign(options, hibpOptions); + return Object.assign(options, hibpOptions); + }, + + async req(path, options = {}) { + const url = `${AppConstants.HIBP_API_ROOT}${path}?code=${encodeURIComponent(AppConstants.HIBP_API_TOKEN)}`; + const reqOptions = this._addStandardOptions(options); return await got(url, reqOptions); }, - async getBreachesForEmail(sha1) { + async kAnonReq(path, options = {}) { + // Construct HIBP url and standard headers + const url = `${AppConstants.HIBP_KANON_API_ROOT}${path}?code=${encodeURIComponent(AppConstants.HIBP_KANON_API_TOKEN)}`; + const reqOptions = this._addStandardOptions(options); + return await got(url, reqOptions); + }, + + async loadBreachesIntoApp(app) { + console.log("Loading breaches from HIBP into app.locals"); + try { + const breachesResponse = await this.req("/breaches"); + const breaches = []; + + for (const breach of breachesResponse.body) { + // const breach = breachesResponse.body[breachIndex]; + // purify the description + breach.Description = DOMPurify.sanitize(breach.Description, {ALLOWED_TAGS: []}); + breaches.push(breach); + } + app.locals.breaches = breaches; + } catch (error) { + console.error(error); + } + console.log("Done loading breaches."); + }, + + async getBreachesForEmail(sha1, allBreaches) { let foundBreaches = []; const sha1Prefix = sha1.slice(0, 6).toUpperCase(); const path = `/breachedaccount/range/${sha1Prefix}`; try { - const response = await HIBP.req(path); + const response = await this.kAnonReq(path); // Parse response body, format: // [ // {"hashSuffix":,"websites":[,...]}, @@ -38,13 +69,14 @@ const HIBP = { // ] for (const breachedAccount of response.body) { if (sha1.toUpperCase() === sha1Prefix + breachedAccount.hashSuffix) { - foundBreaches = await DBUtils.getBreachesByNames(breachedAccount.websites); + foundBreaches = allBreaches.filter(breach => breachedAccount.websites.includes(breach.Name)); + break; } } } catch (error) { console.error(error); } - return foundBreaches.filter(({meta}) => meta.IsVerified && !meta.IsRetired && !meta.IsSensitive && !meta.IsSpamList); + return foundBreaches.filter(breach => breach.IsVerified && !breach.IsRetired && !breach.IsSensitive && !breach.IsSpamList); }, async subscribeHash(sha1) { @@ -57,7 +89,7 @@ const HIBP = { let response; try { - response = await HIBP.req(path, options); + response = await this.kAnonReq(path, options); } catch (error) { console.error(error); } diff --git a/routes/scan.js b/routes/scan.js index 2ed0db845da..3ad4a6f05b8 100644 --- a/routes/scan.js +++ b/routes/scan.js @@ -24,7 +24,7 @@ router.post("/", urlEncodedParser, async (req, res) => { if (req.body.signup) { signup = "checkboxChecked"; } - const foundBreaches = await HIBP.getBreachesForEmail(emailHash); + const foundBreaches = await HIBP.getBreachesForEmail(emailHash, req.app.locals.breaches); res.render("scan", { title: "Firefox Breach Alerts: Scan Results", diff --git a/scripts/add-breached-emails.js b/scripts/add-breached-emails.js deleted file mode 100644 index e6d3e022c5a..00000000000 --- a/scripts/add-breached-emails.js +++ /dev/null @@ -1,52 +0,0 @@ -"use strict"; - -const arg = require("arg"); - -require("dotenv").load(); - -const DBUtils = require("../db/utils"); - - -const args = arg({ - "--extraTestEmail": String, - "--help": Boolean, -}); - -if (args["--help"]) { - console.log("Usage: node add-breached-emails.js [--extraTestEmail=<...>]"); - console.log("Adds test[1-3]@test.com emails to LinkedIn, Adobe, and AllMusic breaches."); - console.log("--extraTestEmail also adds the supplied test email address and includes it in the LinkedIn, Adobe, and AllMusic breaches."); - process.exit(); -} - -const sampleBreaches = [ - { - name: "LinkedIn", - emails: [ "test1@test.com", "test2@test.com" ], - }, - { - name: "Adobe", - emails: [ "test2@test.com", "test3@test.com" ], - }, - { - name: "AllMusic", - emails: [ "test3@test.com", "test1@test.com" ], - }, -]; - -(async () => { - for (const sB of sampleBreaches) { - for (const e of sB.emails) { - await DBUtils.addBreachedEmail(sB.name, e); - if (args["--extraTestEmail"]) { - DBUtils.addBreachedEmail(sB.name, args["--extraTestEmail"]); - } - } - } - - const testEmail = "test1@test.com"; - await DBUtils.deleteBreach(999999); - const breach = await DBUtils.getBreachByName("LinkedIn"); - await DBUtils.setBreachedHashNotified(breach, testEmail); - process.exit(); -})(); diff --git a/scripts/load-breaches.js b/scripts/load-breaches.js deleted file mode 100644 index a1b568c7719..00000000000 --- a/scripts/load-breaches.js +++ /dev/null @@ -1,48 +0,0 @@ -"use strict"; - -const got = require("got"); -const createDOMPurify = require("dompurify"); -const { JSDOM } = require("jsdom"); - -const AppConstants = require("../app-constants"); -const DBUtils = require("../db/utils"); -const pkg = require("../package.json"); - -const HIBP_USER_AGENT = `${pkg.name}/${pkg.version}`; - - -const DOMPurify = createDOMPurify((new JSDOM("")).window); - -async function handleBreachesResponse(response) { - try { - const breachesJSON = JSON.parse(response.body); - - for (const breach of breachesJSON) { - // purify the description going into the DB - breach.Description = DOMPurify.sanitize(breach.Description, {ALLOWED_TAGS: []}); - await DBUtils.createBreach(breach.Name, breach); - } - } catch (error) { - console.error(error); - process.exit(1); - } -} - -(async () => { - try { - const breachesResponse = await got( - `${AppConstants.HIBP_API_ROOT}/breaches`, - { - headers: { - "User-Agent": HIBP_USER_AGENT, - }, - } - ); - await handleBreachesResponse(breachesResponse); - } catch (error) { - console.error(error); - process.exit(1); - } - console.log("Done handling breaches response."); - process.exit(); -})(); diff --git a/server.js b/server.js index 3ccd0e4a282..e96f2ca14a0 100644 --- a/server.js +++ b/server.js @@ -9,6 +9,7 @@ const sessions = require("client-sessions"); const EmailUtils = require("./email-utils"); const HBSHelpers = require("./hbs-helpers"); +const HIBP = require("./hibp"); const DockerflowRoutes = require("./routes/dockerflow"); const HibpRoutes = require("./routes/hibp"); @@ -33,6 +34,14 @@ if (app.get("env") !== "dev") { }); } +(async () => { + try { + await HIBP.loadBreachesIntoApp(app); + } catch (error) { + console.error(error); + } +})(); + // Use helmet to set security headers app.use(helmet()); app.use(helmet.contentSecurityPolicy({ diff --git a/views/email/breach_notification.hbs b/views/email/breach_notification.hbs index da110f7aec3..6374657a3dd 100644 --- a/views/email/breach_notification.hbs +++ b/views/email/breach_notification.hbs @@ -1,2 +1,2 @@ {{!< email }} - Uh-oh, {{ email }} was involved in {{ breach.meta.Title }} data breach. + Uh-oh, {{ email }} was involved in {{ breach.Title }} data breach. diff --git a/views/partials/banner_breach.hbs b/views/partials/banner_breach.hbs index 8c0d894efb8..5c0ceaa8a24 100644 --- a/views/partials/banner_breach.hbs +++ b/views/partials/banner_breach.hbs @@ -2,17 +2,17 @@ {{/if}} diff --git a/views/partials/compromised_accounts.hbs b/views/partials/compromised_accounts.hbs index c76ece30e84..d9d27c84763 100644 --- a/views/partials/compromised_accounts.hbs +++ b/views/partials/compromised_accounts.hbs @@ -3,13 +3,13 @@ {{#each foundBreaches }} {{/each}} diff --git a/views/partials/protect_yourself.hbs b/views/partials/protect_yourself.hbs index 641dad3597f..298087692e0 100644 --- a/views/partials/protect_yourself.hbs +++ b/views/partials/protect_yourself.hbs @@ -2,7 +2,7 @@

Protect yourself from hackers.
Start here.

{{#if breach }} -

See if your account was exposed in the {{ breach.meta.Title }} breach or other data breaches.

+

See if your account was exposed in the {{ breach.Title }} breach or other data breaches.

{{else}}

The first step to keeping your online accounts safe is knowing what you’re up against. Enter your email to find out if your accounts have been compromised.

{{/if}}