Merge pull request mozilla#244 from mozilla/load-breaches-on-startup-242

Load breaches on startup 242
smileys2 · Aug 1, 2018 · 7cbcae8 · 7cbcae8
2 parents b21bc23 + 52f1ce0
commit 7cbcae8
Show file tree

Hide file tree

Showing 13 changed files with 68 additions and 289 deletions.
diff --git a/db/migrations/initial_schema.js b/db/migrations/initial_schema.js
@@ -8,23 +8,10 @@ exports.up = knex => {
       table.string("email");
       table.string("verification_token").unique();
       table.boolean("verified").defaultTo(false);
-    })
-    .createTable("breaches", table => {
-      table.increments("id").primary();
-      table.string("name").unique();
-      table.json("meta");
-    })
-    .createTable("breached_hashes", table => {
-      table.increments("id").primary();
-      table.integer("sha1_id").unsigned().references("id").inTable("email_hashes");
-      table.integer("breach_id").unsigned().references("id").inTable("breaches");
-      table.boolean("notified").defaultTo(false);
     });
 };
 
 exports.down = knex => {
   return knex.schema
-    .dropTableIfExists("breached_hashes")
-    .dropTableIfExists("email_hashes")
-    .dropTableIfExists("breaches");
+    .dropTableIfExists("email_hashes");
 };
diff --git a/db/models/breach.js b/db/models/breach.js
diff --git a/db/models/emailhash.js b/db/models/emailhash.js
@@ -1,31 +1,12 @@
 "use strict";
 
 const Model = require("objection").Model;
-const path = require("path");
 
 class EmailHash extends Model {
   // Table name is the only required property.
   static get tableName() {
     return "email_hashes";
   }
-
-  static get relationMappings() {
-    return {
-      breaches: {
-        relation: Model.ManyToManyRelation,
-        modelClass: path.join(__dirname, "breach"),
-        join: {
-          from: "email_hashes.id",
-          through: {
-            from: "breached_hashes.sha1_id",
-            to: "breached_hashes.breach_id",
-            extra: ["notified"],
-          },
-          to: "breaches.id",
-        },
-      },
-    };
-  }
 }
 
 module.exports = EmailHash;
diff --git a/db/utils.js b/db/utils.js
@@ -6,39 +6,16 @@ const Knex = require("knex");
 const knexConfig = require("./knexfile");
 const { Model } = require("objection");
 
-const Breach = require("./models/breach");
 const EmailHash = require("./models/emailhash");
 
-// FIXME: TODO: resolve circular depenency b/w db/utils and hibp
-// const HIBP = require("../hibp");
+const HIBP = require("../hibp");
 const getSha1 = require("../sha1-utils");
 
 const knex = Knex(knexConfig);
 Model.knex(knex);
 
 
 const DBUtils = {
-  async createBreach(name, meta) {
-    try {
-      return await Breach
-        .query()
-        .insert({ name, meta });
-    } catch(e) {
-      console.error(e);
-      if (e.code && e.code === "23505") {
-        // Duplicate error, silently log.
-        console.error(`Duplicate breach: ${name}`);
-        return;
-      }
-
-      throw e;
-    }
-  },
-
-  async deleteBreach(id) {
-    await Breach.query().deleteById(id);
-  },
-
   async addUnverifiedEmailHash(email) {
     return await EmailHash.query().insert(
       {email: email, sha1: getSha1(email), verification_token: uuidv4(), verified: false}
@@ -110,8 +87,7 @@ const DBUtils = {
   },
 
   async verifySubscriber(emailHash) {
-    // FIXME: TODO: resolve circular depenency b/w db/utils and hibp
-    // await HIBP.subscribeHash(emailHash.sha1);
+    await HIBP.subscribeHash(emailHash.sha1);
     return await emailHash.$query().patch({ verified: true }).returning("*");
   },
 
@@ -127,85 +103,10 @@ const DBUtils = {
     });
   },
 
-  async getSubscribersForBreach(breach) {
-    return await breach
-      .$relatedQuery("email_hashes")
-      .whereNotNull("email")
-      .where("notified", false);
-  },
-
   async getSubscribersByHashes(hashes) {
     return await EmailHash.query().whereIn("sha1", hashes).andWhere("verified", "=", true);
   },
 
-  async addBreachedHash(breachName, sha1) {
-    const addedEmailHash = await this._addEmailHash(sha1);
-
-    const breachesByName = await Breach
-      .query()
-      .where("name", breachName);
-
-    if (!breachesByName.length) {
-      return;
-    }
-
-    const breach = breachesByName[0];
-
-    const relatedSha1 = await breach
-      .$relatedQuery("email_hashes")
-      .where("sha1", sha1);
-
-    if (relatedSha1.length) {
-      // Already associated, nothing to do.
-      return;
-    }
-
-    return await breach
-      .$relatedQuery("email_hashes")
-      .relate(addedEmailHash.id);
-  },
-
-  async addBreachedEmail(breachName, email) {
-    return await this.addBreachedHash(breachName, getSha1(email));
-  },
-
-  async setBreachedHashNotified(breach, email) {
-    return await breach
-      .$relatedQuery("email_hashes")
-      .where("sha1", getSha1(email))
-      .patch({ notified: true });
-  },
-
-  async getBreachesForHash(sha1) {
-    return await this._getSha1EntryAndDo(sha1, async aEntry => {
-      return await aEntry
-        .$relatedQuery("breaches")
-        .orderBy("name");
-    }, async () => {
-      return [];
-    });
-  },
-
-  async getBreachesForHashPrefix(sha1Prefix) {
-    return await this._getSha1EntriesFromPrefixAndDo(sha1Prefix, async aEntries => {
-      return aEntries;
-    }, async () => {
-      return [];
-    });
-  },
-
-  getBreachesForEmail(email) {
-    return this.getBreachesForHash(getSha1(email));
-  },
-
-  async getBreachByName(breachName) {
-    return (await Breach.query().where("name", breachName))[0];
-  },
-
-  async getBreachesByNames(breachNames) {
-    return await Breach.query().where("name", "in", breachNames);
-  },
-
 };
 
 module.exports = DBUtils;
diff --git a/hibp.js b/hibp.js
@@ -1,50 +1,82 @@
 "use strict";
 
 const got = require("got");
+const createDOMPurify = require("dompurify");
+const { JSDOM } = require("jsdom");
 
 const AppConstants = require("./app-constants");
-const DBUtils = require("./db/utils");
 const pkg = require("./package.json");
 
 
+const DOMPurify = createDOMPurify((new JSDOM("")).window);
 const HIBP_USER_AGENT = `${pkg.name}/${pkg.version}`;
 
 
 const HIBP = {
-  async req(path, options = {}) {
-    // Construct HIBP url and standard headers
-    const url = `${AppConstants.HIBP_KANON_API_ROOT}${path}?code=${encodeURIComponent(AppConstants.HIBP_KANON_API_TOKEN)}`;
+  _addStandardOptions (options = {}) {
     const hibpOptions = {
       headers: {
         "User-Agent": HIBP_USER_AGENT,
       },
       json: true,
     };
-    const reqOptions = Object.assign(options, hibpOptions);
+    return Object.assign(options, hibpOptions);
+  },
+
+  async req(path, options = {}) {
+    const url = `${AppConstants.HIBP_API_ROOT}${path}?code=${encodeURIComponent(AppConstants.HIBP_API_TOKEN)}`;
+    const reqOptions = this._addStandardOptions(options);
     return await got(url, reqOptions);
   },
 
-  async getBreachesForEmail(sha1) {
+  async kAnonReq(path, options = {}) {
+    // Construct HIBP url and standard headers
+    const url = `${AppConstants.HIBP_KANON_API_ROOT}${path}?code=${encodeURIComponent(AppConstants.HIBP_KANON_API_TOKEN)}`;
+    const reqOptions = this._addStandardOptions(options);
+    return await got(url, reqOptions);
+  },
+
+  async loadBreachesIntoApp(app) {
+    console.log("Loading breaches from HIBP into app.locals");
+    try {
+      const breachesResponse = await this.req("/breaches");
+      const breaches = [];
+
+      for (const breach of breachesResponse.body) {
+        // const breach = breachesResponse.body[breachIndex];
+        // purify the description
+        breach.Description = DOMPurify.sanitize(breach.Description, {ALLOWED_TAGS: []});
+        breaches.push(breach);
+      }
+      app.locals.breaches = breaches;
+    } catch (error) {
+      console.error(error);
+    }
+    console.log("Done loading breaches.");
+  },
+
+  async getBreachesForEmail(sha1, allBreaches) {
     let foundBreaches = [];
     const sha1Prefix = sha1.slice(0, 6).toUpperCase();
     const path = `/breachedaccount/range/${sha1Prefix}`;
 
     try {
-      const response = await HIBP.req(path);
+      const response = await this.kAnonReq(path);
       // Parse response body, format:
       // [
       //   {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
       //   {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
       // ]
       for (const breachedAccount of response.body) {
         if (sha1.toUpperCase() === sha1Prefix + breachedAccount.hashSuffix) {
-          foundBreaches = await DBUtils.getBreachesByNames(breachedAccount.websites);
+          foundBreaches = allBreaches.filter(breach => breachedAccount.websites.includes(breach.Name));
+          break;
         }
       }
     } catch (error) {
       console.error(error);
     }
-    return foundBreaches.filter(({meta}) => meta.IsVerified && !meta.IsRetired && !meta.IsSensitive && !meta.IsSpamList);
+    return foundBreaches.filter(breach => breach.IsVerified && !breach.IsRetired && !breach.IsSensitive && !breach.IsSpamList);
   },
 
   async subscribeHash(sha1) {
@@ -57,7 +89,7 @@ const HIBP = {
 
     let response;
     try {
-      response = await HIBP.req(path, options);
+      response = await this.kAnonReq(path, options);
     } catch (error) {
       console.error(error);
     }

diff --git a/routes/scan.js b/routes/scan.js
@@ -24,7 +24,7 @@ router.post("/", urlEncodedParser, async (req, res) => {
   if (req.body.signup) {
     signup = "checkboxChecked";
   }
-  const foundBreaches = await HIBP.getBreachesForEmail(emailHash);
+  const foundBreaches = await HIBP.getBreachesForEmail(emailHash, req.app.locals.breaches);
 
   res.render("scan", {
     title: "Firefox Breach Alerts: Scan Results",

diff --git a/scripts/add-breached-emails.js b/scripts/add-breached-emails.js