start load-breaches.js

smileys2 · May 15, 2018 · f09209e · f09209e
1 parent cb86a26
commit f09209e
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 24 deletions.
diff --git a/app-constants.js b/app-constants.js
@@ -18,6 +18,7 @@ const kEnvironmentVariables = [
   "OAUTH_PROFILE_URI",
   "OAUTH_CLIENT_ID",
   "OAUTH_CLIENT_SECRET",
+  "HIBP_STUB_API_ROOT",
   "HIBP_API_ROOT",
   "HIBP_API_TOKEN",
   "AWS_ACCESS_KEY_ID",

diff --git a/db/utils.js b/db/utils.js
@@ -18,17 +18,15 @@ Model.knex(knex);
 const DBUtils = {
   async createBreach(name, meta) {
     try {
-      return await Breach
-        .query()
-        .insert({ name, meta });
+      const insertedBreach = await Breach.query().insert({ name, meta });
+      return insertedBreach;
     } catch(e) {
+      console.error(e);
       if (e.code && e.code === "23505") {
         // Duplicate error, silently log.
-        console.log(`Duplicate breach: ${name}`);
+        console.error(`Duplicate breach: ${name}`);
         return;
       }
-
-      throw e;
     }
   },
 

diff --git a/scripts/load-breaches.js b/scripts/load-breaches.js
@@ -0,0 +1,48 @@
+"use strict";
+
+const got = require("got");
+
+const AppConstants = require("../app-constants");
+const DBUtils = require("../db/utils");
+const pkg = require("../package.json");
+
+const HIBP_AUTH = `Bearer ${AppConstants.HIBP_API_TOKEN}`;
+const HIBP_USER_AGENT = `${pkg.name}/${pkg.version}`;
+
+
+async function handleBreachesResponse(response) {
+  try {
+    const breachesJSON = JSON.parse(response.body);
+
+    for (const breach of breachesJSON) {
+      await DBUtils.createBreach(breach.Name, breach);
+    }
+  } catch (error) {
+    debugger;
+    console.error(error);
+    // We can `process.exit()` here since it's a CLI script.
+    // eslint-disable-next-line no-process-exit
+    process.exit(1);
+  }
+}
+
+(async () => {
+  try {
+    const breachesResponse = await got(
+      `${AppConstants.HIBP_API_ROOT}/breaches`,
+      {
+        headers: {
+          "User-Agent": HIBP_USER_AGENT,
+        },
+      }
+    );
+    await handleBreachesResponse(breachesResponse);
+  } catch (error) {
+    console.error(error);
+    // We can `process.exit()` here since it's a CLI script.
+    // eslint-disable-next-line no-process-exit
+    process.exit(1);
+  }
+  console.log("Done handling breaches response.");
+  process.exit();
+})();
diff --git a/tests/fixtures/make-breach-with-emails.js b/tests/fixtures/make-breach-with-emails.js
@@ -9,37 +9,22 @@ const extraTestEmail = process.argv[2];
 
 const sampleBreaches = [
   {
-    name: "Linkedin",
-    meta: {
-      date: "5 May 2012",
-      dataClasses: "Email addresses, Passwords",
-      acCount: "164,611,595",
-    },
+    name: "LinkedIn",
     emails: [ "test1@test.com", "test2@test.com" ],
   },
   {
     name: "Adobe",
-    meta: {
-      date: "4 October 2013",
-      dataClasses: "Email addresses, Password hints, Passwords, Usernames",
-      acCount: "152,445,165",
-    },
     emails: [ "test2@test.com", "test3@test.com" ],
   },
   {
     name: "AllMusic",
-    meta: {
-      date: "6 December 2015",
-      dataClasses: "Email addresses, IP addresses, Passwords, Usernames, Website activity",
-      acCount: "1,436,486",
-    },
     emails: [ "test3@test.com", "test1@test.com" ],
   },
 ];
 
 (async () => {
   for (const sB of sampleBreaches) {
-    await DBUtils.createBreach(sB.name, sB.meta);
+    const breach = await DBUtils.getBreachByName(sB.name);
     for (const e of sB.emails) {
       await DBUtils.addBreachedEmail(sB.name, e);
       if (extraTestEmail) {
@@ -53,7 +38,7 @@ const sampleBreaches = [
   await DBUtils.deleteBreach(999999);
   console.log(`\n\n${testEmail} was found in the following breaches:\n`);
   console.log(foundBreaches.map(b => b.name));
-  const breach = await DBUtils.getBreachByName("Linkedin");
+  const breach = await DBUtils.getBreachByName("LinkedIn");
   console.log(breach);
   await DBUtils.setBreachedHashNotified(breach, testEmail);
   // eslint-disable-next-line no-process-exit