for mozilla#1182: lowercase email addresses for all HIBP calls

for mozilla#1182: script to lower-case sha1 db records
smileys2 · Sep 16, 2019 · e921363 · e921363
1 parent 990ba69
commit e921363
Show file tree

Hide file tree

Showing 6 changed files with 94 additions and 7 deletions.
diff --git a/controllers/user.js b/controllers/user.js
@@ -96,7 +96,7 @@ function _checkForDuplicateEmail(sessionUser, email) {
 
 async function add(req, res) {
   const sessionUser = await _requireSessionUser(req);
-  const email = req.body.email.toLowerCase();
+  const email = req.body.email;
   if (!email || !isemail.validate(email)) {
     throw new FluentError("user-add-invalid-email");
   }

diff --git a/db/DB.js b/db/DB.js
@@ -158,6 +158,7 @@ const DB = {
           const res = await knex("subscribers")
             .update({
               primary_email: email,
+              primary_sha1: getSha1(email.toLowerCase()),
               primary_verified: verified,
               updated_at: knex.fn.now(),
             })
@@ -171,7 +172,7 @@ const DB = {
         // Always add a verification_token value
         const verification_token = uuidv4();
         const res = await knex("subscribers")
-          .insert({ primary_sha1: sha1, primary_email: email, signup_language, primary_verification_token: verification_token, primary_verified: verified })
+          .insert({ primary_sha1: getSha1(email.toLowerCase()), primary_email: email, signup_language, primary_verification_token: verification_token, primary_verified: verified })
           .returning("*");
         return res[0];
       });

diff --git a/scripts/lowercase-all-records.js b/scripts/lowercase-all-records.js
@@ -0,0 +1,42 @@
+"use strict";
+
+
+const Knex = require("knex");
+const knexConfig = require("../db/knexfile");
+const knex = Knex(knexConfig);
+
+const HIBP = require("../hibp");
+const getSha1 = require("../sha1-utils");
+
+
+async function subscribeLowercaseHashToHIBP(emailAddress) {
+  const lowerCasedEmail = emailAddress.toLowerCase();
+  const lowerCasedSha1 = getSha1(lowerCasedEmail);
+  await HIBP.subscribeHash(lowerCasedSha1);
+  return lowerCasedSha1;
+}
+
+
+(async () => {
+  const subRecordsWithUpperChars = await knex("subscribers")
+    .whereRaw("primary_email != lower(primary_email)");
+  for (const subRecord of subRecordsWithUpperChars) {
+    const lowerCasedSha1 = await subscribeLowercaseHashToHIBP(subRecord.primary_email);
+    await knex("subscribers")
+      .update({
+        primary_sha1: lowerCasedSha1,
+      })
+      .where("id", subRecord.id);
+  }
+
+  const emailRecordsWithUpperChars = await knex("email_addresses")
+    .whereRaw("email != lower(email)");
+  for (const emailRecord of emailRecordsWithUpperChars) {
+    const lowerCasedSha1 = await subscribeLowercaseHashToHIBP(emailRecord.email);
+    await knex("email_addresses")
+      .update({
+        sha1: lowerCasedSha1,
+      })
+      .where("id", emailRecord.id);
+  }
+})();
diff --git a/tests/controllers/hibp.test.js b/tests/controllers/hibp.test.js
@@ -116,7 +116,7 @@ test("notify POST for subscriber with no signup_language should default to en",
   LocaleUtils.fluentFormat = jest.fn();
   HIBPLib.subscribeHash = jest.fn();
 
-  const testEmail = "subscriberWithoutLanguage@test.com";
+  const testEmail = "subscriberwithoutlanguage@test.com";
 
   await DB.addSubscriber(testEmail);
 

diff --git a/tests/controllers/user.test.js b/tests/controllers/user.test.js
@@ -30,7 +30,7 @@ function expectResponseRenderedSubpagePartial(resp, partial) {
 
 
 test("user add POST with email adds unverified subscriber and sends verification email", async () => {
-  const testUserAddEmail = "addingNewEmail@test.com";
+  const testUserAddEmail = "addingnewemail@test.com";
   const testSubscriberEmail = "firefoxaccount@test.com";
   const testSubscriber = await DB.getSubscriberByEmail(testSubscriberEmail);
 
@@ -73,6 +73,40 @@ test("user add POST with email adds unverified subscriber and sends verification
 });
 
 
+test("user add POST with upperCaseAddress adds email_address record with lowercaseaddress sha1", async () => {
+  const testUserAddEmail = "addingUpperCaseEmail@test.com";
+  const testSubscriberEmail = "firefoxaccount@test.com";
+  const testSubscriber = await DB.getSubscriberByEmail(testSubscriberEmail);
+
+  // Set up mocks
+  const req = httpMocks.createRequest({
+    method: "POST",
+    url: "/user/add",
+    body: { email: testUserAddEmail },
+    session: { user: testSubscriber },
+    fluentFormat: jest.fn(),
+    headers: {
+      referer: "",
+    },
+  });
+  const resp = httpMocks.createResponse();
+  EmailUtils.sendEmail.mockResolvedValue(true);
+
+  // Call code-under-test
+  await user.add(req, resp);
+
+  // Check expectations
+  expect(resp.statusCode).toEqual(302);
+  expect(testSubscriber.primary_email).toEqual(testSubscriberEmail);
+
+  const testSubscriberEmailAddressRecords = await DB.getUserEmails(testSubscriber.id);
+  const testSubscriberEmailAddresses = testSubscriberEmailAddressRecords.map(record => record.email);
+  expect(testSubscriberEmailAddresses.includes(testUserAddEmail)).toBeTruthy();
+  const testSubscriberEmailAddressHashes = testSubscriberEmailAddressRecords.map(record => record.sha1);
+  expect(testSubscriberEmailAddressHashes.includes(getSha1(testUserAddEmail))).toBeTruthy();
+});
+
+
 test("user resendEmail with valid session and email id resets email_address record and sends new verification email", async () => {
   const testSubscriberEmail = TEST_SUBSCRIBERS.firefox_account.primary_email;
   const testSubscriber = await DB.getSubscriberByEmail(testSubscriberEmail);

diff --git a/tests/db.test.js b/tests/db.test.js
@@ -100,7 +100,7 @@ test("addSubscriber invalid argument", async () => {
 
 
 test("addSubscriber accepts email, language and returns verified subscriber", async () => {
-  const testEmail = "newFirefoxAccount@test.com";
+  const testEmail = "newfirefoxaccount@test.com";
 
   const verifiedSubscriber = await DB.addSubscriber(testEmail);
 
@@ -111,7 +111,7 @@ test("addSubscriber accepts email, language and returns verified subscriber", as
 
 
 test("addSubscriber with existing email updates updated_at", async () => {
-  const testEmail = "newFirefoxAccount@test.com";
+  const testEmail = "newfirefoxaccount@test.com";
 
   let verifiedSubscriber = await DB.addSubscriber(testEmail);
 
@@ -131,6 +131,16 @@ test("addSubscriber with existing email updates updated_at", async () => {
 });
 
 
+test("addSubscriber accepts upperCasedAddress, and returns verified subscriber with lowercase address hash", async () => {
+  const testEmail = "upperCasedAddress@test.com";
+
+  const verifiedSubscriber = await DB.addSubscriber(testEmail);
+
+  expect(verifiedSubscriber.primary_email).toBe(testEmail);
+  expect(verifiedSubscriber.primary_sha1).toBe(getSha1(testEmail.toLowerCase()));
+});
+
+
 test("setBreachesLastShown updates column and returns subscriber", async() => {
   const startingSubscriber = await DB.getSubscriberByEmail("firefoxaccount@test.com");
 
@@ -168,7 +178,7 @@ test("removeSubscriber accepts subscriber and removes everything from subscriber
 
 
 test("removeEmail accepts email and removes from subscribers table", async () => {
-  const testEmail = "removingFirefoxAccount@test.com";
+  const testEmail = "removingfirefoxaccount@test.com";
 
   const verifiedSubscriber = await DB.addSubscriber(testEmail);
   const subscribers = await DB.getSubscribersByHashes([getSha1(testEmail)]);