use knex-paginate to paginate pre-FXA query

smileys2 · Nov 20, 2019 · 01715ad · 01715ad
1 parent ed0036e
commit 01715ad
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 31 deletions.
diff --git a/db/DB.js b/db/DB.js
@@ -4,6 +4,7 @@
 // eslint-disable-next-line node/no-extraneous-require
 const uuidv4 = require("uuid/v4");
 const Knex = require("knex");
+const { attachPaginate } = require("knex-paginate");
 
 const { FluentError } = require("../locale-utils");
 const AppConstants = require("../app-constants");
@@ -16,6 +17,8 @@ const mozlog = require("../log");
 const knexConfig = require("./knexfile");
 
 let knex = Knex(knexConfig);
+attachPaginate();
+
 const log = mozlog("DB");
 
 
@@ -79,8 +82,11 @@ const DB = {
     return subscriberAndEmails;
   },
 
-  async getPreFxaSubscribers() {
-    return await knex("subscribers").whereRaw("(fxa_uid = '') IS NOT FALSE");
+  async getPreFxaSubscribersPage(pagination) {
+    return await knex("subscribers")
+      .whereRaw("(fxa_uid = '') IS NOT FALSE")
+      .paginate(pagination);
+
   },
 
   async getSubscriberByEmail(email) {

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -28,6 +28,7 @@
     "intl-pluralrules": "^0.2.1",
     "isemail": "^3.1.3",
     "knex": "^0.19.5",
+    "knex-paginate": "^1.1.0",
     "lodash": "^4.17.13",
     "mozlog": "^2.2.0",
     "nodemailer": "^4.6.4",

diff --git a/scripts/send-email-to-pre-fxa-subscribers.js b/scripts/send-email-to-pre-fxa-subscribers.js
@@ -1,5 +1,7 @@
 "use strict";
 
+/* eslint-disable no-process-env */
+
 const { negotiateLanguages, acceptedLanguages } = require("fluent-langneg");
 
 const AppConstants = require("../app-constants");
@@ -8,42 +10,52 @@ const EmailHelpers = require("../template-helpers/emails.js");
 const EmailUtils = require("../email-utils");
 const { LocaleUtils } = require ("../locale-utils");
 
+const PAGE_SIZE = process.env.PAGE_SIZE;
+
 
 (async (req) => {
   const localeUtils = LocaleUtils.init();
   EmailUtils.init();
-
-  const subscribers = await DB.getPreFxaSubscribers();
-  console.log(`Found ${subscribers.length} subscriber records with empty fxa_uid.`);
   const notifiedSubscribers = [];
   const utmID = "pre-fxa";
-  for (const subscriber of subscribers) {
-    const signupLanguage = subscriber.signup_language;
-    const subscriberEmail = subscriber.primary_email;
-    const requestedLanguage = signupLanguage ? acceptedLanguages(signupLanguage) : "";
-    const supportedLocales = negotiateLanguages(
-      requestedLanguage,
-      localeUtils.availableLanguages,
-      {defaultLocale: "en"}
-    );
-
-    if (!notifiedSubscribers.includes(subscriberEmail)) {
-      const sendInfo = await EmailUtils.sendEmail(
-        subscriberEmail,
-        LocaleUtils.fluentFormat(supportedLocales, "pre-fxa-subject"), // email subject
-        "default_email", // email template
-        {
-          supportedLocales,
-          SERVER_URL: AppConstants.SERVER_URL,
-          unsubscribeUrl: EmailUtils.getUnsubscribeUrl(subscriber, utmID), // need to test the flow for legacy users who want to unsubscribe
-          ctaHref: EmailHelpers.getPreFxaUtmParams(AppConstants.SERVER_URL, "create-account-button", subscriberEmail),
-          whichPartial: "email_partials/pre-fxa",
-          preFxaEmail: true,
-          email: subscriberEmail,
-        },
+
+  const subscribersResult = await DB.getPreFxaSubscribersPage({ perPage: PAGE_SIZE, currentPage: 1 });
+  console.log(`Found ${subscribersResult.pagination.total} subscriber records with empty fxa_uid.`);
+  console.log(`Will process ${subscribersResult.pagination.lastPage} pages of size ${PAGE_SIZE}.`);
+  const lastPage = subscribersResult.pagination.lastPage;
+
+
+  for (let currentPage = 1; currentPage <= lastPage; currentPage++) {
+    console.log(`Processing page ${currentPage} of ${lastPage}.`);
+    const subscribersPageResult = await DB.getPreFxaSubscribersPage({ perPage: PAGE_SIZE, currentPage });
+    for (const subscriber of subscribersPageResult.data) {
+      const signupLanguage = subscriber.signup_language;
+      const subscriberEmail = subscriber.primary_email;
+      const requestedLanguage = signupLanguage ? acceptedLanguages(signupLanguage) : "";
+      const supportedLocales = negotiateLanguages(
+        requestedLanguage,
+        localeUtils.availableLanguages,
+        {defaultLocale: "en"}
       );
-      notifiedSubscribers.push(subscriberEmail);
-      console.log(`Sent email to ${subscriberEmail}, info: ${JSON.stringify(sendInfo)}`);
+
+      if (!notifiedSubscribers.includes(subscriberEmail)) {
+        const sendInfo = await EmailUtils.sendEmail(
+          subscriberEmail,
+          LocaleUtils.fluentFormat(supportedLocales, "pre-fxa-subject"), // email subject
+          "default_email", // email template
+          {
+            supportedLocales,
+            SERVER_URL: AppConstants.SERVER_URL,
+            unsubscribeUrl: EmailUtils.getUnsubscribeUrl(subscriber, utmID), // need to test the flow for legacy users who want to unsubscribe
+            ctaHref: EmailHelpers.getPreFxaUtmParams(AppConstants.SERVER_URL, "create-account-button", subscriberEmail),
+            whichPartial: "email_partials/pre-fxa",
+            preFxaEmail: true,
+            email: subscriberEmail,
+          },
+        );
+        notifiedSubscribers.push(subscriberEmail);
+        console.log(`Sent email to ${subscriberEmail}, info: ${JSON.stringify(sendInfo)}`);
+      }
     }
   }
   console.log(`Notified subscribers: ${JSON.stringify(notifiedSubscribers)}`);