adding task-level logging

jonleesy · Feb 6, 2021 · d0f7b7d · d0f7b7d
1 parent 06c26dc
commit d0f7b7d
Showing 12 changed files with 342 additions and 64 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -1,4 +1,5 @@
 node_modules
+logs
 package-lock.json
 .env*
 !.env.example

diff --git a/.eslintrc.json b/.eslintrc.json
@@ -20,6 +20,7 @@
         "no-undef": "warn",
         "no-loop-func": "off",
         "arrow-body-style": "off",
-        "no-continue": "off"
+        "no-continue": "off",
+        "camelcase": "off"
     }
 }
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 node_modules
 .env*
-!.env.example
+!.env.example
+logs
diff --git a/helpers/captcha.js b/helpers/captcha.js
@@ -40,9 +40,11 @@ const getCaptchaResult = async (captchaId) => {
   }
 };
 
-exports.solveCaptcha = async (page, captchaSelector, captchaIframeSelector) => {
+exports.solveCaptcha = async ({
+  taskLogger, page, captchaSelector, captchaIframeSelector
+}) => {
   try {
-    console.log('detected captcha, solving');
+    taskLogger.info('detected captcha, solving');
 
     if (!apiKey) {
       throw new Error('You must set an API_KEY_2CAPTCHA in your .env file.');
@@ -62,11 +64,11 @@ exports.solveCaptcha = async (page, captchaSelector, captchaIframeSelector) => {
       const kValue = iframeSrcParams.get('k');
       return kValue;
     }, captchaSelector);
-    console.log('extracted googleKey ', googleKey);
+    taskLogger.info(`extracted googleKey ${googleKey}`);
 
     const captcha = await submitCaptcha(googleKey, context.url());
     const captchaId = captcha.request;
-    console.log('submitted captcha to 2captcha, got id ', captchaId);
+    taskLogger.info(`submitted captcha to 2captcha, got id ${captchaId}`);
 
     await new Promise((resolve) => {
       const interval = setInterval(async () => {
@@ -79,7 +81,7 @@ exports.solveCaptcha = async (page, captchaSelector, captchaIframeSelector) => {
         const captchaAnswer = solvedCaptcha && solvedCaptcha.request;
 
         if (captchaAnswer) {
-          console.log('got captcha result from 2captcha');
+          taskLogger.info('got captcha result from 2captcha, submitting');
           await context.evaluate((captchaAnswerText) => {
             document.querySelector('#g-recaptcha-response').innerHTML = captchaAnswerText;
             const callbackFunction = ___grecaptcha_cfg.clients['0'].K.K.callback;

diff --git a/helpers/cluster.js b/helpers/cluster.js
@@ -6,6 +6,7 @@ const Address = require('../api/Addresses/model');
 
 const { testProxy, createProxyString } = require('./proxies');
 const { sendEmail } = require('./email');
+const Logger = require('./logger');
 
 const sites = require('../sites');
 
@@ -27,11 +28,27 @@ class PuppeteerCluster {
     });
 
     cluster.task(async ({ page, data: taskId }) => {
+      let taskLogger;
       try {
         const data = {};
         data['tasks.id'] = taskId;
         const task = await new Task().findOne(data);
 
+        const {
+          id,
+          shipping_address_id,
+          billing_address_id,
+          site_name,
+          url,
+          size,
+          style_index,
+          shipping_speed_index,
+          auto_solve_captchas,
+          notification_email_address
+        } = task;
+
+        taskLogger = new Logger().startTaskLogger(id);
+
         const proxies = await new Proxy().find({ has_been_used: false });
 
         const validProxy = proxies.find(async (proxy) => {
@@ -43,47 +60,52 @@ class PuppeteerCluster {
           return false;
         });
         const proxy = validProxy ? createProxyString(validProxy) : null;
+        if (proxy) {
+          taskLogger.info('Using proxy', proxy);
+        }
 
         const shippingAddress = await new Address().findOne({
-          id: task.shipping_address_id
+          id: shipping_address_id
         });
         const billingAddress = await new Address().findOne({
-          id: task.billing_address_id
+          id: billing_address_id
         });
 
-        const checkoutComplete = await sites[task.site_name].guestCheckout(
+        const checkoutComplete = await sites[site_name].guestCheckout({
+          taskLogger,
           page,
-          task.url,
+          url,
           proxy,
-          task.style_index,
-          task.size,
+          styleIndex: style_index,
+          size,
           shippingAddress,
-          task.shipping_speed_index,
+          shippingSpeedIndex: shipping_speed_index,
           billingAddress,
-          task.auto_solve_captchas,
-          task.notification_email_address
-        );
+          autoSolveCaptchas: auto_solve_captchas,
+          notificationEmailAddress: notification_email_address
+        });
 
-        const recipient = task.notification_email_address;
+        const recipient = notification_email_address;
         let subject;
         let text;
         if (!checkoutComplete) {
           subject = 'Checkout task unsuccessful';
           text = `
-            The checkout task for ${task.url} size ${task.size} has a checkout error. 
+            The checkout task for ${url} size ${size} has a checkout error. 
             Please open the browser to check on it within 5 minutes.
           `;
         } else {
           subject = 'Checkout task successful';
-          text = `The checkout task for ${task.url} size ${task.size} has completed.`;
+          text = `The checkout task for ${url} size ${size} has completed.`;
         }
-        await sendEmail(recipient, subject, text);
+        await sendEmail({ recipient, subject, text });
+        taskLogger.info(text);
 
         if (!checkoutComplete) {
           await page.waitForTimeout(5 * 60 * 1000);
         }
       } catch (err) {
-        console.error(err.messsage);
+        taskLogger.error(err);
       }
     });
 

diff --git a/helpers/email.js b/helpers/email.js
@@ -1,7 +1,7 @@
 require('dotenv-flow').config();
 const nodemailer = require('nodemailer');
 
-exports.sendEmail = async (to, subject, text) => {
+exports.sendEmail = async ({ recipient: to, subject, text }) => {
   try {
     const transporter = nodemailer.createTransport({
       host: process.env.EMAIL_HOST,

diff --git a/helpers/logger.js b/helpers/logger.js
@@ -0,0 +1,34 @@
+const winston = require('winston');
+const path = require('path');
+const fs = require('fs');
+
+const { format } = winston;
+
+class Logger {
+  constructor() {
+    this.logDir = path.resolve('logs');
+  }
+
+  startTaskLogger(taskId) {
+    if (!fs.existsSync(this.logDir)) {
+      fs.mkdirSync(this.logDir);
+    }
+
+    return winston.createLogger({
+      transports: [
+        new winston.transports.Console(),
+        new winston.transports.File({ filename: path.join(this.logDir, `/${Date.now()}-taskId=${taskId}.log`) })
+      ],
+      format: format.combine(
+        format.errors({ stack: true }),
+        format.printf((info) => {
+          return info.level === 'info'
+            ? `[${new Date().toLocaleString()}] - ${info.message}`
+            : `[${new Date().toLocaleString()}] - ${info.stack}`;
+        })
+      )
+    });
+  }
+}
+
+module.exports = Logger;