reducing update rate of viewer count

holofans · Nov 3, 2020 · 93be6af · 93be6af
1 parent e9b3ce0
commit 93be6af
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 13 deletions.
diff --git a/apps/crawler-youtube/tasks/video-info-api.js b/apps/crawler-youtube/tasks/video-info-api.js
@@ -10,7 +10,7 @@
 require('dotenv').config();
 const moment = require('moment-timezone');
 const { Op } = require('sequelize');
-const { db, youtube, log, GenericError } = require('../../../modules');
+const { db, youtube, log, GenericError, updateIfSignificant } = require('../../../modules');
 const { STATUSES } = require('../../../consts');
 
 const VIDEOS_MAX_QUERY = 50;
@@ -122,8 +122,9 @@ module.exports = async () => {
           saveInfo.live_schedule = ytInfo.liveStreamingDetails.scheduledStartTime || null;
           saveInfo.live_start = ytInfo.liveStreamingDetails.actualStartTime || null;
           saveInfo.live_end = ytInfo.liveStreamingDetails.actualEndTime || null;
+          const oldViewerCount = targetVideo.live_viewers;
           const currentViewers = +ytInfo.liveStreamingDetails.concurrentViewers || null;
-          saveInfo.live_viewers = currentViewers && +(currentViewers.toPrecision(currentViewers > 1000 ? 2 : 1));
+          saveInfo.live_viewers = currentViewers && updateIfSignificant(oldViewerCount, +(currentViewers.toPrecision(currentViewers > 1000 ? 2 : 1)), 300);
           // Get moment objects
           const scheduleMoment = moment(saveInfo.live_schedule);
           const startMoment = moment(saveInfo.live_start);

diff --git a/apps/crawler-youtube/tasks/video-status-api.js b/apps/crawler-youtube/tasks/video-status-api.js
@@ -10,7 +10,7 @@
 require('dotenv').config();
 const moment = require('moment-timezone');
 const { Op } = require('sequelize');
-const { db, youtube, log, GenericError } = require('../../../modules');
+const { db, youtube, log, GenericError, updateIfSignificant } = require('../../../modules');
 const { STATUSES } = require('../../../consts');
 
 module.exports = async () => {
@@ -96,8 +96,9 @@ module.exports = async () => {
           saveInfo.live_schedule = ytInfo.liveStreamingDetails.scheduledStartTime || null;
           saveInfo.live_start = ytInfo.liveStreamingDetails.actualStartTime || null;
           saveInfo.live_end = ytInfo.liveStreamingDetails.actualEndTime || null;
+          const oldViewerCount = targetVideo.live_viewers;
           const currentViewers = +ytInfo.liveStreamingDetails.concurrentViewers || null;
-          saveInfo.live_viewers = currentViewers && +(currentViewers.toPrecision(currentViewers > 1000 ? 2 : 1));
+          saveInfo.live_viewers = currentViewers && updateIfSignificant(oldViewerCount, +(currentViewers.toPrecision(currentViewers > 1000 ? 2 : 1)), 300);
           // Get moment objects
           const scheduleMoment = moment(saveInfo.live_schedule);
           const startMoment = moment(saveInfo.live_start);

diff --git a/modules/hash.js b/modules/hash.js
diff --git a/modules/index.js b/modules/index.js
@@ -3,13 +3,13 @@ const log = require('./logger');
 const youtube = require('./youtube');
 const error = require('./error');
 const memcached = require('./memcached');
-const hashCode = require('./hash');
+const utils = require('./utils');
 
 module.exports = {
   ...error,
   db,
   log,
   youtube,
   memcached,
-  ...hashCode,
+  ...utils,
 };
diff --git a/modules/utils.js b/modules/utils.js
@@ -0,0 +1,14 @@
+function hashCode(s) {
+  let h;
+  // eslint-disable-next-line no-bitwise
+  for (let i = 0; i < s.length; i += 1) h = Math.imul(31, h) + s.charCodeAt(i) | 0;
+  return h;
+}
+
+function updateIfSignificant(oldNumber, newNumber, minDelta) {
+  if (oldNumber == null) return newNumber;
+  if (!oldNumber) return newNumber;
+  if (Math.abs(oldNumber - newNumber) >= minDelta) return newNumber;
+  return oldNumber; // don't update
+}
+module.exports = { hashCode, updateIfSignificant };