add ability to ban certain twitter accounts, mostly bots etc

karpathy · Feb 9, 2017 · e0c6c37 · e0c6c37
1 parent 7d5030b
commit e0c6c37
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 0 deletions.
diff --git a/twitter_daemon.py b/twitter_daemon.py
@@ -75,6 +75,15 @@ def get_latest_or_loop(q):
 print('mongodb tweets collection size:', tweets.count())
 print('mongodb tweets_top collection size:', tweets_top.count())
 
+# load banned accounts
+banned = {}
+if os.path.isfile(Config.banned_path):
+  with open(Config.banned_path, 'r') as f:
+    lines = f.read().split('\n')
+  for l in lines:
+    if l: banned[l] = 1 # mark banned
+  print('banning users:', list(banned.keys()))
+
 # main loop
 db_pids, last_db_load = None, 0
 while True:
@@ -92,6 +101,7 @@ def get_latest_or_loop(q):
     arxiv_pids = [p for p in arxiv_pids if p in db_pids] # filter to those that are in our paper db
     if not arxiv_pids: continue # nothing we know about here, lets move on
     if tweets.find_one({'id':r.id}): continue # we already have this item
+    if r.user.screen_name in banned: continue # banned user, very likely a bot
 
     # create the tweet. intentionally making it flat here without user nesting
     d = parser.parse(r.created_at) # datetime instance

diff --git a/utils.py b/utils.py
@@ -24,6 +24,7 @@ class Config(object):
     database_path = 'as.db'
     serve_cache_path = 'serve_cache.p'
 
+    banned_path = 'banned.txt' # for twitter users who are banned
     tmp_dir = 'tmp'
 
 # Context managers for atomic writes courtesy of