ready to release

tgbot-collection · Feb 6, 2021 · 0d79c18 · 0d79c18
1 parent 0902f43
commit 0d79c18
Show file tree

Hide file tree

Showing 7 changed files with 92 additions and 23 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -3,7 +3,7 @@ FROM python:3.8-alpine
 RUN apk update && apk add  --no-cache tzdata ca-certificates
 COPY requirements.txt /requirements.txt
 RUN pip3 install  --no-cache-dir -r /requirements.txt && rm /requirements.txt
-COPY ./yyetsbot /YYeTsBot/yyetsbot
+COPY . /YYeTsBot
 
 ENV TZ=Asia/Shanghai
 WORKDIR /YYeTsBot/yyetsbot

diff --git a/requirements.txt b/requirements.txt
@@ -4,4 +4,5 @@ beautifulsoup4
 tgbot-ping
 redis
 apscheduler
-pymongo
+pymongo
+tornado
diff --git a/web/index.html b/web/index.html
@@ -62,8 +62,7 @@ <h2>在 <a style="text-decoration: none;color: deepskyblue" href="https://t.me/y
         <h2>大家都在看……</h2>
     </div>
     <h2>有问题联系 <a style="text-decoration: none;color: green" href="https://t.me/BennyThink">Benny小可爱</a></h2>
-    <img id="ferris" alt="a happy crab is wearing a cowboy hat and holding a lasso. 200 success."
-         src="img/200-wrangler-ferris.gif">
+
 </div>
 </body>
 <script src="js/axios.min.js"></script>

diff --git a/web/convert_db.py → web/prepare/convert_db.py b/web/convert_db.py → web/prepare/convert_db.py
diff --git a/web/prepare/dump_kv.py b/web/prepare/dump_kv.py
@@ -0,0 +1,44 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# YYeTsBot - dump_kv.py
+# 2/6/21 18:12
+#
+
+__author__ = "Benny <benny.think@gmail.com>"
+
+import threading
+import requests
+import json
+from concurrent.futures.thread import ThreadPoolExecutor
+
+s = requests.Session()
+
+with open("index.json", ) as f:
+    ids = json.load(f)
+
+chunk = [ids[x:x + 3000] for x in range(0, len(ids), 3000)]
+
+
+def download(c):
+    print("running batch ", c[0])
+    for i in c:
+        data = s.get("https://yyets.dmesg.app/id={}".format(i)).json()
+        with open(f"{i}.json", "w") as f:
+            json.dump(data, f)
+
+
+if __name__ == '__main__':
+    threads = []
+    for part in chunk:
+        # Create 9 threads counting 10-19, 20-29, ... 90-99.
+        thread = threading.Thread(target=download, args=(part,))
+        threads.append(thread)
+
+    # Start them all
+    for thread in threads:
+        thread.start()
+
+    # Wait for all to complete
+    for thread in threads:
+        thread.join()
diff --git a/web/prepare/load_from_kv.py b/web/prepare/load_from_kv.py
@@ -0,0 +1,23 @@
+#!/usr/local/bin/python3
+# coding: utf-8
+
+# YYeTsBot - load_from_kv.py
+# 2/6/21 18:27
+#
+
+__author__ = "Benny <benny.think@gmail.com>"
+
+import pymongo
+import json
+import os
+
+mongo_client = pymongo.MongoClient()
+
+data_files = [i for i in os.listdir("data/") if i.endswith(".json")]
+col = mongo_client["zimuzu"]["yyets"]
+for data_file in data_files:
+    with open(os.path.join("data", data_file)) as f:
+        d = json.load(f)
+        views = int(d["data"]["info"]["views"])
+        d["data"]["info"]["views"] = views
+        col.insert_one(d)
diff --git a/web/server.py b/web/server.py
@@ -7,9 +7,6 @@
 
 __author__ = "Benny <benny.think@gmail.com>"
 
-import json
-import socket
-from platform import uname
 import os
 import contextlib
 from http import HTTPStatus
@@ -20,11 +17,22 @@
 from tornado.concurrent import run_on_executor
 
 enable_pretty_logging()
-client = pymongo.MongoClient()
-db = client["zimuzu"]
+
+mongo_host = os.getenv("mongo") or "localhost"
+
+
+class Mongo:
+    def __init__(self):
+        self.client = pymongo.MongoClient(host=mongo_host, connect=False)
+        self.db = self.client["zimuzu"]
+
+    def __del__(self):
+        self.client.close()
 
 
 class BaseHandler(web.RequestHandler):
+    mongo = Mongo()
+
     def data_received(self, chunk):
         pass
 
@@ -45,7 +53,7 @@ def get_resource_data(self):
         param = self.get_query_argument("id")
         with contextlib.suppress(ValueError):
             param = int(param)
-        data = db["yyets"].find_one_and_update(
+        data = self.mongo.db["yyets"].find_one_and_update(
             {"data.info.id": param},
             {'$inc': {'data.info.views': 1}},
             {'_id': False})
@@ -57,7 +65,7 @@ def search_resource(self):
         projection = {'_id': False,
                       'data.info': True,
                       }
-        data = db["yyets"].find({
+        data = self.mongo.db["yyets"].find({
             "$or": [
                 {"data.info.cnname": {'$regex': f'.*{param}.*'}},
                 {"data.info.enname": {'$regex': f'.*{param}.*'}},
@@ -88,7 +96,7 @@ def get_top_resource(self):
                       'data.info': True,
                       }
         if top_type == "all":
-            data = db["yyets"].find({}, projection).sort("data.info.views", pymongo.DESCENDING).limit(10)
+            data = self.mongo.db["yyets"].find({}, projection).sort("data.info.views", pymongo.DESCENDING).limit(10)
         else:
             data = []
         return dict(data=list(data))
@@ -105,7 +113,7 @@ class MetricsHandler(BaseHandler):
     @run_on_executor()
     def set_metrics(self):
         metrics_name = self.get_query_argument("type", "access")
-        db['metrics'].find_one_and_update(
+        self.mongo.db['metrics'].find_one_and_update(
             {'type': metrics_name}, {'$inc': {'count': 1}}
         )
         self.set_status(HTTPStatus.CREATED)
@@ -114,7 +122,7 @@ def set_metrics(self):
     @run_on_executor()
     def get_metrics(self):
         metrics_name = self.get_query_argument("type", "access")
-        return db['metrics'].find_one({'type': metrics_name}, {'_id': False})
+        return self.mongo.db['metrics'].find_one({'type': metrics_name}, {'_id': False})
 
     @gen.coroutine
     def get(self):
@@ -138,23 +146,17 @@ class RunServer:
         (r'/(.*\.html|.*\.js|.*\.css|.*\.png|.*\.jpg|.*\.ico|.*\.gif|.*\.woff2)', web.StaticFileHandler,
          {'path': static_path}),
     ]
-    settings = {
-        "cookie_secret": "5Li05DtnQewDZq1mDVB3HAAhFqUu2vD2USnqezkeu+M=",
-        "xsrf_cookies": False,
-        "autoreload": True,
-        # 'template_path': '.',
-    }
 
-    application = web.Application(handlers)
+    application = web.Application(handlers, xheaders=True)
 
     @staticmethod
     def run_server(port, host, **kwargs):
         tornado_server = httpserver.HTTPServer(RunServer.application, **kwargs)
         tornado_server.bind(port, host)
-        tornado_server.start()
+        tornado_server.start(0)
 
         try:
-            print('Server is running on http://{}:{}'.format("127.0.0.1", port))
+            print('Server is running on http://{}:{}'.format(host, port))
             ioloop.IOLoop.instance().current().start()
         except KeyboardInterrupt:
             ioloop.IOLoop.instance().stop()
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,4 +4,5 @@ beautifulsoup4 @@
     tgbot-ping
     redis
     apscheduler
-    pymongo
+    pymongo
+    tornado