Merge branch 'talebook-master'

BitlikerNas · Jan 21, 2022 · 3e77afb · 3e77afb
2 parents 6012928 + 4b917fa
commit 3e77afb
Show file tree

Hide file tree

Showing 9 changed files with 91 additions and 48 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,5 @@
-# 第一阶段，拉取 node 基础镜像并安装依赖，执行构建
+# ----------------------------------------
+# 第一阶段，拉取 node 基础镜像并安装依赖，执行构建
 FROM node:12-alpine as builder
 MAINTAINER Rex <talebook@foxmail.com>
 
@@ -14,8 +15,9 @@ COPY app/ /app/
 RUN npm run build
 
 
+# ----------------------------------------
 # 第二阶段，构建环境
-FROM talebook/calibre:5
+FROM talebook/calibre:5 as server
 
 # install python packages
 COPY ["requirements.txt", "/tmp/"]
@@ -28,6 +30,20 @@ RUN cp /etc/apt/sources.list /tmp/ && \
 RUN apt-get update && apt-get install -y gettext
 RUN mv /tmp/sources.list /etc/apt/sources.list
 
+
+# ----------------------------------------
+# 测试阶段
+FROM server as test
+COPY webserver/ /var/www/talebook/webserver/
+RUN pip install -i https://mirrors.tencent.com/pypi/simple/ \
+        flake8 pytest mock
+CMD ["pytest", "/var/www/talebook/webserver"]
+
+
+# ----------------------------------------
+# 生产环境
+FROM server as production
+
 # prepare dirs
 RUN mkdir -p /data/log/nginx/ && \
     mkdir -p /data/books/library  && \

diff --git a/Makefile b/Makefile
@@ -16,5 +16,9 @@ push:
 	docker push $(REPO1)
 	docker push $(REPO2)
 
+docker-test:
+	docker build -t talebook/test --target test .
+	docker run --rm --name talebook-docker-test talebook/test
+
 test:
 	pytest webserver
diff --git a/app/src/pages/BookDetail.vue b/app/src/pages/BookDetail.vue
@@ -174,7 +174,7 @@
         <v-col cols=12 sm=6 md=4>
             <v-card outlined>
                 <v-list>
-                    <v-list-item :href="'/read/'+bookid" >
+                    <v-list-item :href="'/read/'+bookid" target="_blank">
                         <v-list-item-avatar large color='primary' >
                             <v-icon dark >import_contacts</v-icon>
                         </v-list-item-avatar>

diff --git a/requirements.txt b/requirements.txt
@@ -2,5 +2,5 @@ jinja2==2.11.3
 social-auth-core==3.3.3
 social-auth-app-tornado==1.0.0
 social-auth-storage-sqlalchemy==1.1.0
-tornado==5.1.1
+tornado==6.1
 bs4
diff --git a/webserver/handlers/book.py b/webserver/handlers/book.py
@@ -2,18 +2,16 @@
 #-*- coding: UTF-8 -*-
 
 import logging
-from plugins.meta import douban
-from plugins.meta import baike
-import urllib
 import subprocess
-import tornado.escape
-from handlers.base import *
+import urllib
 
+import loader
+import tornado.escape
 from calibre.ebooks.metadata import authors_to_string
-from calibre.ebooks.conversion.plumber import Plumber
-from calibre.customize.conversion import OptionRecommendation, DummyReporter
+from handlers.base import *
+from plugins.meta import baike
+from plugins.meta import douban
 
-import loader
 CONF = loader.get_settings()
 
 BOOKNAV = (
@@ -88,26 +86,25 @@ def worker():
         t.start()
     return run
 
+
 def do_ebook_convert(old_path, new_path, log_path):
-    '''convert book, and block, and wait'''
+    """convert book, and block, and wait"""
     args = ['ebook-convert', old_path, new_path]
     if new_path.lower().endswith(".epub"): args += ['--flow-size', '0']
 
-    with open(log_path, "w", 0) as log:
-        cmd = " ".join( "'%s'" % v for v in args)
-        logging.info("CMD: %s" % cmd )
+    with open(log_path, "w") as log:
+        cmd = " ".join("'%s'" % v for v in args)
+        logging.info("CMD: %s" % cmd)
         p = subprocess.Popen(args, stdout=log, stderr=subprocess.PIPE)
-        err = ""
-        while p.poll() == None:
-            _, e = p.communicate()
-            err += e
-        logging.info("ebook-convert finish: %s" % new_path)
-
-        if err:
-            log.write(err)
+        try:
+            _, stde = p.communicate(timeout=100)
+            logging.info("ebook-convert finish: %s, err: %s" % (new_path, bytes.decode(stde)))
+        except subprocess.TimeoutExpired:
+            p.kill()
+            logging.info("ebook-convert timeout: %s" % new_path)
             log.write(u"\n服务器处理异常，请在QQ群里联系管理员。\n[FINISH]")
-            return (False, err)
-        return (True, "")
+            return False, ""
+        return True, ""
 
 class Index(BaseHandler):
     def fmt(self, b):
@@ -432,7 +429,7 @@ def get(self):
         title = _(u'新书推荐') % vars()
         category = "recents"
         ids = self.books_by_timestamp()
-        return self.render_book_list([], vars(), ids=ids);
+        return self.render_book_list([], vars(), ids=ids)
 
 class SearchBook(ListHandler):
     def get(self):
@@ -443,7 +440,7 @@ def get(self):
         title = _(u'搜索：%(name)s') % vars()
         ids = self.cache.search(name)
         search_query = name
-        return self.render_book_list([], vars(), ids);
+        return self.render_book_list([], vars(), ids)
 
 class HotBook(ListHandler):
     def get(self):
@@ -477,7 +474,6 @@ def convert(s):
             return {'err': 'permission', 'msg': _(u'无权操作')}
 
         import re
-        from calibre.ebooks.metadata import MetaInformation
         postfile = self.request.files['ebook'][0]
         name = postfile['filename']
         name = re.sub(r'[\x80-\xFF]+', convert, name)
@@ -543,7 +539,14 @@ def get(self, id):
             self.extract_book(book, fpath, fmt)
             return self.html_page('book/read.html', vars())
 
-        raise web.HTTPError(404, reason=_(u"抱歉，在线阅读器暂不支持该格式的书籍") )
+        if 'fmt_pdf' in book:
+            path = book['fmt_pdf']
+            self.set_header('Content-Type', 'application/pdf')
+            with open(path, 'rb') as f:
+                self.write(f.read())
+            return
+
+        raise web.HTTPError(404, reason=_(u"抱歉，在线阅读器暂不支持该格式的书籍"))
 
     @background
     def extract_book(self, book, fpath, fmt):
@@ -559,7 +562,7 @@ def extract_book(self, book, fpath, fmt):
         if fmt != "epub":
             new_fmt = "epub"
             new_path = os.path.join(CONF["convert_path"], 'book-%s-%s.%s'%(book['id'], int(time.time()), new_fmt) )
-            logging.error('convert book: %s => %s' % ( fpath, new_path));
+            logging.info('convert book: %s => %s, progress: %s' % (fpath, new_path, progress_file))
             os.chdir('/tmp/')
 
             ok, err = do_ebook_convert(fpath, new_path, progress_file)
@@ -569,6 +572,7 @@ def extract_book(self, book, fpath, fmt):
 
             with open(new_path, "rb") as f:
                 self.db.add_format(book['id'], new_fmt, f, index_is_id=True)
+                logging.info('add new book: %s', new_path)
             fpath = new_path
 
         # extract to dir
@@ -643,8 +647,8 @@ def do_send_mail(self, book, mail_to, fmt, fpath):
         author = authors_to_string(book['authors'] if book['authors'] else [_(u'佚名')])
         title = book['title'] if book['title'] else _(u"无名书籍")
         fname = u'%s - %s.%s'%(title, author, fmt)
-        with open(fpath) as f:
-            fdata = r.read()
+        with open(fpath, 'rb') as f:
+            fdata = f.read()
 
         site_title = CONF['site_title']
         mail_from = self.settings['smtp_username']

diff --git a/webserver/loader.py b/webserver/loader.py
@@ -12,6 +12,14 @@ def clear(self):
         for key in list(self.keys()):
             self.pop(key)
 
+    def set_store_path(self):
+        p = self.get('settings_path', "").strip()
+        if not os.path.isdir(p):
+            p = os.path.dirname(__file__)
+        if sys.path[0] != p:
+            sys.path.insert(0, p)
+        return p
+
     def loadfile(self):
         try:
             import settings
@@ -21,8 +29,7 @@ def loadfile(self):
             logging.error(traceback.format_exc())
             pass
 
-        self.settings_path = self.get('settings_path', None)
-        if self.settings_path: sys.path.insert(0, self.settings_path)
+        self.set_store_path()
 
         try:
             import auto
@@ -47,8 +54,7 @@ def dumpfile(self, filename="auto.py"):
 }
 '''
 
-        d = self.settings_path
-        if not d: d = os.path.dirname(__file__)
+        d = self.set_store_path()
         py = os.path.join(d, filename)
         pyc = os.path.join(d, filename+"c")
         logging.error("saving settings file: %s" % py)

diff --git a/webserver/plugins/meta/baike/api.py b/webserver/plugins/meta/baike/api.py
@@ -6,7 +6,7 @@
 Do not modify it unless you know what you are doing.
 """
 
-import os, re, json, logging, datetime, io
+import os, re, json, logging, io
 from urllib.request import urlopen
 from .baidubaike.baidubaike import Page
 
@@ -33,6 +33,7 @@ def _baike(self, title):
 
     def _metadata(self, baike):
         from calibre.ebooks.metadata.book.base import Metadata
+        from calibre.utils.date import utcnow, strptime
 
         info = baike.get_info()
         logging.debug("\n".join( "%s:\t%s" % v for v in info.items()))
@@ -47,8 +48,8 @@ def _metadata(self, baike):
         mi.author_sort = mi.authors[0]
         mi.isbn      = BAIKE_ISBN
         mi.tags      = baike.get_tags()
-        mi.pubdate   = datetime.datetime.now()
-        mi.timestamp = datetime.datetime.now()
+        mi.pubdate   = utcnow()
+        mi.timestamp = utcnow()
         mi.cover_url = baike.get_image()
         mi.comments  = re.sub(r'\[\d+\]$', "", baike.get_summary() )
         mi.website   = baike.http.url
@@ -64,7 +65,7 @@ def _metadata(self, baike):
 
         if u'完结' in info.get(u'连载状态', ""):
             day = re.findall('\d*-\d*-\d*', info[u'连载状态'])
-            try: mi.pubdate = datetime.datetime.strptime(day[0], '%Y-%m-%d')
+            try: mi.pubdate = strptime(day[0], '%Y-%m-%d')
             except: pass
         return mi
 

diff --git a/webserver/plugins/meta/douban.py b/webserver/plugins/meta/douban.py
@@ -122,6 +122,7 @@ def _metadata(self, book):
         if not authors: authors = [ u'佚名' ]
 
         from calibre.ebooks.metadata.book.base import Metadata
+        from calibre.utils.date import utcnow
         mi = Metadata(book['title'])
         mi.authors     = authors
         mi.author_sort = mi.authors[0]
@@ -132,7 +133,7 @@ def _metadata(self, book):
         mi.tags        = [ t['name'] for t in book['tags'] ][:8]
         mi.rating      = int(float(book['rating']['average']))
         mi.pubdate     = str2date(book['pubdate'])
-        mi.timestamp   = datetime.datetime.now()
+        mi.timestamp   = utcnow()
         mi.douban_author_intro = book['author_intro']
         mi.douban_subtitle = book.get('subtitle', None)
         mi.website     = "https://book.douban.com/subject/%s/" % book['id']

diff --git a/webserver/unittest/test_app.py b/webserver/unittest/test_app.py
@@ -6,7 +6,7 @@
 
 testdir = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.dirname(testdir))
-import server, models, settings
+import server, models
 
 
 _app = None
@@ -424,11 +424,13 @@ def test_admin_settings(self):
         self.assertEqual(d['err'], 'ok')
         self.assertTrue(len(d['settings']) > 10)
 
-        req = {"settings_path": "/tmp/", "site_title": "abc", "not_work": "en"}
-        d = self.json("/api/admin/settings", method="POST", body=json.dumps(req))
-        self.assertEqual(d['err'], 'ok')
-        self.assertEqual(d['rsp']['site_title'], 'abc')
-        self.assertTrue('not_work' not in d['rsp'])
+        import loader
+        with mock.patch.object(loader.SettingsLoader, 'set_store_path', return_value='/tmp/') as m:
+            req = {"site_title": "abc", "not_work": "en"}
+            d = self.json("/api/admin/settings", method="POST", body=json.dumps(req))
+            self.assertEqual(d['err'], 'ok')
+            self.assertEqual(d['rsp']['site_title'], 'abc')
+            self.assertTrue('not_work' not in d['rsp'])
 
 class TestOpds(TestApp):
     @classmethod
@@ -508,6 +510,15 @@ def test_opds_without_login(self):
         self.assertEqual(rsp.code, 401)
         server.CONF['INVITE_MODE'] = False
 
+class TestConvert(TestApp):
+    def test_convert(self):
+        import handlers
+        fin = testdir + '/library/Han Han/Ta De Guo (5)/Ta De Guo - Han Han.epub'
+        fout = '/tmp/output.mobi'
+        flog = '/tmp/output.log'
+        ok, msg = handlers.book.do_ebook_convert(fin, fout, flog)
+        self.assertEqual(ok, True)
+
 
 def setUpModule():
     setup_server()