refactor some code and fix some errors

tbdsux · Sep 2, 2022 · 97ff753 · 97ff753
1 parent ee2556d
commit 97ff753
Show file tree

Hide file tree

Showing 8 changed files with 79 additions and 76 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -11,5 +11,6 @@
     },
     "python.analysis.extraPaths": [
         "venv/lib"
-    ]
+    ],
+    "python.linting.mypyEnabled": true
 }
diff --git a/api/fetch.py b/api/fetch.py
@@ -23,7 +23,9 @@ def _get_main_container(self) -> None:
         self.info["title"] = container.find("h1", class_="film-title").find("a").text
 
         # RATING (could be either N/A or with number)
-        self.info["rating"] = self._handle_rating(container.find("div", class_="col-film-rating").find("div"))
+        self.info["rating"] = self._handle_rating(
+            container.find("div", class_="col-film-rating").find("div")
+        )
 
         # POSTER
         self.info["poster"] = self._get_poster(container)
@@ -136,7 +138,9 @@ def _get_main_container(self) -> None:
                         "link": urljoin(MYDRAMALIST_WEBSITE, _raw_title["href"]),
                         "name": _raw_title.text,
                     },
-                    "rating": self._handle_rating(i.find("td", class_="text-center").find(class_="text-sm"))
+                    "rating": self._handle_rating(
+                        i.find("td", class_="text-center").find(class_="text-sm")
+                    ),
                 }
 
                 _raw_role = i.find("td", class_="role")
@@ -256,7 +260,9 @@ def _get_main_container(self) -> None:
                 __temp_review["reviewer"] = {
                     "name": i.find("a").text.strip(),
                     "user_link": urljoin(MYDRAMALIST_WEBSITE, i.find("a")["href"]),
-                    "user_image": self._get_poster(i),
+                    "user_image": self._get_poster(i).replace(
+                        "1t", "1c"
+                    ),  # replace 1t to 1c so that it will return a bigger image than the smaller one
                     "info": i.find("div", class_="user-stats").text.strip(),
                 }
 
@@ -267,11 +273,15 @@ def _get_main_container(self) -> None:
                     "div", class_="rating-overall"
                 )
 
-                __temp_review["review"] = (
-                    i.find("div", class_=re.compile("review-body"))
-                    .text.replace(__temp_review_ratings.text.strip(), "")
-                    .strip()
+                __temp_review_container = i.find(
+                    "div", class_=re.compile("review-body")
                 )
+                for i in __temp_review_container.find_all("br"):
+                    i.replace_with("\n")
+
+                __temp_review["review"] = __temp_review_container.text.replace(
+                    __temp_review_ratings.text.strip(), ""
+                ).strip()
 
                 __temp_review["ratings"] = {
                     "overall": float(

diff --git a/api/main.py b/api/main.py
@@ -41,8 +41,11 @@ async def fetch_cast(drama_id: str, response: Response) -> Dict[str, Any]:
 
 
 @app.get("/id/{drama_id}/reviews")
-async def fetch_reviews(drama_id: str, response: Response) -> Dict[str, Any]:
-    code, r = await fetch_func(query=f"{drama_id}/reviews", t="reviews")
+async def fetch_reviews(
+    drama_id: str, response: Response, page: int = 1
+) -> Dict[str, Any]:
+
+    code, r = await fetch_func(query=f"{drama_id}/reviews?page={page}", t="reviews")
 
     response.status_code = code
     return r

diff --git a/api/parser.py b/api/parser.py
@@ -1,4 +1,4 @@
-from typing import Dict, List, Any, Tuple, Type, TypeVar, Union
+from typing import Dict, List, Any, Type, TypeVar, Union
 
 from api import MYDRAMALIST_WEBSITE
 
@@ -32,7 +32,7 @@ def __init__(self, soup: BeautifulSoup, query: str, code: int, ok: bool) -> None
         self.soup = soup
         self.query = query
         self.status_code = code
-        self.url = ""
+        self.ok = ok
 
     @classmethod
     async def scrape(cls: Type[T], query: str, t: str) -> T:
@@ -45,7 +45,7 @@ async def scrape(cls: Type[T], query: str, t: str) -> T:
             url = ScrapeTypes[t] + query
 
         ok = True
-        code = 0
+        code = 500  # default to 500, internal server error
         soup = None
 
         try:
@@ -54,22 +54,17 @@ async def scrape(cls: Type[T], query: str, t: str) -> T:
             resp = scraper.get(url, headers=Parser.headers)
 
             # set the main soup var
-            soup = BeautifulSoup(resp.text, "lxml")
+            soup = BeautifulSoup(resp.text, "html5lib")
 
             # set the status code
             code = resp.status_code
+            ok = resp.status_code == 200
+
         except Exception:
             ok = False
 
         return cls(soup, query, code, ok)
 
-    def check(self) -> Tuple[int, bool]:
-        """Checks the status_code and returns it."""
-        if self.status_code == 200:
-            return 200, True
-
-        return self.status_code, False
-
     # get page err, if possible
     def res_get_err(self) -> Dict[str, Any]:
         container = self.soup.find("div", class_="app-body")

diff --git a/api/utils.py b/api/utils.py
@@ -17,13 +17,12 @@ def error(code: int, description: str) -> Dict[str, Any]:
 # search function
 async def search_func(query: str) -> Tuple[int, Dict[str, Any]]:
     f = await Search.scrape(query=query, t="search")
-    code, ok = f.check()
-    if not ok:
-        return code, error(code, "An unexpected error occurred.")
+    if not f.ok:
+        return f.status_code, error(f.status_code, "An unexpected error occurred.")
     else:
         f._get_search_results()
 
-    return code, f.search()
+    return f.status_code, f.search()
 
 
 fs = {
@@ -40,10 +39,9 @@ async def fetch_func(query: str, t: str) -> Tuple[int, Dict[str, Any]]:
         raise Exception("Invalid Error")
 
     f = await fs[t].scrape(query=query, t="page")
-    code, ok = f.check()
-    if not ok:
-        return code, error(code, "An unexpected error occurred.")
+    if not f.ok:
+        return f.status_code, error(f.status_code, "An unexpected error occurred.")
     else:
         f._get()
 
-    return code, f.fetch()
+    return f.status_code, f.fetch()
diff --git a/mypy.ini b/mypy.ini
@@ -7,13 +7,16 @@ disallow_incomplete_defs = True
 check_untyped_defs = True 
 disallow_untyped_decorators = True 
 no_implicit_optional = True
+show_error_codes = True
 warn_redundant_casts = True 
 warn_unused_ignores = True
 warn_return_any = True 
 # implicit_reexport = False
 strict_equality = True
+no-strict-optional=False
 # --strict end
 
+
 [mypy-tests.*]
 ignore_missing_imports = True
 check_untyped_defs = True

diff --git a/requirements.txt b/requirements.txt
@@ -1,46 +1,36 @@
-appdirs
-asgiref
-astroid
-attrs
-beautifulsoup4
-black
-bs4
-certifi
-chardet
-click
-cloudscraper
-fastapi
-flake8
-h11
-idna
-importlib-metadata
-iniconfig
-isort
-lazy-object-proxy
-lxml
-mccabe
-mypy
-mypy-extensions
-packaging
-pathspec
-pluggy
-py
-pycodestyle
-pydantic
-pyflakes
-pylint
-pyparsing
-pytest
-regex
-requests
-requests-toolbelt
-rope
-soupsieve
-starlette
-toml
-typed-ast==1.4.3
-typing-extensions==3.10.0.0
-urllib3
-uvicorn
-wrapt
-zipp
+anyio==3.6.1
+attrs==22.1.0
+beautifulsoup4==4.11.1
+black==22.8.0
+certifi==2022.6.15
+charset-normalizer==2.1.1
+click==8.1.3
+cloudscraper==1.2.64
+fastapi==0.81.0
+flake8==5.0.4
+h11==0.13.0
+idna==3.3
+iniconfig==1.1.1
+lxml==4.9.1
+mccabe==0.7.0
+mypy==0.971
+mypy-extensions==0.4.3
+packaging==21.3
+pathspec==0.10.0
+platformdirs==2.5.2
+pluggy==1.0.0
+py==1.11.0
+pycodestyle==2.9.1
+pydantic==1.10.1
+pyflakes==2.5.0
+pyparsing==3.0.9
+pytest==7.1.2
+requests==2.28.1
+requests-toolbelt==0.9.1
+sniffio==1.2.0
+soupsieve==2.3.2.post1
+starlette==0.19.1
+tomli==2.0.1
+typing_extensions==4.3.0
+urllib3==1.26.12
+uvicorn==0.18.3
diff --git a/tests/test_main.py b/tests/test_main.py
@@ -15,7 +15,7 @@ def test_sample_drama() -> None:
     dramas = [
         {
             "id": "65743-alien-girl-chai-xiao-qi-2",
-            "title": "My Girlfriend is an Alien 2",
+            "title": "My Girlfriend Is an Alien 2",
         },
         {"id": "58953-mouse", "title": "Mouse"},
     ]
@@ -24,6 +24,9 @@ def test_sample_drama() -> None:
         r = client.get(f"/id/{i['id']}")
 
         assert r.status_code == 200
+
+        print(r.json()["data"])
+
         assert r.json()["data"]["title"] == i["title"]