0.3.13, ignore broken images

yaroslaff · Apr 12, 2023 · 9b0deed · 9b0deed
1 parent 295ffba
commit 9b0deed
Show file tree

Hide file tree

Showing 7 changed files with 40 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -31,3 +31,9 @@
 
 # 0.3.11
 - fixed --workdir / --resume in docker image
+
+# 0.3.13
+- Catch errors (and not crash) with broken images
+- Load nudenet classifier only when needed
+
+
diff --git a/bin/nudecrawler b/bin/nudecrawler
@@ -13,7 +13,7 @@ import requests
 from dotenv import load_dotenv
 
 import nudecrawler 
-from nudecrawler import Page, Unbuffered
+from nudecrawler import Page, Unbuffered, load
 from nudecrawler.page import  get_processed_images, context_fields
 from nudecrawler.version import version
 from nudecrawler.cache import cache
@@ -434,6 +434,9 @@ def main():
             print(f"# No cache file {stats['cache_path']}, start with empty cache")
 
     # processing could start here
+
+    nudecrawler.load()
+
     # --url1 
     if args.url1:
         p = analyse(args.url1)

diff --git a/build-docker.sh b/build-docker.sh
@@ -7,15 +7,25 @@ echo mode: $MODE
 
 # sudo docker build --build-arg VERSION=0.3.10 -t yaroslaff/nudecrawler:0.3.10 -f docker/Dockerfile .
 
+VERSION=`python -c 'import nudecrawler.version; print(nudecrawler.version.version)'`
+echo version: $VERSION
+
 if [ "$MODE" == "dev" ]
 then
     echo build development mode
     sudo docker build -t yaroslaff/nudecrawler:dev -f docker/DevDockerfile .
-else
+elif [ "$MODE" == "publish" ]
+then
+    echo publish version $VERSION
+    python3 setup.py bdist_wheel sdist
+    twine upload dist/nudecrawler*$VERSION*
+
     echo build version $MODE
     echo === $MODE
-    sudo docker build --build-arg VERSION=$MODE -t yaroslaff/nudecrawler:$MODE -f docker/Dockerfile .
+    sudo docker build --build-arg VERSION=$VERSION -t yaroslaff/nudecrawler:$VERSION -f docker/Dockerfile .
 
     echo === LATEST
-    sudo docker build --build-arg VERSION=$MODE -t yaroslaff/nudecrawler:latest -f docker/Dockerfile .
+    sudo docker build --build-arg VERSION=$VERSION -t yaroslaff/nudecrawler:latest -f docker/Dockerfile .
+else
+    echo dev of publish?
 fi
diff --git a/nudecrawler/__init__.py b/nudecrawler/__init__.py
@@ -1,3 +1,7 @@
 from .page import Page
 from .remoteimage import RemoteImage
 from .unbuffered import Unbuffered
+
+def load():
+    remoteimage._load()
+
diff --git a/nudecrawler/page.py b/nudecrawler/page.py
@@ -295,14 +295,7 @@ def is_nude(self, url):
             try:
                 self.do_detect_image(url)
             except Exception as e:
-                if verbose.send_bugreports:
-                    verbose.bugreport(page_url=self.url, image_url=url, detector=self.detect_image, exception=str(e))
-                else:
-                    print("Please run with --bugreport option to send automatical and anonymous bugreport (I will not see your IP)")
-                    print("Detector:", self.detect_image)
-                    print("Problem page:", self.url)
-                    print("Problem image:", url)
-                sys.exit(1)
+                printv("Broken image:", url)
             return
 
 

diff --git a/nudecrawler/remoteimage.py b/nudecrawler/remoteimage.py
@@ -13,13 +13,18 @@
 
 # detector_address = 'http://localhost:9191/api/v1/detect'
 
-try:
-    from nudenet import NudeClassifier
-    print("Loading nudenet classifier....")
-    nudenet_classifier = NudeClassifier()
+nudenet_classifier = None
+
+def _load():
+    global nudenet_classifier
+    try:
+        from nudenet import NudeClassifier
+        print("Loading nudenet classifier....")
+        nudenet_classifier = NudeClassifier()
+    except ModuleNotFoundError:
+        nudenet_classifier = None
+
 
-except ModuleNotFoundError:
-    nudenet_classifier = None
 
 class RemoteImage:
     def __init__(self, url):        

diff --git a/nudecrawler/version.py b/nudecrawler/version.py
@@ -1 +1 @@
-version="0.3.12"
+version="0.3.13"