Scan threshold and auto-deploy to docker (#7)

* Allow 0 threshold * automatic versioning * Add the pragma only for sqlite
db0 · Sep 20, 2023 · 5433153 · 5433153
1 parent a4108ee
commit 5433153
Show file tree

Hide file tree

Showing 28 changed files with 116 additions and 50 deletions.
diff --git a/.env_template b/.env_template
@@ -21,4 +21,5 @@ POSTGRES_URI="postgresql://pictrsafety:ThatSecretPGSQLPass@127.0.0.1/pictrsafety
 # If this amount of images are waiting for scanning, start retuning "OK" for each new scan after that
 # This will prevent pict-rs from rejecting each and every upload because your scanning workers crashed or something
 # and the admin is AFK
+# Set this value to 0 to never bypass the scan.
 SCAN_BYPASS_THRESHOLD=10
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,60 @@
+name: pictrs-safety new release
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+  pull-requests: read
+
+jobs:
+  build-n-publish:
+    name: pictrs-safety new release
+    runs-on: ubuntu-latest
+    steps:
+    - name: "✔️ Checkout"
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: "📣 Release on push"
+      id: release
+      uses: rymndhng/release-on-push-action@master
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        bump_version_scheme: norelease
+        use_github_release_notes: true
+    - name: "✏️ Generate release changelog"
+      if: ${{ steps.release.outputs.version != '' }}
+      uses: heinrichreimer/github-changelog-generator-action@v2.3
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+    - name: "💾 Commit new version"
+      if: ${{ steps.release.outputs.version != '' }}
+      run: |
+          version=${{ steps.release.outputs.version }}
+          sed -i "s/^PICTRS_SAFETY_VERSION = .*/PICTRS_SAFETY_VERSION = ${version}/g" ./pictrs_safety_api/consts.py
+          git config user.email github-actions@github.com
+          git config user.name github-actions
+          git commit -am 'version incremented'
+          git push
+    - name: Login to GitHub Container Registry
+      if: ${{ steps.release.outputs.version != '' }}
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    # - name: Free disk space
+    #   if: ${{ steps.release.outputs.version != '' }}
+    #   uses: jlumbroso/free-disk-space@main
+    #   with:
+    #     tool-cache: false
+    - name: "Build and Publish Container to Github Container Registry"
+      if: ${{ steps.release.outputs.version != '' }}
+      run: |
+        docker build -t ghcr.io/db0/pictrs-safety:${{ steps.release.outputs.version }} .
+        docker push ghcr.io/db0/pictrs-safety:${{ steps.release.outputs.version }}
+        docker push ghcr.io/db0/pictrs-safety:latest
diff --git a/.gitignore b/.gitignore
@@ -161,4 +161,6 @@ cython_debug/
 
 lemmy_safety.db
 lemmy_safety.db-journal
-api_test*
+api_test*
+images
+fedi_safety_api*.bz2
diff --git a/fedi_safety_api/apis/models/__init__.py → CHANGELOG.md b/fedi_safety_api/apis/models/__init__.py → CHANGELOG.md
diff --git a/api_server.py b/api_server.py
@@ -5,8 +5,8 @@
 
 load_dotenv()
 
-from fedi_safety_api.argparser import args
-from fedi_safety_api.flask import APP
+from pictrs_safety_api.argparser import args
+from pictrs_safety_api.flask import APP
 from loguru import logger
 
 if __name__ == "__main__":

diff --git a/fedi_safety_api/apis/v1/__init__.py b/fedi_safety_api/apis/v1/__init__.py
diff --git a/fedi_safety_api/classes/__init__.py b/fedi_safety_api/classes/__init__.py
diff --git a/fedi_safety_api/consts.py b/fedi_safety_api/consts.py
diff --git a/fedi_safety_api/__init__.py → pictrs_safety_api/__init__.py b/fedi_safety_api/__init__.py → pictrs_safety_api/__init__.py
@@ -2,12 +2,12 @@
 import socket
 from uuid import uuid4
 
-from fedi_safety_api.logger import logger
-from fedi_safety_api.flask import APP
-from fedi_safety_api.routes import * 
-from fedi_safety_api.apis import apiv1
-from fedi_safety_api.argparser import args
-from fedi_safety_api.consts import FEDI_SAFETY_VERSION
+from pictrs_safety_api.logger import logger
+from pictrs_safety_api.flask import APP
+from pictrs_safety_api.routes import * 
+from pictrs_safety_api.apis import apiv1
+from pictrs_safety_api.argparser import args
+from pictrs_safety_api.consts import PICTRS_SAFETY_VERSION
 import hashlib
 
 APP.register_blueprint(apiv1)
@@ -18,7 +18,7 @@ def after_request(response):
     response.headers["Access-Control-Allow-Origin"] = "*"
     response.headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS, PUT, DELETE, PATCH"
     response.headers["Access-Control-Allow-Headers"] = "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, apikey, Client-Agent, X-Fields"
-    response.headers["Fedi-Safety-API-Node"] = f"{socket.gethostname()}:{args.port}:{FEDI_SAFETY_VERSION}"
+    response.headers["Fedi-Safety-API-Node"] = f"{socket.gethostname()}:{args.port}:{PICTRS_SAFETY_VERSION}"
     try:
         etag = hashlib.sha1(response.get_data()).hexdigest()
     except RuntimeError:

diff --git a/fedi_safety_api/apis/__init__.py → pictrs_safety_api/apis/__init__.py b/fedi_safety_api/apis/__init__.py → pictrs_safety_api/apis/__init__.py
diff --git a/fedi_safety_api/apis/apiv1.py → pictrs_safety_api/apis/apiv1.py b/fedi_safety_api/apis/apiv1.py → pictrs_safety_api/apis/apiv1.py
@@ -2,7 +2,7 @@
 from flask_restx import Api
 from importlib import import_module
 
-from fedi_safety_api.apis.v1 import api as v1
+from pictrs_safety_api.apis.v1 import api as v1
 
 blueprint = Blueprint('apiv1', __name__, url_prefix='/api')
 api = Api(blueprint,

diff --git a/fedi_safety_api/database/__init__.py → pictrs_safety_api/apis/models/__init__.py b/fedi_safety_api/database/__init__.py → pictrs_safety_api/apis/models/__init__.py
diff --git a/fedi_safety_api/apis/models/v1.py → pictrs_safety_api/apis/models/v1.py b/fedi_safety_api/apis/models/v1.py → pictrs_safety_api/apis/models/v1.py
diff --git a/pictrs_safety_api/apis/v1/__init__.py b/pictrs_safety_api/apis/v1/__init__.py
@@ -0,0 +1,5 @@
+import pictrs_safety_api.apis.v1.base as base
+from pictrs_safety_api.apis.v1.base import api
+
+api.add_resource(base.Scan, "/scan/<string:pictrs_id>")
+api.add_resource(base.Pop, "/pop")
diff --git a/fedi_safety_api/apis/v1/base.py → pictrs_safety_api/apis/v1/base.py b/fedi_safety_api/apis/v1/base.py → pictrs_safety_api/apis/v1/base.py
@@ -7,16 +7,16 @@
 from werkzeug.datastructures import FileStorage
 from flask import request, send_file
 from flask_restx import Namespace, Resource, reqparse
-from fedi_safety_api.flask import cache, db
+from pictrs_safety_api.flask import cache, db
 from loguru import logger
-from fedi_safety_api.classes.request import ScanRequest
-from fedi_safety_api.database import functions as database
-from fedi_safety_api import exceptions as e
-from fedi_safety_api import enums
+from pictrs_safety_api.classes.request import ScanRequest
+from pictrs_safety_api.database import functions as database
+from pictrs_safety_api import exceptions as e
+from pictrs_safety_api import enums
 
 api = Namespace('v1', 'API Version 1' )
 
-from fedi_safety_api.apis.models.v1 import Models
+from pictrs_safety_api.apis.models.v1 import Models
 
 models = Models(api)
 
@@ -44,14 +44,15 @@ def post(self,pictrs_id):
         '''Scan an image
         '''
         # I don't get why this is not using the import from earlier...
-        from fedi_safety_api import exceptions as e
+        from pictrs_safety_api import exceptions as e
         if pictrs_id == "IPADDR":
             if request.remote_addr not in json.loads(os.getenv("KNOWN_PICTRS_IPS","[]"))\
                     and not self.is_private_ip(request.remote_addr):
                 raise e.Unauthorized("You are not authorized to use this service", f"Unauthorized IP: {request.remote_addr}")
         elif pictrs_id not in json.loads(os.getenv("KNOWN_PICTRS_IDS", "[]")):
             raise e.Unauthorized("You are not authorized to use this service", f"Unauthorized ID: {pictrs_id}")
-        if database.count_waiting_scan_requests() > int(os.getenv("SCAN_BYPASS_THRESHOLD", 10)):
+        scan_threshold = int(os.getenv("SCAN_BYPASS_THRESHOLD", 10))
+        if scan_threshold > 0 and database.count_waiting_scan_requests() > scan_threshold:
             return {"message": "Image OK"}, 200 
         self.args = self.post_parser.parse_args()
         file = self.args["file"]
@@ -143,6 +144,7 @@ def get(self):
         '''Pick up an image to safety validate
         '''
         # I don't get why this is not using the import from earlier...
+        logger.debug(request.remote_addr)
         self.args = self.get_parser.parse_args()
         if os.getenv("FEDIVERSE_SAFETY_WORKER_AUTH") != self.args.apikey:
             raise e.Forbidden("Access Denied")

diff --git a/fedi_safety_api/argparser.py → pictrs_safety_api/argparser.py b/fedi_safety_api/argparser.py → pictrs_safety_api/argparser.py
diff --git a/pictrs_safety_api/classes/__init__.py b/pictrs_safety_api/classes/__init__.py
@@ -0,0 +1,8 @@
+from pictrs_safety_api.flask import db, APP
+
+# Importing for DB creation
+from pictrs_safety_api.classes.request import ScanRequest
+
+with APP.app_context():
+
+    db.create_all()
diff --git a/fedi_safety_api/classes/request.py → pictrs_safety_api/classes/request.py b/fedi_safety_api/classes/request.py → pictrs_safety_api/classes/request.py
@@ -3,8 +3,8 @@
 from sqlalchemy import Enum, UniqueConstraint
 
 from loguru import logger
-from fedi_safety_api.flask import db
-from fedi_safety_api import enums
+from pictrs_safety_api.flask import db
+from pictrs_safety_api import enums
 
 
 class ScanRequest(db.Model):

diff --git a/pictrs_safety_api/consts.py b/pictrs_safety_api/consts.py
@@ -0,0 +1 @@
+PICTRS_SAFETY_VERSION = "1.1.0"
diff --git a/pictrs_safety_api/database/__init__.py b/pictrs_safety_api/database/__init__.py
diff --git a/fedi_safety_api/database/functions.py → pictrs_safety_api/database/functions.py b/fedi_safety_api/database/functions.py → pictrs_safety_api/database/functions.py
@@ -5,11 +5,11 @@
 from datetime import datetime, timedelta
 from sqlalchemy import func, or_, and_, not_, Boolean
 from sqlalchemy.orm import noload
-from fedi_safety_api.flask import db
-from fedi_safety_api.utils import hash_api_key
+from pictrs_safety_api.flask import db
+from pictrs_safety_api.utils import hash_api_key
 from sqlalchemy.orm import joinedload
-from fedi_safety_api.classes.request import ScanRequest
-from fedi_safety_api import enums
+from pictrs_safety_api.classes.request import ScanRequest
+from pictrs_safety_api import enums
 
 def find_waiting_scan_request():
     query = ScanRequest.query.filter(

diff --git a/fedi_safety_api/enums.py → pictrs_safety_api/enums.py b/fedi_safety_api/enums.py → pictrs_safety_api/enums.py
diff --git a/fedi_safety_api/exceptions.py → pictrs_safety_api/exceptions.py b/fedi_safety_api/exceptions.py → pictrs_safety_api/exceptions.py
diff --git a/fedi_safety_api/flask.py → pictrs_safety_api/flask.py b/fedi_safety_api/flask.py → pictrs_safety_api/flask.py
@@ -36,6 +36,14 @@
 if not SQLITE_MODE:
     with APP.app_context():
         logger.debug("pool size = {}".format(db.engine.pool.size()))
+else:
+    @event.listens_for(Engine, "connect")
+    def set_sqlite_pragma(dbapi_connection, connection_record):
+        cursor = dbapi_connection.cursor()
+        cursor.execute("PRAGMA journal_mode=WAL;")
+        cursor.close()
+        logger.debug("Set pragma to wal")
+
 logger.init_ok("Fedi Safety Database", status="Started")
 
 # Allow local workstation run
@@ -47,10 +55,3 @@
     cache = Cache(config=cache_config)
     cache.init_app(APP)
     logger.init_warn("Flask Cache", status="SimpleCache")
-
-@event.listens_for(Engine, "connect")
-def set_sqlite_pragma(dbapi_connection, connection_record):
-    cursor = dbapi_connection.cursor()
-    cursor.execute("PRAGMA journal_mode=WAL;")
-    cursor.close()
-    logger.info("Set pragma to wal")
diff --git a/fedi_safety_api/logger.py → pictrs_safety_api/logger.py b/fedi_safety_api/logger.py → pictrs_safety_api/logger.py
@@ -1,7 +1,7 @@
 import sys
 from functools import partialmethod
 from loguru import logger
-from fedi_safety_api.argparser import args
+from pictrs_safety_api.argparser import args
 
 STDOUT_LEVELS = ["GENERATION", "PROMPT"]
 INIT_LEVELS = ["INIT", "INIT_OK", "INIT_WARN", "INIT_ERR"]

diff --git a/fedi_safety_api/routes.py → pictrs_safety_api/routes.py b/fedi_safety_api/routes.py → pictrs_safety_api/routes.py
@@ -1,8 +1,8 @@
 from flask import render_template, redirect, url_for, request
 from markdown import markdown
 from loguru import logger
-from fedi_safety_api.flask import APP
-import fedi_safety_api.exceptions as e
+from pictrs_safety_api.flask import APP
+import pictrs_safety_api.exceptions as e
 
 @logger.catch(reraise=True)
 @APP.route('/')

diff --git a/fedi_safety_api/templates/index.md → pictrs_safety_api/templates/index.md b/fedi_safety_api/templates/index.md → pictrs_safety_api/templates/index.md
diff --git a/fedi_safety_api/utils.py → pictrs_safety_api/utils.py b/fedi_safety_api/utils.py → pictrs_safety_api/utils.py
@@ -9,7 +9,7 @@
 from datetime import datetime
 import dateutil.relativedelta
 from loguru import logger
-from fedi_safety_api.flask import SQLITE_MODE
+from pictrs_safety_api.flask import SQLITE_MODE
 
 random.seed(random.SystemRandom().randint(0, 2**32 - 1))