improve discord logging

mhorst00 · Oct 14, 2022 · 9f41fa8 · 9f41fa8
1 parent 0d459da
commit 9f41fa8
Show file tree

Hide file tree

Showing 2 changed files with 113 additions and 33 deletions.
diff --git a/mining/discord_logging.py b/mining/discord_logging.py
@@ -1,5 +1,7 @@
 import logging
 import os
+from shutil import ExecError
+from typing import Any
 from discord import SyncWebhook, File
 from datetime import datetime
 
@@ -11,11 +13,51 @@
 # webhook.send("<#1020308171140628480> Hello there!") #Mention Channel
 # k.send("<@> Hello there!") #Mention Person
 
-WEBHOOK_LOGGING_URL = os.environ["WEBHOOK_LOGGING_URL"]
-WEBHOOK_ERROR_URL = os.environ["WEBHOOK_ERROR_URL"]
 
-webhookLogging = SyncWebhook.from_url(WEBHOOK_LOGGING_URL)
-webhookError = SyncWebhook.from_url(WEBHOOK_ERROR_URL)
+WEBHOOK_LOGGING_URL = os.environ.get("WEBHOOK_LOGGING_URL", "")
+WEBHOOK_ERROR_URL = os.environ.get("WEBHOOK_ERROR_URL", "")
+
+WEBHOOK_ERROR_ENABLED = True
+WEBHOOK_LOGGING_ENABLED = True
+
+FILE_LOGGING_ENABLED = False
+FILE_ERROR_ENABLED = False
+
+WEBHOOK_ERROR: SyncWebhook
+WEBHOOK_LOGGING: SyncWebhook
+
+
+def initialise():
+    global WEBHOOK_ERROR
+    global WEBHOOK_ERROR_ENABLED
+    global FILE_ERROR_ENABLED
+    global WEBHOOK_LOGGING_ENABLED
+    global FILE_LOGGING_ENABLED
+    global WEBHOOK_LOGGING
+    # if url is not present disable the corresponding logging
+    if WEBHOOK_ERROR_URL == "":
+        logging.warning("WEBHOOK_ERROR_URL is not set, deactivating error logging")
+        WEBHOOK_ERROR_ENABLED = False
+        FILE_ERROR_ENABLED = True
+    else:
+        try:
+            WEBHOOK_ERROR = SyncWebhook.from_url(WEBHOOK_ERROR_URL)
+            FILE_ERROR_ENABLED = False
+        except Exception as err:
+            logging.warning(err)
+            FILE_ERROR_ENABLED = True
+
+    if WEBHOOK_LOGGING_URL == "":
+        logging.warning("WEBHOOK_LOGGING_URL is not set, deactivating info logging")
+        WEBHOOK_LOGGING_ENABLED = False
+        FILE_LOGGING_ENABLED = True
+    else:
+        try:
+            WEBHOOK_LOGGING = SyncWebhook.from_url(WEBHOOK_LOGGING_URL)
+            FILE_LOGGING_ENABLED = False
+        except Exception as err:
+            logging.warning(err)
+            FILE_LOGGING_ENABLED = True
 
 
 def writeFile(filename: str, messages):
@@ -38,52 +80,74 @@ def readFileForWebhooks(filename: str):
     return file
 
 
-def finishLogging(numberOfTrips: int, numberOfBytes: int):
-    webhookLogging.send("Import has finished")
-    webhookLogging.send("Number of trips: " + str(numberOfTrips))
-    webhookLogging.send("Size in bytes: " + str(numberOfBytes))
+def deleteFile(filename: str):
+    currentDirectory = os.getcwd()
+    os.chdir("logs")
+    os.remove(filename)
+    os.chdir(currentDirectory)
 
-    sendMessages()
 
+def finishLogging(numberOfTrips: int, numberOfBytes: int):
+    global FILE_ERROR_ENABLED
+    global FILE_LOGGING_ENABLED
+    # send basic information about result
+    if WEBHOOK_LOGGING_ENABLED:
+        WEBHOOK_LOGGING.send("Import has finished")
+        WEBHOOK_LOGGING.send("Number of trips: " + str(numberOfTrips))
+        WEBHOOK_LOGGING.send("Size in bytes: " + str(numberOfBytes))
 
-def sendMessages():
     currentTime = str(datetime.now()).replace(" ", "_")
 
+    # send detailed logs
     if len(infoList) > 0:
         infoFn = "info_" + currentTime + ".txt"
         writeFile(infoFn, infoList)
-        message = "There were " + str(len(infoList)) + " Infos:"
-        file = readFileForWebhooks(infoFn)
-        webhookLogging.send(message, file=file)
+        # send logs to Discord if enabled
+        if WEBHOOK_LOGGING_ENABLED:
+            message = "There were " + str(len(infoList)) + " Infos:"
+            file = readFileForWebhooks(infoFn)
+            WEBHOOK_LOGGING.send(message, file=file)
+        # only keep file if webhook is not functioning
+        if not FILE_LOGGING_ENABLED:
+            deleteFile(infoFn)
+
     if len(warningList) > 0:
         warningFn = "warning_" + currentTime + ".txt"
         writeFile(warningFn, warningList)
-        message = "There were " + str(len(warningList)) + " Warnings:"
-        file = readFileForWebhooks(warningFn)
-        webhookLogging.send(message, file=file)
+        if WEBHOOK_LOGGING_ENABLED:
+            message = "There were " + str(len(warningList)) + " Warnings:"
+            file = readFileForWebhooks(warningFn)
+            WEBHOOK_LOGGING.send(message, file=file)
+        if not FILE_LOGGING_ENABLED:
+            deleteFile(warningFn)
+
     if len(errorList) > 0:
         errorFn = "error_" + currentTime + ".txt"
         writeFile(errorFn, errorList)
-        message = "There were " + str(len(errorList)) + " Errors:"
-        file = readFileForWebhooks(errorFn)
-        webhookError.send(message, file=file)
+        if WEBHOOK_ERROR_ENABLED:
+            message = "There were " + str(len(errorList)) + " Errors:"
+            file = readFileForWebhooks(errorFn)
+            WEBHOOK_ERROR.send(message, file=file)
+        if not FILE_ERROR_ENABLED:
+            deleteFile(errorFn)
 
 
 def info(message: str):
     logging.info(message)
-    infoList.append(message)
+    infoList.append(str(datetime.now()).replace(" ", "_") + ": " + message)
     # webhookLogging.send("INFO: " + message)
 
 
 def warning(message: str):
     logging.warn(message)
-    warningList.append(message)
+    warningList.append(str(datetime.now()).replace(" ", "_") + ": " + message)
     # webhookLogging.send("WARNING: " + message)
 
 
 def error(message: str):
+    global WEBHOOK_ERROR
     logging.error(message)
-    errorList.append(message)
-    webhookError.send(
+    errorList.append(str(datetime.now()).replace(" ", "_") + ": " + message)
+    WEBHOOK_ERROR.send(
         "<@&1020311126313009233> " + "ERROR: " + message,
     )
diff --git a/mining/download.py b/mining/download.py
@@ -9,19 +9,21 @@
 from retry import retry
 
 
-@retry(Exception, delay=1, tries=5)
+@retry(TypeError, delay=2, tries=3)
 def get_trips_with_retries(
     start: str, destination: str, time: datetime, session: requests.Session
 ):
     try:
         trips = vvspy.get_trips(
             start, destination, time, session=session, limit=5, timeout=(3.05, 6.1)
         )
-        if trips is None:
-            raise Exception("trips is none")
+
     except Exception as err:
         discord_logging.warning(str(err))
-        raise Exception("Request timeout")
+        # raise Exception("Request timeout")
+    if trips is None:
+        raise TypeError("trips was null")
+        # raise Exception("trips is none")
     return trips
 
 
@@ -34,14 +36,25 @@ def get_all_trips_from_station(start: str, stations: list[str], time: datetime):
                 trips = get_trips_with_retries(
                     start, destination, time, session=session
                 )
-                for i in trips:
-                    if isinstance(i, vvspy.obj.Trip):
-                        trip = i.raw
-                        results.append(trip)
+                if trips is not None:
+                    for i in trips:
+                        if isinstance(i, vvspy.obj.Trip):
+                            trip = i.raw
+                            results.append(trip)
+                else:
+                    discord_logging.info(
+                        "trips is None for:"
+                        + utils.station_id_to_name(start)
+                        + utils.station_id_to_name(destination)
+                    )
             except Exception as err:
                 discord_logging.warning(
-                    str(err) + utils.station_id_to_name(start),
-                    utils.station_id_to_name(destination) + str(type(trip)),
+                    str(err)
+                    + " "
+                    + utils.station_id_to_name(start)
+                    + utils.station_id_to_name(destination)
+                    + str(type(trips))
+                    + str(type(trip)),
                 )
     return results
 
@@ -82,12 +95,15 @@ def get_all_station_departures(stations: list[str], curr_time: datetime):
     return station_delays
 
 
+discord_logging.initialise()
 curr_time = datetime.now()
 stations = utils.read_station_ids_csv("vvs_sbahn_haltestellen_2022.csv")
 
 print(utils.station_id_to_name(stations[14]))
 trips = get_all_trips(stations, curr_time)
 # trips = get_all_station_departures(stations, curr_time)
+time_for_execute = datetime.now() - curr_time
 print("Number of trips: ", len(trips))
 print("Size in bytes: ", sys.getsizeof(trips))
+print("Executed in: ", time_for_execute)
 discord_logging.finishLogging(len(trips), sys.getsizeof(trips))