Optimize and fix database config handling in general

bunkerity · TheophileDiot · Jun 14, 2024 · Jun 7, 2024 · Jun 8, 2024 · Jun 8, 2024
commit 4803c4997e1b94c25c7ae90d8ef13bff5668d913
diff --git a/src/common/core/misc/jobs/anonymous-report.py b/src/common/core/misc/jobs/anonymous-report.py
@@ -41,7 +41,7 @@
         if key not in ("version", "integration", "database_version", "is_pro"):
             data.pop(key, None)
 
-    db_config = JOB.db.get_config(methods=True, with_drafts=True)
+    db_config = JOB.db.get_non_default_settings(methods=True, with_drafts=True)
     services = db_config.get("SERVER_NAME", {"value": ""})["value"].split(" ")
     multisite = db_config.get("MULTISITE", {"value": "no"})["value"] == "yes"
 
@@ -85,7 +85,7 @@
 
     data["non_default_settings"] = {}
     for setting, setting_data in db_config.items():
-        if isinstance(setting_data, dict) and setting_data["method"] != "default":
+        if isinstance(setting_data, dict):
             for server in services:
                 if setting.startswith(server + "_"):
                     setting = setting[len(server) + 1 :]  # noqa: E203

diff --git a/src/common/db/Database.py b/src/common/db/Database.py
@@ -1533,102 +1533,15 @@ def save_custom_configs(
 
         return message
 
-    def get_config(self, global_only: bool = False, methods: bool = False, with_drafts: bool = False) -> Dict[str, Any]:
-        """Get the config from the database"""
-        with self.__db_session() as session:
-            config = {}
-            multisite = []
-            for setting in (
-                session.query(Settings)
-                .with_entities(
-                    Settings.id,
-                    Settings.context,
-                    Settings.default,
-                    Settings.multiple,
-                )
-                .order_by(Settings.order)
-            ):
-                default = setting.default or ""
-                config[setting.id] = default if not methods else {"value": default, "global": True, "method": "default"}
-
-                for global_value in (
-                    session.query(Global_values).with_entities(Global_values.value, Global_values.suffix, Global_values.method).filter_by(setting_id=setting.id)
-                ):
-                    config[setting.id + (f"_{global_value.suffix}" if setting.multiple and global_value.suffix > 0 else "")] = (
-                        global_value.value
-                        if not methods
-                        else {
-                            "value": global_value.value,
-                            "global": True,
-                            "method": global_value.method,
-                        }
-                    )
-
-                if setting.context == "multisite":
-                    multisite.append(setting.id)
-
-            is_multisite = config.get("MULTISITE", {"value": "no"})["value"] == "yes" if methods else config.get("MULTISITE", "no") == "yes"
-
-            services = session.query(Services).with_entities(Services.id, Services.is_draft)
-            if not with_drafts:
-                services = services.filter_by(is_draft=False)
-
-            if not global_only and is_multisite:
-                for service in services:
-                    config[f"{service.id}_IS_DRAFT"] = "yes" if service.is_draft else "no"
-                    if methods:
-                        config[f"{service.id}_IS_DRAFT"] = {"value": config[f"{service.id}_IS_DRAFT"], "global": False, "method": "default"}
-
-                    checked_settings = []
-                    for key, value in config.copy().items():
-                        original_key = key
-                        if self.suffix_rx.search(key):
-                            key = key[: -len(str(key.split("_")[-1])) - 1]
-
-                        if key not in multisite:
-                            continue
-                        elif f"{service.id}_{original_key}" not in config:
-                            config[f"{service.id}_{original_key}"] = value
-
-                        if original_key not in checked_settings:
-                            checked_settings.append(original_key)
-                        else:
-                            continue
-
-                        for service_setting in (
-                            session.query(Services_settings)
-                            .with_entities(
-                                Services_settings.value,
-                                Services_settings.suffix,
-                                Services_settings.method,
-                            )
-                            .filter_by(service_id=service.id, setting_id=key)
-                        ):
-                            value = service_setting.value
-                            if key == "SERVER_NAME" and service.id not in value.split(" "):
-                                value = f"{service.id} {value}".strip()
-
-                            config[f"{service.id}_{key}" + (f"_{service_setting.suffix}" if service_setting.suffix > 0 else "")] = (
-                                value
-                                if not methods
-                                else {
-                                    "value": value,
-                                    "global": False,
-                                    "method": service_setting.method,
-                                }
-                            )
-
-            servers = " ".join(service.id for service in services)
-            config["SERVER_NAME"] = servers if not methods else {"value": servers, "global": True, "method": "default"}
-
-            return config
-
-    def get_filtered_config(
+    def get_non_default_settings(
         self,
         global_only: bool = False,
         methods: bool = False,
         with_drafts: bool = False,
         filtered_settings: Optional[Union[List[str], Set[str], Tuple[str]]] = None,
+        *,
+        original_config: Optional[Dict[str, Any]] = None,
+        original_multisite: Optional[Set[str]] = None,
     ) -> Dict[str, Any]:
         """Get the config from the database"""
         filtered_settings = set(filtered_settings or [])
@@ -1637,14 +1550,15 @@ def get_filtered_config(
             filtered_settings.update(("SERVER_NAME", "MULTISITE"))
 
         with self.__db_session() as session:
-            config = {}
+            config = original_config or {}
+            multisite = original_multisite or set()
 
             # Define the join operation
             j = join(Settings, Global_values, Settings.id == Global_values.setting_id)
 
             # Define the select statement
             stmt = (
-                db_select(Settings.id, Settings.multiple, Global_values.value, Global_values.suffix, Global_values.method)
+                db_select(Settings.id.label("setting_id"), Settings.context, Settings.multiple, Global_values.value, Global_values.suffix, Global_values.method)
                 .select_from(j)
                 .order_by(Settings.order)
             )
@@ -1655,78 +1569,111 @@ def get_filtered_config(
             # Execute the query and fetch all results
             results = session.execute(stmt).fetchall()
 
-            for setting in results:
-                key = setting.id + (f"_{setting.suffix}" if setting.multiple and setting.suffix else "")
-                config[key] = setting.value if not methods else {"value": setting.value, "global": True, "method": setting.method}
+            for global_value in results:
+                setting_id = global_value.setting_id + (f"_{global_value.suffix}" if global_value.multiple and global_value.suffix > 0 else "")
+                config[setting_id] = global_value.value if not methods else {"value": global_value.value, "global": True, "method": global_value.method}
+                if global_value.context == "multisite":
+                    multisite.add(setting_id)
 
-            is_multisite = config.get("MULTISITE", {"value": "no"})["value"] == "yes" if methods else config.get("MULTISITE", "no") == "yes"
+            services = session.query(Services).with_entities(Services.id, Services.is_draft)
 
-            # Define the join operation
-            j = join(Services, Services_settings, Services.id == Services_settings.service_id)
-            j = j.join(Settings, Settings.id == Services_settings.setting_id)
+            if not with_drafts:
+                services = services.filter_by(is_draft=False)
 
-            # Define the select statement
-            stmt = db_select(
-                Services.id.label("service_id"),
-                Services.is_draft,
-                Settings.id.label("setting_id"),
-                Settings.context,
-                Settings.default,
-                Settings.multiple,
-                Services_settings.value,
-                Services_settings.suffix,
-                Services_settings.method,
-            ).select_from(j)
+            servers = ""
+            for service in services:
+                config[f"{service.id}_IS_DRAFT"] = "yes" if service.is_draft else "no"
+                if methods:
+                    config[f"{service.id}_IS_DRAFT"] = {"value": config[f"{service.id}_IS_DRAFT"], "global": False, "method": "default"}
+                for key in multisite:
+                    config[f"{service.id}_{key}"] = config[key]
+                servers += f"{service.id} "
+            servers = servers.strip()
 
-            if not with_drafts:
-                stmt = stmt.where(Services.is_draft == False)  # noqa: E712
+            config["SERVER_NAME"] = servers if not methods else {"value": servers, "global": True, "method": "default"}
 
-            if filtered_settings:
-                stmt = stmt.where(Services_settings.setting_id.in_(filtered_settings))
+            if not global_only and (config.get("MULTISITE", {"value": "no"})["value"] == "yes" if methods else config.get("MULTISITE", "no") == "yes"):
+                # Define the join operation
+                j = join(Services, Services_settings, Services.id == Services_settings.service_id)
+                j = j.join(Settings, Settings.id == Services_settings.setting_id)
 
-            # Execute the query and fetch all results
-            results = session.execute(stmt).fetchall()
+                # Define the select statement
+                stmt = (
+                    db_select(
+                        Services.id.label("service_id"),
+                        Settings.id.label("setting_id"),
+                        Settings.multiple,
+                        Services_settings.value,
+                        Services_settings.suffix,
+                        Services_settings.method,
+                    )
+                    .select_from(j)
+                    .order_by(Services.id, Settings.order)
+                )
 
-            if not global_only and is_multisite:
-                for result in results:
-                    if f"{result.service_id}_IS_DRAFT" not in config:
-                        config[f"{result.service_id}_IS_DRAFT"] = "yes" if result.is_draft else "no"
-                        if methods:
-                            config[f"{result.service_id}_IS_DRAFT"] = {"value": config[f"{result.service_id}_IS_DRAFT"], "global": False, "method": "default"}
+                if not with_drafts:
+                    stmt = stmt.where(Services.is_draft == False)  # noqa: E712
 
-                    key = result.setting_id
-                    original_key = key
-                    if self.suffix_rx.search(key):
-                        key = key[: -len(str(key.split("_")[-1])) - 1]
+                if filtered_settings:
+                    stmt = stmt.where(Settings.id.in_(filtered_settings))
 
-                    if result.context != "multisite":
-                        continue
-                    elif f"{result.service_id}_{original_key}" not in config:
-                        config[f"{result.service_id}_{original_key}"] = result.default
+                # Execute the query and fetch all results
+                results = session.execute(stmt).fetchall()
 
+                for result in results:
                     value = result.value
-                    if key == "SERVER_NAME" and not search(r"(^| )" + escape(result.service_id) + r"($| )", value):
-                        value = f"{result.service_id} {value}".strip()
-
-                    config[f"{result.service_id}_{key}" + (f"_{result.suffix}" if result.multiple and result.suffix else "")] = (
-                        value
-                        if not methods
-                        else {
-                            "value": value,
-                            "global": False,
-                            "method": result.method,
-                        }
+
+                    if result.setting_id == "SERVER_NAME" and not search(r"^" + escape(result.service_id) + r"( |$)", value):
+                        split = set(value.split(" "))
+                        split.discard(result.service_id)
+                        value = result.service_id + " " + " ".join(split)
+
+                    config[f"{result.service_id}_{result.setting_id}" + (f"_{result.suffix}" if result.multiple and result.suffix else "")] = (
+                        value if not methods else {"value": value, "global": False, "method": result.method}
                     )
 
-            services = session.query(Services).with_entities(Services.id)
+            return config
 
-            if not with_drafts:
-                services = services.filter_by(is_draft=False)
+    def get_config(
+        self,
+        global_only: bool = False,
+        methods: bool = False,
+        with_drafts: bool = False,
+        filtered_settings: Optional[Union[List[str], Set[str], Tuple[str]]] = None,
+    ) -> Dict[str, Any]:
+        """Get the config from the database"""
+        with self.__db_session() as session:
+            config = {}
+            multisite = set()
 
-            servers = " ".join(service.id for service in services)
-            config["SERVER_NAME"] = servers if not methods else {"value": servers, "global": True, "method": "default"}
+            query = (
+                session.query(Settings)
+                .with_entities(
+                    Settings.id,
+                    Settings.context,
+                    Settings.default,
+                    Settings.multiple,
+                )
+                .order_by(Settings.order)
+            )
 
-            return config
+            if filtered_settings:
+                query = query.filter(Settings.id.in_(filtered_settings))
+
+            for setting in query:
+                default = setting.default or ""
+                config[setting.id] = default if not methods else {"value": default, "global": True, "method": "default"}
+                if setting.context == "multisite":
+                    multisite.add(setting.id)
+
+        return self.get_non_default_settings(
+            global_only=global_only,
+            methods=methods,
+            with_drafts=with_drafts,
+            filtered_settings=filtered_settings,
+            original_config=config,
+            original_multisite=multisite,
+        )
 
     def get_custom_configs(self) -> List[Dict[str, Any]]:
         """Get the custom configs from the database"""

diff --git a/src/ui/main.py b/src/ui/main.py
@@ -753,9 +753,9 @@ def account():
             variable = {}
             variable["PRO_LICENSE_KEY"] = request.form["license"]
 
-            error = app.config["CONFIG"].check_variables(variable)
+            variable = app.config["CONFIG"].check_variables(variable, {"PRO_LICENSE_KEY": request.form["license"]})
 
-            if error:
+            if not variable:
                 return redirect_flash_error("The license key variable checks returned error", "account", True)
 
             # Force job to contact PRO API
@@ -963,7 +963,7 @@ def services():
         if "SERVER_NAME" not in variables:
             variables["SERVER_NAME"] = variables["OLD_SERVER_NAME"]
 
-        config = app.config["CONFIG"].get_config(with_drafts=True, filtered_settings=variables.keys())
+        config = app.config["DB"].get_config(methods=True, with_drafts=True)
         server_name = variables["SERVER_NAME"].split(" ")[0]
         was_draft = config.get(f"{server_name}_IS_DRAFT", {"value": "no"})["value"] == "yes"
         operation = request.form["operation"]
@@ -997,15 +997,15 @@ def services():
             del variables["OLD_SERVER_NAME"]
 
             # Edit check fields and remove already existing ones
-            for variable, value in deepcopy(variables).items():
+            for variable, value in variables.copy().items():
                 if (
                     variable in variables
                     and variable != "SERVER_NAME"
                     and value == config.get(f"{server_name}_{variable}" if request.form["operation"] == "edit" else variable, {"value": None})["value"]
                 ):
                     del variables[variable]
 
-            variables = app.config["CONFIG"].check_variables(variables)
+            variables = app.config["CONFIG"].check_variables(variables, config)
 
             if (
                 was_draft == is_draft
@@ -1024,7 +1024,7 @@ def services():
 
             is_request_params(["SERVER_NAME"], "services", True)
 
-            variables = app.config["CONFIG"].check_variables({"SERVER_NAME": request.form["SERVER_NAME"]})
+            variables = app.config["CONFIG"].check_variables({"SERVER_NAME": request.form["SERVER_NAME"]}, config)
 
             if not variables:
                 error_message(f"Error while deleting the service {request.form['SERVER_NAME']}")
@@ -1138,12 +1138,13 @@ def global_config():
         del variables["csrf_token"]
 
         # Edit check fields and remove already existing ones
-        config = app.config["CONFIG"].get_config(with_drafts=True, filtered_settings=variables.keys())
+        config = app.config["DB"].get_config(methods=True, with_drafts=True)
         services = config["SERVER_NAME"]["value"].split(" ")
         for variable, value in variables.copy().items():
             setting = config.get(variable, {"value": None, "global": True})
             if setting["global"] and value == setting["value"]:
                 del variables[variable]
+                continue
 
         variables = app.config["CONFIG"].check_variables(variables, config)
 

diff --git a/src/ui/src/Config.py b/src/ui/src/Config.py
@@ -92,7 +92,7 @@ def get_config(
         dict
             The nginx variables env file as a dict
         """
-        return self.__db.get_filtered_config(global_only=global_only, methods=methods, with_drafts=with_drafts, filtered_settings=filtered_settings)
+        return self.__db.get_non_default_settings(global_only=global_only, methods=methods, with_drafts=with_drafts, filtered_settings=filtered_settings)
 
     def get_services(self, methods: bool = True, with_drafts: bool = False) -> list[dict]:
         """Get nginx's services

diff --git a/src/ui/src/Instances.py b/src/ui/src/Instances.py
@@ -152,7 +152,7 @@ def get_instances(self, override_instances=None) -> list[Instance]:
         instances = []
         # Override case : only return instances from DB
         if override_instances is None:
-            config = self.__db.get_filtered_config(global_only=True, filtered_settings=("OVERRIDE_INSTANCES",))
+            config = self.__db.get_non_default_settings(global_only=True, filtered_settings=("OVERRIDE_INSTANCES",))
             override_instances = config.get("OVERRIDE_INSTANCES", "") != ""
         if override_instances:
             for instance in self.__db.get_instances():