From c100e1c0445704f30572009290d9813588ef20a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Mon, 17 Jun 2024 12:09:24 +0200 Subject: [PATCH 1/6] chore: Refactor Database.py to improve setting existence check --- src/common/db/Database.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/common/db/Database.py b/src/common/db/Database.py index d3ad8af54..77d162574 100644 --- a/src/common/db/Database.py +++ b/src/common/db/Database.py @@ -394,9 +394,20 @@ def is_setting(self, setting: str, *, multisite: bool = False) -> bool: """Check if the setting exists in the database and optionally if it's multisite""" with self.__db_session() as session: try: - if multisite: - return session.query(Settings).filter_by(id=setting, context="multisite").first() is not None - return session.query(Settings).filter_by(id=setting).first() is not None + multiple = False + if self.suffix_rx.search(setting): + setting = setting.rsplit("_", 1)[0] + multiple = True + + db_setting = session.query(Settings).filter_by(id=setting).first() + + if not db_setting: + return False + elif multisite and db_setting.context != "multisite": + return False + elif multiple and db_setting.multiple is None: + return False + return True except (ProgrammingError, OperationalError): return False From 52c37d72d11e25ecdcee70eb60d5a20a24b0213f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Mon, 17 Jun 2024 12:09:38 +0200 Subject: [PATCH 2/6] chore: Improve Config.py by handling invalid multisite settings and adding debug logs --- src/autoconf/Config.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/autoconf/Config.py b/src/autoconf/Config.py index d599dacec..c1970e1c0 100644 --- a/src/autoconf/Config.py +++ b/src/autoconf/Config.py @@ -46,8 +46,12 @@ def __get_full_env(self) -> dict: if not server_name: continue for variable, value in service.items(): - if self._db.is_setting(variable, multisite=True): - config[f"{server_name}_{variable}"] = value + if variable.startswith("CUSTOM_CONF") or not variable.isupper(): + continue + if not self._db.is_setting(variable, multisite=True): + self.__logger.warning(f"Variable {variable}: {value} is not a valid multisite setting, ignoring it") + continue + config[f"{server_name}_{variable}"] = value config["SERVER_NAME"] += f" {server_name}" config["SERVER_NAME"] = config["SERVER_NAME"].strip() return config @@ -134,6 +138,7 @@ def apply( # update instances in database if "instances" in changes: + self.__logger.debug(f"Updating instances in database: {self.__instances}") err = self._db.update_instances(self.__instances, changed=False) if err: self.__logger.error(f"Failed to update instances: {err}") @@ -141,6 +146,7 @@ def apply( # save config to database changed_plugins = [] if "config" in changes: + self.__logger.debug(f"Saving config in database: {self.__config}") err = self._db.save_config(self.__config, "autoconf", changed=False) if isinstance(err, str): success = False @@ -149,6 +155,7 @@ def apply( # save custom configs to database if "custom_configs" in changes: + self.__logger.debug(f"Saving custom configs in database: {custom_configs}") err = self._db.save_custom_configs(custom_configs, "autoconf", changed=False) if err: success = False From 788ca5945ce188de71bf18ad5ebf65deeb189056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Mon, 17 Jun 2024 12:34:52 +0200 Subject: [PATCH 3/6] chore: Refactor generate_external_plugins function to simplify code and improve readability --- src/scheduler/main.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/src/scheduler/main.py b/src/scheduler/main.py index 1b4a3a318..c5458f36c 100644 --- a/src/scheduler/main.py +++ b/src/scheduler/main.py @@ -165,6 +165,15 @@ def send_nginx_custom_configs(sent_path: Path = CUSTOM_CONFIGS_PATH): logger.info(f"Successfully sent {sent_path} folder") +def send_nginx_external_plugins(sent_path: Path = EXTERNAL_PLUGINS_PATH): + assert SCHEDULER is not None, "SCHEDULER is not defined" + logger.info(f"Sending {sent_path} folder ...") + if not SCHEDULER.send_files(sent_path.as_posix(), "/pro_plugins" if sent_path.as_posix().endswith("/pro/plugins") else "/plugins"): + logger.error(f"Error while sending {sent_path} folder") + else: + logger.info(f"Successfully sent {sent_path} folder") + + def listen_for_instances_reload(): from docker import DockerClient @@ -236,15 +245,14 @@ def generate_custom_configs(configs: Optional[List[Dict[str, Any]]] = None, *, o send_nginx_custom_configs(original_path) -def generate_external_plugins(plugins: Optional[List[Dict[str, Any]]] = None, *, original_path: Union[Path, str] = EXTERNAL_PLUGINS_PATH): +def generate_external_plugins(original_path: Union[Path, str] = EXTERNAL_PLUGINS_PATH): if not isinstance(original_path, Path): original_path = Path(original_path) - pro = "pro" in original_path.parts + pro = original_path.as_posix().endswith("/pro/plugins") - if not plugins: - assert SCHEDULER is not None - plugins = SCHEDULER.db.get_plugins(_type="pro" if pro else "external", with_data=True) - assert plugins is not None, "Couldn't get plugins from database" + assert SCHEDULER is not None + plugins = SCHEDULER.db.get_plugins(_type="pro" if pro else "external", with_data=True) + assert plugins is not None, "Couldn't get plugins from database" # Remove old external/pro plugins files logger.info(f"Removing old/changed {'pro ' if pro else ''}external plugins files ...") @@ -299,10 +307,7 @@ def generate_external_plugins(plugins: Optional[List[Dict[str, Any]]] = None, *, if SCHEDULER and SCHEDULER.apis: logger.info(f"Sending {'pro ' if pro else ''}external plugins to BunkerWeb") - ret = SCHEDULER.send_files(original_path, "/pro_plugins" if original_path.as_posix().endswith("/pro/plugins") else "/plugins") - - if not ret: - logger.error(f"Sending {'pro ' if pro else ''}external plugins failed, configuration will not work as expected...") + send_nginx_external_plugins(original_path) def generate_caches(): @@ -388,7 +393,7 @@ def run_in_slave_mode(): threads = [ Thread(target=generate_custom_configs), Thread(target=generate_external_plugins), - Thread(target=generate_external_plugins, kwargs={"original_path": PRO_PLUGINS_PATH}), + Thread(target=generate_external_plugins, args=(PRO_PLUGINS_PATH,)), Thread(target=generate_caches), ] @@ -613,6 +618,7 @@ def check_plugin_changes(_type: Literal["external", "pro"] = "external"): | ({"jobs": jobs} if jobs else {}) ) + changes = False if tmp_external_plugins: changes = {hash(dict_to_frozenset(d)) for d in tmp_external_plugins} != {hash(dict_to_frozenset(d)) for d in db_plugins} @@ -623,8 +629,10 @@ def check_plugin_changes(_type: Literal["external", "pro"] = "external"): logger.error(f"Couldn't save some manually added {_type} plugins to database: {err}") except BaseException as e: logger.error(f"Error while saving {_type} plugins to database: {e}") + else: + return send_nginx_external_plugins(plugin_path) - generate_external_plugins(SCHEDULER.db.get_plugins(_type=_type, with_data=True), original_path=plugin_path) + generate_external_plugins(plugin_path) threads.extend([Thread(target=check_plugin_changes, args=("external",)), Thread(target=check_plugin_changes, args=("pro",))]) @@ -648,7 +656,7 @@ def check_plugin_changes(_type: Literal["external", "pro"] = "external"): threads.clear() if changes["pro_plugins_changed"]: - threads.append(Thread(target=generate_external_plugins, kwargs={"original_path": PRO_PLUGINS_PATH})) + threads.append(Thread(target=generate_external_plugins, args=(PRO_PLUGINS_PATH,))) if changes["external_plugins_changed"]: threads.append(Thread(target=generate_external_plugins)) @@ -959,12 +967,12 @@ def check_plugin_changes(_type: Literal["external", "pro"] = "external"): if PLUGINS_NEED_GENERATION: CHANGES.append("external_plugins") - generate_external_plugins(SCHEDULER.db.get_plugins(_type="external", with_data=True)) + generate_external_plugins() SCHEDULER.update_jobs() if PRO_PLUGINS_NEED_GENERATION: CHANGES.append("pro_plugins") - generate_external_plugins(SCHEDULER.db.get_plugins(_type="pro", with_data=True), original_path=PRO_PLUGINS_PATH) + generate_external_plugins(PRO_PLUGINS_PATH) SCHEDULER.update_jobs() if CONFIG_NEED_GENERATION: From c15954f573d178b9487c8a5270028ee167960a89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Mon, 17 Jun 2024 12:57:32 +0200 Subject: [PATCH 4/6] chore: Add automatic page refresh after 2 seconds in loading page --- src/bw/loading/index.html | 9 +++++++++ src/common/confs/default-server-http.conf | 10 +++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/bw/loading/index.html b/src/bw/loading/index.html index 81c11d77e..60a746a35 100644 --- a/src/bw/loading/index.html +++ b/src/bw/loading/index.html @@ -256,6 +256,15 @@ + {-raw-} + + {-raw-} +