Skip to content

Commit

Permalink
Optimize and fix database config handling in general
Browse files Browse the repository at this point in the history
  • Loading branch information
@TheophileDiot
TheophileDiot committed Jun 14, 2024
1 parent 8d8cc8b commit 4803c49
Show file tree
Hide file tree
Showing 5 changed files with 108 additions and 160 deletions.
4 changes: 2 additions & 2 deletions src/common/core/misc/jobs/anonymous-report.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
if key not in ("version", "integration", "database_version", "is_pro"):
data.pop(key, None)

db_config = JOB.db.get_config(methods=True, with_drafts=True)
db_config = JOB.db.get_non_default_settings(methods=True, with_drafts=True)
services = db_config.get("SERVER_NAME", {"value": ""})["value"].split(" ")
multisite = db_config.get("MULTISITE", {"value": "no"})["value"] == "yes"

Expand Down Expand Up @@ -85,7 +85,7 @@

data["non_default_settings"] = {}
for setting, setting_data in db_config.items():
if isinstance(setting_data, dict) and setting_data["method"] != "default":
if isinstance(setting_data, dict):
for server in services:
if setting.startswith(server + "_"):
setting = setting[len(server) + 1 :] # noqa: E203
Expand Down
245 changes: 96 additions & 149 deletions src/common/db/Database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1533,102 +1533,15 @@ def save_custom_configs(

return message

def get_config(self, global_only: bool = False, methods: bool = False, with_drafts: bool = False) -> Dict[str, Any]:
"""Get the config from the database"""
with self.__db_session() as session:
config = {}
multisite = []
for setting in (
session.query(Settings)
.with_entities(
Settings.id,
Settings.context,
Settings.default,
Settings.multiple,
)
.order_by(Settings.order)
):
default = setting.default or ""
config[setting.id] = default if not methods else {"value": default, "global": True, "method": "default"}

for global_value in (
session.query(Global_values).with_entities(Global_values.value, Global_values.suffix, Global_values.method).filter_by(setting_id=setting.id)
):
config[setting.id + (f"_{global_value.suffix}" if setting.multiple and global_value.suffix > 0 else "")] = (
global_value.value
if not methods
else {
"value": global_value.value,
"global": True,
"method": global_value.method,
}
)

if setting.context == "multisite":
multisite.append(setting.id)

is_multisite = config.get("MULTISITE", {"value": "no"})["value"] == "yes" if methods else config.get("MULTISITE", "no") == "yes"

services = session.query(Services).with_entities(Services.id, Services.is_draft)
if not with_drafts:
services = services.filter_by(is_draft=False)

if not global_only and is_multisite:
for service in services:
config[f"{service.id}_IS_DRAFT"] = "yes" if service.is_draft else "no"
if methods:
config[f"{service.id}_IS_DRAFT"] = {"value": config[f"{service.id}_IS_DRAFT"], "global": False, "method": "default"}

checked_settings = []
for key, value in config.copy().items():
original_key = key
if self.suffix_rx.search(key):
key = key[: -len(str(key.split("_")[-1])) - 1]

if key not in multisite:
continue
elif f"{service.id}_{original_key}" not in config:
config[f"{service.id}_{original_key}"] = value

if original_key not in checked_settings:
checked_settings.append(original_key)
else:
continue

for service_setting in (
session.query(Services_settings)
.with_entities(
Services_settings.value,
Services_settings.suffix,
Services_settings.method,
)
.filter_by(service_id=service.id, setting_id=key)
):
value = service_setting.value
if key == "SERVER_NAME" and service.id not in value.split(" "):
value = f"{service.id} {value}".strip()

config[f"{service.id}_{key}" + (f"_{service_setting.suffix}" if service_setting.suffix > 0 else "")] = (
value
if not methods
else {
"value": value,
"global": False,
"method": service_setting.method,
}
)

servers = " ".join(service.id for service in services)
config["SERVER_NAME"] = servers if not methods else {"value": servers, "global": True, "method": "default"}

return config

def get_filtered_config(
def get_non_default_settings(
self,
global_only: bool = False,
methods: bool = False,
with_drafts: bool = False,
filtered_settings: Optional[Union[List[str], Set[str], Tuple[str]]] = None,
*,
original_config: Optional[Dict[str, Any]] = None,
original_multisite: Optional[Set[str]] = None,
) -> Dict[str, Any]:
"""Get the config from the database"""
filtered_settings = set(filtered_settings or [])
Expand All @@ -1637,14 +1550,15 @@ def get_filtered_config(
filtered_settings.update(("SERVER_NAME", "MULTISITE"))

with self.__db_session() as session:
config = {}
config = original_config or {}
multisite = original_multisite or set()

# Define the join operation
j = join(Settings, Global_values, Settings.id == Global_values.setting_id)

# Define the select statement
stmt = (
db_select(Settings.id, Settings.multiple, Global_values.value, Global_values.suffix, Global_values.method)
db_select(Settings.id.label("setting_id"), Settings.context, Settings.multiple, Global_values.value, Global_values.suffix, Global_values.method)
.select_from(j)
.order_by(Settings.order)
)
Expand All @@ -1655,78 +1569,111 @@ def get_filtered_config(
# Execute the query and fetch all results
results = session.execute(stmt).fetchall()

for setting in results:
key = setting.id + (f"_{setting.suffix}" if setting.multiple and setting.suffix else "")
config[key] = setting.value if not methods else {"value": setting.value, "global": True, "method": setting.method}
for global_value in results:
setting_id = global_value.setting_id + (f"_{global_value.suffix}" if global_value.multiple and global_value.suffix > 0 else "")
config[setting_id] = global_value.value if not methods else {"value": global_value.value, "global": True, "method": global_value.method}
if global_value.context == "multisite":
multisite.add(setting_id)

is_multisite = config.get("MULTISITE", {"value": "no"})["value"] == "yes" if methods else config.get("MULTISITE", "no") == "yes"
services = session.query(Services).with_entities(Services.id, Services.is_draft)

# Define the join operation
j = join(Services, Services_settings, Services.id == Services_settings.service_id)
j = j.join(Settings, Settings.id == Services_settings.setting_id)
if not with_drafts:
services = services.filter_by(is_draft=False)

# Define the select statement
stmt = db_select(
Services.id.label("service_id"),
Services.is_draft,
Settings.id.label("setting_id"),
Settings.context,
Settings.default,
Settings.multiple,
Services_settings.value,
Services_settings.suffix,
Services_settings.method,
).select_from(j)
servers = ""
for service in services:
config[f"{service.id}_IS_DRAFT"] = "yes" if service.is_draft else "no"
if methods:
config[f"{service.id}_IS_DRAFT"] = {"value": config[f"{service.id}_IS_DRAFT"], "global": False, "method": "default"}
for key in multisite:
config[f"{service.id}_{key}"] = config[key]
servers += f"{service.id} "
servers = servers.strip()

if not with_drafts:
stmt = stmt.where(Services.is_draft == False) # noqa: E712
config["SERVER_NAME"] = servers if not methods else {"value": servers, "global": True, "method": "default"}

if filtered_settings:
stmt = stmt.where(Services_settings.setting_id.in_(filtered_settings))
if not global_only and (config.get("MULTISITE", {"value": "no"})["value"] == "yes" if methods else config.get("MULTISITE", "no") == "yes"):
# Define the join operation
j = join(Services, Services_settings, Services.id == Services_settings.service_id)
j = j.join(Settings, Settings.id == Services_settings.setting_id)

# Execute the query and fetch all results
results = session.execute(stmt).fetchall()
# Define the select statement
stmt = (
db_select(
Services.id.label("service_id"),
Settings.id.label("setting_id"),
Settings.multiple,
Services_settings.value,
Services_settings.suffix,
Services_settings.method,
)
.select_from(j)
.order_by(Services.id, Settings.order)
)

if not global_only and is_multisite:
for result in results:
if f"{result.service_id}_IS_DRAFT" not in config:
config[f"{result.service_id}_IS_DRAFT"] = "yes" if result.is_draft else "no"
if methods:
config[f"{result.service_id}_IS_DRAFT"] = {"value": config[f"{result.service_id}_IS_DRAFT"], "global": False, "method": "default"}
if not with_drafts:
stmt = stmt.where(Services.is_draft == False) # noqa: E712

key = result.setting_id
original_key = key
if self.suffix_rx.search(key):
key = key[: -len(str(key.split("_")[-1])) - 1]
if filtered_settings:
stmt = stmt.where(Settings.id.in_(filtered_settings))

if result.context != "multisite":
continue
elif f"{result.service_id}_{original_key}" not in config:
config[f"{result.service_id}_{original_key}"] = result.default
# Execute the query and fetch all results
results = session.execute(stmt).fetchall()

for result in results:
value = result.value
if key == "SERVER_NAME" and not search(r"(^| )" + escape(result.service_id) + r"($| )", value):
value = f"{result.service_id} {value}".strip()

config[f"{result.service_id}_{key}" + (f"_{result.suffix}" if result.multiple and result.suffix else "")] = (
value
if not methods
else {
"value": value,
"global": False,
"method": result.method,
}

if result.setting_id == "SERVER_NAME" and not search(r"^" + escape(result.service_id) + r"( |$)", value):
split = set(value.split(" "))
split.discard(result.service_id)
value = result.service_id + " " + " ".join(split)

config[f"{result.service_id}_{result.setting_id}" + (f"_{result.suffix}" if result.multiple and result.suffix else "")] = (
value if not methods else {"value": value, "global": False, "method": result.method}
)

services = session.query(Services).with_entities(Services.id)
return config

if not with_drafts:
services = services.filter_by(is_draft=False)
def get_config(
self,
global_only: bool = False,
methods: bool = False,
with_drafts: bool = False,
filtered_settings: Optional[Union[List[str], Set[str], Tuple[str]]] = None,
) -> Dict[str, Any]:
"""Get the config from the database"""
with self.__db_session() as session:
config = {}
multisite = set()

servers = " ".join(service.id for service in services)
config["SERVER_NAME"] = servers if not methods else {"value": servers, "global": True, "method": "default"}
query = (
session.query(Settings)
.with_entities(
Settings.id,
Settings.context,
Settings.default,
Settings.multiple,
)
.order_by(Settings.order)
)

return config
if filtered_settings:
query = query.filter(Settings.id.in_(filtered_settings))

for setting in query:
default = setting.default or ""
config[setting.id] = default if not methods else {"value": default, "global": True, "method": "default"}
if setting.context == "multisite":
multisite.add(setting.id)

return self.get_non_default_settings(
global_only=global_only,
methods=methods,
with_drafts=with_drafts,
filtered_settings=filtered_settings,
original_config=config,
original_multisite=multisite,
)

def get_custom_configs(self) -> List[Dict[str, Any]]:
"""Get the custom configs from the database"""
Expand Down
15 changes: 8 additions & 7 deletions src/ui/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -753,9 +753,9 @@ def account():
variable = {}
variable["PRO_LICENSE_KEY"] = request.form["license"]

error = app.config["CONFIG"].check_variables(variable)
variable = app.config["CONFIG"].check_variables(variable, {"PRO_LICENSE_KEY": request.form["license"]})

if error:
if not variable:
return redirect_flash_error("The license key variable checks returned error", "account", True)

# Force job to contact PRO API
Expand Down Expand Up @@ -963,7 +963,7 @@ def services():
if "SERVER_NAME" not in variables:
variables["SERVER_NAME"] = variables["OLD_SERVER_NAME"]

config = app.config["CONFIG"].get_config(with_drafts=True, filtered_settings=variables.keys())
config = app.config["DB"].get_config(methods=True, with_drafts=True)
server_name = variables["SERVER_NAME"].split(" ")[0]
was_draft = config.get(f"{server_name}_IS_DRAFT", {"value": "no"})["value"] == "yes"
operation = request.form["operation"]
Expand Down Expand Up @@ -997,15 +997,15 @@ def services():
del variables["OLD_SERVER_NAME"]

# Edit check fields and remove already existing ones
for variable, value in deepcopy(variables).items():
for variable, value in variables.copy().items():
if (
variable in variables
and variable != "SERVER_NAME"
and value == config.get(f"{server_name}_{variable}" if request.form["operation"] == "edit" else variable, {"value": None})["value"]
):
del variables[variable]

variables = app.config["CONFIG"].check_variables(variables)
variables = app.config["CONFIG"].check_variables(variables, config)

if (
was_draft == is_draft
Expand All @@ -1024,7 +1024,7 @@ def services():

is_request_params(["SERVER_NAME"], "services", True)

variables = app.config["CONFIG"].check_variables({"SERVER_NAME": request.form["SERVER_NAME"]})
variables = app.config["CONFIG"].check_variables({"SERVER_NAME": request.form["SERVER_NAME"]}, config)

if not variables:
error_message(f"Error while deleting the service {request.form['SERVER_NAME']}")
Expand Down Expand Up @@ -1138,12 +1138,13 @@ def global_config():
del variables["csrf_token"]

# Edit check fields and remove already existing ones
config = app.config["CONFIG"].get_config(with_drafts=True, filtered_settings=variables.keys())
config = app.config["DB"].get_config(methods=True, with_drafts=True)
services = config["SERVER_NAME"]["value"].split(" ")
for variable, value in variables.copy().items():
setting = config.get(variable, {"value": None, "global": True})
if setting["global"] and value == setting["value"]:
del variables[variable]
continue

variables = app.config["CONFIG"].check_variables(variables, config)

Expand Down
2 changes: 1 addition & 1 deletion src/ui/src/Config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def get_config(
dict
The nginx variables env file as a dict
"""
return self.__db.get_filtered_config(global_only=global_only, methods=methods, with_drafts=with_drafts, filtered_settings=filtered_settings)
return self.__db.get_non_default_settings(global_only=global_only, methods=methods, with_drafts=with_drafts, filtered_settings=filtered_settings)

def get_services(self, methods: bool = True, with_drafts: bool = False) -> list[dict]:
"""Get nginx's services
Expand Down
2 changes: 1 addition & 1 deletion src/ui/src/Instances.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ def get_instances(self, override_instances=None) -> list[Instance]:
instances = []
# Override case : only return instances from DB
if override_instances is None:
config = self.__db.get_filtered_config(global_only=True, filtered_settings=("OVERRIDE_INSTANCES",))
config = self.__db.get_non_default_settings(global_only=True, filtered_settings=("OVERRIDE_INSTANCES",))
override_instances = config.get("OVERRIDE_INSTANCES", "") != ""
if override_instances:
for instance in self.__db.get_instances():
Expand Down

0 comments on commit 4803c49

Please sign in to comment.