fix bwcli when using redis sentinels, init work on linux ha and init …

…work on static instances
DitZii · Apr 7, 2024 · 21be7c1 · 21be7c1
1 parent 936600a
commit 21be7c1
Show file tree

Hide file tree

Showing 8 changed files with 280 additions and 118 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,17 +2,16 @@
 
 ## v1.5.7 - ????/??/??
 
-- [LINUX] Fix potential issues when removing the bunkerweb package
 - [BUGFIX] Fix rare error when the cache is not properly initialized and jobs are executed
-- [FEATURE] Add an automatic renaming of old database tables when upgrading to a new version in order to avoid data loss
-- [FEATURE] Add the possibility to add custom bwcli commands in plugins
+- [BUGFIX] Fix bug when downloading new mmdb files
+- [BUGFIX] Remove potential false positives with ModSecurity on the jobs page of the web UI
+- [BUGFIX] Fix bwcli not working with Redis sentinel
+- [BUGFIX] Fix potential issues when removing the bunkerweb Linux package
 - [FEATURE] Add backup plugin to backup and restore easily the database
 - [FEATURE] Add LETS_ENCRYPT_CLEAR_OLD_CERTS setting to control if old certificates should be removed when generating Let's Encrypt certificates (default is no)
 - [FEATURE] Add DISABLE_DEFAULT_SERVER_STRICT_SNI setting to allow/block requests when SNI is unknown or unset (default is no)
-- [MISC] Remove potential false positives with ModSecurity on the jobs page of the web UI
-- [MISC] Fix rare bug when downloading new mmdb files
-- [DOCUMENTATION] Add procedure to follow when upgrading from 1.5.7+
-- [DOCUMENTATION] Add documentation about the procedure to follow when upgrading from a version prior to 1.5.0
+- [DOCUMENTATION] Add upgrade procedure for 1.5.7+
+- [MISC] Support custom bwcli commands using plugins
 - [DEPS] Updated LuaJIT version to v2.1-20240314
 
 ## v1.5.6 - 2024/03/25

diff --git a/src/common/cli/CLI.py b/src/common/cli/CLI.py
@@ -77,7 +77,8 @@ def __init__(self):
         if self.__use_redis:
             self.__logger.info("Fetching redis configuration")
             redis_host = self.__get_variable("REDIS_HOST")
-            if redis_host:
+            sentinel_hosts = self.__get_variable("REDIS_SENTINEL_HOSTS")
+            if redis_host or sentinel_hosts:
                 redis_port = self.__get_variable("REDIS_PORT", "6379")
                 assert isinstance(redis_port, str), "REDIS_PORT is not a string"
                 if not redis_port.isdigit():
@@ -107,8 +108,6 @@ def __init__(self):
                     redis_keepalive_pool = "10"
                 redis_keepalive_pool = int(redis_keepalive_pool)
 
-                self.__logger.info("Redis configuration is valid")
-
                 redis_ssl = self.__get_variable("REDIS_SSL", "no") == "yes"
                 username = self.__get_variable("REDIS_USERNAME", None) or None
                 password = self.__get_variable("REDIS_PASSWORD", None) or None
@@ -177,7 +176,7 @@ def __init__(self):
                     self.__use_redis = False
                 self.__logger.info("Connected to redis")
             else:
-                self.__logger.error("USE_REDIS is set to yes but REDIS_HOST is not set, disabling redis")
+                self.__logger.error("USE_REDIS is set to yes but REDIS_HOST or REDIS_SENTINEL_HOSTS is not set, disabling redis")
                 self.__use_redis = False
 
         if self.__integration == "linux":

diff --git a/src/common/gen/Configurator.py b/src/common/gen/Configurator.py
@@ -197,6 +197,8 @@ def get_config(self) -> Dict[str, Any]:
                     "NJS_VERSION",
                     "PKG_RELEASE",
                     "DOCKER_HOST",
+                    "SLAVE_MODE",
+                    "MASTER_MODE",
                 )
             ):
                 self.__logger.warning(f"Ignoring variable {variable} : {err}")

diff --git a/src/common/settings.json b/src/common/settings.json
@@ -317,12 +317,12 @@
       "emerg"
     ]
   },
-  "STATIC_INSTANCES": {
+  "OVERRIDE_INSTANCES": {
     "context": "global",
     "default": "",
-    "help": "List of additional BunkerWeb instances separated with spaces (format : fqdn-or-ip:5000 fqdn-or-ip:5000)",
-    "id": "static-instances",
-    "label": "Static instances",
+    "help": "List of BunkerWeb instances separated with spaces (format : fqdn-or-ip:5000 fqdn-or-ip:5000)",
+    "id": "override-instances",
+    "label": "Override instances",
     "regex": "^.*$",
     "type": "text"
   }

diff --git a/src/linux/scripts/start.sh b/src/linux/scripts/start.sh
@@ -100,98 +100,104 @@ function start() {
         log "SYSTEMCTL" "ℹ️" "Created dummy variables.env file"
     fi
 
+    # Create PID folder
+    if [ ! -f /var/run/bunkerweb ] ; then
+        mkdir -p /var/run/bunkerweb
+        chown nginx:nginx /var/run/bunkerweb
+    fi
+
     # Stop scheduler if it's running
     stop_scheduler
 
     # Stop nginx if it's running
     stop_nginx
 
-    # Generate temp conf for jobs and start nginx
-    DNS_RESOLVERS="$(grep "^DNS_RESOLVERS=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$DNS_RESOLVERS" = "" ] ; then
-        DNS_RESOLVERS="8.8.8.8 8.8.4.4"
-    fi
-    API_LISTEN_IP="$(grep "^API_LISTEN_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$API_LISTEN_IP" = "" ] ; then
-        API_LISTEN_IP="127.0.0.1"
-    fi
-    API_HTTP_PORT="$(grep "^API_HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$API_HTTP_PORT" = "" ] ; then
-        API_HTTP_PORT="5000"
-    fi
-    API_SERVER_NAME="$(grep "^API_SERVER_NAME=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$API_SERVER_NAME" = "" ] ; then
-        API_SERVER_NAME="bwapi"
-    fi
-    API_WHITELIST_IP="$(grep "^API_WHITELIST_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$API_WHITELIST_IP" = "" ] ; then
-        API_WHITELIST_IP="127.0.0.0/8"
-    fi
-    USE_REAL_IP="$(grep "^USE_REAL_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$USE_REAL_IP" = "" ] ; then
-        USE_REAL_IP="no"
-    fi
-    USE_PROXY_PROTOCOL="$(grep "^USE_PROXY_PROTOCOL=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$USE_PROXY_PROTOCOL" = "" ] ; then
-        USE_PROXY_PROTOCOL="no"
-    fi
-    REAL_IP_FROM="$(grep "^REAL_IP_FROM=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$REAL_IP_FROM" = "" ] ; then
-        REAL_IP_FROM="192.168.0.0/16 172.16.0.0/12 10.0.0.0/8"
-    fi
-    REAL_IP_HEADER="$(grep "^REAL_IP_HEADER=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$REAL_IP_HEADER" = "" ] ; then
-        REAL_IP_HEADER="X-Forwarded-For"
-    fi
-    HTTP_PORT="$(grep "^HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$HTTP_PORT" = "" ] ; then
-        HTTP_PORT="80"
-    fi
-    HTTPS_PORT="$(grep "^HTTPS_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$HTTPS_PORT" = "" ] ; then
-        HTTPS_PORT="443"
-    fi
-    MODSECURITY_CRS_VERSION="$(grep "^MODSECURITY_CRS_VERSION=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
-    if [ "$MODSECURITY_CRS_VERSION" = "" ] ; then
-        MODSECURITY_CRS_VERSION="4"
-    fi
-    sudo -E -u nginx -g nginx /bin/bash -c "echo -ne 'IS_LOADING=yes\nUSE_BUNKERNET=no\nSEND_ANONYMOUS_REPORT=no\nSERVER_NAME=\nMODSECURITY_CRS_VERSION=${MODSECURITY_CRS_VERSION}\nDNS_RESOLVERS=${DNS_RESOLVERS}\nAPI_HTTP_PORT=${API_HTTP_PORT}\nAPI_LISTEN_IP=${API_LISTEN_IP}\nAPI_SERVER_NAME=${API_SERVER_NAME}\nAPI_WHITELIST_IP=${API_WHITELIST_IP}\nUSE_REAL_IP=${USE_REAL_IP}\nUSE_PROXY_PROTOCOL=${USE_PROXY_PROTOCOL}\nREAL_IP_FROM=${REAL_IP_FROM}\nREAL_IP_HEADER=${REAL_IP_HEADER}\nHTTP_PORT=${HTTP_PORT}\nHTTPS_PORT=${HTTPS_PORT}\n' > /var/tmp/bunkerweb/tmp.env"
-    sudo -E -u nginx -g nginx /bin/bash -c "PYTHONPATH=/usr/share/bunkerweb/deps/python/ /usr/share/bunkerweb/gen/main.py --variables /var/tmp/bunkerweb/tmp.env --no-linux-reload"
-    # shellcheck disable=SC2181
-    if [ $? -ne 0 ] ; then
-        log "SYSTEMCTL" "❌" "Error while generating config from /var/tmp/bunkerweb/tmp.env"
-        exit 1
-    fi
-
-    if [ ! -f /var/run/bunkerweb ] ; then
-        mkdir -p /var/run/bunkerweb
-        chown nginx:nginx /var/run/bunkerweb
-    fi
+    # Check if we are in slave/master mode
+    export MASTER_MODE="$(grep "^MASTER_MODE=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+    export SLAVE_MODE="$(grep "^SLAVE_MODE=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
 
-    # Start nginx
-    log "SYSTEMCTL" "ℹ️" "Starting nginx ..."
-    sudo -E -u nginx -g nginx /usr/sbin/nginx -e /var/log/bunkerweb/error.log
-    # shellcheck disable=SC2181
-    if [ $? -ne 0 ] ; then
-        log "SYSTEMCTL" "❌" "Error while executing temp nginx"
-        exit 1
-    fi
-    count=0
-    while [ $count -lt 10 ] ; do
-        check="$(curl -s -H "Host: healthcheck.bunkerweb.io" http://127.0.0.1:6000/healthz 2>&1)"
+    if [ "$MASTER_MODE" != "yes" ] ; then
+        # Generate temp conf for jobs and start nginx
+        DNS_RESOLVERS="$(grep "^DNS_RESOLVERS=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$DNS_RESOLVERS" = "" ] ; then
+            DNS_RESOLVERS="8.8.8.8 8.8.4.4"
+        fi
+        API_LISTEN_IP="$(grep "^API_LISTEN_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$API_LISTEN_IP" = "" ] ; then
+            API_LISTEN_IP="127.0.0.1"
+        fi
+        API_HTTP_PORT="$(grep "^API_HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$API_HTTP_PORT" = "" ] ; then
+            API_HTTP_PORT="5000"
+        fi
+        API_SERVER_NAME="$(grep "^API_SERVER_NAME=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$API_SERVER_NAME" = "" ] ; then
+            API_SERVER_NAME="bwapi"
+        fi
+        API_WHITELIST_IP="$(grep "^API_WHITELIST_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$API_WHITELIST_IP" = "" ] ; then
+            API_WHITELIST_IP="127.0.0.0/8"
+        fi
+        USE_REAL_IP="$(grep "^USE_REAL_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$USE_REAL_IP" = "" ] ; then
+            USE_REAL_IP="no"
+        fi
+        USE_PROXY_PROTOCOL="$(grep "^USE_PROXY_PROTOCOL=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$USE_PROXY_PROTOCOL" = "" ] ; then
+            USE_PROXY_PROTOCOL="no"
+        fi
+        REAL_IP_FROM="$(grep "^REAL_IP_FROM=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$REAL_IP_FROM" = "" ] ; then
+            REAL_IP_FROM="192.168.0.0/16 172.16.0.0/12 10.0.0.0/8"
+        fi
+        REAL_IP_HEADER="$(grep "^REAL_IP_HEADER=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$REAL_IP_HEADER" = "" ] ; then
+            REAL_IP_HEADER="X-Forwarded-For"
+        fi
+        HTTP_PORT="$(grep "^HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$HTTP_PORT" = "" ] ; then
+            HTTP_PORT="80"
+        fi
+        HTTPS_PORT="$(grep "^HTTPS_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$HTTPS_PORT" = "" ] ; then
+            HTTPS_PORT="443"
+        fi
+        MODSECURITY_CRS_VERSION="$(grep "^MODSECURITY_CRS_VERSION=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
+        if [ "$MODSECURITY_CRS_VERSION" = "" ] ; then
+            MODSECURITY_CRS_VERSION="3"
+        fi
+        sudo -E -u nginx -g nginx /bin/bash -c "echo -ne 'IS_LOADING=yes\nUSE_BUNKERNET=no\nSEND_ANONYMOUS_REPORT=no\nSERVER_NAME=\nMODSECURITY_CRS_VERSION=${MODSECURITY_CRS_VERSION}\nDNS_RESOLVERS=${DNS_RESOLVERS}\nAPI_HTTP_PORT=${API_HTTP_PORT}\nAPI_LISTEN_IP=${API_LISTEN_IP}\nAPI_SERVER_NAME=${API_SERVER_NAME}\nAPI_WHITELIST_IP=${API_WHITELIST_IP}\nUSE_REAL_IP=${USE_REAL_IP}\nUSE_PROXY_PROTOCOL=${USE_PROXY_PROTOCOL}\nREAL_IP_FROM=${REAL_IP_FROM}\nREAL_IP_HEADER=${REAL_IP_HEADER}\nHTTP_PORT=${HTTP_PORT}\nHTTPS_PORT=${HTTPS_PORT}\n' > /var/tmp/bunkerweb/tmp.env"
+        sudo -E -u nginx -g nginx /bin/bash -c "PYTHONPATH=/usr/share/bunkerweb/deps/python/ /usr/share/bunkerweb/gen/main.py --variables /var/tmp/bunkerweb/tmp.env --no-linux-reload"
         # shellcheck disable=SC2181
-        if [ $? -eq 0 ] && [ "$check" = "ok" ] ; then
-            break
+        if [ $? -ne 0 ] ; then
+            log "SYSTEMCTL" "❌" "Error while generating config from /var/tmp/bunkerweb/tmp.env"
+            exit 1
         fi
-        count=$((count + 1))
-        sleep 1
-        log "SYSTEMCTL" "ℹ️" "Waiting for nginx to start ..."
-    done
-    if [ $count -ge 10 ] ; then
-        log "SYSTEMCTL" "❌" "nginx is not started"
-        exit 1
+        # Start nginx
+        log "SYSTEMCTL" "ℹ️" "Starting nginx ..."
+        sudo -E -u nginx -g nginx /usr/sbin/nginx -e /var/log/bunkerweb/error.log
+        # shellcheck disable=SC2181
+        if [ $? -ne 0 ] ; then
+            log "SYSTEMCTL" "❌" "Error while executing temp nginx"
+            exit 1
+        fi
+        count=0
+        while [ $count -lt 10 ] ; do
+            check="$(curl -s -H "Host: healthcheck.bunkerweb.io" http://127.0.0.1:6000/healthz 2>&1)"
+            # shellcheck disable=SC2181
+            if [ $? -eq 0 ] && [ "$check" = "ok" ] ; then
+                break
+            fi
+            count=$((count + 1))
+            sleep 1
+            log "SYSTEMCTL" "ℹ️" "Waiting for nginx to start ..."
+        done
+        if [ $count -ge 10 ] ; then
+            log "SYSTEMCTL" "❌" "nginx is not started"
+            exit 1
+        fi
+        log "SYSTEMCTL" "ℹ️" "nginx started ..."
     fi
-    log "SYSTEMCTL" "ℹ️" "nginx started ..."
 
     # Execute scheduler
     log "SYSTEMCTL" "ℹ️ " "Executing scheduler ..."