feat(*): env var NO_MULTIPROCESSING

Signed-off-by: Rongrong <i@rong.moe>
xlmm · May 27, 2022 · 3c39136 · 3c39136
1 parent d87e6ba
commit 3c39136
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 24 deletions.
diff --git a/.env.sample b/.env.sample
@@ -35,5 +35,6 @@ TELEGRAPH_TOKEN="
 #LAZY_MEDIA_VALIDATION=1  # default: 0
 #MANAGER_PRIVILEGED=1  # default: 0
 #NO_UVLOOP=1  # default: 0
+#NO_MULTIPROCESSING=1  # default: 0
 #DEBUG=1  # debug logging, default: 0
 # ↑------ Advanced settings ------↑ #
diff --git a/docker-compose.yml.sample b/docker-compose.yml.sample
@@ -45,5 +45,6 @@ services:
       #- LAZY_MEDIA_VALIDATION=1  # default: 0
       #- MANAGER_PRIVILEGED=1  # default: 0
       #- NO_UVLOOP=1  # default: 0
+      #- NO_MULTIPROCESSING=1  # default: 0
       #- DEBUG=1  # debug logging, default: 0
 # ↑------ Advanced settings ------↑ #
diff --git a/docs/advanced-settings.md b/docs/advanced-settings.md
@@ -50,19 +50,20 @@
 | `TABLE_TO_IMAGE`     | Convert tables to image (causing higher CPU load) or just drop them?      | `1`                                           | `0`                                                    |
 | `MANAGER_PRIVILEGED` | Allow the bot manager to manipulate any users' subscriptions or not? [^8] | `1`                                           | `0`                                                    |
 | `NO_UVLOOP`          | Never enable `uvloop` (even if it is found) or not?                       | `1`                                           | `0`                                                    |
+| `NO_MULTIPROCESSING` | Limit the process count to `1` or not (up to `min(3, CPU_COUNT)`)? [^9]   | `1`                                           | `0`                                                    |
 | `DEBUG`              | Enable debug logging or not?                                              | `1`                                           | `0`                                                    |
 
 ## Manager options
 
 > Manager options are options stored in the database. The bot manager can change it by using the `/set_option` command.
 
-| Key                          | Description                                              | Example                         | Default          |
-|------------------------------|----------------------------------------------------------|---------------------------------|------------------|
-| `default_interval`           | Default feed monitoring interval [^9]                    | `5`                             | `10`             |
-| `minimal_interval`           | Minimal feed monitoring interval [^10]                   | `10`                            | `5`              |
-| `user_sub_limit`             | Subscription number limit for ordinary user [^11]        | `150`                           | `-1` (unlimited) |
-| `channel_or_group_sub_limit` | Subscription number limit for channel or group [^11]     | `150`                           | `-1` (unlimited) |
-| `sub_limit_reached_message`  | Additional message attached to the limit reached warning | `https://t.me/RSStT_Channel/58` |                  |
+| Key                          | Description                                                | Example                         | Default          |
+|------------------------------|------------------------------------------------------------|---------------------------------|------------------|
+| `default_interval`           | Default feed monitoring interval [^10]                     | `15`                            | `10`             |
+| `minimal_interval`           | Minimal feed monitoring interval [^11] [^12]               | `10`                            | `5`              |
+| `user_sub_limit`             | Subscription number limit for ordinary user [^13] [^12]    | `150`                           | `-1` (unlimited) |
+| `channel_or_group_sub_limit` | Subscription number limit for channel or group [^13] [^12] | `150`                           | `-1` (unlimited) |
+| `sub_limit_reached_message`  | Additional message attached to the limit reached warning   | `https://t.me/RSStT_Channel/58` |                  |
 
 [^1]: Refresh the page every time you get a new token. If you have a lot of subscriptions, make sure to get at least 5 tokens.
 [^2]: Can be a list, separated by `;`, `,`, `(space)`, `(linebreak)`, or `(tab)`
@@ -72,6 +73,8 @@
 [^6]: Use with caution. Help cut down network traffic further. If enabled, RSStT no longer fetches media and validates it. Effectively disable long-pic detection and partially disable icon detection.
 [^7]: Ref: [https://tortoise-orm.readthedocs.io/en/latest/databases.html](). Note that Railway.app will automatically fill this env variable.
 [^8]: Use with caution. If enabled, the bot manager can bypass the permission check before manipulating any users'/channels'/groups' subscriptions. The command format is like `/sub @username`, `/sub +9999999999` (ordinary user) or `/sub -1009999999999` (channel/group). Should only be used temporarily and be disabled after finishing the manipulation. This option is considered safe for bot users since the bot manager can always manipulate their subscriptions by manipulating the database manually.
-[^9]: After a user subscribes to a feed, the default monitoring interval is applied.
-[^10]: The minimal monitoring interval a user can set for a subscription. Note that the bot manager will not be limited by this value.
-[^11]: Once reached the limit, no more subscriptions can be created. However, existing subscriptions will not be removed even if reaching the limit. As a bot manager, you can enable `MANAGER_PRIVILEGED` mode to manually unsubscribe their subscriptions.
+[^9]: Only valid when there are more than 1 CPU cores, otherwise the process count is always `1`. Using multiprocessing helps improve the performance on multicore CPUs but consumes more memory. Enable it to disable multiprocessing if you are concerned about memory consumption.
+[^10]: After a user subscribes to a feed, the default monitoring interval is applied.
+[^11]: The minimal monitoring interval a user can set for a subscription.
+[^12]: The bot manager will not be limited by this value.
+[^13]: Once reached the limit, no more subscriptions can be created. However, existing subscriptions will not be removed even if reaching the limit. As a bot manager, you can enable `MANAGER_PRIVILEGED` mode to manually unsubscribe their subscriptions.
diff --git a/src/__init__.py b/src/__init__.py
@@ -3,7 +3,7 @@
 from . import env  # the event loop and basic configurations are initialized in env, so import it first
 from . import pool  # the process pool need to be initialized once the event loop is ready to reduce memory consumption
 
-pool.init(env.loop)
+pool.init()
 
 import asyncio
 from functools import partial

diff --git a/src/env.py b/src/env.py
@@ -242,6 +242,7 @@ def __list_parser(var: Optional[str]) -> list[str]:
 TRAFFIC_SAVING: Final = __bool_parser(os.environ.get('TRAFFIC_SAVING'))
 LAZY_MEDIA_VALIDATION: Final = __bool_parser(os.environ.get('LAZY_MEDIA_VALIDATION'))
 NO_UVLOOP: Final = __bool_parser(os.environ.get('NO_UVLOOP'))
+NO_MULTIPROCESSING: Final = __bool_parser(os.environ.get('NO_MULTIPROCESSING'))
 DEBUG: Final = __bool_parser(os.environ.get('DEBUG'))
 __configure_logging(  # config twice to make .env file work
     level=colorlog.DEBUG if DEBUG else colorlog.INFO,

diff --git a/src/pool.py b/src/pool.py
@@ -2,20 +2,17 @@
 Various pools.
 """
 from __future__ import annotations
-from typing import Optional
 
 import os
-from asyncio import AbstractEventLoop
 from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
 from time import sleep
 from signal import signal, SIGINT
 
+from . import env
+
 CPU_COUNT = os.cpu_count()
 AVAIL_CPU_COUNT = len(os.sched_getaffinity(0))
-PROCESS_COUNT = min(AVAIL_CPU_COUNT, 3)
-LOOP: Optional[AbstractEventLoop] = None
-
-del os
+PROCESS_COUNT = 1 if env.NO_MULTIPROCESSING else min(AVAIL_CPU_COUNT, 3)
 
 # Asyncio executor. Either a thread pool or a process pool.
 aioExecutor = (
@@ -30,17 +27,13 @@ async def run_async(func, *args):
     """
     Run a CPU-consuming function asynchronously.
     """
-    return await LOOP.run_in_executor(aioExecutor, func, *args)
+    return await env.loop.run_in_executor(aioExecutor, func, *args)
 
 
-def init(loop: AbstractEventLoop):
-    global LOOP
-
-    if CPU_COUNT > 1:
+def init():
+    if PROCESS_COUNT > 1:
         [aioExecutor.submit(sleep, 0.01 * (i + 1)) for i in range(PROCESS_COUNT - 1)]
 
-    LOOP = loop
-
 
 def shutdown():
     aioExecutor.shutdown()