From b934061a66f6f1083364f1de75b06246b7529b4e Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Tue, 10 Dec 2024 02:44:41 +0100 Subject: [PATCH] WebHost: Set Generator memory limit to 4GiB (#4319) * WebHost: Set Generator memory limit to 4GiB * WebHost: make generator memory limit configurable, better naming * Update WebHostLib/__init__.py Co-authored-by: Fabian Dill * Update docs/webhost configuration sample.yaml --------- Co-authored-by: Fabian Dill (cherry picked from commit 4a5ba756b6d0d26b40a8c3ca5bb3ea3e1ed931b3) --- WebHostLib/__init__.py | 2 ++ WebHostLib/autolauncher.py | 21 ++++++++++++++++++--- docs/webhost configuration sample.yaml | 10 ++++++++-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/WebHostLib/__init__.py b/WebHostLib/__init__.py index dbe2182b0747..9fad499fa802 100644 --- a/WebHostLib/__init__.py +++ b/WebHostLib/__init__.py @@ -39,6 +39,8 @@ app.config["JOB_THRESHOLD"] = 1 # after what time in seconds should generation be aborted, freeing the queue slot. Can be set to None to disable. app.config["JOB_TIME"] = 600 +# memory limit for generator processes in bytes +app.config["GENERATOR_MEMORY_LIMIT"] = 4294967296 app.config['SESSION_PERMANENT'] = True # waitress uses one thread for I/O, these are for processing of views that then get sent diff --git a/WebHostLib/autolauncher.py b/WebHostLib/autolauncher.py index 08a1309ebc73..8ba093e014c5 100644 --- a/WebHostLib/autolauncher.py +++ b/WebHostLib/autolauncher.py @@ -6,6 +6,7 @@ import typing from datetime import timedelta, datetime from threading import Event, Thread +from typing import Any from uuid import UUID from pony.orm import db_session, select, commit @@ -53,7 +54,21 @@ def launch_generator(pool: multiprocessing.pool.Pool, generation: Generation): generation.state = STATE_STARTED -def init_db(pony_config: dict): +def init_generator(config: dict[str, Any]) -> None: + try: + import resource + except ModuleNotFoundError: + pass # unix only module + else: + # set soft limit for memory to from config (default 4GiB) + soft_limit = config["GENERATOR_MEMORY_LIMIT"] + old_limit, hard_limit = resource.getrlimit(resource.RLIMIT_AS) + if soft_limit != old_limit: + resource.setrlimit(resource.RLIMIT_AS, (soft_limit, hard_limit)) + logging.debug(f"Changed AS mem limit {old_limit} -> {soft_limit}") + del resource, soft_limit, hard_limit + + pony_config = config["PONY"] db.bind(**pony_config) db.generate_mapping() @@ -105,8 +120,8 @@ def keep_running(): try: with Locker("autogen"): - with multiprocessing.Pool(config["GENERATORS"], initializer=init_db, - initargs=(config["PONY"],), maxtasksperchild=10) as generator_pool: + with multiprocessing.Pool(config["GENERATORS"], initializer=init_generator, + initargs=(config,), maxtasksperchild=10) as generator_pool: with db_session: to_start = select(generation for generation in Generation if generation.state == STATE_STARTED) diff --git a/docs/webhost configuration sample.yaml b/docs/webhost configuration sample.yaml index afb87b399643..93094f1ce73f 100644 --- a/docs/webhost configuration sample.yaml +++ b/docs/webhost configuration sample.yaml @@ -27,8 +27,14 @@ # If you wish to deploy, uncomment the following line and set it to something not easily guessable. # SECRET_KEY: "Your secret key here" -# TODO -#JOB_THRESHOLD: 2 +# Slot limit to post a generation to Generator process pool instead of rolling directly in WebHost process +#JOB_THRESHOLD: 1 + +# After what time in seconds should generation be aborted, freeing the queue slot. Can be set to None to disable. +#JOB_TIME: 600 + +# Memory limit for Generator processes in bytes, -1 for unlimited. Currently only works on Linux. +#GENERATOR_MEMORY_LIMIT: 4294967296 # waitress uses one thread for I/O, these are for processing of view that get sent #WAITRESS_THREADS: 10