From 22865b89b81761b77ead604ca0445c08971dae4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ho=CC=88=C3=9Fl?= Date: Sat, 7 Dec 2024 11:33:27 +0100 Subject: [PATCH] Usage within supervisord --- README.md | 18 ++++++++++- commands/BackgroundJobController.php | 44 ++++++++++++++++++++++----- components/BackgroundJobScheduler.php | 23 ++++++++------ docs/supervisor.conf | 6 ++++ models/settings/AntragsgruenApp.php | 1 + 5 files changed, 74 insertions(+), 18 deletions(-) create mode 100644 docs/supervisor.conf diff --git a/README.md b/README.md index dd54216e1..f5a97402c 100644 --- a/README.md +++ b/README.md @@ -280,7 +280,7 @@ Instead of "antragsgruen_sites", a custom plugin managing the authentication and Redis can be used to cache the changes in amendments, user sessions, and many other aspects of the site. To enable redis, simply add a `redis` configuration key to the `config.json` and point it to your setup: -Add the following settings to your config.json (and adapt them to your needs): +Add the following settings to your `config.json` (and adapt them to your needs): ```json { "redis": { @@ -292,6 +292,22 @@ Add the following settings to your config.json (and adapt them to your needs): } ``` +### Enable background job processing + +Some processes that are potentially blocking or long-running can be executed as background jobs, by using a permanently running worker-job that executes these jobs asynchonously. + +The following example on how to run the background job processor uses [Supervisord](http://supervisord.org), but it is just as possible running it via any other process manager. +- Copy [supervisor.conf](docs/supervisor.conf) to your supervisord configuration directory, modify it to your needs, and run it. +- Enable background jobs by adding the following settings to your `config.json`: + +```json +{ + "backgroundJobs": true +} +``` + +Currently, this only affects the sending of e-mails. + ### File-based View Caching (very large consultations) Antragsgrün already does a decent amount of caching by default, and even more when enabling Redis. An even more aggressive caching mode that caches some fully rendered HTML pages and PDFs can be enabled by enabling the following option in the `config.json`: diff --git a/commands/BackgroundJobController.php b/commands/BackgroundJobController.php index 29dbab556..e50236f29 100644 --- a/commands/BackgroundJobController.php +++ b/commands/BackgroundJobController.php @@ -12,17 +12,30 @@ */ class BackgroundJobController extends Controller { - private const MAX_EVENTS = 1000; - private const MAX_RUNTIME_SECONDS = 600; - private const MAX_MEMORY_USAGE = 64_000_000; + private const DEFAULT_MAX_EVENTS = 1000; + private const DEFAULT_MAX_RUNTIME_SECONDS = 600; + private const DEFAULT_MAX_MEMORY_USAGE = 64_000_000; + + protected int $maxEvents = self::DEFAULT_MAX_EVENTS; + protected int $maxRuntimeSeconds = self::DEFAULT_MAX_RUNTIME_SECONDS; + protected int $maxMemoryUsage = self::DEFAULT_MAX_MEMORY_USAGE; + + public function options($actionID): array + { + return ['maxEvents', 'maxRuntimeSeconds', 'maxMemoryUsage']; + } /** * Runs the background job processor - * - * @throws \yii\db\Exception + * Options: + * --max-runtime-seconds 600 + * --max-events 1000 + * --max-memory-usage 64000000 */ public function actionRun(): void { + echo "Starting background job processor at: " . (new \DateTimeImmutable())->format("Y-m-d H:i:s.u") . "\n"; + $connection = \Yii::$app->getDb(); $connection->enableLogging = false; @@ -35,12 +48,27 @@ public function actionRun(): void usleep(100_000); } } + + echo "Stopping background job processor at: " . (new \DateTimeImmutable())->format("Y-m-d H:i:s.u") . "\n"; } private function needsRestart(BackgroundJobProcessor $processor): bool { - return $processor->getProcessedEvents() >= self::MAX_EVENTS - || $processor->getRuntimeInSeconds() >= self::MAX_RUNTIME_SECONDS - || memory_get_peak_usage() >= self::MAX_MEMORY_USAGE; + if ($processor->getProcessedEvents() >= $this->maxEvents) { + echo "Stopping because maximum number of processed events has been reached.\n"; + return true; + } + + if ($processor->getRuntimeInSeconds() >= $this->maxRuntimeSeconds) { + echo "Stopping because maximum runtime has been reached.\n"; + return true; + } + + if (memory_get_peak_usage() >= $this->maxMemoryUsage) { + echo "Stopping because maximum memory usage has been reached.\n"; + return true; + } + + return false; } } diff --git a/components/BackgroundJobScheduler.php b/components/BackgroundJobScheduler.php index b24ceb2fd..b130a7d83 100644 --- a/components/BackgroundJobScheduler.php +++ b/components/BackgroundJobScheduler.php @@ -5,19 +5,24 @@ namespace app\components; use app\models\backgroundJobs\IBackgroundJob; +use app\models\settings\AntragsgruenApp; class BackgroundJobScheduler { public static function executeOrScheduleJob(IBackgroundJob $job): void { - \Yii::$app->getDb()->createCommand( - 'INSERT INTO backgroundJob (`siteId`, `consultationId`, `type`, `dateCreation`, `payload`) VALUES (:siteId, :consultationId, :type, NOW(), :payload)', - [ - ':siteId' => $job->getSite()?->id, - ':consultationId' => $job->getConsultation()?->id, - ':type' => $job->getTypeId(), - ':payload' => $job->toJson(), - ] - )->execute(); + if (AntragsgruenApp::getInstance()->backgroundJobs) { + \Yii::$app->getDb()->createCommand( + 'INSERT INTO `backgroundJob` (`siteId`, `consultationId`, `type`, `dateCreation`, `payload`) VALUES (:siteId, :consultationId, :type, NOW(), :payload)', + [ + ':siteId' => $job->getSite()?->id, + ':consultationId' => $job->getConsultation()?->id, + ':type' => $job->getTypeId(), + ':payload' => $job->toJson(), + ] + )->execute(); + } else { + $job->execute(); + } } } diff --git a/docs/supervisor.conf b/docs/supervisor.conf new file mode 100644 index 000000000..2111d33b9 --- /dev/null +++ b/docs/supervisor.conf @@ -0,0 +1,6 @@ +[program:antragsgruen-background-jobs] +command=/var/www/antragsgruen/yii background-job/run +redirect_stderr=true +user=www-data +environment=ANTRAGSGRUEN_CONFIG="/var/www/antragsgruen/config/config.json" +autorestart=true diff --git a/models/settings/AntragsgruenApp.php b/models/settings/AntragsgruenApp.php index 76bfea5ff..b92398e05 100644 --- a/models/settings/AntragsgruenApp.php +++ b/models/settings/AntragsgruenApp.php @@ -34,6 +34,7 @@ class AntragsgruenApp implements \JsonSerializable public bool $confirmEmailAddresses = true; public bool $enforceTwoFactorAuthentication = false; public bool $dataPrivacyCheckbox = false; + public bool $backgroundJobs = false; public string $mailFromName = 'Antragsgrün'; public string $mailFromEmail = ''; public ?string $mailDefaultReplyTo = null;