From d33e4807d7d9c7fd8cfe93c98f7c76f42b73f495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ho=CC=88=C3=9Fl?= Date: Sun, 1 Dec 2024 12:58:47 +0100 Subject: [PATCH 1/9] Data stucture for background jobs --- assets/db/create.sql | 40 ++++++++++++++ assets/db/data.sql | 3 +- assets/db/delete.sql | 1 + commands/BackgroundJobController.php | 45 ++++++++++++++++ components/BackgroundJobProcessor.php | 52 +++++++++++++++++++ migrations/m241201_100317_background_jobs.php | 37 +++++++++++++ 6 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 commands/BackgroundJobController.php create mode 100644 components/BackgroundJobProcessor.php create mode 100644 migrations/m241201_100317_background_jobs.php diff --git a/assets/db/create.sql b/assets/db/create.sql index 2d0b246e9..b145bfab8 100644 --- a/assets/db/create.sql +++ b/assets/db/create.sql @@ -815,6 +815,22 @@ CREATE TABLE `###TABLE_PREFIX###votingQuestion` ( `votingData` text DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +-- +-- Table structure for table `backgroundJob` +-- + +CREATE TABLE `###TABLE_PREFIX###backgroundJob` ( + `id` bigint UNSIGNED NOT NULL, + `siteId` int DEFAULT NULL, + `consultationId` int DEFAULT NULL, + `type` varchar(150) NOT NULL, + `dateCreation` timestamp NOT NULL, + `dateStarted` timestamp NULL DEFAULT NULL, + `dateUpdated` timestamp NULL DEFAULT NULL, + `dateFinished` timestamp NULL DEFAULT NULL, + `payload` mediumblob NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + -- -- Indexes for dumped tables -- @@ -1155,6 +1171,16 @@ ALTER TABLE `###TABLE_PREFIX###votingQuestion` ADD KEY `fk_question_block` (`votingBlockId`), ADD KEY `fk_question_consultation` (`consultationId`); +-- +-- Indexes for table `backgroundJob` +-- +ALTER TABLE `###TABLE_PREFIX###backgroundJob` + ADD PRIMARY KEY (`id`), + ADD KEY `fk_background_site` (`siteId`), + ADD KEY `fk_background_consultation` (`consultationId`), + ADD KEY `ix_background_pending` (`dateStarted`,`id`), + ADD KEY `ix_background_todelete` (`dateFinished`); + -- -- AUTO_INCREMENT for dumped tables -- @@ -1320,6 +1346,13 @@ ALTER TABLE `###TABLE_PREFIX###votingBlock` -- ALTER TABLE `###TABLE_PREFIX###votingQuestion` MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; + +-- +-- AUTO_INCREMENT for table `backgroundJob` +-- +ALTER TABLE `###TABLE_PREFIX###backgroundJob` + MODIFY `id` bigint UNSIGNED NOT NULL AUTO_INCREMENT; + -- -- Constraints for dumped tables -- @@ -1629,6 +1662,13 @@ ALTER TABLE `###TABLE_PREFIX###votingQuestion` ADD CONSTRAINT `fk_question_block` FOREIGN KEY (`votingBlockId`) REFERENCES `###TABLE_PREFIX###votingBlock` (`id`), ADD CONSTRAINT `fk_question_consultation` FOREIGN KEY (`consultationId`) REFERENCES `###TABLE_PREFIX###consultation` (`id`); +-- +-- Constraints for table `backgroundJob` +-- +ALTER TABLE `###TABLE_PREFIX###backgroundJob` + ADD CONSTRAINT `fk_background_consultation` FOREIGN KEY (`consultationId`) REFERENCES `###TABLE_PREFIX###consultation` (`id`), + ADD CONSTRAINT `fk_background_site` FOREIGN KEY (`siteId`) REFERENCES `###TABLE_PREFIX###site` (`id`); + SET SQL_MODE = @OLD_SQL_MODE; SET FOREIGN_KEY_CHECKS = @OLD_FOREIGN_KEY_CHECKS; SET UNIQUE_CHECKS = @OLD_UNIQUE_CHECKS; diff --git a/assets/db/data.sql b/assets/db/data.sql index 87de93c71..d29bfa84b 100644 --- a/assets/db/data.sql +++ b/assets/db/data.sql @@ -89,7 +89,8 @@ INSERT INTO `migration` (`version`, `apply_time`) VALUES ('m240427_090527_motion_status_index', '1714209051'), ('m240830_181716_user_secret_key', '1725041937'), ('m241013_105549_pages_files', '1728817360'), - ('m241027_074032_pages_policies', '1730015023') + ('m241027_074032_pages_policies', '1730015023'), + ('m241201_100317_background_jobs', '1733052690') ; SET SQL_MODE = @OLD_SQL_MODE; diff --git a/assets/db/delete.sql b/assets/db/delete.sql index f5dea068e..23eadaaa5 100644 --- a/assets/db/delete.sql +++ b/assets/db/delete.sql @@ -47,6 +47,7 @@ DROP TABLE IF EXISTS `###TABLE_PREFIX###consultationMotionType`; DROP TABLE IF EXISTS `###TABLE_PREFIX###consultationSettingsTag`; DROP TABLE IF EXISTS `###TABLE_PREFIX###consultationLog`; DROP TABLE IF EXISTS `###TABLE_PREFIX###texTemplate`; +DROP TABLE IF EXISTS `###TABLE_PREFIX###backgroundJob`; SET SQL_MODE=@OLD_SQL_MODE; SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS; diff --git a/commands/BackgroundJobController.php b/commands/BackgroundJobController.php new file mode 100644 index 000000000..003b72028 --- /dev/null +++ b/commands/BackgroundJobController.php @@ -0,0 +1,45 @@ +getDb()); + while (!$this->needsRestart($processor)) { + $row = $processor->getJobAndSetStarted(); + if ($row) { + $processor->processRow($row); + } else { + usleep(100000); + } + + file_put_contents('/tmp/memory_usage.log', memory_get_peak_usage() . "\n", FILE_APPEND); + } + } + + private function needsRestart(BackgroundJobProcessor $processor): bool + { + return $processor->getProcessedEvents() >= self::MAX_EVENTS + || $processor->getRuntimeInSeconds() >= self::MAX_RUNTIME_SECONDS + || memory_get_peak_usage() >= self::MAX_MEMORY_USAGE; + } +} diff --git a/components/BackgroundJobProcessor.php b/components/BackgroundJobProcessor.php new file mode 100644 index 000000000..aeb98bb76 --- /dev/null +++ b/components/BackgroundJobProcessor.php @@ -0,0 +1,52 @@ +connection = $connection; + $this->startedAt = new \DateTimeImmutable(); + } + + public function getJobAndSetStarted(): ?array { + $foundRow = null; + + $this->connection->transaction(function () use (&$foundRow) { + $command = $this->connection->createCommand('SELECT * FROM backgroundJob WHERE dateStarted IS NULL ORDER BY id ASC LIMIT 0,1 FOR UPDATE'); + $foundRows = $command->queryAll(); + if (empty($foundRows)) { + return; + } + + $foundRow = $foundRows[0]; + $this->connection->createCommand('UPDATE backgroundJob SET dateStarted = NOW() WHERE id = :id', ['id' => $foundRow['id']])->execute(); + }); + + return $foundRow; + } + + public function processRow(array $row): void + { + echo "Processing row: " . $row['id'] . "\n"; + sleep(2); + $this->connection->createCommand('UPDATE backgroundJob SET dateFinished = NOW() WHERE id = :id', ['id' => $row['id']])->execute(); + } + + public function getProcessedEvents(): int { + return $this->processedEvents; + } + + public function getRuntimeInSeconds(): int { + return (new \DateTimeImmutable())->getTimestamp() - $this->startedAt->getTimestamp(); + } +} diff --git a/migrations/m241201_100317_background_jobs.php b/migrations/m241201_100317_background_jobs.php new file mode 100644 index 000000000..214827ff7 --- /dev/null +++ b/migrations/m241201_100317_background_jobs.php @@ -0,0 +1,37 @@ +createTable('backgroundJob', [ + 'id' => 'BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY', + 'siteId' => 'INTEGER DEFAULT NULL', + 'consultationId' => 'INTEGER DEFAULT NULL', + 'type' => 'VARCHAR(150) NOT NULL', + 'dateCreation' => 'TIMESTAMP NOT NULL', + 'dateStarted' => 'TIMESTAMP DEFAULT NULL', + 'dateUpdated' => 'TIMESTAMP DEFAULT NULL', + 'dateFinished' => 'TIMESTAMP DEFAULT NULL', + 'payload' => 'MEDIUMBLOB NOT NULL', + ]); + + $this->addForeignKey('fk_background_site', 'backgroundJob', 'siteId', 'site', 'id'); + $this->addForeignKey('fk_background_consultation', 'backgroundJob', 'consultationId', 'consultation', 'id'); + $this->createIndex('ix_background_pending', 'backgroundJob', ['dateStarted', 'id'], false); + $this->createIndex('ix_background_todelete', 'backgroundJob', 'dateFinished', false); + } + + /** + * {@inheritdoc} + */ + public function safeDown(): void + { + $this->dropTable('backgroundJob'); + } +} From fb7996c8f19836dbf37ce2fb04cb644e0a5a4860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ho=CC=88=C3=9Fl?= Date: Sun, 1 Dec 2024 13:04:06 +0100 Subject: [PATCH 2/9] Disable SQL logging --- commands/BackgroundJobController.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/commands/BackgroundJobController.php b/commands/BackgroundJobController.php index 003b72028..3daf65c22 100644 --- a/commands/BackgroundJobController.php +++ b/commands/BackgroundJobController.php @@ -23,7 +23,10 @@ class BackgroundJobController extends Controller */ public function actionRun(): void { - $processor = new BackgroundJobProcessor(\Yii::$app->getDb()); + $connection = \Yii::$app->getDb(); + $connection->enableLogging = false; + + $processor = new BackgroundJobProcessor($connection); while (!$this->needsRestart($processor)) { $row = $processor->getJobAndSetStarted(); if ($row) { @@ -32,7 +35,7 @@ public function actionRun(): void usleep(100000); } - file_put_contents('/tmp/memory_usage.log', memory_get_peak_usage() . "\n", FILE_APPEND); + file_put_contents('/tmp/memory_usage.log', date("Y-m-d H:i:s") . ": " . memory_get_peak_usage() . "\n", FILE_APPEND); } } From c9887b770fa6185d8ae6d5f6f19210025bcccd93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ho=CC=88=C3=9Fl?= Date: Sun, 1 Dec 2024 16:29:27 +0100 Subject: [PATCH 3/9] First draft of background worker --- assets/db/create.sql | 2 +- commands/BackgroundJobController.php | 4 +- components/BackgroundJobProcessor.php | 35 ++++-- components/BackgroundJobScheduler.php | 23 ++++ components/mail/Tools.php | 94 +++------------ migrations/m241201_100317_background_jobs.php | 2 +- models/backgroundJobs/IBackgroundJob.php | 80 +++++++++++++ models/backgroundJobs/SendNotification.php | 112 ++++++++++++++++++ 8 files changed, 263 insertions(+), 89 deletions(-) create mode 100644 components/BackgroundJobScheduler.php create mode 100644 models/backgroundJobs/IBackgroundJob.php create mode 100644 models/backgroundJobs/SendNotification.php diff --git a/assets/db/create.sql b/assets/db/create.sql index b145bfab8..6dac9e4fe 100644 --- a/assets/db/create.sql +++ b/assets/db/create.sql @@ -828,7 +828,7 @@ CREATE TABLE `###TABLE_PREFIX###backgroundJob` ( `dateStarted` timestamp NULL DEFAULT NULL, `dateUpdated` timestamp NULL DEFAULT NULL, `dateFinished` timestamp NULL DEFAULT NULL, - `payload` mediumblob NOT NULL + `payload` mediumtext NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- diff --git a/commands/BackgroundJobController.php b/commands/BackgroundJobController.php index 3daf65c22..29dbab556 100644 --- a/commands/BackgroundJobController.php +++ b/commands/BackgroundJobController.php @@ -32,10 +32,8 @@ public function actionRun(): void if ($row) { $processor->processRow($row); } else { - usleep(100000); + usleep(100_000); } - - file_put_contents('/tmp/memory_usage.log', date("Y-m-d H:i:s") . ": " . memory_get_peak_usage() . "\n", FILE_APPEND); } } diff --git a/components/BackgroundJobProcessor.php b/components/BackgroundJobProcessor.php index aeb98bb76..b977f500e 100644 --- a/components/BackgroundJobProcessor.php +++ b/components/BackgroundJobProcessor.php @@ -4,6 +4,7 @@ namespace app\components; +use app\models\backgroundJobs\IBackgroundJob; use yii\db\Connection; class BackgroundJobProcessor @@ -18,10 +19,10 @@ public function __construct(Connection $connection) { $this->startedAt = new \DateTimeImmutable(); } - public function getJobAndSetStarted(): ?array { - $foundRow = null; + public function getJobAndSetStarted(): ?IBackgroundJob { + $foundJob = null; - $this->connection->transaction(function () use (&$foundRow) { + $this->connection->transaction(function () use (&$foundJob) { $command = $this->connection->createCommand('SELECT * FROM backgroundJob WHERE dateStarted IS NULL ORDER BY id ASC LIMIT 0,1 FOR UPDATE'); $foundRows = $command->queryAll(); if (empty($foundRows)) { @@ -30,16 +31,34 @@ public function getJobAndSetStarted(): ?array { $foundRow = $foundRows[0]; $this->connection->createCommand('UPDATE backgroundJob SET dateStarted = NOW() WHERE id = :id', ['id' => $foundRow['id']])->execute(); + + $foundJob = IBackgroundJob::fromJson( + intval($foundRow['id']), + $foundRow['type'], + ($foundRow['siteId'] > 0 ? $foundRow['siteId'] : null), + ($foundRow['consultationId'] > 0 ? $foundRow['consultationId'] : null), + $foundRow['payload'] + ); }); - return $foundRow; + return $foundJob; } - public function processRow(array $row): void + public function processRow(IBackgroundJob $job): void { - echo "Processing row: " . $row['id'] . "\n"; - sleep(2); - $this->connection->createCommand('UPDATE backgroundJob SET dateFinished = NOW() WHERE id = :id', ['id' => $row['id']])->execute(); + echo "Processing row: " . $job->getId() . "\n"; + + $this->connection->createCommand( + 'UPDATE backgroundJob SET dateUpdated = NOW() WHERE id = :id', + ['id' => $job->getId()] + )->execute(); + + $job->execute(); + + $this->connection->createCommand( + 'UPDATE backgroundJob SET dateFinished = NOW() WHERE id = :id', + ['id' => $job->getId()] + )->execute(); } public function getProcessedEvents(): int { diff --git a/components/BackgroundJobScheduler.php b/components/BackgroundJobScheduler.php new file mode 100644 index 000000000..b24ceb2fd --- /dev/null +++ b/components/BackgroundJobScheduler.php @@ -0,0 +1,23 @@ +getDb()->createCommand( + 'INSERT INTO backgroundJob (`siteId`, `consultationId`, `type`, `dateCreation`, `payload`) VALUES (:siteId, :consultationId, :type, NOW(), :payload)', + [ + ':siteId' => $job->getSite()?->id, + ':consultationId' => $job->getConsultation()?->id, + ':type' => $job->getTypeId(), + ':payload' => $job->toJson(), + ] + )->execute(); + } +} diff --git a/components/mail/Tools.php b/components/mail/Tools.php index f568d1ccf..e8fc8b8ff 100644 --- a/components/mail/Tools.php +++ b/components/mail/Tools.php @@ -4,11 +4,10 @@ namespace app\components\mail; -use app\components\RequestContext; +use app\components\BackgroundJobScheduler; +use app\models\backgroundJobs\SendNotification; use app\models\settings\AntragsgruenApp; -use app\models\db\{Consultation, EMailLog, IMotion, User}; -use app\models\exceptions\{MailNotSent, ServerConfiguration}; -use Symfony\Component\Mailer\Exception\TransportExceptionInterface; +use app\models\db\{Consultation, IMotion, User}; class Tools { @@ -53,10 +52,6 @@ public static function getDefaultReplyTo(?IMotion $imotion = null, ?Consultation return $replyTo; } - /** - * @throws MailNotSent - * @throws ServerConfiguration - */ public static function sendWithLog( int $mailType, ?Consultation $fromConsultation, @@ -70,79 +65,26 @@ public static function sendWithLog( ?string $replyTo = null ): void { $params = AntragsgruenApp::getInstance(); - $mailer = Base::createMailer($params->mailService); - if (!$mailer) { - throw new MailNotSent('E-Mail not configured'); - } - - $sendTextPlain = ($noLogReplaces ? str_replace( - array_keys($noLogReplaces), - array_values($noLogReplaces), - $textPlain - ) : $textPlain); - $sendTextHtml = ($noLogReplaces ? str_replace( - array_keys($noLogReplaces), - array_values($noLogReplaces), - $textHtml - ) : $textHtml); - $fromEmail = $params->mailFromEmail; if (!$fromName) { - $fromName = static::getDefaultMailFromName($fromConsultation); + $fromName = Tools::getDefaultMailFromName($fromConsultation); } if (!$replyTo) { - $replyTo = static::getDefaultReplyTo(null, $fromConsultation); - } - - $exception = null; - $messageId = ''; - try { - $message = $mailer->createMessage( - $subject, - $sendTextPlain, - $sendTextHtml, - $fromName, - $fromEmail, - $replyTo, - $fromConsultation - ); - $result = $mailer->send($message, $toEmail); - if (is_string($result)) { - $status = EMailLog::STATUS_SENT; - $messageId = $result; - } else { - $status = $result; - } - } catch (TransportExceptionInterface $e) { - $status = EMailLog::STATUS_DELIVERY_ERROR; - $exception = $e; + $replyTo = Tools::getDefaultReplyTo(null, $fromConsultation); } - $obj = new EMailLog(); - if ($toPersonId) { - $obj->toUserId = $toPersonId; - } - if ($fromConsultation) { - $obj->fromSiteId = $fromConsultation->siteId; - } - $obj->toEmail = $toEmail; - $obj->type = $mailType; - $obj->fromEmail = $fromName . ' <' . $fromEmail . '>'; - $obj->subject = mb_substr($subject, 0, 190); - $obj->text = $textPlain; - $obj->dateSent = date('Y-m-d H:i:s'); - $obj->status = $status; - $obj->messageId = $messageId; - $obj->save(); - - if ($exception) { - \Yii::error($exception->getMessage()); - throw new MailNotSent($exception->getMessage()); - } - - if (YII_ENV === 'test') { - $pre = RequestContext::getSession()->getFlash('email', ''); - RequestContext::getSession()->setFlash('email', $pre . 'E-Mail sent to: ' . $toEmail . " (Type $mailType)\n"); - } + BackgroundJobScheduler::executeOrScheduleJob(new SendNotification( + $fromConsultation, + $mailType, + $toEmail, + $toPersonId, + $subject, + $textPlain, + $textHtml, + $noLogReplaces, + $fromEmail, + $fromName, + $replyTo + )); } } diff --git a/migrations/m241201_100317_background_jobs.php b/migrations/m241201_100317_background_jobs.php index 214827ff7..2919893bb 100644 --- a/migrations/m241201_100317_background_jobs.php +++ b/migrations/m241201_100317_background_jobs.php @@ -18,7 +18,7 @@ public function safeUp(): void 'dateStarted' => 'TIMESTAMP DEFAULT NULL', 'dateUpdated' => 'TIMESTAMP DEFAULT NULL', 'dateFinished' => 'TIMESTAMP DEFAULT NULL', - 'payload' => 'MEDIUMBLOB NOT NULL', + 'payload' => 'MEDIUMTEXT NOT NULL', ]); $this->addForeignKey('fk_background_site', 'backgroundJob', 'siteId', 'site', 'id'); diff --git a/models/backgroundJobs/IBackgroundJob.php b/models/backgroundJobs/IBackgroundJob.php new file mode 100644 index 000000000..47d17fb4e --- /dev/null +++ b/models/backgroundJobs/IBackgroundJob.php @@ -0,0 +1,80 @@ +> + */ + public static function getAllBackgroundJobs(): array + { + return [ + SendNotification::TYPE_ID => SendNotification::class, + ]; + } + + public function toJson(): string + { + $serializer = Tools::getSerializer(); + + return $serializer->serialize($this, 'json'); + } + + /** + * @Ignore + */ + public function getConsultation(): ?Consultation + { + return $this->consultation; + } + + /** + * @Ignore + */ + public function getSite(): ?Site + { + return $this->site; + } + + /** + * @Ignore + */ + public function getId(): ?int + { + return $this->id; + } + + public static function fromJson(int $id, string $typeId, ?int $siteId, ?int $consultationId, string $json): IBackgroundJob + { + $serializer = Tools::getSerializer(); + + $class = self::getAllBackgroundJobs()[$typeId]; + + /** @var IBackgroundJob $job */ + $job = $serializer->deserialize($json, $class, 'json'); + $job->id = $id; + if ($siteId !== null) { + $job->site = Site::findOne(['id' => $siteId]); + } + if ($consultationId !== null) { + $job->consultation = Consultation::findOne(['id' => $consultationId]); + } + + return $job; + } +} diff --git a/models/backgroundJobs/SendNotification.php b/models/backgroundJobs/SendNotification.php new file mode 100644 index 000000000..44a382517 --- /dev/null +++ b/models/backgroundJobs/SendNotification.php @@ -0,0 +1,112 @@ +consultation = $consultation; + } + + public function getTypeId(): string + { + return self::TYPE_ID; + } + + public function execute(): void + { + file_put_contents('/tmp/sendmail.log', 'SEND MAIL: ' . $this->toEmail . PHP_EOL, FILE_APPEND); + + $params = AntragsgruenApp::getInstance(); + $mailer = Base::createMailer($params->mailService); + if (!$mailer) { + throw new MailNotSent('E-Mail not configured'); + } + + $sendTextPlain = ($this->noLogReplaces ? str_replace( + array_keys($this->noLogReplaces), + array_values($this->noLogReplaces), + $this->textPlain + ) : $this->textPlain); + $sendTextHtml = ($this->noLogReplaces ? str_replace( + array_keys($this->noLogReplaces), + array_values($this->noLogReplaces), + $this->textHtml + ) : $this->textHtml); + + $exception = null; + $messageId = ''; + try { + $message = $mailer->createMessage( + $this->subject, + $sendTextPlain, + $sendTextHtml, + $this->fromName, + $this->fromEmail, + $this->replyTo, + $this->consultation + ); + $result = $mailer->send($message, $this->toEmail); + if (is_string($result)) { + $status = EMailLog::STATUS_SENT; + $messageId = $result; + } else { + $status = $result; + } + } catch (TransportExceptionInterface $e) { + $status = EMailLog::STATUS_DELIVERY_ERROR; + $exception = $e; + } + + $obj = new EMailLog(); + if ($this->toPersonId) { + $obj->toUserId = $this->toPersonId; + } + if ($this->consultation) { + $obj->fromSiteId = $this->consultation->siteId; + } + $obj->toEmail = $this->toEmail; + $obj->type = $this->mailType; + $obj->fromEmail = $this->fromName . ' <' . $this->fromEmail . '>'; + $obj->subject = mb_substr($this->subject, 0, 190); + $obj->text = $this->textPlain; + $obj->dateSent = date('Y-m-d H:i:s'); + $obj->status = $status; + $obj->messageId = $messageId; + $obj->save(); + + if ($exception) { + \Yii::error($exception->getMessage()); + throw new MailNotSent($exception->getMessage()); + } + + if (YII_ENV === 'test') { + $pre = RequestContext::getSession()->getFlash('email', ''); + RequestContext::getSession()->setFlash('email', $pre . 'E-Mail sent to: ' . $this->toEmail . " (Type $this->mailType)\n"); + } + } +} From e1fdba4ca8dcfafa067b5a8978c6b0b4cc3db9b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ho=CC=88=C3=9Fl?= Date: Sun, 1 Dec 2024 17:53:02 +0100 Subject: [PATCH 4/9] Handle errors in background jobs --- assets/db/create.sql | 3 ++- components/BackgroundJobProcessor.php | 21 ++++++++++++------- components/SitePurger.php | 10 +++++++++ migrations/m241201_100317_background_jobs.php | 1 + models/backgroundJobs/SendNotification.php | 2 -- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/assets/db/create.sql b/assets/db/create.sql index 6dac9e4fe..98e9d67f6 100644 --- a/assets/db/create.sql +++ b/assets/db/create.sql @@ -828,7 +828,8 @@ CREATE TABLE `###TABLE_PREFIX###backgroundJob` ( `dateStarted` timestamp NULL DEFAULT NULL, `dateUpdated` timestamp NULL DEFAULT NULL, `dateFinished` timestamp NULL DEFAULT NULL, - `payload` mediumtext NOT NULL + `payload` mediumtext NOT NULL, + `error` text DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- diff --git a/components/BackgroundJobProcessor.php b/components/BackgroundJobProcessor.php index b977f500e..f55c6cf0f 100644 --- a/components/BackgroundJobProcessor.php +++ b/components/BackgroundJobProcessor.php @@ -46,19 +46,24 @@ public function getJobAndSetStarted(): ?IBackgroundJob { public function processRow(IBackgroundJob $job): void { - echo "Processing row: " . $job->getId() . "\n"; - $this->connection->createCommand( 'UPDATE backgroundJob SET dateUpdated = NOW() WHERE id = :id', ['id' => $job->getId()] )->execute(); - $job->execute(); - - $this->connection->createCommand( - 'UPDATE backgroundJob SET dateFinished = NOW() WHERE id = :id', - ['id' => $job->getId()] - )->execute(); + try { + $job->execute(); + + $this->connection->createCommand( + 'UPDATE backgroundJob SET dateFinished = NOW() WHERE id = :id', + ['id' => $job->getId()] + )->execute(); + } catch (\Throwable $exception) { + $this->connection->createCommand( + 'UPDATE backgroundJob SET error = :error WHERE id = :id', + [':error' => $exception->getMessage() . PHP_EOL . $exception->getTraceAsString(), ':id' => $job->getId()] + )->execute(); + } } public function getProcessedEvents(): int { diff --git a/components/SitePurger.php b/components/SitePurger.php index e30975047..0af6f76fd 100644 --- a/components/SitePurger.php +++ b/components/SitePurger.php @@ -136,6 +136,11 @@ public static function purgeConsultation(int $consultationId): void [':conId' => $consultationId] )->execute(); + $connection->createCommand( + 'DELETE FROM backgroundJob WHERE consultationId = :conId', + [':conId' => $consultationId] + )->execute(); + $connection->createCommand( 'UPDATE site SET currentConsultationId = NULL WHERE currentConsultationId = :conId', [':conId' => $consultationId] @@ -164,6 +169,11 @@ public static function purgeSite(int $siteId): void [':siteId' => $siteId] )->execute(); + $connection->createCommand( + 'DELETE FROM backgroundJob WHERE siteId = :siteId', + [':siteId' => $siteId] + )->execute(); + $connection->createCommand( 'DELETE FROM site WHERE id = :siteId', [':siteId' => $siteId] diff --git a/migrations/m241201_100317_background_jobs.php b/migrations/m241201_100317_background_jobs.php index 2919893bb..135d58e2b 100644 --- a/migrations/m241201_100317_background_jobs.php +++ b/migrations/m241201_100317_background_jobs.php @@ -19,6 +19,7 @@ public function safeUp(): void 'dateUpdated' => 'TIMESTAMP DEFAULT NULL', 'dateFinished' => 'TIMESTAMP DEFAULT NULL', 'payload' => 'MEDIUMTEXT NOT NULL', + 'error' => 'TEXT DEFAULT NULL', ]); $this->addForeignKey('fk_background_site', 'backgroundJob', 'siteId', 'site', 'id'); diff --git a/models/backgroundJobs/SendNotification.php b/models/backgroundJobs/SendNotification.php index 44a382517..ea0e83caa 100644 --- a/models/backgroundJobs/SendNotification.php +++ b/models/backgroundJobs/SendNotification.php @@ -39,8 +39,6 @@ public function getTypeId(): string public function execute(): void { - file_put_contents('/tmp/sendmail.log', 'SEND MAIL: ' . $this->toEmail . PHP_EOL, FILE_APPEND); - $params = AntragsgruenApp::getInstance(); $mailer = Base::createMailer($params->mailService); if (!$mailer) { From 78134ed3163d34b9fb0accf6ebfffdecd1fda49a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ho=CC=88=C3=9Fl?= Date: Sat, 7 Dec 2024 11:33:27 +0100 Subject: [PATCH 5/9] Usage within supervisord --- README.md | 18 ++++++++++- commands/BackgroundJobController.php | 44 ++++++++++++++++++++++----- components/BackgroundJobScheduler.php | 23 ++++++++------ docs/supervisor.conf | 6 ++++ models/settings/AntragsgruenApp.php | 1 + 5 files changed, 74 insertions(+), 18 deletions(-) create mode 100644 docs/supervisor.conf diff --git a/README.md b/README.md index dd54216e1..f5a97402c 100644 --- a/README.md +++ b/README.md @@ -280,7 +280,7 @@ Instead of "antragsgruen_sites", a custom plugin managing the authentication and Redis can be used to cache the changes in amendments, user sessions, and many other aspects of the site. To enable redis, simply add a `redis` configuration key to the `config.json` and point it to your setup: -Add the following settings to your config.json (and adapt them to your needs): +Add the following settings to your `config.json` (and adapt them to your needs): ```json { "redis": { @@ -292,6 +292,22 @@ Add the following settings to your config.json (and adapt them to your needs): } ``` +### Enable background job processing + +Some processes that are potentially blocking or long-running can be executed as background jobs, by using a permanently running worker-job that executes these jobs asynchonously. + +The following example on how to run the background job processor uses [Supervisord](http://supervisord.org), but it is just as possible running it via any other process manager. +- Copy [supervisor.conf](docs/supervisor.conf) to your supervisord configuration directory, modify it to your needs, and run it. +- Enable background jobs by adding the following settings to your `config.json`: + +```json +{ + "backgroundJobs": true +} +``` + +Currently, this only affects the sending of e-mails. + ### File-based View Caching (very large consultations) Antragsgrün already does a decent amount of caching by default, and even more when enabling Redis. An even more aggressive caching mode that caches some fully rendered HTML pages and PDFs can be enabled by enabling the following option in the `config.json`: diff --git a/commands/BackgroundJobController.php b/commands/BackgroundJobController.php index 29dbab556..e50236f29 100644 --- a/commands/BackgroundJobController.php +++ b/commands/BackgroundJobController.php @@ -12,17 +12,30 @@ */ class BackgroundJobController extends Controller { - private const MAX_EVENTS = 1000; - private const MAX_RUNTIME_SECONDS = 600; - private const MAX_MEMORY_USAGE = 64_000_000; + private const DEFAULT_MAX_EVENTS = 1000; + private const DEFAULT_MAX_RUNTIME_SECONDS = 600; + private const DEFAULT_MAX_MEMORY_USAGE = 64_000_000; + + protected int $maxEvents = self::DEFAULT_MAX_EVENTS; + protected int $maxRuntimeSeconds = self::DEFAULT_MAX_RUNTIME_SECONDS; + protected int $maxMemoryUsage = self::DEFAULT_MAX_MEMORY_USAGE; + + public function options($actionID): array + { + return ['maxEvents', 'maxRuntimeSeconds', 'maxMemoryUsage']; + } /** * Runs the background job processor - * - * @throws \yii\db\Exception + * Options: + * --max-runtime-seconds 600 + * --max-events 1000 + * --max-memory-usage 64000000 */ public function actionRun(): void { + echo "Starting background job processor at: " . (new \DateTimeImmutable())->format("Y-m-d H:i:s.u") . "\n"; + $connection = \Yii::$app->getDb(); $connection->enableLogging = false; @@ -35,12 +48,27 @@ public function actionRun(): void usleep(100_000); } } + + echo "Stopping background job processor at: " . (new \DateTimeImmutable())->format("Y-m-d H:i:s.u") . "\n"; } private function needsRestart(BackgroundJobProcessor $processor): bool { - return $processor->getProcessedEvents() >= self::MAX_EVENTS - || $processor->getRuntimeInSeconds() >= self::MAX_RUNTIME_SECONDS - || memory_get_peak_usage() >= self::MAX_MEMORY_USAGE; + if ($processor->getProcessedEvents() >= $this->maxEvents) { + echo "Stopping because maximum number of processed events has been reached.\n"; + return true; + } + + if ($processor->getRuntimeInSeconds() >= $this->maxRuntimeSeconds) { + echo "Stopping because maximum runtime has been reached.\n"; + return true; + } + + if (memory_get_peak_usage() >= $this->maxMemoryUsage) { + echo "Stopping because maximum memory usage has been reached.\n"; + return true; + } + + return false; } } diff --git a/components/BackgroundJobScheduler.php b/components/BackgroundJobScheduler.php index b24ceb2fd..b130a7d83 100644 --- a/components/BackgroundJobScheduler.php +++ b/components/BackgroundJobScheduler.php @@ -5,19 +5,24 @@ namespace app\components; use app\models\backgroundJobs\IBackgroundJob; +use app\models\settings\AntragsgruenApp; class BackgroundJobScheduler { public static function executeOrScheduleJob(IBackgroundJob $job): void { - \Yii::$app->getDb()->createCommand( - 'INSERT INTO backgroundJob (`siteId`, `consultationId`, `type`, `dateCreation`, `payload`) VALUES (:siteId, :consultationId, :type, NOW(), :payload)', - [ - ':siteId' => $job->getSite()?->id, - ':consultationId' => $job->getConsultation()?->id, - ':type' => $job->getTypeId(), - ':payload' => $job->toJson(), - ] - )->execute(); + if (AntragsgruenApp::getInstance()->backgroundJobs) { + \Yii::$app->getDb()->createCommand( + 'INSERT INTO `backgroundJob` (`siteId`, `consultationId`, `type`, `dateCreation`, `payload`) VALUES (:siteId, :consultationId, :type, NOW(), :payload)', + [ + ':siteId' => $job->getSite()?->id, + ':consultationId' => $job->getConsultation()?->id, + ':type' => $job->getTypeId(), + ':payload' => $job->toJson(), + ] + )->execute(); + } else { + $job->execute(); + } } } diff --git a/docs/supervisor.conf b/docs/supervisor.conf new file mode 100644 index 000000000..2111d33b9 --- /dev/null +++ b/docs/supervisor.conf @@ -0,0 +1,6 @@ +[program:antragsgruen-background-jobs] +command=/var/www/antragsgruen/yii background-job/run +redirect_stderr=true +user=www-data +environment=ANTRAGSGRUEN_CONFIG="/var/www/antragsgruen/config/config.json" +autorestart=true diff --git a/models/settings/AntragsgruenApp.php b/models/settings/AntragsgruenApp.php index 76bfea5ff..b92398e05 100644 --- a/models/settings/AntragsgruenApp.php +++ b/models/settings/AntragsgruenApp.php @@ -34,6 +34,7 @@ class AntragsgruenApp implements \JsonSerializable public bool $confirmEmailAddresses = true; public bool $enforceTwoFactorAuthentication = false; public bool $dataPrivacyCheckbox = false; + public bool $backgroundJobs = false; public string $mailFromName = 'Antragsgrün'; public string $mailFromEmail = ''; public ?string $mailDefaultReplyTo = null; From 3b3677afdab45718bc5798dca091fd60004d7907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ho=CC=88=C3=9Fl?= Date: Sun, 22 Dec 2024 15:19:45 +0100 Subject: [PATCH 6/9] Health checks for background jobs --- README.md | 6 +++- components/BackgroundJobScheduler.php | 32 ++++++++++++++++++- config/urls.php | 3 +- controllers/ManagerController.php | 37 +++++++++++++++++++--- models/backgroundJobs/SendNotification.php | 3 +- models/settings/AntragsgruenApp.php | 5 ++- 6 files changed, 76 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f5a97402c..6ac497db5 100644 --- a/README.md +++ b/README.md @@ -298,11 +298,15 @@ Some processes that are potentially blocking or long-running can be executed as The following example on how to run the background job processor uses [Supervisord](http://supervisord.org), but it is just as possible running it via any other process manager. - Copy [supervisor.conf](docs/supervisor.conf) to your supervisord configuration directory, modify it to your needs, and run it. +- Create an API key for the health checks (optional) and its hash (via `password_encode($password, PASSWORD_DEFAULT)`). - Enable background jobs by adding the following settings to your `config.json`: ```json { - "backgroundJobs": true + "backgroundJobs": { + "notifications": true + }, + "healthCheckKey": "$2y$12$...." } ``` diff --git a/components/BackgroundJobScheduler.php b/components/BackgroundJobScheduler.php index b130a7d83..7a39acfb9 100644 --- a/components/BackgroundJobScheduler.php +++ b/components/BackgroundJobScheduler.php @@ -9,9 +9,11 @@ class BackgroundJobScheduler { + public const HEALTH_MAX_AGE_SECONDS = 120; + public static function executeOrScheduleJob(IBackgroundJob $job): void { - if (AntragsgruenApp::getInstance()->backgroundJobs) { + if (isset(AntragsgruenApp::getInstance()->backgroundJobs['notifications']) && AntragsgruenApp::getInstance()->backgroundJobs['notifications']) { \Yii::$app->getDb()->createCommand( 'INSERT INTO `backgroundJob` (`siteId`, `consultationId`, `type`, `dateCreation`, `payload`) VALUES (:siteId, :consultationId, :type, NOW(), :payload)', [ @@ -25,4 +27,32 @@ public static function executeOrScheduleJob(IBackgroundJob $job): void $job->execute(); } } + + /** + * @return array{healthy: bool, data: array} + */ + public static function getDiagnostics(): array + { + $command = \Yii::$app->getDb()->createCommand('SELECT MIN(dateCreation) minAge, COUNT(*) num FROM backgroundJob WHERE dateStarted IS NULL'); + $result = $command->queryAll()[0]; + $unstarted = [ + 'num' => intval($result['num']), + 'age' => time() - Tools::dateSql2timestamp($result['minAge']), + ]; + + $command = \Yii::$app->getDb()->createCommand('SELECT MIN(dateCreation) minAge, COUNT(*) num FROM backgroundJob WHERE dateFinished IS NULL'); + $result = $command->queryAll()[0]; + $unfinished = [ + 'num' => intval($result['num']), + 'age' => time() - Tools::dateSql2timestamp($result['minAge']), + ]; + + return [ + 'healthy' => ($unstarted['age'] <= self::HEALTH_MAX_AGE_SECONDS && $unfinished['age'] <= self::HEALTH_MAX_AGE_SECONDS), + 'data' => [ + 'unstarted' => $unstarted, + 'unfinished' => $unfinished, + ], + ]; + } } diff --git a/config/urls.php b/config/urls.php index 5e8ff5712..c433c8bb8 100644 --- a/config/urls.php +++ b/config/urls.php @@ -81,9 +81,10 @@ $dom . 'page/' => 'pages/show-page', $dom . 'page//save' => 'pages/save-page', $dom . 'page//delete' => 'pages/delete-page', - $dom . 'admin/<_a:(siteconfig)>' => 'manager/<_a>', + $dom . 'admin/<_a:(siteconfig|health)>' => 'manager/<_a>', $restBase => 'consultation/rest-site', + $restBase . '/health' => '/manager/health', $restBaseCon => 'consultation/rest', $restBaseCon . '/proposed-procedure' => 'consultation/proposed-procedure-rest', $restBaseCon . '/motion/' => '/motion/rest', diff --git a/controllers/ManagerController.php b/controllers/ManagerController.php index 7496d34f4..25fa57f2b 100644 --- a/controllers/ManagerController.php +++ b/controllers/ManagerController.php @@ -1,25 +1,31 @@ id, ['siteconfig'])) { + if (in_array($action->id, [self::VIEW_ID_SITECONFIG, self::VIEW_ID_HEALTH])) { // No cookieValidationKey is set in the beginning RequestContext::getWebApplication()->request->enableCookieValidation = false; return parent::beforeAction($action); } - if (!$this->getParams()->multisiteMode && !in_array($action->id, ['siteconfig'])) { + if (!$this->getParams()->multisiteMode && !in_array($action->id, [self::VIEW_ID_SITECONFIG, self::VIEW_ID_HEALTH])) { return false; } @@ -118,4 +124,27 @@ public function actionSiteconfig(): ResponseInterface 'makeEditabeCommand' => $makeEditabeCommand, ])); } + + public function actionHealth(): RestApiResponse + { + $pwdHash = AntragsgruenApp::getInstance()->healthCheckKey; + if ($pwdHash === null) { + return new RestApiResponse(404, ['success' => false, 'error' => 'Health checks not activated']); + } + if ($this->getHttpHeader('X-API-Key') === null || !password_verify($this->getHttpHeader('X-API-Key'), $pwdHash)) { + return new RestApiResponse(401, ['success' => false, 'error' => 'No or invalid X-API-Key given']); + } + + $backgroundJobs = BackgroundJobScheduler::getDiagnostics(); + $healthy = $backgroundJobs['healthy']; + + return new RestApiResponse( + ($healthy ? 200 : 500), + [ + 'success' => true, + 'healthy' => $healthy, + 'backgroundJobs' => $backgroundJobs['data'], + ] + ); + } } diff --git a/models/backgroundJobs/SendNotification.php b/models/backgroundJobs/SendNotification.php index ea0e83caa..409046d15 100644 --- a/models/backgroundJobs/SendNotification.php +++ b/models/backgroundJobs/SendNotification.php @@ -6,8 +6,7 @@ use app\components\mail\Base; use app\components\RequestContext; -use app\models\db\Consultation; -use app\models\db\EMailLog; +use app\models\db\{Consultation, EMailLog}; use app\models\exceptions\MailNotSent; use app\models\settings\AntragsgruenApp; use Symfony\Component\Mailer\Exception\TransportExceptionInterface; diff --git a/models/settings/AntragsgruenApp.php b/models/settings/AntragsgruenApp.php index b92398e05..c046f5550 100644 --- a/models/settings/AntragsgruenApp.php +++ b/models/settings/AntragsgruenApp.php @@ -34,7 +34,6 @@ class AntragsgruenApp implements \JsonSerializable public bool $confirmEmailAddresses = true; public bool $enforceTwoFactorAuthentication = false; public bool $dataPrivacyCheckbox = false; - public bool $backgroundJobs = false; public string $mailFromName = 'Antragsgrün'; public string $mailFromEmail = ''; public ?string $mailDefaultReplyTo = null; @@ -59,6 +58,7 @@ class AntragsgruenApp implements \JsonSerializable public string $mode = 'production'; // [production | sandbox] public ?string $updateKey = null; public ?string $jwtPrivateKey = null; + public ?string $healthCheckKey = null; // A hash generated with password_hash(..., PASSWORD_DEFAULT) /** @var array{mode: string, ignoredIps: string[], difficulty: string} */ public array $captcha = [ @@ -78,6 +78,9 @@ class AntragsgruenApp implements \JsonSerializable /** @var array{installationId: string, wsUri: string, stompJsUri: string, rabbitMqUri: string, rabbitMqExchangeName: string, rabbitMqUsername: string, rabbitMqPassword: string}|null */ public ?array $live = null; + /** @var array{notifications?: bool}|null */ + public ?array $backgroundJobs = null; + public static function getInstance(): AntragsgruenApp { /** @var AntragsgruenApp $app */ From 607e8a2a84ba8553fd7c5d8739be67fc06707b7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ho=CC=88=C3=9Fl?= Date: Sun, 22 Dec 2024 15:51:40 +0100 Subject: [PATCH 7/9] Healthcheck bugfixes --- components/BackgroundJobScheduler.php | 13 ++++++++++--- controllers/ManagerController.php | 17 +++++++---------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/components/BackgroundJobScheduler.php b/components/BackgroundJobScheduler.php index 7a39acfb9..b074fab0d 100644 --- a/components/BackgroundJobScheduler.php +++ b/components/BackgroundJobScheduler.php @@ -29,22 +29,29 @@ public static function executeOrScheduleJob(IBackgroundJob $job): void } /** - * @return array{healthy: bool, data: array} + * @return array{healthy: bool|null, data: array} */ public static function getDiagnostics(): array { + if (!isset(AntragsgruenApp::getInstance()->backgroundJobs['notifications']) || !AntragsgruenApp::getInstance()->backgroundJobs['notifications']) { + return [ + 'healthy' => null, + 'data' => [], + ]; + } + $command = \Yii::$app->getDb()->createCommand('SELECT MIN(dateCreation) minAge, COUNT(*) num FROM backgroundJob WHERE dateStarted IS NULL'); $result = $command->queryAll()[0]; $unstarted = [ 'num' => intval($result['num']), - 'age' => time() - Tools::dateSql2timestamp($result['minAge']), + 'age' => ($result['minAge'] ? (time() - Tools::dateSql2timestamp($result['minAge'])) : 0), ]; $command = \Yii::$app->getDb()->createCommand('SELECT MIN(dateCreation) minAge, COUNT(*) num FROM backgroundJob WHERE dateFinished IS NULL'); $result = $command->queryAll()[0]; $unfinished = [ 'num' => intval($result['num']), - 'age' => time() - Tools::dateSql2timestamp($result['minAge']), + 'age' => ($result['minAge'] ? (time() - Tools::dateSql2timestamp($result['minAge'])) : 0), ]; return [ diff --git a/controllers/ManagerController.php b/controllers/ManagerController.php index 25fa57f2b..7f833f956 100644 --- a/controllers/ManagerController.php +++ b/controllers/ManagerController.php @@ -136,15 +136,12 @@ public function actionHealth(): RestApiResponse } $backgroundJobs = BackgroundJobScheduler::getDiagnostics(); - $healthy = $backgroundJobs['healthy']; - - return new RestApiResponse( - ($healthy ? 200 : 500), - [ - 'success' => true, - 'healthy' => $healthy, - 'backgroundJobs' => $backgroundJobs['data'], - ] - ); + $healthy = $backgroundJobs['healthy'] !== false; + + return new RestApiResponse(200, [ + 'success' => true, + 'healthy' => $healthy, + 'backgroundJobs' => $backgroundJobs['data'], + ]); } } From 1d6a508d2d7cf08dac3e145955dc59ea0e90318b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ho=CC=88=C3=9Fl?= Date: Sun, 22 Dec 2024 19:01:55 +0100 Subject: [PATCH 8/9] Clean up background jobs --- History.md | 4 ++++ README.md | 3 ++- commands/BackgroundJobController.php | 14 ++++++++++++++ components/BackgroundJobScheduler.php | 10 ++++++++++ plugins/antragsgruen_sites/Module.php | 1 + 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/History.md b/History.md index 7b1af8751..62764d4c7 100644 --- a/History.md +++ b/History.md @@ -1,5 +1,9 @@ # Version history +## Version 4.16.0 [not released yet] + +- An optional mechanism for background job execution is introduced, making it possible to send e-mails asynchronously (therefore not blocking regular requests). + ### Version 4.15.1 [not released yet] - Next to the status-dropdown for motions and amendments, there is now a link to a reference page, explaining the uses of these different statuses. diff --git a/README.md b/README.md index 6ac497db5..b3a444300 100644 --- a/README.md +++ b/README.md @@ -299,7 +299,8 @@ Some processes that are potentially blocking or long-running can be executed as The following example on how to run the background job processor uses [Supervisord](http://supervisord.org), but it is just as possible running it via any other process manager. - Copy [supervisor.conf](docs/supervisor.conf) to your supervisord configuration directory, modify it to your needs, and run it. - Create an API key for the health checks (optional) and its hash (via `password_encode($password, PASSWORD_DEFAULT)`). -- Enable background jobs by adding the following settings to your `config.json`: +- Enable background jobs by adding the following settings to your `config.json`. +- Set up a cronjob to clean the database at least once a day by executing `yii background-job/cleanup`. ```json { diff --git a/commands/BackgroundJobController.php b/commands/BackgroundJobController.php index e50236f29..8bbd86289 100644 --- a/commands/BackgroundJobController.php +++ b/commands/BackgroundJobController.php @@ -5,6 +5,8 @@ namespace app\commands; use app\components\BackgroundJobProcessor; +use app\components\BackgroundJobScheduler; +use Yii; use yii\console\Controller; /** @@ -16,6 +18,8 @@ class BackgroundJobController extends Controller private const DEFAULT_MAX_RUNTIME_SECONDS = 600; private const DEFAULT_MAX_MEMORY_USAGE = 64_000_000; + private const MAX_RETENTION_PERIOD_HOURS = 24 * 3; + protected int $maxEvents = self::DEFAULT_MAX_EVENTS; protected int $maxRuntimeSeconds = self::DEFAULT_MAX_RUNTIME_SECONDS; protected int $maxMemoryUsage = self::DEFAULT_MAX_MEMORY_USAGE; @@ -71,4 +75,14 @@ private function needsRestart(BackgroundJobProcessor $processor): bool return false; } + + /** + * Cleans up old tasks from database + */ + public function actionCleanup(): void + { + $deletedJobs = BackgroundJobScheduler::cleanup(self::MAX_RETENTION_PERIOD_HOURS); + + echo "Deleted $deletedJobs jobs.\n"; + } } diff --git a/components/BackgroundJobScheduler.php b/components/BackgroundJobScheduler.php index b074fab0d..504e15379 100644 --- a/components/BackgroundJobScheduler.php +++ b/components/BackgroundJobScheduler.php @@ -62,4 +62,14 @@ public static function getDiagnostics(): array ], ]; } + + public static function cleanup(int $maxHageHours): int + { + $command = \Yii::$app->getDb()->createCommand( + 'DELETE FROM backgroundJob WHERE dateFinished < NOW() - INTERVAL :hours HOUR', + [':hours' => $maxHageHours] + ); + + return $command->execute(); + } } diff --git a/plugins/antragsgruen_sites/Module.php b/plugins/antragsgruen_sites/Module.php index 2d66e8184..dc113a213 100644 --- a/plugins/antragsgruen_sites/Module.php +++ b/plugins/antragsgruen_sites/Module.php @@ -15,6 +15,7 @@ public static function getManagerUrlRoutes(string $domainPlain): array return [ $domainPlain => 'antragsgruen_sites/manager/index', $domainPlain . '/help/<_a:(' . $helpPaths . ')>' => 'antragsgruen_sites/manager/<_a>', + $domainPlain . '/rest/health' => '/manager/health', $domainPlain . '/<_a:(' . $domPlainPaths . ')>' => 'antragsgruen_sites/manager/<_a>', ]; } From 68346673f93858224b283a84beb691ed870afaa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ho=CC=88=C3=9Fl?= Date: Wed, 25 Dec 2024 10:09:43 +0100 Subject: [PATCH 9/9] Set site for notifications --- models/backgroundJobs/SendNotification.php | 1 + 1 file changed, 1 insertion(+) diff --git a/models/backgroundJobs/SendNotification.php b/models/backgroundJobs/SendNotification.php index 409046d15..88062f282 100644 --- a/models/backgroundJobs/SendNotification.php +++ b/models/backgroundJobs/SendNotification.php @@ -29,6 +29,7 @@ public function __construct( public ?string $replyTo ) { $this->consultation = $consultation; + $this->site = $consultation?->site; } public function getTypeId(): string