Data stucture for background jobs

assets/db/create.sql

@@ -815,6 +815,22 @@ CREATE TABLE `###TABLE_PREFIX###votingQuestion` (
   `votingData` text DEFAULT NULL
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
+--
+-- Table structure for table `backgroundJob`
+--
+
+CREATE TABLE `###TABLE_PREFIX###backgroundJob` (
+  `id` bigint UNSIGNED NOT NULL,
+  `siteId` int DEFAULT NULL,
+  `consultationId` int DEFAULT NULL,
+  `type` varchar(150) NOT NULL,
+  `dateCreation` timestamp NOT NULL,
+  `dateStarted` timestamp NULL DEFAULT NULL,
+  `dateUpdated` timestamp NULL DEFAULT NULL,
+  `dateFinished` timestamp NULL DEFAULT NULL,
+  `payload` mediumblob NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
 --
 -- Indexes for dumped tables
 --
@@ -1155,6 +1171,16 @@ ALTER TABLE `###TABLE_PREFIX###votingQuestion`
   ADD KEY `fk_question_block` (`votingBlockId`),
   ADD KEY `fk_question_consultation` (`consultationId`);
 
+--
+-- Indexes for table `backgroundJob`
+--
+ALTER TABLE `###TABLE_PREFIX###backgroundJob`
+  ADD PRIMARY KEY (`id`),
+  ADD KEY `fk_background_site` (`siteId`),
+  ADD KEY `fk_background_consultation` (`consultationId`),
+  ADD KEY `ix_background_pending` (`dateStarted`,`id`),
+  ADD KEY `ix_background_todelete` (`dateFinished`);
+
 --
 -- AUTO_INCREMENT for dumped tables
 --
@@ -1320,6 +1346,13 @@ ALTER TABLE `###TABLE_PREFIX###votingBlock`
 --
 ALTER TABLE `###TABLE_PREFIX###votingQuestion`
   MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+
+--
+-- AUTO_INCREMENT for table `backgroundJob`
+--
+ALTER TABLE `###TABLE_PREFIX###backgroundJob`
+  MODIFY `id` bigint UNSIGNED NOT NULL AUTO_INCREMENT;
+
 --
 -- Constraints for dumped tables
 --
@@ -1629,6 +1662,13 @@ ALTER TABLE `###TABLE_PREFIX###votingQuestion`
   ADD CONSTRAINT `fk_question_block` FOREIGN KEY (`votingBlockId`) REFERENCES `###TABLE_PREFIX###votingBlock` (`id`),
   ADD CONSTRAINT `fk_question_consultation` FOREIGN KEY (`consultationId`) REFERENCES `###TABLE_PREFIX###consultation` (`id`);
 
+--
+-- Constraints for table `backgroundJob`
+--
+ALTER TABLE `###TABLE_PREFIX###backgroundJob`
+    ADD CONSTRAINT `fk_background_consultation` FOREIGN KEY (`consultationId`) REFERENCES `###TABLE_PREFIX###consultation` (`id`),
+    ADD CONSTRAINT `fk_background_site` FOREIGN KEY (`siteId`) REFERENCES `###TABLE_PREFIX###site` (`id`);
+
 SET SQL_MODE = @OLD_SQL_MODE;
 SET FOREIGN_KEY_CHECKS = @OLD_FOREIGN_KEY_CHECKS;
 SET UNIQUE_CHECKS = @OLD_UNIQUE_CHECKS;

assets/db/data.sql

@@ -89,7 +89,8 @@ INSERT INTO `migration` (`version`, `apply_time`) VALUES
   ('m240427_090527_motion_status_index', '1714209051'),
   ('m240830_181716_user_secret_key', '1725041937'),
   ('m241013_105549_pages_files', '1728817360'),
-  ('m241027_074032_pages_policies', '1730015023')
+  ('m241027_074032_pages_policies', '1730015023'),
+  ('m241201_100317_background_jobs', '1733052690')
 ;
 
 SET SQL_MODE = @OLD_SQL_MODE;

assets/db/delete.sql

@@ -47,6 +47,7 @@ DROP TABLE IF EXISTS `###TABLE_PREFIX###consultationMotionType`;
 DROP TABLE IF EXISTS `###TABLE_PREFIX###consultationSettingsTag`;
 DROP TABLE IF EXISTS `###TABLE_PREFIX###consultationLog`;
 DROP TABLE IF EXISTS `###TABLE_PREFIX###texTemplate`;
+DROP TABLE IF EXISTS `###TABLE_PREFIX###backgroundJob`;
 
 SET SQL_MODE=@OLD_SQL_MODE;
 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;

commands/BackgroundJobController.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\commands;
+
+use app\components\BackgroundJobProcessor;
+use yii\console\Controller;
+
+/**
+ * Run background commends
+ */
+class BackgroundJobController extends Controller
+{
+    private const MAX_EVENTS = 1000;
+    private const MAX_RUNTIME_SECONDS = 600;
+    private const MAX_MEMORY_USAGE = 64_000_000;
+
+    /**
+     * Runs the background job processor
+     *
+     * @throws \yii\db\Exception
+     */
+    public function actionRun(): void
+    {
+        $processor = new BackgroundJobProcessor(\Yii::$app->getDb());
+        while (!$this->needsRestart($processor)) {
+            $row = $processor->getJobAndSetStarted();
+            if ($row) {
+                $processor->processRow($row);
+            } else {
+                usleep(100000);
+            }
+
+            file_put_contents('/tmp/memory_usage.log', memory_get_peak_usage() . "\n", FILE_APPEND);
+        }
+    }
+
+    private function needsRestart(BackgroundJobProcessor $processor): bool
+    {
+        return $processor->getProcessedEvents() >= self::MAX_EVENTS
+            || $processor->getRuntimeInSeconds() >= self::MAX_RUNTIME_SECONDS
+            || memory_get_peak_usage() >= self::MAX_MEMORY_USAGE;
+    }
+}

components/BackgroundJobProcessor.php

@@ -0,0 +1,52 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\components;
+
+use yii\db\Connection;
+
+class BackgroundJobProcessor
+{
+    private Connection $connection;
+
+    private int $processedEvents = 0;
+    private \DateTimeImmutable $startedAt;
+
+    public function __construct(Connection $connection) {
+        $this->connection = $connection;
+        $this->startedAt = new \DateTimeImmutable();
+    }
+
+    public function getJobAndSetStarted(): ?array {
+        $foundRow = null;
+
+        $this->connection->transaction(function () use (&$foundRow) {
+            $command = $this->connection->createCommand('SELECT * FROM backgroundJob WHERE dateStarted IS NULL ORDER BY id ASC LIMIT 0,1 FOR UPDATE');
+            $foundRows = $command->queryAll();
+            if (empty($foundRows)) {
+                return;
+            }
+
+            $foundRow = $foundRows[0];
+            $this->connection->createCommand('UPDATE backgroundJob SET dateStarted = NOW() WHERE id = :id', ['id' => $foundRow['id']])->execute();
+        });
+
+        return $foundRow;
+    }
+
+    public function processRow(array $row): void
+    {
+        echo "Processing row: " . $row['id'] . "\n";
+        sleep(2);
+        $this->connection->createCommand('UPDATE backgroundJob SET dateFinished = NOW() WHERE id = :id', ['id' => $row['id']])->execute();
+    }
+
+    public function getProcessedEvents(): int {
+        return $this->processedEvents;
+    }
+
+    public function getRuntimeInSeconds(): int {
+        return (new \DateTimeImmutable())->getTimestamp() - $this->startedAt->getTimestamp();
+    }
+}

migrations/m241201_100317_background_jobs.php

@@ -0,0 +1,37 @@
+<?php
+
+use yii\db\Migration;
+
+class m241201_100317_background_jobs extends Migration
+{
+    /**
+     * {@inheritdoc}
+     */
+    public function safeUp(): void
+    {
+        $this->createTable('backgroundJob', [
+            'id' => 'BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY',
+            'siteId' => 'INTEGER DEFAULT NULL',
+            'consultationId' => 'INTEGER DEFAULT NULL',
+            'type' => 'VARCHAR(150) NOT NULL',
+            'dateCreation' => 'TIMESTAMP NOT NULL',
+            'dateStarted' => 'TIMESTAMP DEFAULT NULL',
+            'dateUpdated' => 'TIMESTAMP DEFAULT NULL',
+            'dateFinished' => 'TIMESTAMP DEFAULT NULL',
+            'payload' => 'MEDIUMBLOB NOT NULL',
+        ]);
+
+        $this->addForeignKey('fk_background_site', 'backgroundJob', 'siteId', 'site', 'id');
+        $this->addForeignKey('fk_background_consultation', 'backgroundJob', 'consultationId', 'consultation', 'id');
+        $this->createIndex('ix_background_pending', 'backgroundJob', ['dateStarted', 'id'], false);
+        $this->createIndex('ix_background_todelete', 'backgroundJob', 'dateFinished', false);
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function safeDown(): void
+    {
+        $this->dropTable('backgroundJob');
+    }
+}