From c51f6e0ba8b8e976509e7e1eecab909368c63f4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ho=CC=88=C3=9Fl?= Date: Sun, 1 Dec 2024 12:58:47 +0100 Subject: [PATCH] Data stucture for background jobs --- assets/db/create.sql | 40 ++++++++++++++ assets/db/data.sql | 3 +- assets/db/delete.sql | 1 + commands/BackgroundJobController.php | 45 ++++++++++++++++ components/BackgroundJobProcessor.php | 52 +++++++++++++++++++ migrations/m241201_100317_background_jobs.php | 37 +++++++++++++ 6 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 commands/BackgroundJobController.php create mode 100644 components/BackgroundJobProcessor.php create mode 100644 migrations/m241201_100317_background_jobs.php diff --git a/assets/db/create.sql b/assets/db/create.sql index 2d0b246e9..b145bfab8 100644 --- a/assets/db/create.sql +++ b/assets/db/create.sql @@ -815,6 +815,22 @@ CREATE TABLE `###TABLE_PREFIX###votingQuestion` ( `votingData` text DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +-- +-- Table structure for table `backgroundJob` +-- + +CREATE TABLE `###TABLE_PREFIX###backgroundJob` ( + `id` bigint UNSIGNED NOT NULL, + `siteId` int DEFAULT NULL, + `consultationId` int DEFAULT NULL, + `type` varchar(150) NOT NULL, + `dateCreation` timestamp NOT NULL, + `dateStarted` timestamp NULL DEFAULT NULL, + `dateUpdated` timestamp NULL DEFAULT NULL, + `dateFinished` timestamp NULL DEFAULT NULL, + `payload` mediumblob NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + -- -- Indexes for dumped tables -- @@ -1155,6 +1171,16 @@ ALTER TABLE `###TABLE_PREFIX###votingQuestion` ADD KEY `fk_question_block` (`votingBlockId`), ADD KEY `fk_question_consultation` (`consultationId`); +-- +-- Indexes for table `backgroundJob` +-- +ALTER TABLE `###TABLE_PREFIX###backgroundJob` + ADD PRIMARY KEY (`id`), + ADD KEY `fk_background_site` (`siteId`), + ADD KEY `fk_background_consultation` (`consultationId`), + ADD KEY `ix_background_pending` (`dateStarted`,`id`), + ADD KEY `ix_background_todelete` (`dateFinished`); + -- -- AUTO_INCREMENT for dumped tables -- @@ -1320,6 +1346,13 @@ ALTER TABLE `###TABLE_PREFIX###votingBlock` -- ALTER TABLE `###TABLE_PREFIX###votingQuestion` MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; + +-- +-- AUTO_INCREMENT for table `backgroundJob` +-- +ALTER TABLE `###TABLE_PREFIX###backgroundJob` + MODIFY `id` bigint UNSIGNED NOT NULL AUTO_INCREMENT; + -- -- Constraints for dumped tables -- @@ -1629,6 +1662,13 @@ ALTER TABLE `###TABLE_PREFIX###votingQuestion` ADD CONSTRAINT `fk_question_block` FOREIGN KEY (`votingBlockId`) REFERENCES `###TABLE_PREFIX###votingBlock` (`id`), ADD CONSTRAINT `fk_question_consultation` FOREIGN KEY (`consultationId`) REFERENCES `###TABLE_PREFIX###consultation` (`id`); +-- +-- Constraints for table `backgroundJob` +-- +ALTER TABLE `###TABLE_PREFIX###backgroundJob` + ADD CONSTRAINT `fk_background_consultation` FOREIGN KEY (`consultationId`) REFERENCES `###TABLE_PREFIX###consultation` (`id`), + ADD CONSTRAINT `fk_background_site` FOREIGN KEY (`siteId`) REFERENCES `###TABLE_PREFIX###site` (`id`); + SET SQL_MODE = @OLD_SQL_MODE; SET FOREIGN_KEY_CHECKS = @OLD_FOREIGN_KEY_CHECKS; SET UNIQUE_CHECKS = @OLD_UNIQUE_CHECKS; diff --git a/assets/db/data.sql b/assets/db/data.sql index 87de93c71..d29bfa84b 100644 --- a/assets/db/data.sql +++ b/assets/db/data.sql @@ -89,7 +89,8 @@ INSERT INTO `migration` (`version`, `apply_time`) VALUES ('m240427_090527_motion_status_index', '1714209051'), ('m240830_181716_user_secret_key', '1725041937'), ('m241013_105549_pages_files', '1728817360'), - ('m241027_074032_pages_policies', '1730015023') + ('m241027_074032_pages_policies', '1730015023'), + ('m241201_100317_background_jobs', '1733052690') ; SET SQL_MODE = @OLD_SQL_MODE; diff --git a/assets/db/delete.sql b/assets/db/delete.sql index f5dea068e..23eadaaa5 100644 --- a/assets/db/delete.sql +++ b/assets/db/delete.sql @@ -47,6 +47,7 @@ DROP TABLE IF EXISTS `###TABLE_PREFIX###consultationMotionType`; DROP TABLE IF EXISTS `###TABLE_PREFIX###consultationSettingsTag`; DROP TABLE IF EXISTS `###TABLE_PREFIX###consultationLog`; DROP TABLE IF EXISTS `###TABLE_PREFIX###texTemplate`; +DROP TABLE IF EXISTS `###TABLE_PREFIX###backgroundJob`; SET SQL_MODE=@OLD_SQL_MODE; SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS; diff --git a/commands/BackgroundJobController.php b/commands/BackgroundJobController.php new file mode 100644 index 000000000..003b72028 --- /dev/null +++ b/commands/BackgroundJobController.php @@ -0,0 +1,45 @@ +getDb()); + while (!$this->needsRestart($processor)) { + $row = $processor->getJobAndSetStarted(); + if ($row) { + $processor->processRow($row); + } else { + usleep(100000); + } + + file_put_contents('/tmp/memory_usage.log', memory_get_peak_usage() . "\n", FILE_APPEND); + } + } + + private function needsRestart(BackgroundJobProcessor $processor): bool + { + return $processor->getProcessedEvents() >= self::MAX_EVENTS + || $processor->getRuntimeInSeconds() >= self::MAX_RUNTIME_SECONDS + || memory_get_peak_usage() >= self::MAX_MEMORY_USAGE; + } +} diff --git a/components/BackgroundJobProcessor.php b/components/BackgroundJobProcessor.php new file mode 100644 index 000000000..aeb98bb76 --- /dev/null +++ b/components/BackgroundJobProcessor.php @@ -0,0 +1,52 @@ +connection = $connection; + $this->startedAt = new \DateTimeImmutable(); + } + + public function getJobAndSetStarted(): ?array { + $foundRow = null; + + $this->connection->transaction(function () use (&$foundRow) { + $command = $this->connection->createCommand('SELECT * FROM backgroundJob WHERE dateStarted IS NULL ORDER BY id ASC LIMIT 0,1 FOR UPDATE'); + $foundRows = $command->queryAll(); + if (empty($foundRows)) { + return; + } + + $foundRow = $foundRows[0]; + $this->connection->createCommand('UPDATE backgroundJob SET dateStarted = NOW() WHERE id = :id', ['id' => $foundRow['id']])->execute(); + }); + + return $foundRow; + } + + public function processRow(array $row): void + { + echo "Processing row: " . $row['id'] . "\n"; + sleep(2); + $this->connection->createCommand('UPDATE backgroundJob SET dateFinished = NOW() WHERE id = :id', ['id' => $row['id']])->execute(); + } + + public function getProcessedEvents(): int { + return $this->processedEvents; + } + + public function getRuntimeInSeconds(): int { + return (new \DateTimeImmutable())->getTimestamp() - $this->startedAt->getTimestamp(); + } +} diff --git a/migrations/m241201_100317_background_jobs.php b/migrations/m241201_100317_background_jobs.php new file mode 100644 index 000000000..214827ff7 --- /dev/null +++ b/migrations/m241201_100317_background_jobs.php @@ -0,0 +1,37 @@ +createTable('backgroundJob', [ + 'id' => 'BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY', + 'siteId' => 'INTEGER DEFAULT NULL', + 'consultationId' => 'INTEGER DEFAULT NULL', + 'type' => 'VARCHAR(150) NOT NULL', + 'dateCreation' => 'TIMESTAMP NOT NULL', + 'dateStarted' => 'TIMESTAMP DEFAULT NULL', + 'dateUpdated' => 'TIMESTAMP DEFAULT NULL', + 'dateFinished' => 'TIMESTAMP DEFAULT NULL', + 'payload' => 'MEDIUMBLOB NOT NULL', + ]); + + $this->addForeignKey('fk_background_site', 'backgroundJob', 'siteId', 'site', 'id'); + $this->addForeignKey('fk_background_consultation', 'backgroundJob', 'consultationId', 'consultation', 'id'); + $this->createIndex('ix_background_pending', 'backgroundJob', ['dateStarted', 'id'], false); + $this->createIndex('ix_background_todelete', 'backgroundJob', 'dateFinished', false); + } + + /** + * {@inheritdoc} + */ + public function safeDown(): void + { + $this->dropTable('backgroundJob'); + } +}