From 7f9999ffb97a922f07d947e01ae369c95b84e497 Mon Sep 17 00:00:00 2001 From: Adam Rusinowski Date: Mon, 9 Dec 2024 17:20:43 +0100 Subject: [PATCH 1/4] added commands to purge queues and test queues, cc --- .gitattributes | 45 ++++ README.md | 34 ++- src/Command/NotifyCommand.php | 12 +- ...{PurgeCommand.php => PurgeLogsCommand.php} | 17 +- src/Command/PurgeQueueCommand.php | 195 ++++++++++++++++++ src/Command/TestQueueCommand.php | 117 +++++++++++ src/Mailer/TestQueueMailer.php | 50 +++++ src/QueueMonitorPlugin.php | 24 +-- src/Service/EnqueueClientService.php | 54 +++++ templates/email/html/test.php | 5 + templates/email/text/test.php | 5 + 11 files changed, 535 insertions(+), 23 deletions(-) create mode 100644 .gitattributes rename src/Command/{PurgeCommand.php => PurgeLogsCommand.php} (89%) create mode 100644 src/Command/PurgeQueueCommand.php create mode 100644 src/Command/TestQueueCommand.php create mode 100644 src/Mailer/TestQueueMailer.php create mode 100644 src/Service/EnqueueClientService.php create mode 100644 templates/email/html/test.php create mode 100644 templates/email/text/test.php diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..aa4dad0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,45 @@ +# Define the line ending behavior of the different file extensions +# Set default behavior, in case users don't have core.autocrlf set. +* text text=auto eol=lf + +.php diff=php + +# Declare files that will always have CRLF line endings on checkout. +*.bat eol=crlf + +# Declare files that will always have LF line endings on checkout. +*.pem eol=lf + +# Denote all files that are truly binary and should not be modified. +*.png binary +*.jpg binary +*.gif binary +*.ico binary +*.mo binary +*.pdf binary +*.phar binary +*.woff binary +*.woff2 binary +*.ttf binary +*.otf binary +*.eot binary + +# Remove files for archives generated using `git archive` +.github export-ignore +.phive export-ignore +contrib export-ignore +tests/test_app export-ignore +tests/TestCase export-ignore + +.editorconfig export-ignore +.gitattributes export-ignore +.gitignore export-ignore +.mailmap export-ignore +.stickler.yml export-ignore +Makefile export-ignore +phpcs.xml export-ignore +phpstan.neon.dist export-ignore +phpstan-baseline.neon export-ignore +phpunit.xml.dist export-ignore +psalm.xml export-ignore +psalm-baseline.xml export-ignore diff --git a/README.md b/README.md index 61078e8..4466f87 100644 --- a/README.md +++ b/README.md @@ -97,17 +97,43 @@ For each queue configuration add `listener` setting To set up notifications when there are long running or possible stuck jobs please use command ```shell -bin/cake queue_monitor notify +bin/cake queue-monitor notify ``` This command will send notification emails to recipients specified in `QueueMonitor.notificationRecipients`. Best is -to use it as a cronjob +to use it as a cronjob. -## Purge command +## Test Enqueue command + +To quickly test if all queues are running correctly please run this command (replace `your-email@domain.com` with working +email address: +```shell +bin/cake queue-monitor test-enqueue your-email@domain.com +``` + +This command will send the command through all configured queues. + +## Purge queues command +To purge the content of a specified queue you can use the purge queue command: +```shell +bin/cake queue-monitor purge-queue your-queue-name + +``` + +The above command will remove all pending queue jobs from the specified queue. + +To purge all queues you can use command: + +```shell +bin/cake queue-monitor purge-queue --all + +``` + +## Purge Logs command The logs table may grow overtime, to keep it slim you can use the purge command: ```shell -bin/cake queue_monitor purge +bin/cake queue-monitor purge-logs ``` This command will purge logs older than value specified in `QueueMonitor.purgeLogsOlderThanDays`, the value is in diff --git a/src/Command/NotifyCommand.php b/src/Command/NotifyCommand.php index aef878d..abe8077 100644 --- a/src/Command/NotifyCommand.php +++ b/src/Command/NotifyCommand.php @@ -48,7 +48,15 @@ public function __construct( */ public static function defaultName(): string { - return 'queue_monitor notify'; + return 'queue-monitor notify'; + } + + /** + * @inheritDoc + */ + public static function getDescription(): string + { + return __('Queue Monitoring notifier'); } /** @@ -57,7 +65,7 @@ public static function defaultName(): string public function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser { return parent::buildOptionParser($parser) - ->setDescription(__('Queue Monitoring notifier')); + ->setDescription(self::getDescription()); } /** diff --git a/src/Command/PurgeCommand.php b/src/Command/PurgeLogsCommand.php similarity index 89% rename from src/Command/PurgeCommand.php rename to src/Command/PurgeLogsCommand.php index a60d1d3..5977681 100644 --- a/src/Command/PurgeCommand.php +++ b/src/Command/PurgeLogsCommand.php @@ -25,9 +25,9 @@ use function Cake\I18n\__; /** - * Purge command. + * Purge Logs command. */ -final class PurgeCommand extends Command +final class PurgeLogsCommand extends Command { use DisableTrait; use LogTrait; @@ -48,7 +48,15 @@ public function __construct( */ public static function defaultName(): string { - return 'queue_monitor purge'; + return 'queue-monitor purge-logs'; + } + + /** + * @inheritDoc + */ + public static function getDescription(): string + { + return __('Queue Monitoring log purger'); } /** @@ -57,7 +65,7 @@ public static function defaultName(): string public function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser { return parent::buildOptionParser($parser) - ->setDescription(__('Queue Monitoring log purger')); + ->setDescription(self::getDescription()); } /** @@ -73,6 +81,7 @@ public function execute(Arguments $args, ConsoleIo $io) return self::CODE_SUCCESS; } + $purgeLogsOlderThanDays = (int)Configure::read( 'QueueMonitor.purgeLogsOlderThanDays', self::DEFAULT_PURGE_DAYS_OLD diff --git a/src/Command/PurgeQueueCommand.php b/src/Command/PurgeQueueCommand.php new file mode 100644 index 0000000..c671d71 --- /dev/null +++ b/src/Command/PurgeQueueCommand.php @@ -0,0 +1,195 @@ +setDescription(self::getDescription()) + ->addArgument('queue-config', [ + 'help' => __('Queue configuration key'), + ]) + ->addOption('all', [ + 'help' => __('All messages will be purged'), + 'short' => 'a', + 'boolean' => true, + 'default' => false, + ]) + ->addOption('yes', [ + 'short' => 'y', + 'boolean' => true, + 'default' => false, + 'help' => __('Yes - skip confirmation prompt'), + ]); + } + + /** + * @inheritDoc + */ + public function execute(Arguments $args, ConsoleIo $io) + { + if ($this->isDisabled()) { + $this->log( + 'Logs were not purged because Queue Monitor is disabled.', + LogLevel::WARNING + ); + + return self::CODE_SUCCESS; + } + + if ($args->getOption('all')) { + $this->checkConfirmation( + __('Are you sure you want to purge messages from all queues?'), + $args, + $io + ); + + collection($this->getConfiguredQueues())->each(function (string $queueConfig) use ($io): void { + try { + $this->enqueueClientService->purgeQueue($queueConfig); + $io->success(__('Queue `{0}` purged successfully', $queueConfig)); + } catch (QueueMonitorException $e) { + $io->error(__('Unable to purge queue `{0}`, reason: {1}', $queueConfig, $e->getMessage())); + } + }); + } else { + $queueConfig = $args->getArgument('queue-config'); + + if (!$this->validateQueueConfig($queueConfig)) { + $io->error(__('Queue configuration key is invalid')); + $configuredQueues = $this->getConfiguredQueues(); + if ($configuredQueues) { + $io->error(__('Valid configuration keys are: {0}', implode(', ', $configuredQueues))); + } else { + $io->error(__('There are no queue configurations')); + } + $this->displayHelp($this->getOptionParser(), $args, $io); + + return self::CODE_ERROR; + } + + $this->checkConfirmation( + __('Are you sure you want to purge messages from specified queue?'), + $args, + $io + ); + + try { + $this->enqueueClientService->purgeQueue($queueConfig); + $io->success(__('Queue `{0}` purged successfully', $queueConfig)); + + return self::CODE_SUCCESS; + } catch (QueueMonitorException $e) { + $io->error(__('Unable to purge queue `{0}`, reason: {1}', $queueConfig, $e->getMessage())); + + return self::CODE_ERROR; + } + } + } + + /** + * Validate queue config + */ + private function validateQueueConfig(?string $queueConfig): bool + { + if (empty($queueConfig)) { + return false; + } + + $validQueueConfigs = $this->getConfiguredQueues(); + + if (!in_array($queueConfig, $validQueueConfigs, true)) { + return false; + } + + return true; + } + + /** + * Get configured queues + */ + private function getConfiguredQueues(): array + { + return array_keys(Configure::read('Queue', [])); + } + + /** + * Check confirmation + */ + private function checkConfirmation(string $prompt, Arguments $args, ConsoleIo $io): void + { + if (!$args->getOption('yes')) { + $confirmation = $io->askChoice( + $prompt, + [ + __('yes'), + __('no'), + ], + __('no') + ); + + if ($confirmation === __('no')) { + $io->abort(__('Aborting')); + } + } + } +} diff --git a/src/Command/TestQueueCommand.php b/src/Command/TestQueueCommand.php new file mode 100644 index 0000000..ebc855c --- /dev/null +++ b/src/Command/TestQueueCommand.php @@ -0,0 +1,117 @@ +setDescription(self::getDescription()) + ->addArgument($this::ARGUMENT_EMAIL, [ + 'help' => __('Email to send to'), + 'required' => true, + ]); + } + + /** + * @inheritDoc + */ + public function execute(Arguments $args, ConsoleIo $io) + { + if ($this->isDisabled()) { + $this->log( + 'Test Enqueue was not performed because Queue Monitor is disabled.', + LogLevel::WARNING + ); + + return self::CODE_SUCCESS; + } + + $email = $args->getArgument(self::ARGUMENT_EMAIL); + if (!Validation::email($email)) { + $io->error(__('Invalid email')); + + return $this::CODE_ERROR; + } + + collection(Configure::read('Queue', [])) + ->each(function ( + array $queueConfig, + string $queueConfigKey + ) use ( + $email, + $io + ): void { + /** @var \CakeDC\QueueMonitor\Mailer\TestQueueMailer $mailer */ + $mailer = $this->getMailer('CakeDC/QueueMonitor.TestQueue'); + /** @uses \CakeDC\QueueMonitor\Mailer\TestQueueMailer::testQueue() */ + $mailer->push( + action: $mailer::SEND_TEST_QUEUE, + args: [ + $email, + $queueConfigKey, + ], + options: [ + 'config' => $queueConfigKey, + ] + ); + $io->info(__( + 'Queued test email `{0}` in queue `{1}`', + $email, + $queueConfigKey + )); + }); + + return $this::CODE_SUCCESS; + } +} diff --git a/src/Mailer/TestQueueMailer.php b/src/Mailer/TestQueueMailer.php new file mode 100644 index 0000000..4f4c6ae --- /dev/null +++ b/src/Mailer/TestQueueMailer.php @@ -0,0 +1,50 @@ +setProfile(Configure::read('QueueMonitor.mailerConfig', 'default')) + ->setTo($emailAddress) + ->setSubject(__('Test enqueue from queue `{0}`', $queueConfig)) + ->setEmailFormat(Message::MESSAGE_BOTH) + ->viewBuilder() + ->disableAutoLayout() + ->setTemplate('QueueMonitor.test_enqueue'); + } +} diff --git a/src/QueueMonitorPlugin.php b/src/QueueMonitorPlugin.php index 3a98a02..9a0bd7b 100644 --- a/src/QueueMonitorPlugin.php +++ b/src/QueueMonitorPlugin.php @@ -12,11 +12,12 @@ */ namespace CakeDC\QueueMonitor; -use Cake\Console\CommandCollection; use Cake\Core\BasePlugin; use Cake\Core\ContainerInterface; use CakeDC\QueueMonitor\Command\NotifyCommand; -use CakeDC\QueueMonitor\Command\PurgeCommand; +use CakeDC\QueueMonitor\Command\PurgeLogsCommand; +use CakeDC\QueueMonitor\Command\PurgeQueueCommand; +use CakeDC\QueueMonitor\Service\EnqueueClientService; use CakeDC\QueueMonitor\Service\QueueMonitoringService; /** @@ -39,27 +40,24 @@ class QueueMonitorPlugin extends BasePlugin */ protected bool $middlewareEnabled = false; - /** - * @inheritDoc - */ - public function console(CommandCollection $commands): CommandCollection - { - return parent::console($commands) - ->add('queue_monitor purge', PurgeCommand::class) - ->add('queue_monitor notify', NotifyCommand::class); - } - /** * @inheritDoc */ public function services(ContainerInterface $container): void { $container->add(QueueMonitoringService::class); + $container->addShared(EnqueueClientService::class); + $container - ->add(PurgeCommand::class) + ->add(PurgeLogsCommand::class) ->addArguments([ QueueMonitoringService::class, ]); + $container + ->add(PurgeQueueCommand::class) + ->addArguments([ + EnqueueClientService::class, + ]); $container ->add(NotifyCommand::class) ->addArguments([ diff --git a/src/Service/EnqueueClientService.php b/src/Service/EnqueueClientService.php new file mode 100644 index 0000000..0c7eb01 --- /dev/null +++ b/src/Service/EnqueueClientService.php @@ -0,0 +1,54 @@ +getDriver()->getContext(); + $queueName = $this->getEnqueueInternalQueueName($simpleClient->getDriver()->getConfig()); + $queue = $context->createQueue($queueName); + $context->purgeQueue($queue); + } catch (PurgeQueueNotSupportedException $e) { + throw new QueueMonitorException($e->getMessage(), $e->getCode(), $e); + } + } + + /** + * Get enqueue internal queue name + */ + private function getEnqueueInternalQueueName(Config $enqueueClientConfig): string + { + return implode('.', [ + $enqueueClientConfig->getPrefix(), + $enqueueClientConfig->getApp(), + $enqueueClientConfig->getDefaultQueue(), + ]); + } +} diff --git a/templates/email/html/test.php b/templates/email/html/test.php new file mode 100644 index 0000000..515332a --- /dev/null +++ b/templates/email/html/test.php @@ -0,0 +1,5 @@ + + +Test email diff --git a/templates/email/text/test.php b/templates/email/text/test.php new file mode 100644 index 0000000..515332a --- /dev/null +++ b/templates/email/text/test.php @@ -0,0 +1,5 @@ + + +Test email From c466f9801b5c8ddbe1de6659e181c6c4a50054ed Mon Sep 17 00:00:00 2001 From: Adam Rusinowski Date: Mon, 9 Dec 2024 17:58:39 +0100 Subject: [PATCH 2/4] fixed test queue template path --- src/Mailer/TestQueueMailer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Mailer/TestQueueMailer.php b/src/Mailer/TestQueueMailer.php index 4f4c6ae..447324a 100644 --- a/src/Mailer/TestQueueMailer.php +++ b/src/Mailer/TestQueueMailer.php @@ -45,6 +45,6 @@ public function testQueue(string $emailAddress, ?string $queueConfig = 'default' ->setEmailFormat(Message::MESSAGE_BOTH) ->viewBuilder() ->disableAutoLayout() - ->setTemplate('QueueMonitor.test_enqueue'); + ->setTemplate('CakeDC/QueueMonitor.test'); } } From d57f67dfade08b3dc3a12bec6491ba87aea40e85 Mon Sep 17 00:00:00 2001 From: Adam Rusinowski Date: Mon, 9 Dec 2024 21:41:36 +0100 Subject: [PATCH 3/4] fix psalm issues --- src/Command/PurgeQueueCommand.php | 8 ++++---- src/Mailer/TestQueueMailer.php | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Command/PurgeQueueCommand.php b/src/Command/PurgeQueueCommand.php index c671d71..c963be8 100644 --- a/src/Command/PurgeQueueCommand.php +++ b/src/Command/PurgeQueueCommand.php @@ -96,7 +96,7 @@ public function execute(Arguments $args, ConsoleIo $io) return self::CODE_SUCCESS; } - if ($args->getOption('all')) { + if ($args->getOption('all') === true) { $this->checkConfirmation( __('Are you sure you want to purge messages from all queues?'), $args, @@ -134,7 +134,7 @@ public function execute(Arguments $args, ConsoleIo $io) ); try { - $this->enqueueClientService->purgeQueue($queueConfig); + $this->enqueueClientService->purgeQueue((string)$queueConfig); $io->success(__('Queue `{0}` purged successfully', $queueConfig)); return self::CODE_SUCCESS; @@ -151,7 +151,7 @@ public function execute(Arguments $args, ConsoleIo $io) */ private function validateQueueConfig(?string $queueConfig): bool { - if (empty($queueConfig)) { + if (is_null($queueConfig) || !strlen($queueConfig)) { return false; } @@ -177,7 +177,7 @@ private function getConfiguredQueues(): array */ private function checkConfirmation(string $prompt, Arguments $args, ConsoleIo $io): void { - if (!$args->getOption('yes')) { + if ($args->getOption('yes') === false) { $confirmation = $io->askChoice( $prompt, [ diff --git a/src/Mailer/TestQueueMailer.php b/src/Mailer/TestQueueMailer.php index 447324a..b4378c2 100644 --- a/src/Mailer/TestQueueMailer.php +++ b/src/Mailer/TestQueueMailer.php @@ -16,6 +16,7 @@ use Cake\Mailer\Mailer; use Cake\Mailer\Message; use Cake\Queue\Mailer\QueueTrait; +use function Cake\I18n\__; /** * Test Queue Mailer From cd50827dde939976028aaf46e0cb61bc4cc4ed6a Mon Sep 17 00:00:00 2001 From: Adam Rusinowski Date: Tue, 10 Dec 2024 10:58:42 +0100 Subject: [PATCH 4/4] changed defaults, updated docs and changelog --- CHANGELOG.md | 20 +++++++++++++ README.md | 49 ++++++++++---------------------- src/Command/PurgeLogsCommand.php | 2 +- 3 files changed, 36 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7536ff..71bb20c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,26 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.1.0] - 2024-12-10 +* added commands to purge queues and test queues +* fixed test queue template path +* changed `queue_monitor purge` to `queue-monitor purge-logs` +* changed `queue_monitor notify` to `queue-monitor notify +* changed default configuration of `purgeLogsOlderThanDays` to 7 days instead of 30 + +## [2.0.2] - 2024-12-09 +* ported fixes and enhancement from version 1.x +* fixed handling `QueueMonitor.purgeLogsOlderThanDays` in Purge command +* added `QueueMonitor.disabled` option +* added support for disabling queue monitor commands +* updated README.md +* decreased log level when queue monitor is disabled +* replaced saveOrFail with save + +## [2.0.1] - 2024-05-14 +### Fixed +- Removed TimestampBehavior from LogsTable to avoid problems when TimestampBehavior is overriden + ## [2.0] - 2024-04-17 ### Added diff --git a/README.md b/README.md index 4466f87..85ec9b2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## Versions and branches | CakePHP | CakeDC Queue Monitor Plugin | Tag | Notes | |:-------:|:--------------------------------------------------------------------------:|:------------:|:-------| -| ^5.0 | [2.0.0](https://github.com/CakeDC/cakephp-queue-monitor/tree/2.next-cake5) | 2.next-cake5 | stable | +| ^5.0 | [2.1.0](https://github.com/CakeDC/cakephp-queue-monitor/tree/2.next-cake5) | 2.next-cake5 | stable | | ^4.4 | [1.0.0](https://github.com/CakeDC/cakephp-queue-monitor/tree/1.next-cake4) | 1.next-cake4 | stable | ## Overview @@ -26,28 +26,14 @@ composer require cakedc/queue-monitor ``` ## Configuration - -Add QueueMonitorPlugin to your `Application::bootstrap`: -```php -use Cake\Http\BaseApplication; -use CakeDC\QueueMonitor\QueueMonitorPlugin; - -class Application extends BaseApplication -{ - // ... - - public function bootstrap(): void - { - parent::bootstrap(); - - $this->addPlugin(QueueMonitorPlugin::class); - } - - // ... -} - +Add QueueMonitorPlugin to your application by running command: +```shell +bin/cake plugin load CakeDC/QueueMonitor +``` +Run the required migrations +```shell +bin/cake migrations migrate -p CakeDC/QueueMonitor ``` - Set up the QueueMonitor configuration in your `config/app_local.php`: ```php // ... @@ -58,16 +44,15 @@ Set up the QueueMonitor configuration in your `config/app_local.php`: // mailer config, the default is `default` mailer, you can ommit // this setting if you use default value - 'mailerConfig' => 'myCustomMailer', + 'mailerConfig' => 'default', - // the default is 30 minutes, you can ommit this setting if you - // use the default value - 'longJobInMinutes' => 45, + // the default is 30 minutes, you can ommit this setting if you use the default value + 'longJobInMinutes' => 30, - // the default is 30 days, you can ommit this setting if you + // the default is 7 days, you can ommit this setting if you use the default value // its advised to set this value correctly after queue usage analysis to avoid // high space usage in db - 'purgeLogsOlderThanDays' => 10, + 'purgeLogsOlderThanDays' => 7, // comma separated list of recipients of notification about long running queue jobs 'notificationRecipients' => 'recipient1@yourdomain.com,recipient2@yourdomain.com,recipient3@yourdomain.com', @@ -75,11 +60,6 @@ Set up the QueueMonitor configuration in your `config/app_local.php`: // ... ``` -Run the required migrations -```shell -bin/cake migrations migrate -p CakeDC/QueueMonitor -``` - For each queue configuration add `listener` setting ```php // ... @@ -95,7 +75,8 @@ For each queue configuration add `listener` setting ## Notification command -To set up notifications when there are long running or possible stuck jobs please use command +To set up notifications when there are jobs running for a long time or jobs that may be stuck and blocking the queue +please use command: ```shell bin/cake queue-monitor notify ``` diff --git a/src/Command/PurgeLogsCommand.php b/src/Command/PurgeLogsCommand.php index 5977681..3d1b9ac 100644 --- a/src/Command/PurgeLogsCommand.php +++ b/src/Command/PurgeLogsCommand.php @@ -32,7 +32,7 @@ final class PurgeLogsCommand extends Command use DisableTrait; use LogTrait; - private const DEFAULT_PURGE_DAYS_OLD = 30; + private const DEFAULT_PURGE_DAYS_OLD = 7; /** * Constructor