Allow execution of each migration in a separate process

kaliop-uk · Oct 7, 2016 · c2cc3d8 · c2cc3d8
1 parent 4fbdccc
commit c2cc3d8
Show file tree

Hide file tree

Showing 3 changed files with 124 additions and 26 deletions.
diff --git a/Command/MigrateCommand.php b/Command/MigrateCommand.php
@@ -8,12 +8,20 @@
 use Symfony\Component\Console\Input\InputOption;
 use Kaliop\eZMigrationBundle\API\Value\MigrationDefinition;
 use Kaliop\eZMigrationBundle\API\Value\Migration;
+use Symfony\Component\Process\ProcessBuilder;
+use Symfony\Component\Process\PhpExecutableFinder;
 
 /**
  * Command to execute the available migration definitions.
  */
 class MigrateCommand extends AbstractCommand
 {
+    // in between QUIET and NORMAL
+    const VERBOSITY_CHILD = 0.5;
+    /** @var OutputInterface $output */
+    protected $output;
+    protected $verbosity = OutputInterface::VERBOSITY_NORMAL;
+
     /**
      * Set up the command.
      *
@@ -33,11 +41,14 @@ protected function configure()
                 InputOption::VALUE_OPTIONAL | InputOption::VALUE_IS_ARRAY,
                 "The directory or file to load the migration definitions from"
             )
+            // nb: when adding options, remember to forward them to sub-commands executed in 'separate-process' mode
             ->addOption('default-language', null, InputOption::VALUE_REQUIRED, "Default language code that will be used if no language is provided in migration steps")
             ->addOption('ignore-failures', null, InputOption::VALUE_NONE, "Keep executing migrations even if one fails")
             ->addOption('clear-cache', null, InputOption::VALUE_NONE, "Clear the cache after the command finishes")
             ->addOption('no-interaction', 'n', InputOption::VALUE_NONE, "Do not ask any interactive question")
             ->addOption('no-transactions', 'u', InputOption::VALUE_NONE, "Do not use a repository transaction to wrap each migration. Unsafe, but needed for legacy slot handlers")
+            ->addOption('separate-process', 'p', InputOption::VALUE_NONE, "Use a separate php process to run each migration. Safe if your migration leak memory. A tad slower")
+            ->addOption('child', null, InputOption::VALUE_NONE, "*DO NOT USE* Internal option for when forking separate processes")
             ->setHelp(<<<EOT
 The <info>kaliop:migration:migrate</info> command loads and executes migrations:
 
@@ -56,11 +67,16 @@ protected function configure()
      * @param InputInterface $input
      * @param OutputInterface $output
      * @return null|int null or 0 if everything went fine, or an error code
-     *
-     * @todo Add functionality to work with specified version files not just directories.
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        $this->setOutput($output);
+        $this->setVerbosity($output->getVerbosity());
+
+        if ($input->getOption('child')) {
+            $this->setVerbosity(self::VERBOSITY_CHILD);
+        }
+
         $migrationsService = $this->getMigrationService();
 
         $paths = $input->getOption('path');
@@ -98,7 +114,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
             return;
         }
 
-        $output->writeln("\n <info>==</info> Migrations to be executed\n");
+        $this->writeln("\n <info>==</info> Migrations to be executed\n");
 
         $data = array();
         $i = 1;
@@ -114,13 +130,16 @@ protected function execute(InputInterface $input, OutputInterface $output)
             );
         }
 
-        $table = $this->getHelperSet()->get('table');
-        $table
-            ->setHeaders(array('#', 'Migration', 'Notes'))
-            ->setRows($data);
-        $table->render($output);
+        if (!$input->getOption('child')) {
+            $table = $this->getHelperSet()->get('table');
+            $table
+                ->setHeaders(array('#', 'Migration', 'Notes'))
+                ->setRows($data);
+            $table->render($output);
+        }
+
+        $this->writeln('');
 
-        $output->writeln('');
         // ask user for confirmation to make changes
         if ($input->isInteractive() && !$input->getOption('no-interaction')) {
             $dialog = $this->getHelperSet()->get('dialog');
@@ -134,7 +153,30 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 return 0;
             }
         } else {
-            $output->writeln("=============================================\n");
+            $this->writeln("=============================================\n");
+        }
+
+        if ($input->getOption('separate-process')) {
+            $builder = new ProcessBuilder();
+            $executableFinder = new PhpExecutableFinder();
+            if (false !== $php = $executableFinder->find()) {
+                $builder->setPrefix($php);
+            }
+            // mandatory args and options
+            $builderArgs = array(
+                $_SERVER['argv'][0], // sf console
+                $_SERVER['argv'][1], // name of sf command
+                '--env=' . $this->getContainer()->get('kernel')->getEnvironment(), // sf env
+                '--child'
+            );
+            // 'optional' options
+            // note: options 'clear-cache', 'ignore-failures' and 'no-transactions' we never propagate
+            if ($input->getOption('default-language')) {
+                $builderArgs[]='--default-language='.$input->getOption('default-language');
+            }
+            if ($input->getOption('no-transactions')) {
+                $builderArgs[]='--no-transactions';
+            }
         }
 
         foreach($toExecute as $name => $migrationDefinition) {
@@ -145,24 +187,49 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 continue;
             }
 
-            $output->writeln("<info>Processing $name</info>");
-
-            try {
-                $migrationsService->executeMigration(
-                    $migrationDefinition,
-                    !$input->getOption('no-transactions'),
-                    $input->getOption('default-language')
-                );
-            } catch(\Exception $e) {
-                if ($input->getOption('ignore-failures')) {
-                    $output->writeln("\n<error>Migration failed! Reason: " . $e->getMessage() . "</error>\n");
-                    continue;
+            $this->writeln("<info>Processing $name</info>");
+
+            if ($input->getOption('separate-process')) {
+
+                $process = $builder
+                    ->setArguments(array_merge($builderArgs, array('--path=' . $migrationDefinition->path)))
+                    ->getProcess();
+
+                $this->writeln('<info>Executing: ' . $process->getCommandLine() . '</info>', OutputInterface::VERBOSITY_VERBOSE);
+
+                $process->run();
+
+                $output->write($process->getOutput());
+                if (!$process->isSuccessful()) {
+                    $err = $process->getErrorOutput();
+                    if ($input->getOption('ignore-failures')) {
+                        $output->writeln("\n<error>Migration failed! Reason: " . $err . "</error>\n");
+                        continue;
+                    }
+                    $output->writeln("\n<error>Migration aborted! Reason: " . $err . "</error>");
+                    return 1;
                 }
-                $output->writeln("\n<error>Migration aborted! Reason: " . $e->getMessage() . "</error>");
-                return 1;
+
+            } else {
+
+                try {
+                    $migrationsService->executeMigration(
+                        $migrationDefinition,
+                        !$input->getOption('no-transactions'),
+                        $input->getOption('default-language')
+                    );
+                } catch(\Exception $e) {
+                    if ($input->getOption('ignore-failures')) {
+                        $output->writeln("\n<error>Migration failed! Reason: " . $e->getMessage() . "</error>\n");
+                        continue;
+                    }
+                    $output->writeln("\n<error>Migration aborted! Reason: " . $e->getMessage() . "</error>");
+                    return 1;
+                }
+
             }
 
-            $output->writeln('');
+            $this->writeln('');
         }
 
         if ($input->getOption('clear-cache')) {
@@ -171,4 +238,27 @@ protected function execute(InputInterface $input, OutputInterface $output)
             $command->run($inputArray, $output);
         }
     }
+
+    /**
+     * Small tricks to allow us to lower verbosity between NORMAL and QUIET and have a decent writeln API, even with old SF versions
+     * @param $message
+     * @param int $verbosity
+     */
+    protected function writeln($message, $verbosity=OutputInterface::VERBOSITY_NORMAL)
+    {
+        if ($this->verbosity >= $verbosity) {
+            $this->output->writeln($message);
+        }
+    }
+
+    protected function setOutput(OutputInterface $output)
+    {
+        $this->output = $output;
+    }
+
+    protected function setVerbosity($verbosity)
+    {
+        $this->verbosity = $verbosity;
+    }
+
 }
diff --git a/WHATSNEW.md b/WHATSNEW.md
@@ -1,3 +1,10 @@
+Version 2.2.0
+=============
+
+* New: the 'migrate' command learned a `--separate-process` option, to run each migration it its own separate
+    php process. This should help when running many migrations in a single pass, and there are f.e. memory leaks.
+
+
 Version 2.1.0
 =============
 

diff --git a/composer.json b/composer.json
@@ -25,7 +25,8 @@
         "php": ">=5.4",
         "ezsystems/ezpublish-kernel": ">=5.3|>=2014.03",
         "ext-pdo": "*",
-        "nikic/php-parser": "2.*"
+        "nikic/php-parser": "2.*",
+        "symfony/process": "*"
     },
     "require-dev": {
         "mikey179/vfsStream": "~1.2.0",