feat: add zstd and lz4 compression support and auto compression detec…

…t on db import
netz98 · Nov 7, 2024 · 53cfbf5 · 53cfbf5
1 parent b718615
commit 53cfbf5
Show file tree

Hide file tree

Showing 8 changed files with 266 additions and 11 deletions.
diff --git a/src/N98/Magento/Command/Database/AbstractDatabaseCommand.php b/src/N98/Magento/Command/Database/AbstractDatabaseCommand.php
@@ -101,9 +101,9 @@ protected function getCompressionHelp()
      * @return Compressor
      * @deprecated Since 1.1.12; use AbstractCompressor::create() instead
      */
-    protected function getCompressor($type)
+    protected function getCompressor($type, InputInterface $input)
     {
-        return AbstractCompressor::create($type);
+        return AbstractCompressor::create($type, $input);
     }
 
     /**

diff --git a/src/N98/Magento/Command/Database/Compressor/AbstractCompressor.php b/src/N98/Magento/Command/Database/Compressor/AbstractCompressor.php
@@ -4,34 +4,73 @@
 
 use InvalidArgumentException;
 use N98\Util\OperatingSystem;
+use Symfony\Component\Console\Input\InputInterface;
 
 /**
  * Class AbstractCompressor
  * @package N98\Magento\Command\Database\Compressor
  */
 abstract class AbstractCompressor implements Compressor
 {
+    /**
+     * @param InputInterface $input
+     */
+    public function __construct(InputInterface $input)
+    {
+    }
+
     /**
      * @param string $type
+     * @param InputInterface $input
      * @return AbstractCompressor
      * @throws InvalidArgumentException
      */
-    public static function create($type)
+    public static function create($type, InputInterface $input)
     {
         switch ($type) {
             case null:
             case 'none':
-                return new Uncompressed();
+                return new Uncompressed($input);
 
             case 'gz':
             case 'gzip':
-                return new Gzip();
+                return new Gzip($input);
+
+            case 'zstd':
+                return new Zstandard($input);
+
+            case 'lz4':
+                return new LZ4($input);
 
             default:
                 throw new InvalidArgumentException("Compression type '{$type}' is not supported.");
         }
     }
 
+    /**
+     * @param string $filename
+     * @return string|null
+     */
+    public static function tryGetCompressionType(string $filename)
+    {
+        switch (true) {
+            case str_ends_with($filename, '.sql'):
+                return 'none';
+            case str_ends_with($filename, '.sql.zstd'):
+            case str_ends_with($filename, '.tar.zstd'):
+                return 'zstd';
+            case str_ends_with($filename, '.sql.lz4'):
+            case str_ends_with($filename, '.tar.lz4'):
+                return 'lz4';
+            case str_ends_with($filename, '.sql.gz'):
+            case str_ends_with($filename, '.tgz'):
+            case str_ends_with($filename, '.gz'):
+                return 'gzip';
+            default:
+                return null;
+        }
+    }
+
     /**
      * Returns the command line for compressing the dump file.
      *

diff --git a/src/N98/Magento/Command/Database/Compressor/LZ4.php b/src/N98/Magento/Command/Database/Compressor/LZ4.php
@@ -0,0 +1,89 @@
+<?php
+
+namespace N98\Magento\Command\Database\Compressor;
+
+/**
+ * Class LZ4
+ * @package N98\Magento\Command\Database\Compressor
+ */
+class LZ4 extends AbstractCompressor
+{
+    /**
+     * Returns the command line for compressing the dump file.
+     *
+     * @param string $command
+     * @param bool $pipe
+     * @return string
+     */
+    public function getCompressingCommand($command, $pipe = true)
+    {
+        if ($pipe) {
+            return $command . ' | lz4 -c ';
+        } else {
+            return sprintf(
+                "tar -I 'lz4' -cf %s",
+                $command,
+            );
+        }
+    }
+
+    /**
+     * Returns the command line for decompressing the dump file.
+     *
+     * @param string $command
+     * @param string $fileName Filename (shell argument escaped)
+     * @param bool $pipe
+     * @return string
+     */
+    public function getDecompressingCommand($command, $fileName, $pipe = true)
+    {
+        if ($pipe) {
+            if ($this->hasPipeViewer()) {
+                return 'pv -cN lz4 ' . escapeshellarg($fileName) . ' | lz4 -d | pv -cN mysql | ' . $command;
+            }
+
+            return 'lz4 -dc < ' . escapeshellarg($fileName) . ' | ' . $command;
+        } else {
+            if ($this->hasPipeViewer()) {
+                return 'pv -cN tar -zxf ' . escapeshellarg($fileName) . ' && pv -cN mysql | ' . $command;
+            }
+
+            return 'tar -zxf ' . escapeshellarg($fileName) . ' -C ' . dirname($fileName) . ' && ' . $command . ' < '
+                . escapeshellarg(substr($fileName, 0, -4));
+        }
+    }
+
+    /**
+     * Returns the file name for the compressed dump file.
+     *
+     * @param string $fileName
+     * @param bool $pipe
+     * @return string
+     */
+    public function getFileName($fileName, $pipe = true)
+    {
+        if ($fileName === null) {
+            $fileName = '';
+        }
+
+        if (!strlen($fileName)) {
+            return $fileName;
+        }
+
+        if ($pipe) {
+            if (substr($fileName, -4, 4) === '.lz4') {
+                return $fileName;
+            } elseif (substr($fileName, -4, 4) === '.sql') {
+                $fileName .= '.lz4';
+            } else {
+                $fileName .= '.sql.lz4';
+            }
+        } elseif (substr($fileName, -8, 8) === '.tar.lz4') {
+            return $fileName;
+        } else {
+            $fileName .= '.tar.lz4';
+        }
+
+        return $fileName;
+    }
+}
diff --git a/src/N98/Magento/Command/Database/Compressor/Zstandard.php b/src/N98/Magento/Command/Database/Compressor/Zstandard.php
@@ -0,0 +1,110 @@
+<?php
+
+namespace N98\Magento\Command\Database\Compressor;
+
+use Symfony\Component\Console\Input\InputInterface;
+
+/**
+ * Class Zstandard
+ * @package N98\Magento\Command\Database\Compressor
+ */
+class Zstandard extends AbstractCompressor
+{
+    protected int $compressionLevel;
+
+    protected string $extraArgs;
+
+    public function __construct(InputInterface $input)
+    {
+        $this->compressionLevel = (int)$input->getOption('zstd-level');
+        $this->extraArgs = (string)$input->getOption('zstd-extra-args');
+
+        parent::__construct($input);
+    }
+
+    /**
+     * Returns the command line for compressing the dump file.
+     *
+     * @param string $command
+     * @param bool $pipe
+     * @return string
+     */
+    public function getCompressingCommand($command, $pipe = true)
+    {
+        if ($pipe) {
+            return sprintf(
+                "%s | zstd -c -%s %s",
+                $command,
+                $this->compressionLevel,
+                $this->extraArgs,
+            );
+        } else {
+            return sprintf(
+                "tar -I 'zstd %s -%s' -cf %s",
+                $this->extraArgs,
+                $this->compressionLevel,
+                $command,
+            );
+        }
+    }
+
+    /**
+     * Returns the command line for decompressing the dump file.
+     *
+     * @param string $command
+     * @param string $fileName Filename (shell argument escaped)
+     * @param bool $pipe
+     * @return string
+     */
+    public function getDecompressingCommand($command, $fileName, $pipe = true)
+    {
+        if ($pipe) {
+            if ($this->hasPipeViewer()) {
+                return 'pv -cN zstd ' . escapeshellarg($fileName) . ' | zstd -d | pv -cN mysql | ' . $command;
+            }
+
+            return 'zstd -dc < ' . escapeshellarg($fileName) . ' | ' . $command;
+        } else {
+            if ($this->hasPipeViewer()) {
+                return 'pv -cN tar -zxf ' . escapeshellarg($fileName) . ' && pv -cN mysql | ' . $command;
+            }
+
+            return 'tar -zxf ' . escapeshellarg($fileName) . ' -C ' . dirname($fileName) . ' && ' . $command . ' < '
+                . escapeshellarg(substr($fileName, 0, -4));
+        }
+    }
+
+    /**
+     * Returns the file name for the compressed dump file.
+     *
+     * @param string $fileName
+     * @param bool $pipe
+     * @return string
+     */
+    public function getFileName($fileName, $pipe = true)
+    {
+        if ($fileName === null) {
+            $fileName = '';
+        }
+
+        if (!strlen($fileName)) {
+            return $fileName;
+        }
+
+        if ($pipe) {
+            if (substr($fileName, -5, 5) === '.zstd') {
+                return $fileName;
+            } elseif (substr($fileName, -4, 4) === '.sql') {
+                $fileName .= '.zstd';
+            } else {
+                $fileName .= '.sql.zstd';
+            }
+        } elseif (substr($fileName, -9, 9) === '.tar.zstd') {
+            return $fileName;
+        } else {
+            $fileName .= '.tar.zstd';
+        }
+
+        return $fileName;
+    }
+}
diff --git a/src/N98/Magento/Command/Database/DumpCommand.php b/src/N98/Magento/Command/Database/DumpCommand.php
@@ -39,6 +39,8 @@ protected function configure()
         $this
             ->setName('db:dump')
             ->addArgument('filename', InputArgument::OPTIONAL, 'Dump filename')
+            ->addOption('zstd-level', null, InputOption::VALUE_OPTIONAL, '', 10)
+            ->addOption('zstd-extra-args', null, InputOption::VALUE_OPTIONAL, '', '')
             ->addOption(
                 'add-time',
                 't',
@@ -296,7 +298,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
     private function createExecs(InputInterface $input, OutputInterface $output)
     {
         $execs = new Execs('mysqldump');
-        $execs->setCompression($input->getOption('compression'));
+        $execs->setCompression($input->getOption('compression'), $input);
         $execs->setFileName($this->getFileName($input, $output, $execs->getCompressor()));
 
         if (!$input->getOption('no-single-transaction')) {

diff --git a/src/N98/Magento/Command/Database/Execs.php b/src/N98/Magento/Command/Database/Execs.php
@@ -6,6 +6,7 @@
 namespace N98\Magento\Command\Database;
 
 use N98\Magento\Command\Database\Compressor\AbstractCompressor;
+use Symfony\Component\Console\Input\InputInterface;
 
 /**
  * One or multiple commands to execute, with support for Compressors
@@ -47,9 +48,9 @@ public function __construct($command = null)
     /**
      * @param string $type of compression: "gz" | "gzip" | "none" | null
      */
-    public function setCompression($type)
+    public function setCompression($type, InputInterface $input)
     {
-        $this->compressor = AbstractCompressor::create($type);
+        $this->compressor = AbstractCompressor::create($type, $input);
     }
 
     /**

diff --git a/src/N98/Magento/Command/Database/ImportCommand.php b/src/N98/Magento/Command/Database/ImportCommand.php
@@ -24,7 +24,9 @@ protected function configure()
         $this
             ->setName('db:import')
             ->addArgument('filename', InputArgument::OPTIONAL, 'Dump filename')
-            ->addOption('compression', 'c', InputOption::VALUE_REQUIRED, 'The compression of the specified file')
+            ->addOption('compression', 'c', InputOption::VALUE_OPTIONAL, 'The compression of the specified file')
+            ->addOption('zstd-level', null, InputOption::VALUE_OPTIONAL, '', 10)
+            ->addOption('zstd-extra-args', null, InputOption::VALUE_OPTIONAL, '', '')
             ->addOption('only-command', null, InputOption::VALUE_NONE, 'Print only mysql command. Do not execute')
             ->addOption('only-if-empty', null, InputOption::VALUE_NONE, 'Imports only if database is empty')
             ->addOption(
@@ -126,7 +128,17 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
         $fileName = $this->checkFilename($input);
 
-        $compressor = AbstractCompressor::create($input->getOption('compression'));
+        if ($input->getOption('compression')) {
+            $compression = $input->getOption('compression');
+        } else {
+            $compression = AbstractCompressor::tryGetCompressionType($fileName);
+
+            if ($compression == null) {
+                throw new \RuntimeException("Could not guess compression type or the file is in a format that is not supported.");
+            }
+        }
+
+        $compressor = AbstractCompressor::create($compression, $input);
 
         $exec = 'mysql ';
         if ($input->getOption('force')) {

diff --git a/tests/N98/Magento/Command/Database/ExecsTest.php b/tests/N98/Magento/Command/Database/ExecsTest.php
@@ -8,6 +8,7 @@
 use N98\Magento\Command\Database\Compressor\AbstractCompressor;
 use N98\Magento\Command\Database\Compressor\Uncompressed;
 use PHPUnit\Framework\TestCase;
+use Symfony\Component\Console\Input\InputInterface;
 
 /**
  * Class ExecsTest
@@ -31,9 +32,10 @@ public function creation()
      */
     public function facade()
     {
+        $input = $this->createMock(InputInterface::class);
         $execs = new Execs('foo');
         $this->assertInstanceOf(Uncompressed::class, $execs->getCompressor());
-        $execs->setCompression('gzip');
+        $execs->setCompression('gzip', $input);
         $this->assertInstanceOf(AbstractCompressor::class, $execs->getCompressor());
         $this->assertNull($execs->getFileName());
         $execs->setFileName('output.sql');