feat: add zstd and lz4 compression support and auto compression detec…

.ddev/config.yaml

@@ -167,7 +167,7 @@ nodejs_version: "16"
 # The mailhog port is not normally bound on the host at all, instead being routed
 # through ddev-router, but it can be bound directly to localhost if specified here.
 
-# webimage_extra_packages: [php7.4-tidy, php-bcmath]
+webimage_extra_packages: ['lz4', 'zstd']
 # Extra Debian packages that are needed in the webimage can be added here
 
 # dbimage_extra_packages: [telnet,netcat]

.github/workflows/linux-setup.sh

@@ -9,5 +9,8 @@ set -x
 # bats (for testing)
 git clone --branch v1.2.1 https://github.com/bats-core/bats-core.git /tmp/bats-core && pushd /tmp/bats-core >/dev/null && sudo ./install.sh /usr/local
 
+sudo apt-get update
+sudo apt-get -y install zstd lz4
+
 # Show info to simplify debugging
 lsb_release -a

src/N98/Magento/Command/Database/AbstractDatabaseCommand.php

@@ -89,8 +89,8 @@ protected function getCompressionHelp()
         $messages = [];
         $messages[] = '';
         $messages[] = '<info>Compression option</info>';
-        $messages[] = ' Supported compression: gzip';
-        $messages[] = ' The gzip cli tool has to be installed.';
+        $messages[] = ' Supported compression: gzip, lz4, zstd';
+        $messages[] = ' The gzip/lz4/zstd cli tool has to be installed.';
         $messages[] = ' Additionally, for data-to-csv option tar cli tool has to be installed too.';
 
         return implode(PHP_EOL, $messages);
@@ -101,9 +101,9 @@ protected function getCompressionHelp()
      * @return Compressor
      * @deprecated Since 1.1.12; use AbstractCompressor::create() instead
      */
-    protected function getCompressor($type)
+    protected function getCompressor($type, InputInterface $input)
     {
-        return AbstractCompressor::create($type);
+        return AbstractCompressor::create($type, $input);
     }
 
     /**

src/N98/Magento/Command/Database/Compressor/AbstractCompressor.php

@@ -3,35 +3,75 @@
 namespace N98\Magento\Command\Database\Compressor;
 
 use InvalidArgumentException;
+use N98\Util\BinaryString;
 use N98\Util\OperatingSystem;
+use Symfony\Component\Console\Input\InputInterface;
 
 /**
  * Class AbstractCompressor
  * @package N98\Magento\Command\Database\Compressor
  */
 abstract class AbstractCompressor implements Compressor
 {
+    /**
+     * @param InputInterface|null $input
+     */
+    public function __construct(InputInterface $input = null)
+    {
+    }
+
     /**
      * @param string $type
+     * @param InputInterface $input
      * @return AbstractCompressor
      * @throws InvalidArgumentException
      */
-    public static function create($type)
+    public static function create($type, InputInterface $input = null)
     {
         switch ($type) {
             case null:
             case 'none':
-                return new Uncompressed();
+                return new Uncompressed($input);
 
             case 'gz':
             case 'gzip':
-                return new Gzip();
+                return new Gzip($input);
+
+            case 'zstd':
+                return new Zstandard($input);
+
+            case 'lz4':
+                return new LZ4($input);
 
             default:
                 throw new InvalidArgumentException("Compression type '{$type}' is not supported.");
         }
     }
 
+    /**
+     * @param string $filename
+     * @return string|null
+     */
+    public static function tryGetCompressionType(string $filename)
+    {
+        switch (true) {
+            case BinaryString::endsWith($filename, '.sql'):
+                return 'none';
+            case BinaryString::endsWith($filename, '.sql.zstd'):
+            case BinaryString::endsWith($filename, '.tar.zstd'):
+                return 'zstd';
+            case BinaryString::endsWith($filename, '.sql.lz4'):
+            case BinaryString::endsWith($filename, '.tar.lz4'):
+                return 'lz4';
+            case BinaryString::endsWith($filename, '.sql.gz'):
+            case BinaryString::endsWith($filename, '.tgz'):
+            case BinaryString::endsWith($filename, '.gz'):
+                return 'gzip';
+            default:
+                return null;
+        }
+    }
+
     /**
      * Returns the command line for compressing the dump file.
      *

src/N98/Magento/Command/Database/Compressor/LZ4.php

@@ -0,0 +1,89 @@
+<?php
+
+namespace N98\Magento\Command\Database\Compressor;
+
+/**
+ * Class LZ4
+ * @package N98\Magento\Command\Database\Compressor
+ */
+class LZ4 extends AbstractCompressor
+{
+    /**
+     * Returns the command line for compressing the dump file.
+     *
+     * @param string $command
+     * @param bool $pipe
+     * @return string
+     */
+    public function getCompressingCommand($command, $pipe = true)
+    {
+        if ($pipe) {
+            return $command . ' | lz4 -c ';
+        } else {
+            return sprintf(
+                "tar -I 'lz4' -cf %s",
+                $command,
+            );
+        }
+    }
+
+    /**
+     * Returns the command line for decompressing the dump file.
+     *
+     * @param string $command
+     * @param string $fileName Filename (shell argument escaped)
+     * @param bool $pipe
+     * @return string
+     */
+    public function getDecompressingCommand($command, $fileName, $pipe = true)
+    {
+        if ($pipe) {
+            if ($this->hasPipeViewer()) {
+                return 'pv -cN lz4 ' . escapeshellarg($fileName) . ' | lz4 -d | pv -cN mysql | ' . $command;
+            }
+
+            return 'lz4 -dc < ' . escapeshellarg($fileName) . ' | ' . $command;
+        } else {
+            if ($this->hasPipeViewer()) {
+                return 'pv -cN tar -zxf ' . escapeshellarg($fileName) . ' && pv -cN mysql | ' . $command;
+            }
+
+            return 'tar -zxf ' . escapeshellarg($fileName) . ' -C ' . dirname($fileName) . ' && ' . $command . ' < '
+                . escapeshellarg(substr($fileName, 0, -4));
+        }
+    }
+
+    /**
+     * Returns the file name for the compressed dump file.
+     *
+     * @param string $fileName
+     * @param bool $pipe
+     * @return string
+     */
+    public function getFileName($fileName, $pipe = true)
+    {
+        if ($fileName === null) {
+            $fileName = '';
+        }
+
+        if (!strlen($fileName)) {
+            return $fileName;
+        }
+
+        if ($pipe) {
+            if (substr($fileName, -4, 4) === '.lz4') {
+                return $fileName;
+            } elseif (substr($fileName, -4, 4) === '.sql') {
+                $fileName .= '.lz4';
+            } else {
+                $fileName .= '.sql.lz4';
+            }
+        } elseif (substr($fileName, -8, 8) === '.tar.lz4') {
+            return $fileName;
+        } else {
+            $fileName .= '.tar.lz4';
+        }
+
+        return $fileName;
+    }
+}

src/N98/Magento/Command/Database/Compressor/Zstandard.php

@@ -0,0 +1,113 @@
+<?php
+
+namespace N98\Magento\Command\Database\Compressor;
+
+use Symfony\Component\Console\Input\InputInterface;
+
+/**
+ * Class Zstandard
+ * @package N98\Magento\Command\Database\Compressor
+ */
+class Zstandard extends AbstractCompressor
+{
+    protected int $compressionLevel;
+
+    protected string $extraArgs;
+
+    /**
+     * @param InputInterface|null $input
+     */
+    public function __construct(InputInterface $input = null)
+    {
+        $this->compressionLevel = $input ? (int)$input->getOption('zstd-level') : 10;
+        $this->extraArgs = $input ? (string)$input->getOption('zstd-extra-args') : '';
+
+        parent::__construct($input);
+    }
+
+    /**
+     * Returns the command line for compressing the dump file.
+     *
+     * @param string $command
+     * @param bool $pipe
+     * @return string
+     */
+    public function getCompressingCommand($command, $pipe = true)
+    {
+        if ($pipe) {
+            return sprintf(
+                "%s | zstd -c -%s %s",
+                $command,
+                $this->compressionLevel,
+                $this->extraArgs,
+            );
+        } else {
+            return sprintf(
+                "tar -I 'zstd %s -%s' -cf %s",
+                $this->extraArgs,
+                $this->compressionLevel,
+                $command,
+            );
+        }
+    }
+
+    /**
+     * Returns the command line for decompressing the dump file.
+     *
+     * @param string $command
+     * @param string $fileName Filename (shell argument escaped)
+     * @param bool $pipe
+     * @return string
+     */
+    public function getDecompressingCommand($command, $fileName, $pipe = true)
+    {
+        if ($pipe) {
+            if ($this->hasPipeViewer()) {
+                return 'pv -cN zstd ' . escapeshellarg($fileName) . ' | zstd -d | pv -cN mysql | ' . $command;
+            }
+
+            return 'zstd -dc < ' . escapeshellarg($fileName) . ' | ' . $command;
+        } else {
+            if ($this->hasPipeViewer()) {
+                return 'pv -cN tar -zxf ' . escapeshellarg($fileName) . ' && pv -cN mysql | ' . $command;
+            }
+
+            return 'tar -zxf ' . escapeshellarg($fileName) . ' -C ' . dirname($fileName) . ' && ' . $command . ' < '
+                . escapeshellarg(substr($fileName, 0, -4));
+        }
+    }
+
+    /**
+     * Returns the file name for the compressed dump file.
+     *
+     * @param string $fileName
+     * @param bool $pipe
+     * @return string
+     */
+    public function getFileName($fileName, $pipe = true)
+    {
+        if ($fileName === null) {
+            $fileName = '';
+        }
+
+        if (!strlen($fileName)) {
+            return $fileName;
+        }
+
+        if ($pipe) {
+            if (substr($fileName, -5, 5) === '.zstd') {
+                return $fileName;
+            } elseif (substr($fileName, -4, 4) === '.sql') {
+                $fileName .= '.zstd';
+            } else {
+                $fileName .= '.sql.zstd';
+            }
+        } elseif (substr($fileName, -9, 9) === '.tar.zstd') {
+            return $fileName;
+        } else {
+            $fileName .= '.tar.zstd';
+        }
+
+        return $fileName;
+    }
+}

src/N98/Magento/Command/Database/DumpCommand.php

@@ -39,6 +39,8 @@ protected function configure()
         $this
             ->setName('db:dump')
             ->addArgument('filename', InputArgument::OPTIONAL, 'Dump filename')
+            ->addOption('zstd-level', null, InputOption::VALUE_OPTIONAL, 'Set the level of compression the higher the smaller the result', 10)
+            ->addOption('zstd-extra-args', null, InputOption::VALUE_OPTIONAL, 'Set custom extra options that zstd supports', '')
             ->addOption(
                 'add-time',
                 't',
@@ -296,7 +298,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
     private function createExecs(InputInterface $input, OutputInterface $output)
     {
         $execs = new Execs('mysqldump');
-        $execs->setCompression($input->getOption('compression'));
+        $execs->setCompression($input->getOption('compression'), $input);
         $execs->setFileName($this->getFileName($input, $output, $execs->getCompressor()));
 
         if (!$input->getOption('no-single-transaction')) {

src/N98/Magento/Command/Database/Execs.php

@@ -6,6 +6,7 @@
 namespace N98\Magento\Command\Database;
 
 use N98\Magento\Command\Database\Compressor\AbstractCompressor;
+use Symfony\Component\Console\Input\InputInterface;
 
 /**
  * One or multiple commands to execute, with support for Compressors
@@ -47,9 +48,9 @@ public function __construct($command = null)
     /**
      * @param string $type of compression: "gz" | "gzip" | "none" | null
      */
-    public function setCompression($type)
+    public function setCompression($type, InputInterface $input = null)
     {
-        $this->compressor = AbstractCompressor::create($type);
+        $this->compressor = AbstractCompressor::create($type, $input);
     }
 
     /**

src/N98/Magento/Command/Database/ImportCommand.php

@@ -24,7 +24,9 @@ protected function configure()
         $this
             ->setName('db:import')
             ->addArgument('filename', InputArgument::OPTIONAL, 'Dump filename')
-            ->addOption('compression', 'c', InputOption::VALUE_REQUIRED, 'The compression of the specified file')
+            ->addOption('compression', 'c', InputOption::VALUE_OPTIONAL, 'The compression of the specified file')
+            ->addOption('zstd-level', null, InputOption::VALUE_OPTIONAL, '', 10)
+            ->addOption('zstd-extra-args', null, InputOption::VALUE_OPTIONAL, '', '')
             ->addOption('only-command', null, InputOption::VALUE_NONE, 'Print only mysql command. Do not execute')
             ->addOption('only-if-empty', null, InputOption::VALUE_NONE, 'Imports only if database is empty')
             ->addOption(
@@ -126,7 +128,17 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
         $fileName = $this->checkFilename($input);
 
-        $compressor = AbstractCompressor::create($input->getOption('compression'));
+        if ($input->getOption('compression')) {
+            $compression = $input->getOption('compression');
+        } else {
+            $compression = AbstractCompressor::tryGetCompressionType($fileName);
+
+            if ($compression == null) {
+                throw new \RuntimeException("Could not guess compression type or the file is in a format that is not supported.");
+            }
+        }
+
+        $compressor = AbstractCompressor::create($compression, $input);
 
         $exec = 'mysql ';
         if ($input->getOption('force')) {