From 790fd866fd99dc9fc0da49b3c306dccd8e064a25 Mon Sep 17 00:00:00 2001 From: ResuBaka Date: Thu, 7 Nov 2024 13:40:15 +0100 Subject: [PATCH] feat: add zstd and lz4 compression support and auto compression detect on db import --- .../Database/AbstractDatabaseCommand.php | 4 +- .../Compressor/AbstractCompressor.php | 41 ++++++- .../Command/Database/Compressor/LZ4.php | 89 ++++++++++++++ .../Command/Database/Compressor/Zstandard.php | 110 ++++++++++++++++++ .../Magento/Command/Database/DumpCommand.php | 4 +- src/N98/Magento/Command/Database/Execs.php | 5 +- .../Command/Database/ImportCommand.php | 16 ++- .../Magento/Command/Database/ExecsTest.php | 4 +- 8 files changed, 262 insertions(+), 11 deletions(-) create mode 100644 src/N98/Magento/Command/Database/Compressor/LZ4.php create mode 100644 src/N98/Magento/Command/Database/Compressor/Zstandard.php diff --git a/src/N98/Magento/Command/Database/AbstractDatabaseCommand.php b/src/N98/Magento/Command/Database/AbstractDatabaseCommand.php index 967569f37..c0fcac658 100644 --- a/src/N98/Magento/Command/Database/AbstractDatabaseCommand.php +++ b/src/N98/Magento/Command/Database/AbstractDatabaseCommand.php @@ -101,9 +101,9 @@ protected function getCompressionHelp() * @return Compressor * @deprecated Since 1.1.12; use AbstractCompressor::create() instead */ - protected function getCompressor($type) + protected function getCompressor($type, InputInterface $input) { - return AbstractCompressor::create($type); + return AbstractCompressor::create($type, $input); } /** diff --git a/src/N98/Magento/Command/Database/Compressor/AbstractCompressor.php b/src/N98/Magento/Command/Database/Compressor/AbstractCompressor.php index c251cab06..b6d6cdb74 100644 --- a/src/N98/Magento/Command/Database/Compressor/AbstractCompressor.php +++ b/src/N98/Magento/Command/Database/Compressor/AbstractCompressor.php @@ -3,7 +3,9 @@ namespace N98\Magento\Command\Database\Compressor; use InvalidArgumentException; +use N98\Magento\Command\Database\AbstractDatabaseCommand; use N98\Util\OperatingSystem; +use Symfony\Component\Console\Input\InputInterface; /** * Class AbstractCompressor @@ -11,27 +13,60 @@ */ abstract class AbstractCompressor implements Compressor { + public function __construct(InputInterface $input) {} + /** * @param string $type + * @param InputInterface $input * @return AbstractCompressor * @throws InvalidArgumentException */ - public static function create($type) + public static function create($type, InputInterface $input) { switch ($type) { case null: case 'none': - return new Uncompressed(); + return new Uncompressed($input); case 'gz': case 'gzip': - return new Gzip(); + return new Gzip($input); + + case 'zstd': + return new Zstandard($input); + + case 'lz4': + return new LZ4($input); default: throw new InvalidArgumentException("Compression type '{$type}' is not supported."); } } + /** + * @param string $filename + * @return string|null + */ + public static function tryGetCompressionType(string $filename) + { + switch (true) { + case str_ends_with($filename, '.sql'): + return 'none'; + case str_ends_with($filename, '.sql.zstd'): + case str_ends_with($filename, '.tar.zstd'): + return 'zstd'; + case str_ends_with($filename, '.sql.lz4'): + case str_ends_with($filename, '.tar.lz4'): + return 'lz4'; + case str_ends_with($filename, '.sql.gz'): + case str_ends_with($filename, '.tgz'): + case str_ends_with($filename, '.gz'): + return 'gzip'; + default: + return null; + } + } + /** * Returns the command line for compressing the dump file. * diff --git a/src/N98/Magento/Command/Database/Compressor/LZ4.php b/src/N98/Magento/Command/Database/Compressor/LZ4.php new file mode 100644 index 000000000..ccd528205 --- /dev/null +++ b/src/N98/Magento/Command/Database/Compressor/LZ4.php @@ -0,0 +1,89 @@ +hasPipeViewer()) { + return 'pv -cN lz4 ' . escapeshellarg($fileName) . ' | lz4 -d | pv -cN mysql | ' . $command; + } + + return 'lz4 -dc < ' . escapeshellarg($fileName) . ' | ' . $command; + } else { + if ($this->hasPipeViewer()) { + return 'pv -cN tar -zxf ' . escapeshellarg($fileName) . ' && pv -cN mysql | ' . $command; + } + + return 'tar -zxf ' . escapeshellarg($fileName) . ' -C ' . dirname($fileName) . ' && ' . $command . ' < ' + . escapeshellarg(substr($fileName, 0, -4)); + } + } + + /** + * Returns the file name for the compressed dump file. + * + * @param string $fileName + * @param bool $pipe + * @return string + */ + public function getFileName($fileName, $pipe = true) + { + if ($fileName === null) { + $fileName = ''; + } + + if (!strlen($fileName)) { + return $fileName; + } + + if ($pipe) { + if (substr($fileName, -4, 4) === '.lz4') { + return $fileName; + } elseif (substr($fileName, -4, 4) === '.sql') { + $fileName .= '.lz4'; + } else { + $fileName .= '.sql.lz4'; + } + } elseif (substr($fileName, -8, 8) === '.tar.lz4') { + return $fileName; + } else { + $fileName .= '.tar.lz4'; + } + + return $fileName; + } +} diff --git a/src/N98/Magento/Command/Database/Compressor/Zstandard.php b/src/N98/Magento/Command/Database/Compressor/Zstandard.php new file mode 100644 index 000000000..dd1c4ef51 --- /dev/null +++ b/src/N98/Magento/Command/Database/Compressor/Zstandard.php @@ -0,0 +1,110 @@ +compressionLevel = (int)$input->getOption('zstd-level'); + $this->extraArgs = (string)$input->getOption('zstd-extra-args'); + + parent::__construct($input); + } + + /** + * Returns the command line for compressing the dump file. + * + * @param string $command + * @param bool $pipe + * @return string + */ + public function getCompressingCommand($command, $pipe = true) + { + if ($pipe) { + return sprintf( + "%s | zstd -c -%s %s", + $command, + $this->compressionLevel, + $this->extraArgs, + ); + } else { + return sprintf( + "tar -I 'zstd %s -%s' -cf %s", + $this->extraArgs, + $this->compressionLevel, + $command, + ); + } + } + + /** + * Returns the command line for decompressing the dump file. + * + * @param string $command + * @param string $fileName Filename (shell argument escaped) + * @param bool $pipe + * @return string + */ + public function getDecompressingCommand($command, $fileName, $pipe = true) + { + if ($pipe) { + if ($this->hasPipeViewer()) { + return 'pv -cN zstd ' . escapeshellarg($fileName) . ' | zstd -d | pv -cN mysql | ' . $command; + } + + return 'zstd -dc < ' . escapeshellarg($fileName) . ' | ' . $command; + } else { + if ($this->hasPipeViewer()) { + return 'pv -cN tar -zxf ' . escapeshellarg($fileName) . ' && pv -cN mysql | ' . $command; + } + + return 'tar -zxf ' . escapeshellarg($fileName) . ' -C ' . dirname($fileName) . ' && ' . $command . ' < ' + . escapeshellarg(substr($fileName, 0, -4)); + } + } + + /** + * Returns the file name for the compressed dump file. + * + * @param string $fileName + * @param bool $pipe + * @return string + */ + public function getFileName($fileName, $pipe = true) + { + if ($fileName === null) { + $fileName = ''; + } + + if (!strlen($fileName)) { + return $fileName; + } + + if ($pipe) { + if (substr($fileName, -5, 5) === '.zstd') { + return $fileName; + } elseif (substr($fileName, -4, 4) === '.sql') { + $fileName .= '.zstd'; + } else { + $fileName .= '.sql.zstd'; + } + } elseif (substr($fileName, -9, 9) === '.tar.zstd') { + return $fileName; + } else { + $fileName .= '.tar.zstd'; + } + + return $fileName; + } +} diff --git a/src/N98/Magento/Command/Database/DumpCommand.php b/src/N98/Magento/Command/Database/DumpCommand.php index 5af22df36..76b0ad4ac 100644 --- a/src/N98/Magento/Command/Database/DumpCommand.php +++ b/src/N98/Magento/Command/Database/DumpCommand.php @@ -39,6 +39,8 @@ protected function configure() $this ->setName('db:dump') ->addArgument('filename', InputArgument::OPTIONAL, 'Dump filename') + ->addOption('zstd-level', null, InputOption::VALUE_OPTIONAL, '', 10) + ->addOption('zstd-extra-args', null, InputOption::VALUE_OPTIONAL, '', '') ->addOption( 'add-time', 't', @@ -296,7 +298,7 @@ protected function execute(InputInterface $input, OutputInterface $output) private function createExecs(InputInterface $input, OutputInterface $output) { $execs = new Execs('mysqldump'); - $execs->setCompression($input->getOption('compression')); + $execs->setCompression($input->getOption('compression'), $input); $execs->setFileName($this->getFileName($input, $output, $execs->getCompressor())); if (!$input->getOption('no-single-transaction')) { diff --git a/src/N98/Magento/Command/Database/Execs.php b/src/N98/Magento/Command/Database/Execs.php index e174f1bf6..0b4799fd6 100644 --- a/src/N98/Magento/Command/Database/Execs.php +++ b/src/N98/Magento/Command/Database/Execs.php @@ -6,6 +6,7 @@ namespace N98\Magento\Command\Database; use N98\Magento\Command\Database\Compressor\AbstractCompressor; +use Symfony\Component\Console\Input\InputInterface; /** * One or multiple commands to execute, with support for Compressors @@ -47,9 +48,9 @@ public function __construct($command = null) /** * @param string $type of compression: "gz" | "gzip" | "none" | null */ - public function setCompression($type) + public function setCompression($type, InputInterface $input) { - $this->compressor = AbstractCompressor::create($type); + $this->compressor = AbstractCompressor::create($type, $input); } /** diff --git a/src/N98/Magento/Command/Database/ImportCommand.php b/src/N98/Magento/Command/Database/ImportCommand.php index febe41ab5..bf282885a 100644 --- a/src/N98/Magento/Command/Database/ImportCommand.php +++ b/src/N98/Magento/Command/Database/ImportCommand.php @@ -24,7 +24,9 @@ protected function configure() $this ->setName('db:import') ->addArgument('filename', InputArgument::OPTIONAL, 'Dump filename') - ->addOption('compression', 'c', InputOption::VALUE_REQUIRED, 'The compression of the specified file') + ->addOption('compression', 'c', InputOption::VALUE_OPTIONAL, 'The compression of the specified file') + ->addOption('zstd-level', null, InputOption::VALUE_OPTIONAL, '', 10) + ->addOption('zstd-extra-args', null, InputOption::VALUE_OPTIONAL, '', '') ->addOption('only-command', null, InputOption::VALUE_NONE, 'Print only mysql command. Do not execute') ->addOption('only-if-empty', null, InputOption::VALUE_NONE, 'Imports only if database is empty') ->addOption( @@ -126,7 +128,17 @@ protected function execute(InputInterface $input, OutputInterface $output) $fileName = $this->checkFilename($input); - $compressor = AbstractCompressor::create($input->getOption('compression')); + if ($input->getOption('compression')) { + $compression = $input->getOption('compression'); + } else { + $compression = AbstractCompressor::tryGetCompressionType($fileName); + + if ($compression == null) { + throw new \RuntimeException("Could not guess compression type or the file is in a format that is not supported."); + } + } + + $compressor = AbstractCompressor::create($compression, $input); $exec = 'mysql '; if ($input->getOption('force')) { diff --git a/tests/N98/Magento/Command/Database/ExecsTest.php b/tests/N98/Magento/Command/Database/ExecsTest.php index 7c77ec961..a2d63b3a8 100644 --- a/tests/N98/Magento/Command/Database/ExecsTest.php +++ b/tests/N98/Magento/Command/Database/ExecsTest.php @@ -8,6 +8,7 @@ use N98\Magento\Command\Database\Compressor\AbstractCompressor; use N98\Magento\Command\Database\Compressor\Uncompressed; use PHPUnit\Framework\TestCase; +use Symfony\Component\Console\Input\InputInterface; /** * Class ExecsTest @@ -31,9 +32,10 @@ public function creation() */ public function facade() { + $input = $this->createMock(InputInterface::class); $execs = new Execs('foo'); $this->assertInstanceOf(Uncompressed::class, $execs->getCompressor()); - $execs->setCompression('gzip'); + $execs->setCompression('gzip', $input); $this->assertInstanceOf(AbstractCompressor::class, $execs->getCompressor()); $this->assertNull($execs->getFileName()); $execs->setFileName('output.sql');