Merge pull request shlinkio#24 from acelaya-forks/feature/csv-import

Feature/csv import

CHANGELOG.md

@@ -4,12 +4,14 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org).
 
-## [Unreleased]
+## [2.2.0] - 2021-02-06
 ### Added
 * [#21](https://github.com/shlinkio/shlink-importer/issues/21) Added support to import URL `title` prop.
+* [#5](https://github.com/shlinkio/shlink-importer/issues/5) Added support to import from a standard CSV file.
 
 ### Changed
 * Migrated build to Github Actions.
+* [#23](https://github.com/shlinkio/shlink-importer/issues/23) Increased required MSI to 75%.
 
 ### Deprecated
 * *Nothing*

README.md

@@ -16,7 +16,15 @@ This module can be installed using composer:
 
 ## Supported import sources
 
-* Bit.ly
+#### Bit.ly
+
+It imports using the API v4. The only required param is an [access token](https://bitly.is/accesstoken).
+
+#### Standard CSV
+
+It parses a CSV file with the `Long URL` and `Short code` columns. It can optionally contain `Domain`, `Title` and `Tags`, being the latter a pipe-separated list of items (`foo|bar|baz`).
+
+Column names can have spaces and have any combination of upper and lowercase.
 
 ## Usage
 

composer.json

@@ -14,10 +14,11 @@
     "require": {
         "php": "^7.4 || ^8.0",
         "ext-json": "*",
-        "laminas/laminas-servicemanager": "^3.4",
-        "lstrojny/functional-php": "^1.14",
+        "laminas/laminas-servicemanager": "^3.6.4",
+        "league/csv": "^9.6",
+        "lstrojny/functional-php": "^1.15",
         "shlinkio/shlink-config": "^1.0",
-        "symfony/console": "^5.1"
+        "symfony/console": "^5.2"
     },
     "require-dev": {
         "guzzlehttp/guzzle": "^7.2",
@@ -57,7 +58,7 @@
         "test": "phpdbg -qrr vendor/bin/phpunit --order-by=random --testdox --colors=always",
         "test:ci": "@test --coverage-clover=build/clover.xml --coverage-xml=build/coverage-xml --log-junit=build/junit.xml",
         "test:pretty": "@test --coverage-html build/coverage-html",
-        "infect": "infection --threads=4 --min-msi=70 --log-verbosity=default --only-covered",
+        "infect": "infection --threads=4 --min-msi=75 --log-verbosity=default --only-covered",
         "infect:ci": "@infect --coverage=build --skip-initial-tests",
         "infect:show": "@infect --show-mutations",
         "infect:show:ci": "@infect --show-mutations --coverage=build",

config/dependencies.config.php

@@ -33,27 +33,31 @@
     'cli' => [
         'importer_strategies' => [
             'factories' => [
-                Strategy\BitlyApiImporter::class => ConfigAbstractFactory::class,
+                Sources\Bitly\BitlyApiImporter::class => ConfigAbstractFactory::class,
+                Sources\Csv\CsvImporter::class => InvokableFactory::class,
             ],
 
             'aliases' => [
-                Strategy\ImportSources::BITLY => Strategy\BitlyApiImporter::class,
+                Sources\ImportSources::BITLY => Sources\Bitly\BitlyApiImporter::class,
+                Sources\ImportSources::CSV => Sources\Csv\CsvImporter::class,
             ],
         ],
 
         'params_console_helpers' => [
             'factories' => [
-                Params\ConsoleHelper\BitlyApiParamsConsoleHelper::class => InvokableFactory::class,
+                Sources\Bitly\BitlyApiParamsConsoleHelper::class => InvokableFactory::class,
+                Sources\Csv\CsvParamsConsoleHelper::class => InvokableFactory::class,
             ],
 
             'aliases' => [
-                Strategy\ImportSources::BITLY => Params\ConsoleHelper\BitlyApiParamsConsoleHelper::class,
+                Sources\ImportSources::BITLY => Sources\Bitly\BitlyApiParamsConsoleHelper::class,
+                Sources\ImportSources::CSV => Sources\Csv\CsvParamsConsoleHelper::class,
             ],
         ],
     ],
 
     ConfigAbstractFactory::class => [
-        Strategy\BitlyApiImporter::class => [ClientInterface::class, RequestFactoryInterface::class],
+        Sources\Bitly\BitlyApiImporter::class => [ClientInterface::class, RequestFactoryInterface::class],
         Command\ImportCommand::class => [
             Strategy\ImporterStrategyManager::class,
             Params\ConsoleHelper\ConsoleHelperManager::class,

src/Command/ImportCommand.php

@@ -9,9 +9,9 @@
 use Shlinkio\Shlink\Importer\ImportedLinksProcessorInterface;
 use Shlinkio\Shlink\Importer\Params\ConsoleHelper\ConsoleHelperManagerInterface;
 use Shlinkio\Shlink\Importer\Params\ConsoleHelper\ParamsConsoleHelperInterface;
+use Shlinkio\Shlink\Importer\Sources\ImportSources;
 use Shlinkio\Shlink\Importer\Strategy\ImporterStrategyInterface;
 use Shlinkio\Shlink\Importer\Strategy\ImporterStrategyManagerInterface;
-use Shlinkio\Shlink\Importer\Strategy\ImportSources;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
@@ -47,8 +47,8 @@ protected function configure(): void
             ->setName(self::NAME)
             ->setDescription('Allows to import short URLs from third party sources')
             ->addArgument('source', InputArgument::REQUIRED, sprintf(
-                'The source from which you want to import. Supported sources: ["%s"]',
-                implode('", "', ImportSources::getAll()),
+                'The source from which you want to import. Supported sources: [<info>%s</info>]',
+                implode('</info>, <info>', ImportSources::getAll()),
             ));
     }
 

src/Exception/InvalidSourceException.php

@@ -5,7 +5,7 @@
 namespace Shlinkio\Shlink\Importer\Exception;
 
 use InvalidArgumentException;
-use Shlinkio\Shlink\Importer\Strategy\ImportSources;
+use Shlinkio\Shlink\Importer\Sources\ImportSources;
 
 use function implode;
 use function sprintf;

src/Exception/BitlyApiException.php → src/Sources/Bitly/BitlyApiException.php

@@ -2,7 +2,9 @@
 
 declare(strict_types=1);
 
-namespace Shlinkio\Shlink\Importer\Exception;
+namespace Shlinkio\Shlink\Importer\Sources\Bitly;
+
+use Shlinkio\Shlink\Importer\Exception\ImportException;
 
 use function sprintf;
 

src/Strategy/BitlyApiImporter.php → src/Sources/Bitly/BitlyApiImporter.php

@@ -2,17 +2,19 @@
 
 declare(strict_types=1);
 
-namespace Shlinkio\Shlink\Importer\Strategy;
+namespace Shlinkio\Shlink\Importer\Sources\Bitly;
 
 use JsonException;
 use Psr\Http\Client\ClientExceptionInterface;
 use Psr\Http\Client\ClientInterface;
 use Psr\Http\Message\RequestFactoryInterface;
-use Shlinkio\Shlink\Importer\Exception\BitlyApiException;
 use Shlinkio\Shlink\Importer\Exception\ImportException;
-use Shlinkio\Shlink\Importer\Model\BitlyApiProgressTracker;
 use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl;
-use Shlinkio\Shlink\Importer\Params\BitlyApiParams;
+use Shlinkio\Shlink\Importer\Sources\Bitly\BitlyApiException;
+use Shlinkio\Shlink\Importer\Sources\Bitly\BitlyApiParams;
+use Shlinkio\Shlink\Importer\Sources\Bitly\BitlyApiProgressTracker;
+use Shlinkio\Shlink\Importer\Sources\ImportSources;
+use Shlinkio\Shlink\Importer\Strategy\ImporterStrategyInterface;
 use Shlinkio\Shlink\Importer\Util\DateHelpersTrait;
 use Throwable;
 

src/Params/BitlyApiParams.php → src/Sources/Bitly/BitlyApiParams.php

@@ -2,9 +2,9 @@
 
 declare(strict_types=1);
 
-namespace Shlinkio\Shlink\Importer\Params;
+namespace Shlinkio\Shlink\Importer\Sources\Bitly;
 
-class BitlyApiParams
+final class BitlyApiParams
 {
     private string $accessToken;
     private bool $importTags;

src/Params/ConsoleHelper/BitlyApiParamsConsoleHelper.php → src/Sources/Bitly/BitlyApiParamsConsoleHelper.php

@@ -2,8 +2,9 @@
 
 declare(strict_types=1);
 
-namespace Shlinkio\Shlink\Importer\Params\ConsoleHelper;
+namespace Shlinkio\Shlink\Importer\Sources\Bitly;
 
+use Shlinkio\Shlink\Importer\Params\ConsoleHelper\ParamsConsoleHelperInterface;
 use Symfony\Component\Console\Style\StyleInterface;
 
 class BitlyApiParamsConsoleHelper implements ParamsConsoleHelperInterface

src/Model/BitlyApiProgressTracker.php → src/Sources/Bitly/BitlyApiProgressTracker.php

@@ -2,11 +2,11 @@
 
 declare(strict_types=1);
 
-namespace Shlinkio\Shlink\Importer\Model;
+namespace Shlinkio\Shlink\Importer\Sources\Bitly;
 
 use DateInterval;
 use DateTimeImmutable;
-use Shlinkio\Shlink\Importer\Params\BitlyApiParams;
+use Shlinkio\Shlink\Importer\Sources\Bitly\BitlyApiParams;
 use Shlinkio\Shlink\Importer\Util\DateHelpersTrait;
 
 use function base64_decode;

src/Sources/Csv/CsvImporter.php

@@ -0,0 +1,89 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Importer\Sources\Csv;
+
+use DateTimeImmutable;
+use DateTimeInterface;
+use League\Csv\Reader;
+use Shlinkio\Shlink\Importer\Exception\ImportException;
+use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl;
+use Shlinkio\Shlink\Importer\Sources\ImportSources;
+use Shlinkio\Shlink\Importer\Strategy\ImporterStrategyInterface;
+
+use function array_filter;
+use function explode;
+use function Functional\reduce_left;
+use function str_replace;
+use function strtolower;
+use function trim;
+
+class CsvImporter implements ImporterStrategyInterface
+{
+    private const TAG_SEPARATOR = '|';
+
+    private ?DateTimeInterface $date;
+
+    public function __construct(?DateTimeInterface $date = null)
+    {
+        $this->date = $date;
+    }
+
+    /**
+     * @return ImportedShlinkUrl[]
+     * @throws ImportException
+     */
+    public function import(array $rawParams): iterable
+    {
+        $params = CsvParams::fromRawParams($rawParams);
+        $now = $this->date ?? new DateTimeImmutable();
+
+        $csvReader = Reader::createFromStream($params->stream())->setDelimiter($params->delimiter())
+                                                                ->setHeaderOffset(0);
+
+        foreach ($csvReader as $record) {
+            $record = $this->remapRecordHeaders($record);
+
+            yield new ImportedShlinkUrl(
+                ImportSources::CSV,
+                $record['longurl'],
+                $this->parseTags($record),
+                $now,
+                $this->nonEmptyValueOrNull($record, 'domain'),
+                $record['shortcode'],
+                $this->nonEmptyValueOrNull($record, 'title'),
+            );
+        }
+    }
+
+    private function remapRecordHeaders(array $record): array
+    {
+        return reduce_left($record, static function ($value, string $index, array $c, array $acc) {
+            $normalizedKey = strtolower(str_replace(' ', '', $index));
+            $acc[$normalizedKey] = $value;
+
+            return $acc;
+        }, []);
+    }
+
+    private function nonEmptyValueOrNull(array $record, string $key): ?string
+    {
+        $value = $record[$key] ?? null;
+        if (empty($value)) {
+            return null;
+        }
+
+        $trimmedValue = trim($value);
+        if (empty($trimmedValue)) {
+            return null;
+        }
+
+        return $trimmedValue;
+    }
+
+    private function parseTags(array $record): array
+    {
+        return array_filter(explode(self::TAG_SEPARATOR, $this->nonEmptyValueOrNull($record, 'tags') ?? ''));
+    }
+}

src/Sources/Csv/CsvParams.php

@@ -0,0 +1,38 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Importer\Sources\Csv;
+
+final class CsvParams
+{
+    /** @var resource */
+    private $stream;
+    private string $delimiter;
+
+    private function __construct()
+    {
+    }
+
+    public static function fromRawParams(array $params): self
+    {
+        $instance = new self();
+        $instance->delimiter = $params['delimiter'] ?? '';
+        $instance->stream = $params['stream'] ?? '';
+
+        return $instance;
+    }
+
+    /**
+     * @return resource
+     */
+    public function stream()
+    {
+        return $this->stream;
+    }
+
+    public function delimiter(): string
+    {
+        return $this->delimiter;
+    }
+}

src/Sources/Csv/CsvParamsConsoleHelper.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Importer\Sources\Csv;
+
+use Shlinkio\Shlink\Importer\Params\ConsoleHelper\ParamsConsoleHelperInterface;
+use Symfony\Component\Console\Style\StyleInterface;
+
+use function fopen;
+
+class CsvParamsConsoleHelper implements ParamsConsoleHelperInterface
+{
+    public function requestParams(StyleInterface $io): array
+    {
+        return [
+            'import_short_codes' => true,
+            'stream' => $io->ask('What\'s the path for the CSV file you want to import', null, [$this, 'pathToStream']),
+            'delimiter' => $io->choice('What\'s the delimiter used to separate values?', [
+                ',' => 'Comma',
+                ';' => 'Semicolon',
+            ], ','),
+        ];
+    }
+
+    /**
+     * @return resource
+     */
+    public function pathToStream(?string $value)
+    {
+        if (empty($value)) {
+            throw InvalidPathException::pathNotProvided();
+        }
+
+        $file = @fopen($value, 'rb');
+        if (! $file) {
+            throw InvalidPathException::pathIsNotFile($value);
+        }
+
+        return $file;
+    }
+}

src/Sources/Csv/InvalidPathException.php

@@ -0,0 +1,23 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Importer\Sources\Csv;
+
+use RuntimeException;
+use Shlinkio\Shlink\Importer\Exception\ExceptionInterface;
+
+use function sprintf;
+
+class InvalidPathException extends RuntimeException implements ExceptionInterface
+{
+    public static function pathNotProvided(): self
+    {
+        return new self('The path of the file is required.');
+    }
+
+    public static function pathIsNotFile(string $path): self
+    {
+        return new self(sprintf('The file "%s" does not seem to exist. Try another one.', $path));
+    }
+}