diff --git a/.travis.yml b/.travis.yml index 2d76f658..5b120993 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ sudo: false env: global: - - COMPOSER_ROOT_VERSION=1.x-dev + - COMPOSER_ROOT_VERSION=1.5.x-dev - CORE_RELEASE=master matrix: diff --git a/composer.json b/composer.json index e111c746..a24345e9 100644 --- a/composer.json +++ b/composer.json @@ -33,7 +33,7 @@ }, "extra": { "branch-alias": { - "1.x-dev": "1.4.x-dev" + "1.x-dev": "1.5.x-dev" }, "installer-name": "silverstripe-assets" }, diff --git a/src/FileMigrationHelper.php b/src/FileMigrationHelper.php index ccdf4fe8..2f17ee20 100644 --- a/src/FileMigrationHelper.php +++ b/src/FileMigrationHelper.php @@ -2,6 +2,20 @@ namespace SilverStripe\Assets; +use Psr\Log\LoggerInterface; +use SilverStripe\Assets\Flysystem\FlysystemAssetStore; +use SilverStripe\Assets\Storage\AssetStore; +use SilverStripe\Core\Config\Config; +use SilverStripe\Core\Config\Configurable; +use SilverStripe\Core\Environment; +use SilverStripe\Core\Injector\Injectable; +use SilverStripe\Core\Injector\Injector; +use SilverStripe\ORM\DataList; +use SilverStripe\ORM\DataObject; +use SilverStripe\ORM\DataQuery; +use SilverStripe\ORM\DB; +use SilverStripe\Versioned\Versioned; + /** * Service to help migrate File dataobjects to the new APL. * @@ -11,4 +25,244 @@ */ class FileMigrationHelper extends \SilverStripe\Assets\Dev\Tasks\FileMigrationHelper { + use Injectable; + use Configurable; + + /** + * If a file fails to validate during migration, delete it. + * If set to false, the record will exist but will not be attached to any filesystem + * item anymore. + * + * @config + * @var bool + */ + private static $delete_invalid_files = true; + + /** + * Specifies the interval for every Nth file looped at which output will be logged. + * + * @config + * @var int + */ + private static $log_interval = 100; + + private static $dependencies = [ + 'logger' => '%$' . LoggerInterface::class, + ]; + + /** @var LoggerInterface|null */ + private $logger; + + /** + * @param LoggerInterface $logger + */ + public function setLogger(LoggerInterface $logger) + { + $this->logger = $logger; + } + + /** + * Perform migration + * + * @param string $base Absolute base path (parent of assets folder). Will default to PUBLIC_PATH + * @return int Number of files successfully migrated + * @throws \Exception + */ + public function run($base = null) + { + if (empty($base)) { + $base = PUBLIC_PATH; + } + + // Check if the File dataobject has a "Filename" field. + // If not, cannot migrate + /** @skipUpgrade */ + if (!DB::get_schema()->hasField('File', 'Filename')) { + return 0; + } + + // Set max time and memory limit + Environment::increaseTimeLimitTo(); + Environment::increaseMemoryLimitTo(); + + // Loop over all files + $processedCount = $migratedCount = 0; + $originalState = null; + if (class_exists(Versioned::class)) { + $originalState = Versioned::get_reading_mode(); + Versioned::set_stage(Versioned::DRAFT); + } + + $query = $this->getFileQuery(); + $total = $query->count(); + foreach ($query as $file) { + // Bypass the accessor and the filename from the column + $filename = $file->getField('Filename'); + + $success = $this->migrateFile($base, $file, $filename); + + if ($processedCount % $this->config()->get('log_interval') === 0) { + if ($this->logger) { + $this->logger->info("Iterated $processedCount out of $total files. Migrated $migratedCount files."); + } + } + + $processedCount++; + if ($success) { + $migratedCount++; + } + } + if (class_exists(Versioned::class)) { + Versioned::set_reading_mode($originalState); + } + return $migratedCount; + } + + /** + * Migrate a single file + * + * @param string $base Absolute base path (parent of assets folder) + * @param File $file + * @param string $legacyFilename + * @return bool True if this file is imported successfully + * @throws \SilverStripe\ORM\ValidationException + */ + protected function migrateFile($base, File $file, $legacyFilename) + { + $useLegacyFilenames = Config::inst()->get(FlysystemAssetStore::class, 'legacy_filenames'); + + // Make sure this legacy file actually exists + $path = $base . '/' . $legacyFilename; + if (!file_exists($path)) { + if ($this->logger) { + $this->logger->warning("$legacyFilename not migrated because the file does not exist ($path)"); + } + return false; + } + + // Remove this file if it violates allowed_extensions + $allowed = array_filter(File::getAllowedExtensions()); + $extension = strtolower($file->getExtension()); + if (!in_array($extension, $allowed)) { + if ($this->config()->get('delete_invalid_files')) { + $file->delete(); + } + if ($this->logger) { + $this->logger->warning("$legacyFilename not migrated because the extension $extension is not a valid extension"); + } + return false; + } + + // Fix file classname if it has a classname that's incompatible with it's extention + if (!$this->validateFileClassname($file, $extension)) { + // We disable validation (if it is enabled) so that we are able to write a corrected + // classname, once that is changed we re-enable it for subsequent writes + $validationEnabled = DataObject::Config()->get('validation_enabled'); + if ($validationEnabled) { + DataObject::Config()->set('validation_enabled', false); + } + $destinationClass = $file->get_class_for_file_extension($extension); + $file = $file->newClassInstance($destinationClass); + $fileID = $file->write(); + if ($validationEnabled) { + DataObject::Config()->set('validation_enabled', true); + } + $file = File::get_by_id($fileID); + } + + // Copy local file into this filesystem + $filename = $file->generateFilename(); + $result = $file->setFromLocalFile( + $path, + $filename, + null, + null, + [ + 'conflict' => $useLegacyFilenames ? + AssetStore::CONFLICT_USE_EXISTING : + AssetStore::CONFLICT_OVERWRITE + ] + ); + + // Move file if the APL changes filename value + if ($result['Filename'] !== $filename) { + $file->setFilename($result['Filename']); + } + + // Save and publish + $file->write(); + if (class_exists(Versioned::class)) { + $file->copyVersionToStage(Versioned::DRAFT, Versioned::LIVE); + } + + if (!$useLegacyFilenames) { + // removing the legacy file since it has been migrated now and not using legacy filenames + $removed = unlink($path); + if (!$removed) { + if ($this->logger) { + $this->logger->warning("$legacyFilename was migrated, but failed to remove the legacy file ($path)"); + } + } + return $removed; + } + return true; + } + + /** + * Check if a file's classname is compatible with it's extension + * + * @param File $file + * @param string $extension + * @return bool + */ + protected function validateFileClassname($file, $extension) + { + $destinationClass = $file->get_class_for_file_extension($extension); + return $file->ClassName === $destinationClass; + } + + /** + * Get list of File dataobjects to import + * + * @return DataList + * @throws \Exception + */ + protected function getFileQuery() + { + $table = DataObject::singleton(File::class)->baseTable(); + // Select all records which have a Filename value, but not FileFilename. + /** @skipUpgrade */ + return File::get() + ->exclude('ClassName', [Folder::class, 'Folder']) + ->filter('FileFilename', array('', null)) + ->where(sprintf( + '"%s"."Filename" IS NOT NULL AND "%s"."Filename" != \'\'', + $table, + $table + )) // Non-orm field + ->alterDataQuery(function (DataQuery $query) use ($table) { + return $query->addSelectFromTable($table, ['Filename']); + }); + } + + /** + * Get map of File IDs to legacy filenames + * + * @deprecated 4.4.0 + * @return array + * @throws \Exception + */ + protected function getFilenameArray() + { + $table = DataObject::singleton(File::class)->baseTable(); + // Convert original query, ensuring the legacy "Filename" is included in the result + /** @skipUpgrade */ + return $this + ->getFileQuery() + ->dataQuery() + ->selectFromTable($table, ['ID', 'Filename']) + ->execute() + ->map(); // map ID to Filename + } + } diff --git a/src/Flysystem/FlysystemAssetStore.php b/src/Flysystem/FlysystemAssetStore.php index f8b11193..57f2eb1e 100644 --- a/src/Flysystem/FlysystemAssetStore.php +++ b/src/Flysystem/FlysystemAssetStore.php @@ -312,10 +312,12 @@ private function applyToFileOnFilesystem(callable $callable, ParsedFileID $parse // Let's try validating the hash of our file if ($parsedFileID->getHash()) { $mainFileID = $strategy->buildFileID($strategy->stripVariant($parsedFileID)); + if (!$fs->has($mainFileID)) { // The main file doesn't exists ... this is kind of weird. continue; } + $stream = $fs->readStream($mainFileID); if (!$this->validateStreamHash($stream, $parsedFileID->getHash())) { continue; @@ -603,6 +605,7 @@ function (ParsedFileID $parsedFileID, Filesystem $fs, FileResolutionStrategy $st $destination = $strategy->buildFileID( $destParsedFileID->setVariant($originParsedFileID->getVariant()) ); + if ($origin !== $destination) { if ($fs->has($destination)) { $fs->delete($origin); @@ -611,6 +614,7 @@ function (ParsedFileID $parsedFileID, Filesystem $fs, FileResolutionStrategy $st } $this->truncateDirectory(dirname($origin), $fs); } + } // Build and parsed non-variant file ID so we can figure out what the new name file name is @@ -760,7 +764,12 @@ public function publish($filename, $hash) ); } - + /** + * Similar to publish, only any existing files that would be overriden by publishing will be moved back to the + * protected store. + * @param $filename + * @param $hash + */ public function swapPublish($filename, $hash) { if ($this->getVisibility($filename, $hash) === AssetStore::VISIBILITY_PUBLIC) { diff --git a/tests/php/Flysystem/FlysystemAssetStoreTest.php b/tests/php/Flysystem/FlysystemAssetStoreTest.php index 15d478b7..66dd7936 100644 --- a/tests/php/Flysystem/FlysystemAssetStoreTest.php +++ b/tests/php/Flysystem/FlysystemAssetStoreTest.php @@ -173,4 +173,18 @@ public function testPublicResolutionStrategy() $strategy = $assetStore->getPublicResolutionStrategy(); $this->assertEquals($expected, $strategy); } + + public function testProtectedResolutionStrategy() + { + $assetStore = new FlysystemAssetStore(); + $strategy = $assetStore->getProtectedResolutionStrategy(); + $expected = Injector::inst()->get(FileResolutionStrategy::class . '.protected'); + $this->assertEquals($expected, $strategy); + + $expected = new FileIDHelperResolutionStrategy(); + $assetStore->setProtectedResolutionStrategy($expected); + $strategy = $assetStore->getProtectedResolutionStrategy(); + $this->assertEquals($expected, $strategy); + } + } diff --git a/tests/php/Storage/AssetStoreTest.php b/tests/php/Storage/AssetStoreTest.php index b700ec33..3d411cf1 100644 --- a/tests/php/Storage/AssetStoreTest.php +++ b/tests/php/Storage/AssetStoreTest.php @@ -894,7 +894,6 @@ public function testVariantWriteNextToFile( $this->assertTrue($fs->has($expectedVariantPath)); } - public function listOfFilesToNormalise() { $public = AssetStore::VISIBILITY_PUBLIC;