diff --git a/_config.php b/_config.php index 56fb6865..96cc4259 100644 --- a/_config.php +++ b/_config.php @@ -10,4 +10,4 @@ // Shortcode parser which only regenerates shortcodes ShortcodeParser::get('regenerator') - ->register('image', [ImageShortcodeProvider::class, 'regenerate_shortcode']); \ No newline at end of file + ->register('image', [ImageShortcodeProvider::class, 'regenerate_shortcode']); diff --git a/src/Dev/Tasks/FileMigrationHelper.php b/src/Dev/Tasks/FileMigrationHelper.php new file mode 100644 index 00000000..f52dda8e --- /dev/null +++ b/src/Dev/Tasks/FileMigrationHelper.php @@ -0,0 +1,351 @@ + '%$' . LoggerInterface::class, + ]; + + /** @var LoggerInterface */ + private $logger; + + /** @var FlysystemAssetStore */ + private $store; + + /** + * If a file fails to validate during migration, delete it. + * If set to false, the record will exist but will not be attached to any filesystem + * item anymore. + * + * @config + * @var bool + */ + private static $delete_invalid_files = true; + + public function __construct() + { + $this->logger = new NullLogger(); + } + + public function setLogger(LoggerInterface $logger) + { + $this->logger = $logger; + } + + /** + * Perform migration + * + * @param string $base Absolute base path (parent of assets folder). Will default to PUBLIC_PATH + * @return int Number of files successfully migrated + */ + public function run($base = null) + { + $this->store = Injector::inst()->get(AssetStore::class); + if (!$this->store instanceof AssetStore || !method_exists($this->store, 'normalisePath')) { + throw new LogicException( + 'FileMigrationHelper: Can not run if the default asset store does not have a `normalisePath` method.' + ); + } + + if (empty($base)) { + $base = PUBLIC_PATH; + } + + // Set max time and memory limit + Environment::increaseTimeLimitTo(); + Environment::increaseMemoryLimitTo(); + + $this->logger->info('MIGRATING SILVERSTRIPE 3 LEGACY FILES'); + $ss3Count = $this->ss3Migration($base); + + $this->logger->info('NORMALISE SILVERTSTRIPE 4 FILES'); + $ss4Count = 0; + if (class_exists(Versioned::class) && File::has_extension(Versioned::class)) { + Versioned::prepopulate_versionnumber_cache(File::class, Versioned::LIVE); + Versioned::prepopulate_versionnumber_cache(File::class, Versioned::DRAFT); + + $this->logger->info('Looking at live files'); + $ss4Count += Versioned::withVersionedMode(function () { + Versioned::set_stage(Versioned::LIVE); + return $this->normaliseAllFiles('on the live stage'); + }); + + $this->logger->info('Looking at draft files'); + $ss4Count += Versioned::withVersionedMode(function () { + Versioned::set_stage(Versioned::DRAFT); + return $this->normaliseAllFiles('on the draft stage', true); + }); + } else { + $ss4Count = $this->normaliseAllFiles(''); + } + + if ($ss4Count > 0) { + $this->logger->info(sprintf('%d files were normalised', $ss4Count)); + } else { + $this->logger->info('No files needed to be normalised'); + } + + return $ss3Count + $ss4Count; + } + + protected function ss3Migration($base) + { + // Check if the File dataobject has a "Filename" field. + // If not, cannot migrate + /** @skipUpgrade */ + if (!DB::get_schema()->hasField('File', 'Filename')) { + return 0; + } + + // Clean up SS3 files + $ss3Count = 0; + $originalState = null; + if (class_exists(Versioned::class) && File::has_extension(Versioned::class)) { + $originalState = Versioned::get_reading_mode(); + Versioned::set_stage(Versioned::DRAFT); + } + + foreach ($this->getLegacyFileQuery() as $file) { + // Bypass the accessor and the filename from the column + $filename = $file->getField('Filename'); + + $success = $this->migrateFile($base, $file, $filename); + if ($success) { + $ss3Count++; + } + } + + // Show summary of results + if ($ss3Count > 0) { + $this->logger->info(sprintf('%d legacy files have been migrated.', $ss3Count)); + } else { + $this->logger->info(sprintf('No SilverStripe 3 files have been migrated.', $ss3Count)); + } + + if (class_exists(Versioned::class)) { + Versioned::set_reading_mode($originalState); + } + + return $ss3Count; + } + + /** + * Migrate a single file + * + * @param string $base Absolute base path (parent of assets folder) + * @param File $file + * @param string $legacyFilename + * @return bool True if this file is imported successfully + */ + protected function migrateFile($base, File $file, $legacyFilename) + { + // Make sure this legacy file actually exists + $path = $base . '/' . $legacyFilename; + if (!file_exists($path)) { + return false; + } + + // Remove this file if it violates allowed_extensions + $allowed = array_filter(File::getAllowedExtensions()); + $extension = strtolower($file->getExtension()); + if (!in_array($extension, $allowed)) { + if ($this->config()->get('delete_invalid_files')) { + $file->delete(); + } + return false; + } + + // Fix file classname if it has a classname that's incompatible with it's extention + if (!$this->validateFileClassname($file, $extension)) { + // We disable validation (if it is enabled) so that we are able to write a corrected + // classname, once that is changed we re-enable it for subsequent writes + $validationEnabled = DataObject::Config()->get('validation_enabled'); + if ($validationEnabled) { + DataObject::Config()->set('validation_enabled', false); + } + $destinationClass = $file->get_class_for_file_extension($extension); + $file = $file->newClassInstance($destinationClass); + $fileID = $file->write(); + if ($validationEnabled) { + DataObject::Config()->set('validation_enabled', true); + } + $file = File::get_by_id($fileID); + } + + // Copy local file into this filesystem + $filename = $file->generateFilename(); + $results = $this->store->normalisePath($filename); + + // Move file if the APL changes filename value + $file->File->Filename = $results['Filename']; + $file->File->Hash = $results['Hash']; + + + // Save and publish + $file->write(); + if (class_exists(Versioned::class)) { + $file->copyVersionToStage(Versioned::DRAFT, Versioned::LIVE); + } + + $this->logger->info(sprintf('* SS3 file %s converted to SS4 format', $file->getFilename())); + if (!empty($results['Operations'])) { + foreach ($results['Operations'] as $origin => $destination) { + $this->logger->info(sprintf(' * %s moved to %s', $origin, $destination)); + } + } + + return true; + } + + /** + * Go through the list of files and make sure each one is at its default location + * @param string $stageString Complement of information to append to the confirmation message + * @param bool $skipIdenticalStages Whatever files that are already present on an other stage should be skipped + * @return int + */ + protected function normaliseAllFiles($stageString, $skipIdenticalStages = false) + { + $count = 0; + + $files = $this->chunk(File::get()->exclude('ClassName', [Folder::class, 'Folder'])); + + /** @var File $file */ + foreach ($files as $file) { + // There's no point doing those checks the live and draft file are the same + if ($skipIdenticalStages && !$file->stagesDiffer()) { + continue; + } + + if (!$this->store->exists($file->File->Filename, $file->File->Hash)) { + $this->logger->warning(sprintf( + 'Can not normalise %s / %s because it does not exists.', + $file->File->Filename, + $file->File->Hash + )); + continue; + } + + $results = $this->store->normalise($file->File->Filename, $file->File->Hash); + if ($results && !empty($results['Operations'])) { + $this->logger->info( + sprintf('* %s has been normalised %s', $file->getFilename(), $stageString) + ); + foreach ($results['Operations'] as $origin => $destination) { + $this->logger->info(sprintf(' * %s moved to %s', $origin, $destination)); + } + $count++; + } + } + + return $count; + } + + /** + * Check if a file's classname is compatible with it's extension + * + * @param File $file + * @param string $extension + * @return bool + */ + protected function validateFileClassname($file, $extension) + { + $destinationClass = $file->get_class_for_file_extension($extension); + return $file->ClassName === $destinationClass; + } + + /** + * Get list of File dataobjects to import + * + * @return DataList + */ + protected function getFileQuery() + { + $table = DataObject::singleton(File::class)->baseTable(); + // Select all records which have a Filename value, but not FileFilename. + /** @skipUpgrade */ + return File::get() + ->exclude('ClassName', [Folder::class, 'Folder']) + ->filter('FileFilename', array('', null)) + ->where(sprintf( + '"%s"."Filename" IS NOT NULL AND "%s"."Filename" != \'\'', + $table, + $table + )) // Non-orm field + ->alterDataQuery(function (DataQuery $query) use ($table) { + return $query->addSelectFromTable($table, ['Filename']); + }); + } + + protected function getLegacyFileQuery() + { + return $this->chunk($this->getFileQuery()); + } + + /** + * Split queries into smaller chunks to avoid using too much memory + * @param DataList $query + * @return Generator + */ + private function chunk(DataList $query) + { + $chunkSize = 100; + $greaterThanID = 0; + $query = $query->limit($chunkSize)->sort('ID'); + while ($chunk = $query->filter('ID:GreaterThan', $greaterThanID)) { + foreach ($chunk as $file) { + yield $file; + } + if ($chunk->count() == 0) { + break; + } + $greaterThanID = $file->ID; + } + } + + /** + * Get map of File IDs to legacy filenames + * + * @deprecated 4.4.0 + * @return array + */ + protected function getFilenameArray() + { + $table = DataObject::singleton(File::class)->baseTable(); + // Convert original query, ensuring the legacy "Filename" is included in the result + /** @skipUpgrade */ + return $this + ->getFileQuery() + ->dataQuery() + ->selectFromTable($table, ['ID', 'Filename']) + ->execute() + ->map(); // map ID to Filename + } +} diff --git a/src/FilenameParsing/FileIDHelperResolutionStrategy.php b/src/FilenameParsing/FileIDHelperResolutionStrategy.php index 0a8a2e2b..f1aa825b 100644 --- a/src/FilenameParsing/FileIDHelperResolutionStrategy.php +++ b/src/FilenameParsing/FileIDHelperResolutionStrategy.php @@ -370,21 +370,22 @@ public function findVariants($tuple, Filesystem $filesystem) $helpers = $this->getResolutionFileIDHelpers(); array_unshift($helpers, $this->getDefaultFileIDHelper()); + /** @var FileIDHelper[] $resolvableHelpers */ + $resolvableHelpers = []; + // Search for a helper that will allow us to find a file - /** @var FileIDHelper $helper */ - $helper = null; - foreach ($helpers as $helperToTry) { - $fileID = $helperToTry->buildFileID( + foreach ($helpers as $helper) { + $fileID = $helper->buildFileID( $parsedFileID->getFilename(), $parsedFileID->getHash() ); - if ($filesystem->has($fileID) && $this->validateHash($helperToTry, $parsedFileID, $filesystem)) { - $helper = $helperToTry; - break; + + if ($filesystem->has($fileID) && $this->validateHash($helper, $parsedFileID, $filesystem)) { + $resolvableHelpers[] = $helper; } } - if ($helper) { + foreach ($resolvableHelpers as $helper) { $folder = $helper->lookForVariantIn($parsedFileID); $possibleVariants = $filesystem->listContents($folder, true); foreach ($possibleVariants as $possibleVariant) { diff --git a/src/FilenameParsing/LegacyFileIDHelper.php b/src/FilenameParsing/LegacyFileIDHelper.php index 20723a48..e585c670 100644 --- a/src/FilenameParsing/LegacyFileIDHelper.php +++ b/src/FilenameParsing/LegacyFileIDHelper.php @@ -64,7 +64,7 @@ public function cleanFilename($filename) /** * @note LegacyFileIDHelper is meant to fail when parsing newer format fileIDs with a variant e.g.: - * `subfolder/abcdef7890/sam__resizeXYZ.jpg`. When parsing fileIDs without a variant, it should return the same + * `subfolder/abcdef7890/sam__resizeXYZ.jpg`. When parsing fileIDs without variant, it should return the same * results as natural paths. */ public function parseFileID($fileID) diff --git a/src/Flysystem/FlysystemAssetStore.php b/src/Flysystem/FlysystemAssetStore.php index 25534864..28eda16f 100644 --- a/src/Flysystem/FlysystemAssetStore.php +++ b/src/Flysystem/FlysystemAssetStore.php @@ -311,6 +311,12 @@ private function applyToFileOnFilesystem(callable $callable, ParsedFileID $parse // Let's try validating the hash of our file if ($parsedFileID->getHash()) { $mainFileID = $strategy->buildFileID($strategy->stripVariant($parsedFileID)); + + if (!$fs->has($mainFileID)) { + // The main file doesn't exists ... this is kind of weird. + continue; + } + $stream = $fs->readStream($mainFileID); if (!$this->validateStreamHash($stream, $parsedFileID->getHash())) { continue; @@ -598,8 +604,15 @@ function (ParsedFileID $parsedFileID, Filesystem $fs, FileResolutionStrategy $st $destination = $strategy->buildFileID( $destParsedFileID->setVariant($originParsedFileID->getVariant()) ); - $fs->rename($origin, $destination); - $this->truncateDirectory(dirname($origin), $fs); + + if ($origin !== $destination) { + if ($fs->has($destination)) { + $fs->delete($origin); + } else { + $fs->rename($origin, $destination); + } + $this->truncateDirectory(dirname($origin), $fs); + } } // Build and parsed non-variant file ID so we can figure out what the new name file name is @@ -629,7 +642,11 @@ function (ParsedFileID $pfid, Filesystem $fs, FileResolutionStrategy $strategy) foreach ($strategy->findVariants($pfid, $fs) as $variantParsedFileID) { $fromFileID = $variantParsedFileID->getFileID(); $toFileID = $strategy->buildFileID($variantParsedFileID->setFilename($newName)); - $fs->copy($fromFileID, $toFileID); + if ($fromFileID !== $toFileID) { + if (!$fs->has($toFileID)) { + $fs->copy($fromFileID, $toFileID); + } + } } return $pfid->setFilename($newName); @@ -656,8 +673,7 @@ protected function deleteFromFilesystem($fileID, Filesystem $filesystem) $deleted = true; } - // Truncate empty dirs - $this->truncateDirectory(dirname($fileID), $filesystem); + return $deleted; } @@ -694,11 +710,12 @@ protected function deleteFromFileStore(ParsedFileID $parsedFileID, Filesystem $f protected function truncateDirectory($dirname, Filesystem $filesystem) { if ($dirname - && ltrim(dirname($dirname), '.') + && ltrim($dirname, '.') && !$this->config()->get('keep_empty_dirs') && !$filesystem->listContents($dirname) ) { $filesystem->deleteDir($dirname); + $this->truncateDirectory(dirname($dirname), $filesystem); } } @@ -844,6 +861,7 @@ public function protect($filename, $hash) * @param string $fileID * @param Filesystem $from * @param Filesystem $to + * @deprecated 1.4.0 */ protected function moveBetweenFilesystems($fileID, Filesystem $from, Filesystem $to) { @@ -1533,4 +1551,51 @@ protected function createErrorResponse($code) return $response; } + + public function normalisePath($fileID) + { + return $this->applyToFileIDOnFilesystem( + function (...$args) { + return $this->normaliseToDefaultPath(...$args); + }, + $fileID + ); + } + + public function normalise($filename, $hash) + { + return $this->applyToFileOnFilesystem( + function (...$args) { + return $this->normaliseToDefaultPath(...$args); + }, + new ParsedFileID($filename, $hash) + ); + } + + /** + * Given a parsed file ID move the matching file and all its variant to the default position as defined by the + * provided startegy. + * @param ParsedFileID $pfid + * @param Filesystem $fs + * @param FileResolutionStrategy $strategy + * @return array List of new file names with the old name as the key + */ + private function normaliseToDefaultPath(ParsedFileID $pfid, Filesystem $fs, FileResolutionStrategy $strategy) + { + $ops = []; + foreach ($strategy->findVariants($pfid, $fs) as $variantPfid) { + $origin = $variantPfid->getFileID(); + $targetVariantFileID = $strategy->buildFileID($variantPfid); + if ($targetVariantFileID !== $origin) { + if ($fs->has($targetVariantFileID)) { + $fs->delete($origin); + } else { + $fs->rename($origin, $targetVariantFileID); + $ops[$origin] = $targetVariantFileID; + } + $this->truncateDirectory(dirname($origin), $fs); + } + } + return array_merge($pfid->getTuple(), ['Operations' => $ops]); + } } diff --git a/src/ImageManipulation.php b/src/ImageManipulation.php index aa4397b4..62912726 100644 --- a/src/ImageManipulation.php +++ b/src/ImageManipulation.php @@ -871,6 +871,8 @@ function (AssetStore $store, $filename, $hash, $variant) use ($callback) { $tuple = $store->setFromStream($result->getStream(), $filename, $hash, $variant); return [$tuple, $result]; } finally { + // Unload the Intervention Image resource so it can be garbaged collected + $res = $backend->setImageResource(null); gc_collect_cycles(); } } @@ -889,6 +891,8 @@ function (AssetStore $store, $filename, $hash, $variant) use ($callback) { return [$tuple, $result]; } finally { + // Unload the Intervention Image resource so it can be garbaged collected + $res = $backend->setImageResource(null); gc_collect_cycles(); } } diff --git a/src/Migration/FileMigrationHelper.php b/src/Migration/FileMigrationHelper.php index 387f7adf..1a355fa5 100644 --- a/src/Migration/FileMigrationHelper.php +++ b/src/Migration/FileMigrationHelper.php @@ -16,14 +16,18 @@ use SilverStripe\ORM\DataObject; use SilverStripe\ORM\DataQuery; use SilverStripe\ORM\DB; +use SilverStripe\ORM\ValidationException; use SilverStripe\Versioned\Versioned; +use SilverStripe\Assets\Dev\Tasks\FileMigrationHelper as BaseFileMigrationHelper; /** * Service to help migrate File dataobjects to the new APL. * * This service does not alter these records in such a way that prevents downgrading back to 3.x + * + * @deprecated 1.4.0 Use \SilverStripe\Assets\Dev\Tasks\FileMigrationHelper instead */ -class FileMigrationHelper +class FileMigrationHelper extends BaseFileMigrationHelper { use Injectable; use Configurable; @@ -44,7 +48,7 @@ class FileMigrationHelper * @config * @var int */ - private static $log_interval = 100; + private static $log_interval = 10; private static $dependencies = [ 'logger' => '%$' . LoggerInterface::class, @@ -98,7 +102,6 @@ public function run($base = null) foreach ($query as $file) { // Bypass the accessor and the filename from the column $filename = $file->getField('Filename'); - $success = $this->migrateFile($base, $file, $filename); if ($processedCount % $this->config()->get('log_interval') === 0) { @@ -140,20 +143,29 @@ protected function migrateFile($base, File $file, $legacyFilename) return false; } - // Remove this file if it violates allowed_extensions - $allowed = array_filter(File::getAllowedExtensions()); - $extension = strtolower($file->getExtension()); - if (!in_array($extension, $allowed)) { + $validationResult = $file->validate(); + if (!$validationResult->isValid()) { if ($this->config()->get('delete_invalid_files')) { $file->delete(); } if ($this->logger) { - $this->logger->warning("$legacyFilename not migrated because the extension $extension is not a valid extension"); + $messages = implode("\n\n", array_map(function ($msg) { + return $msg['message']; + }, $validationResult->getMessages())); + + $this->logger->warning( + sprintf( + "%s was not migrated because the file is not valid. More information: %s", + $legacyFilename, + $messages + ) + ); } return false; } // Fix file classname if it has a classname that's incompatible with it's extention + $extension = $file->getExtension(); if (!$this->validateFileClassname($file, $extension)) { // We disable validation (if it is enabled) so that we are able to write a corrected // classname, once that is changed we re-enable it for subsequent writes @@ -169,6 +181,16 @@ protected function migrateFile($base, File $file, $legacyFilename) } $file = File::get_by_id($fileID); } + + if (!is_readable($path)) { + if ($this->logger) { + $this->logger->warning(sprintf( + 'File at %s is not readable and could not be migrated.', + $path + )); + return false; + } + } // Copy local file into this filesystem $filename = $file->generateFilename(); @@ -190,7 +212,19 @@ protected function migrateFile($base, File $file, $legacyFilename) } // Save and publish - $file->write(); + try { + $file->write(); + } catch (ValidationException $e) { + if ($this->logger) { + $this->logger->error(sprintf( + "File %s could not be migrated due to an error. + This problem likely existed before the migration began. Error: %s", + $legacyFilename, + $e->getMessage() + )); + } + return false; + } if (class_exists(Versioned::class)) { $file->copyVersionToStage(Versioned::DRAFT, Versioned::LIVE); } @@ -207,7 +241,6 @@ protected function migrateFile($base, File $file, $legacyFilename) } return $removed; } - return true; } /** diff --git a/tests/php/Dev/Tasks/FileMigrationHelperTest.php b/tests/php/Dev/Tasks/FileMigrationHelperTest.php new file mode 100644 index 00000000..4875de84 --- /dev/null +++ b/tests/php/Dev/Tasks/FileMigrationHelperTest.php @@ -0,0 +1,170 @@ + array( + Extension::class, + ) + ); + + /** + * get the BASE_PATH for this test + * + * @return string + */ + protected function getBasePath() + { + // Note that the actual filesystem base is the 'assets' subdirectory within this + return ASSETS_PATH . '/FileMigrationHelperTest'; + } + + + public function setUp() + { + Config::nest(); // additional nesting here necessary + Config::modify()->merge(File::class, 'migrate_legacy_file', false); + parent::setUp(); + + // Set backend root to /FileMigrationHelperTest/assets + TestAssetStore::activate('FileMigrationHelperTest/assets'); + + // Ensure that each file has a local record file in this new assets base + $from = __DIR__ . '/../../ImageTest/test-image-low-quality.jpg'; + foreach (File::get()->exclude('ClassName', Folder::class) as $file) { + $dest = TestAssetStore::base_path() . '/' . $file->generateFilename(); + Filesystem::makeFolder(dirname($dest)); + copy($from, $dest); + } + + // Let's create some variants for our images + $from = __DIR__ . '/../../ImageTest/test-image-high-quality.jpg'; + foreach (Image::get() as $file) { + $dest = TestAssetStore::base_path() . '/' . $file->generateFilename(); + $dir = dirname($dest); + $basename = basename($dest); + Filesystem::makeFolder($dir . '/_resampled'); + Filesystem::makeFolder($dir . '/_resampled/resizeXYZ'); + Filesystem::makeFolder($dir . '/_resampled/resizeXYZ/scaleABC'); + copy($from, $dir . '/_resampled/resizeXYZ/' . $basename); + copy($from, $dir . '/_resampled/resizeXYZ/scaleABC/' . $basename); + } + } + + public function tearDown() + { + TestAssetStore::reset(); + Filesystem::removeFolder($this->getBasePath()); + parent::tearDown(); + Config::unnest(); + } + + /** + * Test file migration + */ + public function testMigration() + { + // Prior to migration, check that each file has empty Filename / Hash properties + foreach (File::get()->exclude('ClassName', Folder::class) as $file) { + $filename = $file->generateFilename(); + $this->assertNotEmpty($filename, "File {$file->Name} has a filename"); + $this->assertEmpty($file->File->getFilename(), "File {$file->Name} has no DBFile filename"); + $this->assertEmpty($file->File->getHash(), "File {$file->Name} has no hash"); + $this->assertFalse($file->exists(), "File with name {$file->Name} does not yet exist"); + $this->assertFalse($file->isPublished(), "File is not published yet"); + } + + // + $this->assertFileExists( + TestAssetStore::base_path() . '/ParentFolder/SubFolder/myfile.exe', + 'We should have an invalid file before going into the migration.' + ); + + // Do migration + $helper = new FileMigrationHelper(); + $result = $helper->run($this->getBasePath()); + $this->assertEquals(7, $result); + + // Test that each file exists + foreach (File::get()->exclude('ClassName', Folder::class) as $file) { + /** @var File $file */ + $expectedFilename = $file->generateFilename(); + $filename = $file->File->getFilename(); + $this->assertTrue($file->exists(), "File with name {$filename} exists"); + $this->assertNotEmpty($filename, "File {$file->Name} has a Filename"); + $this->assertEquals($expectedFilename, $filename, "File {$file->Name} has retained its Filename value"); + $this->assertEquals( + '33be1b95cba0358fe54e8b13532162d52f97421c', + $file->File->getHash(), + "File with name {$filename} has the correct hash" + ); + $this->assertTrue($file->isPublished(), "File is published after migration"); + $this->assertGreaterThan(0, $file->getAbsoluteSize()); + } + + // Test that our image variant got moved correctly + foreach (Image::get() as $file) { + $filename = TestAssetStore::base_path() . '/' . $file->getFilename(); + $dir = dirname($filename); + $filename = basename($filename); + $this->assertFileNotExists($dir . '/_resampled'); + $this->assertFileExists($dir . '/' . $filename); + + $filename = preg_replace('#^(.*)\.(.*)$#', '$1__resizeXYZ.$2', $filename); + $this->assertFileExists($dir . '/' . $filename); + + $filename = preg_replace('#^(.*)\.(.*)$#', '$1_scaleABC.$2', $filename); + $this->assertFileExists($dir . '/' . $filename); + } + + // Ensure that invalid file has been removed during migration + $invalidID = $this->idFromFixture(File::class, 'invalid'); + $this->assertNotEmpty($invalidID); + $this->assertNull(File::get()->byID($invalidID)); + + # TODO confirm if we should delete the physical invalid file as well +// $this->assertFileNotExists( +// TestAssetStore::base_path() . '/ParentFolder/SubFolder/myfile.exe' , +// 'Invalid file should have been removed by migration' +// ); + + // Ensure file with invalid filenames have been rename + /** @var File $badname */ + $badname = $this->objFromFixture(File::class, 'badname'); + $this->assertEquals( + 'ParentFolder/bad_name.zip', + $badname->getFilename(), + 'file names with invalid file name should have been cleaned up' + ); + + // SS2.4 considered PDFs to be images. We should convert that back to Regular files + $pdf = File::find('myimage.pdf'); + $this->assertEquals(File::class, $pdf->ClassName, 'Our PDF classnames should have been corrrected'); + } + + public function testMigrationWithLegacyFilenames() + { + Config::modify()->set(FlysystemAssetStore::class, 'legacy_filenames', true); + $this->testMigration(); + } +} diff --git a/tests/php/Migration/FileMigrationHelperTest.yml b/tests/php/Dev/Tasks/FileMigrationHelperTest.yml similarity index 74% rename from tests/php/Migration/FileMigrationHelperTest.yml rename to tests/php/Dev/Tasks/FileMigrationHelperTest.yml index 08b4de20..6172eb02 100644 --- a/tests/php/Migration/FileMigrationHelperTest.yml +++ b/tests/php/Dev/Tasks/FileMigrationHelperTest.yml @@ -10,15 +10,20 @@ SilverStripe\Assets\Image: image2: Name: myimage.jpg ParentID: =>SilverStripe\Assets\Folder.subfolder + fakePdf: + Name: myimage.pdf SilverStripe\Assets\File: file1: - Name: anotherfile.jpg + Name: anotherfile.txt file2: - Name: file.jpg + Name: file.doc ParentID: =>SilverStripe\Assets\Folder.parent file3: - Name: picture.jpg + Name: picture.pdf ParentID: =>SilverStripe\Assets\Folder.subfolder + badname: + Name: bad__name.zip + ParentID: =>SilverStripe\Assets\Folder.parent invalid: Name: myfile.exe ParentID: =>SilverStripe\Assets\Folder.subfolder diff --git a/tests/php/FileMigrationHelperTest/Extension.php b/tests/php/Dev/Tasks/FileMigrationHelperTest/Extension.php similarity index 88% rename from tests/php/FileMigrationHelperTest/Extension.php rename to tests/php/Dev/Tasks/FileMigrationHelperTest/Extension.php index 32c00500..ae6ab591 100644 --- a/tests/php/FileMigrationHelperTest/Extension.php +++ b/tests/php/Dev/Tasks/FileMigrationHelperTest/Extension.php @@ -1,6 +1,6 @@ get(AssetStore::class); + + $hashHelper = new HashFileIDHelper(); + $legacyHelper = new LegacyFileIDHelper(); + + $protected = new FileIDHelperResolutionStrategy(); + $protected->setVersionedStage(Versioned::DRAFT); + $protected->setDefaultFileIDHelper($hashHelper); + $protected->setResolutionFileIDHelpers([$hashHelper]); + + $store->setProtectedResolutionStrategy($protected); + + $public = new FileIDHelperResolutionStrategy(); + $public->setVersionedStage(Versioned::LIVE); + $public->setDefaultFileIDHelper($legacyHelper); + $public->setResolutionFileIDHelpers([$legacyHelper]); + + $store->setPublicResolutionStrategy($public); + } + + protected function defineDestinationStrategy() + { + /** @var FlysystemAssetStore $store */ + $store = Injector::inst()->get(AssetStore::class); + + $hashHelper = new HashFileIDHelper(); + $naturalPath = new NaturalFileIDHelper(); + $legacyHelper = new LegacyFileIDHelper(); + + $protected = new FileIDHelperResolutionStrategy(); + $protected->setVersionedStage(Versioned::DRAFT); + $protected->setDefaultFileIDHelper($hashHelper); + $protected->setResolutionFileIDHelpers([$hashHelper]); + + $store->setProtectedResolutionStrategy($protected); + + $public = new FileIDHelperResolutionStrategy(); + $public->setVersionedStage(Versioned::LIVE); + $public->setDefaultFileIDHelper($hashHelper); + $public->setResolutionFileIDHelpers([$hashHelper, $naturalPath, $legacyHelper]); + + $store->setPublicResolutionStrategy($public); + } +} diff --git a/tests/php/Dev/Tasks/SS4FileMigrationHelperTest.php b/tests/php/Dev/Tasks/SS4FileMigrationHelperTest.php new file mode 100644 index 00000000..06c05ee4 --- /dev/null +++ b/tests/php/Dev/Tasks/SS4FileMigrationHelperTest.php @@ -0,0 +1,310 @@ + array( + Extension::class, + ) + ); + + public function setUp() + { + Config::nest(); // additional nesting here necessary + Config::modify()->merge(File::class, 'migrate_legacy_file', false); + + // Set backend root to /FileMigrationHelperTest/assets + TestAssetStore::activate('FileMigrationHelperTest'); + $this->defineOriginStrategy(); + parent::setUp(); + + // Ensure that each file has a local record file in this new assets base + /** @var File $file */ + foreach (File::get()->filter('ClassName', File::class) as $file) { + $filename = $file->getFilename(); + + // Create an archive version of the file + DBDatetime::set_mock_now('2000-01-01 11:00:00'); + $file->setFromString('Archived content of ' . $filename, $filename); + $file->write(); + $file->publishSingle(); + DBDatetime::clear_mock_now(); + + // Publish a version of the file + $file->setFromString('Published content of ' . $filename, $filename); + $file->write(); + $file->publishSingle(Versioned::DRAFT, Versioned::LIVE); + + // Create a draft of the file + $file->setFromString('Draft content of ' . $filename, $filename); + $file->write(); + } + + // Let's create some variants for our images + /** @var Image $image */ + foreach (Image::get() as $image) { + $filename = $image->getFilename(); + + // Create an archive version of our image with a thumbnail + DBDatetime::set_mock_now('2000-01-01 11:00:00'); + $stream = $this->generateImage('Archived', $filename)->stream($image->getExtension()); + $image->setFromStream(StreamWrapper::getResource($stream), $filename); + $image->write(); + $image->CMSThumbnail(); + $image->publishSingle(); + DBDatetime::clear_mock_now(); + + // Publish a live version of our image with a thumbnail + $stream = $this->generateImage('Published', $filename)->stream($image->getExtension()); + $image->setFromStream(StreamWrapper::getResource($stream), $filename); + $image->write(); + $image->CMSThumbnail(); + $image->publishSingle(); + + // Create a draft version of our images with a thumbnail + $stream = $this->generateImage('Draft', $filename)->stream($image->getExtension()); + $image->setFromStream(StreamWrapper::getResource($stream), $filename); + $image->CMSThumbnail(); + $image->write(); + } + + $this->defineDestinationStrategy(); + } + + /** + * Generate a placeholder image + * @param string $targetedStage + * @param string $filename + * @return \Intervention\Image\Image + */ + private function generateImage($targetedStage, $filename) + { + /** @var ImageManager $imageManager */ + $imageManager = Injector::inst()->create(ImageManager::class); + return $imageManager + ->canvas(400, 300, '#142237') + ->text($targetedStage, 20, 170, function (AbstractFont $font) { + $font->color('#44C8F5'); + })->text($filename, 20, 185, function (AbstractFont $font) { + $font->color('#ffffff'); + })->rectangle(20, 200, 100, 202, function (AbstractShape $shape) { + $shape->background('#DA1052'); + }); + } + + /** + * Called by set up before creating all the fixture entries. Defines the original startegies for the assets store. + */ + protected function defineOriginStrategy() + { + /** @var FlysystemAssetStore $store */ + $store = Injector::inst()->get(AssetStore::class); + + $hashHelper = new HashFileIDHelper(); + + $protected = new FileIDHelperResolutionStrategy(); + $protected->setVersionedStage(Versioned::DRAFT); + $protected->setDefaultFileIDHelper($hashHelper); + $protected->setResolutionFileIDHelpers([$hashHelper]); + + $store->setProtectedResolutionStrategy($protected); + + $public = new FileIDHelperResolutionStrategy(); + $public->setVersionedStage(Versioned::LIVE); + $public->setDefaultFileIDHelper($hashHelper); + $public->setResolutionFileIDHelpers([$hashHelper]); + + $store->setPublicResolutionStrategy($public); + } + + /** + * Called by set up after creating all the fixture entries. Defines the targeted strategies that the + * FileMigrationHelper should move the files to. + */ + protected function defineDestinationStrategy() + { + /** @var FlysystemAssetStore $store */ + $store = Injector::inst()->get(AssetStore::class); + + $hashHelper = new HashFileIDHelper(); + $naturalHelper = new NaturalFileIDHelper(); + + $protected = new FileIDHelperResolutionStrategy(); + $protected->setVersionedStage(Versioned::DRAFT); + $protected->setDefaultFileIDHelper($hashHelper); + $protected->setResolutionFileIDHelpers([$hashHelper, $naturalHelper]); + + $store->setProtectedResolutionStrategy($protected); + + $public = new FileIDHelperResolutionStrategy(); + $public->setVersionedStage(Versioned::LIVE); + $public->setDefaultFileIDHelper($naturalHelper); + $public->setResolutionFileIDHelpers([$hashHelper, $naturalHelper]); + + $store->setPublicResolutionStrategy($public); + } + + public function tearDown() + { + TestAssetStore::reset(); + parent::tearDown(); + Config::unnest(); + } + + public function testMigration() + { + $helper = new FileMigrationHelper(); + $result = $helper->run(TestAssetStore::base_path()); + + // Let's look at our draft files + Versioned::withVersionedMode(function () { + Versioned::set_stage(Versioned::DRAFT); + foreach (File::get()->filter('ClassName', File::class) as $file) { + $this->assertFileAt($file, AssetStore::VISIBILITY_PROTECTED, 'Draft'); + } + + foreach (Image::get() as $image) { + $this->assertImageAt($image, AssetStore::VISIBILITY_PROTECTED, 'Draft'); + } + }); + + // Let's look at our live files + Versioned::withVersionedMode(function () { + Versioned::set_stage(Versioned::LIVE); + + // There's one file with restricted views, the published version of this file will be protected + $restrictedFileID = $this->idFromFixture(File::class, 'restrictedViewFolder-file4'); + $this->lookAtRestrictedFile($restrictedFileID); + + /** @var File $file */ + foreach (File::get()->filter('ClassName', File::class)->exclude('ID', $restrictedFileID) as $file) { + $this->assertFileAt($file, AssetStore::VISIBILITY_PUBLIC, 'Published'); + } + + foreach (Image::get() as $image) { + $this->assertImageAt($image, AssetStore::VISIBILITY_PUBLIC, 'Published'); + } + }); + } + + /** + * Test that this restricted file is protected. This test is in its own method so that transition where this + * scenario can not exist can override it. + * @param $restrictedFileID + */ + protected function lookAtRestrictedFile($restrictedFileID) + { + $restrictedFile = File::get()->byID($restrictedFileID); + $this->assertFileAt($restrictedFile, AssetStore::VISIBILITY_PROTECTED, 'Published'); + } + + /** + * Convenience method to group a bunch of assertions about a regular files + * @param File $file + * @param string $visibility Expected visibility of the file + * @param string $stage Stage that we are testing, will appear in some error messages and in the expected content + */ + protected function assertFileAt(File $file, $visibility, $stage) + { + $ucVisibility = ucfirst($visibility); + $filename = $file->getFilename(); + $hash = $file->getHash(); + /** @var FlysystemAssetStore $store */ + $store = Injector::inst()->get(AssetStore::class); + /** @var Filesystem $fs */ + $fs = call_user_func([$store, "get{$ucVisibility}Filesystem"]); + /** @var FileResolutionStrategy $strategy */ + $strategy = call_user_func([$store, "get{$ucVisibility}ResolutionStrategy"]); + + $this->assertEquals( + $visibility, + $store->getVisibility($filename, $hash), + sprintf('%s version of %s should be %s', $stage, $filename, $visibility) + ); + $expectedURL = $strategy->buildFileID(new ParsedFileID($filename, $hash)); + $this->assertTrue( + $fs->has($expectedURL), + sprintf('%s version of %s should be on %s store under %s', $stage, $filename, $visibility, $expectedURL) + ); + $this->assertEquals( + sprintf('%s content of %s', $stage, $filename), + $fs->read($expectedURL), + sprintf('%s version of %s on %s store has wrong content', $stage, $filename, $visibility) + ); + } + + /** + * Convenience method to group a bunch of assertions about an image + * @param File $file + * @param string $visibility Expected visibility of the file + * @param string $stage Stage that we are testing, will appear in some error messages + */ + protected function assertImageAt(Image $file, $visibility, $stage) + { + $ucVisibility = ucfirst($visibility); + $filename = $file->getFilename(); + $hash = $file->getHash(); + $pfid = new ParsedFileID($filename, $hash); + + /** @var FlysystemAssetStore $store */ + $store = Injector::inst()->get(AssetStore::class); + + /** @var Filesystem $fs */ + $fs = call_user_func([$store, "get{$ucVisibility}Filesystem"]); + /** @var FileResolutionStrategy $strategy */ + $strategy = call_user_func([$store, "get{$ucVisibility}ResolutionStrategy"]); + + $this->assertEquals( + $visibility, + $store->getVisibility($filename, $hash), + sprintf('%s version of %s should be %s', $stage, $filename, $visibility) + ); + + $expectedURL = $strategy->buildFileID($pfid); + $this->assertTrue( + $fs->has($expectedURL), + sprintf('%s version of %s should be on %s store under %s', $stage, $filename, $visibility, $expectedURL) + ); + $expectedURL = $strategy->buildFileID($pfid->setVariant('FillWzEwMCwxMDBd')); + $this->assertTrue( + $fs->has($expectedURL), + sprintf('%s thumbnail of %s should be on %s store under %s', $stage, $filename, $visibility, $expectedURL) + ); + } +} diff --git a/tests/php/Dev/Tasks/SS4KeepArchivedFileMigrationHelperTest.php b/tests/php/Dev/Tasks/SS4KeepArchivedFileMigrationHelperTest.php new file mode 100644 index 00000000..f83bda21 --- /dev/null +++ b/tests/php/Dev/Tasks/SS4KeepArchivedFileMigrationHelperTest.php @@ -0,0 +1,43 @@ +filter('ClassName', File::class) as $file) { + $this->assertFileAt($file, AssetStore::VISIBILITY_PROTECTED, 'Archived'); + } + + foreach (Image::get() as $image) { + $this->assertImageAt($image, AssetStore::VISIBILITY_PROTECTED, 'Archived'); + } + }); + } + + protected function defineOriginStrategy() + { + parent::defineOriginStrategy(); + + File::config()->set('keep_archived_assets', true); + } +} diff --git a/tests/php/Dev/Tasks/SS4LegacyFileMigrationHelperTest.php b/tests/php/Dev/Tasks/SS4LegacyFileMigrationHelperTest.php new file mode 100644 index 00000000..abaa3b66 --- /dev/null +++ b/tests/php/Dev/Tasks/SS4LegacyFileMigrationHelperTest.php @@ -0,0 +1,45 @@ +get(AssetStore::class); + + $naturalHelper = new NaturalFileIDHelper(); + + $protected = new FileIDHelperResolutionStrategy(); + $protected->setVersionedStage(Versioned::DRAFT); + $protected->setDefaultFileIDHelper($naturalHelper); + $protected->setResolutionFileIDHelpers([$naturalHelper]); + + $store->setProtectedResolutionStrategy($protected); + + $public = new FileIDHelperResolutionStrategy(); + $public->setVersionedStage(Versioned::LIVE); + $public->setDefaultFileIDHelper($naturalHelper); + $public->setResolutionFileIDHelpers([$naturalHelper]); + + $store->setPublicResolutionStrategy($public); + } + + protected function lookAtRestrictedFile($restrictedFileID) + { + // Legacy files names did not allow you to have a restricted file in draft and live simultanously + } +} diff --git a/tests/php/FileTest.yml b/tests/php/FileTest.yml index 15f89a46..b635ddc3 100644 --- a/tests/php/FileTest.yml +++ b/tests/php/FileTest.yml @@ -54,11 +54,11 @@ SilverStripe\Assets\Folder: Name: FileTest-folder1-subfolder1 ParentID: =>SilverStripe\Assets\Folder.folder1 restrictedFolder: - Name: FileTest-restricted-folder + Name: restricted-folder CanEditType: OnlyTheseUsers EditorGroups: =>SilverStripe\Security\Group.assetusers restrictedViewFolder: - Name: FileTest-restricted-view-folder + Name: restricted-view-folder CanViewType: OnlyTheseUsers ViewerGroups: =>SilverStripe\Security\Group.assetusers deep-folder: @@ -89,12 +89,12 @@ SilverStripe\Assets\File: Name: File1.txt ParentID: =>SilverStripe\Assets\Folder.folder1 restrictedFolder-file3: - FileFilename: restrictedFolder/File3.txt + FileFilename: restricted-folder/File3.txt FileHash: 55b443b60176235ef09801153cca4e6da7494a0c - Name: File1.txt + Name: File3.txt ParentID: =>SilverStripe\Assets\Folder.restrictedFolder restrictedViewFolder-file4: - FileFilename: restrictedViewFolder/File4.txt + FileFilename: restricted-view-folder/File4.txt FileHash: 55b443b60176235ef09801153cca4e6da7494a0c Name: File4.txt ParentID: =>SilverStripe\Assets\Folder.restrictedViewFolder diff --git a/tests/php/Migration/FileMigrationHelperTest.php b/tests/php/Migration/FileMigrationHelperTest.php deleted file mode 100644 index 22a44579..00000000 --- a/tests/php/Migration/FileMigrationHelperTest.php +++ /dev/null @@ -1,116 +0,0 @@ - array( - Extension::class, - ) - ); - - /** - * get the BASE_PATH for this test - * - * @return string - */ - protected function getBasePath() - { - // Note that the actual filesystem base is the 'assets' subdirectory within this - return ASSETS_PATH . '/FileMigrationHelperTest'; - } - - - public function setUp() - { - parent::setUp(); - - // Set backend root to /FileMigrationHelperTest/assets - TestAssetStore::activate('FileMigrationHelperTest/assets'); - - // Ensure that each file has a local record file in this new assets base - $from = __DIR__ . '/../ImageTest/test-image-low-quality.jpg'; - foreach (File::get()->exclude('ClassName', Folder::class) as $file) { - $dest = TestAssetStore::base_path() . '/' . $file->generateFilename(); - Filesystem::makeFolder(dirname($dest)); - copy($from, $dest); - } - } - - public function tearDown() - { - TestAssetStore::reset(); - Filesystem::removeFolder($this->getBasePath()); - parent::tearDown(); - } - - /** - * Test file migration - */ - public function testMigration() - { - // TODO Fix file migration test by adjusting file migration logic to new behaviour - // added through https://github.com/silverstripe/silverstripe-versioned/issues/177 - -// // Prior to migration, check that each file has empty Filename / Hash properties -// foreach (File::get()->exclude('ClassName', Folder::class) as $file) { -// $filename = $file->generateFilename(); -// $this->assertNotEmpty($filename, "File {$file->Name} has a filename"); -// $this->assertEmpty($file->File->getFilename(), "File {$file->Name} has no DBFile filename"); -// $this->assertEmpty($file->File->getHash(), "File {$file->Name} has no hash"); -// $this->assertFalse($file->exists(), "File with name {$file->Name} does not yet exist"); -// $this->assertFalse($file->isPublished(), "File is not published yet"); -// } -// -// // Do migration -// $helper = new FileMigrationHelper(); -// $result = $helper->run($this->getBasePath()); -// $this->assertEquals(5, $result); -// -// // Test that each file exists -// foreach (File::get()->exclude('ClassName', Folder::class) as $file) { -// /** @var File $file */ -// $expectedFilename = $file->generateFilename(); -// $filename = $file->File->getFilename(); -// $this->assertTrue($file->exists(), "File with name {$filename} exists"); -// $this->assertNotEmpty($filename, "File {$file->Name} has a Filename"); -// $this->assertEquals($expectedFilename, $filename, "File {$file->Name} has retained its Filename value"); -// $this->assertEquals( -// '33be1b95cba0358fe54e8b13532162d52f97421c', -// $file->File->getHash(), -// "File with name {$filename} has the correct hash" -// ); -// $this->assertTrue($file->isPublished(), "File is published after migration"); -// $this->assertGreaterThan(0, $file->getAbsoluteSize()); -// } -// -// // Ensure that invalid file has been removed during migration -// $invalidID = $this->idFromFixture(File::class, 'invalid'); -// $this->assertNotEmpty($invalidID); -// $this->assertNull(File::get()->byID($invalidID)); - } - - public function testMigrationWithLegacyFilenames() - { - Config::modify()->set(FlysystemAssetStore::class, 'legacy_filenames', true); - $this->testMigration(); - } -} diff --git a/tests/php/ProtectedFileControllerTest.php b/tests/php/ProtectedFileControllerTest.php index cc8bfa4a..476007f4 100644 --- a/tests/php/ProtectedFileControllerTest.php +++ b/tests/php/ProtectedFileControllerTest.php @@ -188,8 +188,8 @@ public function testAccessControl() */ public function testFolders() { - $result = $this->get('assets/55b443b601'); - $this->assertResponseEquals(403, null, $result); + $result = $this->get('assets/does-not-exists'); + $this->assertResponseEquals(404, null, $result); $result = $this->get('assets/FileTest-subfolder'); $this->assertResponseEquals(403, null, $result); diff --git a/tests/php/Storage/AssetStoreTest.php b/tests/php/Storage/AssetStoreTest.php index 17f2760a..3d411cf1 100644 --- a/tests/php/Storage/AssetStoreTest.php +++ b/tests/php/Storage/AssetStoreTest.php @@ -3,10 +3,13 @@ namespace SilverStripe\Assets\Tests\Storage; use Exception; +use InvalidArgumentException; use League\Flysystem\Filesystem; use Silverstripe\Assets\Dev\TestAssetStore; use SilverStripe\Assets\File; +use SilverStripe\Assets\FilenameParsing\FileIDHelper; use SilverStripe\Assets\FilenameParsing\HashFileIDHelper; +use SilverStripe\Assets\FilenameParsing\LegacyFileIDHelper; use SilverStripe\Assets\FilenameParsing\NaturalFileIDHelper; use SilverStripe\Assets\FilenameParsing\ParsedFileID; use SilverStripe\Assets\Flysystem\FlysystemAssetStore; @@ -890,4 +893,234 @@ public function testVariantWriteNextToFile( $this->assertTrue($fs->has($expectedVariantPath)); } + + public function listOfFilesToNormalise() + { + $public = AssetStore::VISIBILITY_PUBLIC; + $protected = AssetStore::VISIBILITY_PROTECTED; + + /** @var FileIDHelper $hashHelper */ + $hashHelper = new HashFileIDHelper(); + $naturalHelper = new NaturalFileIDHelper(); + $legacyHelper = new LegacyFileIDHelper(); + + $content = "The quick brown fox jumps over the lazy dog."; + $hash = sha1($content); + $filename = 'folder/file.txt'; + $hashPath = $hashHelper->buildFileID($filename, $hash); + $legacyPath = $legacyHelper->buildFileID($filename, $hash); + + $variant = 'uppercase'; + $vContent = strtoupper($content); + $vNatural = $naturalHelper->buildFileID($filename, $hash, $variant); + $vHash = $hashHelper->buildFileID($filename, $hash, $variant); + $vLegacy = $legacyHelper->buildFileID($filename, $hash, $variant); + + return [ + // Main file only + [$public, [$filename => $content], $filename, $hash, [$filename], [$hashPath, dirname($hashPath)]], + [$public, [$hashPath => $content], $filename, $hash, [$filename], [$hashPath, dirname($hashPath)]], + [$protected, [$filename => $content], $filename, $hash, [$hashPath], [$filename]], + [$protected, [$hashPath => $content], $filename, $hash, [$hashPath], [$filename]], + + // Main File with variant + [ + $public, + [$filename => $content, $vNatural => $vContent], + $filename, + $hash, + [$filename, $vNatural], + [$hashPath, $vHash, dirname($hashPath)] + ], + [ + $public, + [$hashPath => $content, $vHash => $vContent], + $filename, + $hash, + [$filename, $vNatural], + [$hashPath, $vHash, dirname($hashPath)] + ], + [ + $protected, + [$filename => $content, $vNatural => $vContent], + $filename, + $hash, + [$hashPath, $vHash], + [$filename, $vNatural] + ], + [ + $protected, + [$hashPath => $content, $vHash => $vContent], + $filename, + $hash, + [$hashPath, $vHash], + [$filename, $vNatural] + ], + + // SS3 variants ... the protected store doesn't resolve SS3 paths + [ + $public, + [$legacyPath => $content, $vLegacy => $vContent], + $filename, + $hash, + [$filename, $vNatural], + [$vLegacy, dirname($vLegacy), dirname(dirname($vLegacy))] + ], + ]; + } + + /** + * @dataProvider listOfFilesToNormalise + * @param string $fsName + * @param array $contents + * @param string $filename + * @param string $hash + * @param array $expected + * @param array $notExpected + */ + public function testNormalise($fsName, array $contents, $filename, $hash, array $expected, array $notExpected = []) + { + $this->writeDummyFiles($fsName, $contents); + + $results = $this->getBackend()->normalise($filename, $hash); + + $this->assertEquals($filename, $results['Filename']); + $this->assertEquals($hash, $results['Hash']); + + $fs = $this->getFilesystem($fsName); + + foreach ($expected as $expectedFile) { + $this->assertTrue($fs->has($expectedFile), "$expectedFile should exists"); + $this->assertNotEmpty($fs->read($expectedFile), "$expectedFile should be non empty"); + } + + foreach ($notExpected as $notExpectedFile) { + $this->assertFalse($fs->has($notExpectedFile), "$notExpectedFile should NOT exists"); + } + } + + public function listOfFileIDsToNormalise() + { + $public = AssetStore::VISIBILITY_PUBLIC; + $protected = AssetStore::VISIBILITY_PROTECTED; + + /** @var FileIDHelper $hashHelper */ + $hashHelper = new HashFileIDHelper(); + $naturalHelper = new NaturalFileIDHelper(); + $legacyHelper = new LegacyFileIDHelper(); + + $content = "The quick brown fox jumps over the lazy dog."; + $hash = sha1($content); + $filename = 'folder/file.txt'; + $hashPath = $hashHelper->buildFileID($filename, $hash); + $legacyPath = $legacyHelper->buildFileID($filename, $hash); + + $variant = 'uppercase'; + $vContent = strtoupper($content); + $vNatural = $naturalHelper->buildFileID($filename, $hash, $variant); + $vHash = $hashHelper->buildFileID($filename, $hash, $variant); + $vLegacy = $legacyHelper->buildFileID($filename, $hash, $variant); + + return [ + // Main file only + [$public, [$filename => $content], $filename, [$filename], [$hashPath, dirname($hashPath)]], + [$public, [$hashPath => $content], $hashPath, [$filename], [$hashPath, dirname($hashPath)]], + [$protected, [$filename => $content], $filename, [$hashPath], [$filename]], + [$protected, [$hashPath => $content], $hashPath, [$hashPath], [$filename]], + + // Main File with variant + [ + $public, + [$filename => $content, $vNatural => $vContent], + $filename, + [$filename, $vNatural], + [$hashPath, $vHash, dirname($hashPath)] + ], + [ + $public, + [$hashPath => $content, $vHash => $vContent], + $hashPath, + [$filename, $vNatural], + [$hashPath, $vHash, dirname($hashPath)] + ], + [ + $protected, + [$filename => $content, $vNatural => $vContent], + $filename, + [$hashPath, $vHash], + [$filename, $vNatural] + ], + [ + $protected, + [$hashPath => $content, $vHash => $vContent], + $hashPath, + [$hashPath, $vHash], + [$filename, $vNatural] + ], + + // SS3 variants ... the protected store doesn't resolve SS3 paths + [ + $public, + [$legacyPath => $content, $vLegacy => $vContent], + $legacyPath, + [$filename, $vNatural], + [$vLegacy, dirname($vLegacy), dirname(dirname($vLegacy))] + ], + ]; + } + + /** + * @dataProvider listOfFileIDsToNormalise + * @param string $fsName + * @param array $contents + * @param string $fileID + * @param array $expected + * @param array $notExpected + */ + public function testNormalisePath($fsName, array $contents, $fileID, array $expected, array $notExpected = []) + { + $this->writeDummyFiles($fsName, $contents); + + $results = $this->getBackend()->normalisePath($fileID); + + $this->assertEquals('folder/file.txt', $results['Filename']); + $this->assertTrue( + strpos(sha1("The quick brown fox jumps over the lazy dog."), $results['Hash']) === 0 + ); + + $fs = $this->getFilesystem($fsName); + + foreach ($expected as $expectedFile) { + $this->assertTrue($fs->has($expectedFile), "$expectedFile should exists"); + $this->assertNotEmpty($fs->read($expectedFile), "$expectedFile should be non empty"); + } + + foreach ($notExpected as $notExpectedFile) { + $this->assertFalse($fs->has($notExpectedFile), "$notExpectedFile should NOT exists"); + } + } + + /** + * @param $fs + * @return Filesystem + */ + private function getFilesystem($fs) + { + switch (strtolower($fs)) { + case AssetStore::VISIBILITY_PUBLIC: + return $this->getBackend()->getPublicFilesystem(); + case AssetStore::VISIBILITY_PROTECTED: + return $this->getBackend()->getProtectedFilesystem(); + default: + new InvalidArgumentException('getFilesystem(): $fs must be an equal to a know visibility.'); + } + } + + private function writeDummyFiles($fsName, array $contents) + { + $fs = $this->getFilesystem($fsName); + foreach ($contents as $path => $content) { + $fs->write($path, $content); + } + } }