From e0c5ac80ac7b630ca9f53f0d9c057c70e4e5c7db Mon Sep 17 00:00:00 2001 From: Shukri Adams Date: Fri, 27 Sep 2024 14:07:19 +0200 Subject: [PATCH] test fixes --- src/Tetrifact.Core/ArchiveService.cs | 114 +++++++++--------- .../ArchiveService/GetPackageAsArchive.cs | 2 + .../Core/PackagePrune/Prune.cs | 10 +- .../Core/RepositoryCleanService/Clean.cs | 2 +- src/Tetrifact.Tests/Helpers/PackageHelper.cs | 2 +- 5 files changed, 70 insertions(+), 60 deletions(-) diff --git a/src/Tetrifact.Core/ArchiveService.cs b/src/Tetrifact.Core/ArchiveService.cs index 7eaebab..48bda2a 100644 --- a/src/Tetrifact.Core/ArchiveService.cs +++ b/src/Tetrifact.Core/ArchiveService.cs @@ -187,7 +187,7 @@ private async Task Archive7Zip(string packageId, string archivePathTemp) Manifest manifest = _indexReader.GetManifest(packageId); - // copy all files to single Direct + // copy all files to single Directory if (!Directory.Exists(tempDir2)) { _log.LogInformation($"Archive generation : gathering files for package {packageId}"); @@ -195,56 +195,58 @@ private async Task Archive7Zip(string packageId, string archivePathTemp) long cacheUpdateIncrements = manifest.Files.Count / 100; long counter = 0; - manifest.Files.AsParallel().WithDegreeOfParallelism(_settings.ArchiveCPUThreads).ForAll(async delegate (ManifestItem file) + manifest.Files.AsParallel().WithDegreeOfParallelism(_settings.ArchiveCPUThreads).ForAll(delegate (ManifestItem file) { - string targetPath = Path.Join(tempDir1, file.Path); - List knownDirectories = new List(); - if (manifest.IsCompressed) - { - GetFileResponse fileLookup = _indexReader.GetFile(file.Id); - if (fileLookup == null) - throw new Exception($"Failed to find expected package file {file.Id} - repository is likely corrupt"); - - using (var storageArchive = new ZipArchive(fileLookup.Content)) - { - ZipArchiveEntry storageArchiveEntry = storageArchive.Entries[0]; - using (var storageArchiveStream = storageArchiveEntry.Open()) - using (FileStream writeStream = new FileStream(targetPath, FileMode.Create)) - await StreamsHelper.CopyAsync(storageArchiveStream, writeStream, bufSize); - } - } - else - { - GetFileResponse fileLookup = _indexReader.GetFile(file.Id); - if (fileLookup == null) - throw new Exception($"Failed to find expected package file {file.Id}- repository is likely corrupt"); - - string dir = Path.GetDirectoryName(targetPath); - if (!knownDirectories.Contains(dir)) - { - Directory.CreateDirectory(dir); - knownDirectories.Add(dir); - } - - // is this the fastest way of copying? benchmark - using (Stream fileStream = fileLookup.Content) - using (FileStream writeStream = new FileStream(targetPath, FileMode.Create)) - await StreamsHelper.CopyAsync(fileStream, writeStream, bufSize); - } - - counter ++; - - if (cacheUpdateIncrements == 0 || counter % cacheUpdateIncrements == 0) - { - _log.LogInformation($"Gathering file {counter}/{manifest.Files.Count}, package \"{packageId}\"."); - string key = this.GetArchiveProgressKey(packageId); - ArchiveProgressInfo progress = _cache.Get(key); - if (progress != null) - { - progress.FileCopyProgress = ((decimal)counter / (decimal)manifest.Files.Count) * 100; - _cache.Set(key, progress); - } - } + string targetPath = Path.Join(tempDir1, file.Path); + List knownDirectories = new List(); + if (manifest.IsCompressed) + { + GetFileResponse fileLookup = _indexReader.GetFile(file.Id); + if (fileLookup == null) + throw new Exception($"Failed to find expected package file {file.Id} - repository is likely corrupt"); + + using (var storageArchive = new ZipArchive(fileLookup.Content)) + { + ZipArchiveEntry storageArchiveEntry = storageArchive.Entries[0]; + using (var storageArchiveStream = storageArchiveEntry.Open()) + using (FileStream writeStream = new FileStream(targetPath, FileMode.Create)) + // copy async not used here because cannot get this delegate to block asParallel, + StreamsHelper.Copy(storageArchiveStream, writeStream, bufSize); + } + } + else + { + GetFileResponse fileLookup = _indexReader.GetFile(file.Id); + if (fileLookup == null) + throw new Exception($"Failed to find expected package file {file.Id}- repository is likely corrupt"); + + string dir = Path.GetDirectoryName(targetPath); + if (!knownDirectories.Contains(dir)) + { + Directory.CreateDirectory(dir); + knownDirectories.Add(dir); + } + + // is this the fastest way of copying? benchmark + using (Stream fileStream = fileLookup.Content) + using (FileStream writeStream = new FileStream(targetPath, FileMode.Create)) + // copy async not used here because cannot get this delegate to block asParallel, + StreamsHelper.Copy(fileStream, writeStream, bufSize); + } + + counter++; + + if (cacheUpdateIncrements == 0 || counter % cacheUpdateIncrements == 0) + { + _log.LogInformation($"Gathering file {counter}/{manifest.Files.Count}, package \"{packageId}\"."); + string key = this.GetArchiveProgressKey(packageId); + ArchiveProgressInfo progress = _cache.Get(key); + if (progress != null) + { + progress.FileCopyProgress = ((decimal)counter / (decimal)manifest.Files.Count) * 100; + _cache.Set(key, progress); + } + } }); Directory.Move(tempDir1, tempDir2); @@ -282,7 +284,7 @@ private async Task Archive7Zip(string packageId, string archivePathTemp) } } - private async Task ArchiveDefaultMode(string packageId, string archivePathTemp) + private async Task ArchiveDotNetZip(string packageId, string archivePathTemp) { DateTime compressStart = DateTime.Now; @@ -422,12 +424,12 @@ public async Task CreateArchive(string packageId) try { - if (string.IsNullOrEmpty(_settings.ExternaArchivingExecutable)) - await ArchiveDefaultMode(packageId, archivePathTemp); - else + if (_settings.ArchivingMode == ArchivingModes.SevenZip) await Archive7Zip(packageId, archivePathTemp); - - // flip temp file to final path, it is ready for use only when this happens + else + await ArchiveDotNetZip(packageId, archivePathTemp); + + // flip temp file to final path, it is ready for use only when this happens _fileSystem.File.Move(archivePathTemp, archivePath); TimeSpan totalTaken = DateTime.Now - totalStart; _log.LogInformation($"Archive generation : package {packageId} complete, total time {Math.Round(totalTaken.TotalSeconds, 0)} seconds."); diff --git a/src/Tetrifact.Tests/Core/ArchiveService/GetPackageAsArchive.cs b/src/Tetrifact.Tests/Core/ArchiveService/GetPackageAsArchive.cs index 382fcfa..2a558b8 100644 --- a/src/Tetrifact.Tests/Core/ArchiveService/GetPackageAsArchive.cs +++ b/src/Tetrifact.Tests/Core/ArchiveService/GetPackageAsArchive.cs @@ -58,6 +58,8 @@ public async void GetExistingArchive() public async void GetWithSevenZip() { Settings.ExternaArchivingExecutable = Path.Combine(Path.GetFullPath($"../../../../"), "lib", "7za.exe"); + Settings.ArchivingMode = ArchivingModes.SevenZip; + IArchiveService archiveService = TestContext.Get(); TestPackage testPackage = PackageHelper.CreateRandomPackage(); diff --git a/src/Tetrifact.Tests/Core/PackagePrune/Prune.cs b/src/Tetrifact.Tests/Core/PackagePrune/Prune.cs index 5f96335..d0cdef8 100644 --- a/src/Tetrifact.Tests/Core/PackagePrune/Prune.cs +++ b/src/Tetrifact.Tests/Core/PackagePrune/Prune.cs @@ -23,7 +23,13 @@ public Prune() [Fact] public void HappyPath() - { + { + Settings.PruneBrackets = new List(){ + new PruneBracket{ Days=7, Amount = 3 }, + new PruneBracket{ Days=31, Amount = 3 }, + new PruneBracket{ Days=365, Amount = 3 } + }; + // create packages : // packages under week threshold, none of these should not be deleted PackageHelper.CreateNewPackageFiles("under-week-1"); @@ -74,7 +80,7 @@ public void HappyPath() IEnumerable packages = IndexReader.GetAllPackageIds(); - Assert.Equal(14, packages.Count()); + Assert.Equal(14, packages.Count()); // 5 + 3 + 3 + 3 Assert.Equal(3, packages.Where(r => r.StartsWith("above-week-")).Count()); Assert.Equal(3, packages.Where(r => r.StartsWith("above-month-")).Count()); diff --git a/src/Tetrifact.Tests/Core/RepositoryCleanService/Clean.cs b/src/Tetrifact.Tests/Core/RepositoryCleanService/Clean.cs index 18dbc6e..d842034 100644 --- a/src/Tetrifact.Tests/Core/RepositoryCleanService/Clean.cs +++ b/src/Tetrifact.Tests/Core/RepositoryCleanService/Clean.cs @@ -227,7 +227,7 @@ public void EnsureNoLock_Coverage() lockInstance.Lock(ProcessLockCategories.Package_Create, "some-package"); CleanResult result = repoCleaner.Clean(); - Assert.Contains(result.Description, "Package locks found, clean exited before start"); + Assert.Contains("Package locks found, clean exited before start", result.Description); } } } diff --git a/src/Tetrifact.Tests/Helpers/PackageHelper.cs b/src/Tetrifact.Tests/Helpers/PackageHelper.cs index f827676..1273951 100644 --- a/src/Tetrifact.Tests/Helpers/PackageHelper.cs +++ b/src/Tetrifact.Tests/Helpers/PackageHelper.cs @@ -49,7 +49,7 @@ public string CreateNewPackage(ISettings settings, IEnumerable filesCont { IFileSystem filesystem = new FileSystem(); - ITetrifactMemoryCache memcache = _moqHelper.CreateInstanceWithAllMoqed(); + ITetrifactMemoryCache memcache = _context.Get(); IIndexReadService indexReader = new IndexReadService( settings, new TestMemoryCache(),