Skip to content

Commit

Permalink
test fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@shukriadams
shukriadams committed Sep 27, 2024
1 parent 44e1a51 commit e0c5ac8
Show file tree
Hide file tree
Showing 5 changed files with 70 additions and 60 deletions.
114 changes: 58 additions & 56 deletions src/Tetrifact.Core/ArchiveService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,64 +187,66 @@ private async Task Archive7Zip(string packageId, string archivePathTemp)

Manifest manifest = _indexReader.GetManifest(packageId);

// copy all files to single Direct
// copy all files to single Directory
if (!Directory.Exists(tempDir2))
{
_log.LogInformation($"Archive generation : gathering files for package {packageId}");
Directory.CreateDirectory(tempDir1);
long cacheUpdateIncrements = manifest.Files.Count / 100;
long counter = 0;

manifest.Files.AsParallel().WithDegreeOfParallelism(_settings.ArchiveCPUThreads).ForAll(async delegate (ManifestItem file)
manifest.Files.AsParallel().WithDegreeOfParallelism(_settings.ArchiveCPUThreads).ForAll(delegate (ManifestItem file)
{
string targetPath = Path.Join(tempDir1, file.Path);
List<string> knownDirectories = new List<string>();
if (manifest.IsCompressed)
{
GetFileResponse fileLookup = _indexReader.GetFile(file.Id);
if (fileLookup == null)
throw new Exception($"Failed to find expected package file {file.Id} - repository is likely corrupt");

using (var storageArchive = new ZipArchive(fileLookup.Content))
{
ZipArchiveEntry storageArchiveEntry = storageArchive.Entries[0];
using (var storageArchiveStream = storageArchiveEntry.Open())
using (FileStream writeStream = new FileStream(targetPath, FileMode.Create))
await StreamsHelper.CopyAsync(storageArchiveStream, writeStream, bufSize);
}
}
else
{
GetFileResponse fileLookup = _indexReader.GetFile(file.Id);
if (fileLookup == null)
throw new Exception($"Failed to find expected package file {file.Id}- repository is likely corrupt");

string dir = Path.GetDirectoryName(targetPath);
if (!knownDirectories.Contains(dir))
{
Directory.CreateDirectory(dir);
knownDirectories.Add(dir);
}

// is this the fastest way of copying? benchmark
using (Stream fileStream = fileLookup.Content)
using (FileStream writeStream = new FileStream(targetPath, FileMode.Create))
await StreamsHelper.CopyAsync(fileStream, writeStream, bufSize);
}

counter ++;

if (cacheUpdateIncrements == 0 || counter % cacheUpdateIncrements == 0)
{
_log.LogInformation($"Gathering file {counter}/{manifest.Files.Count}, package \"{packageId}\".");
string key = this.GetArchiveProgressKey(packageId);
ArchiveProgressInfo progress = _cache.Get<ArchiveProgressInfo>(key);
if (progress != null)
{
progress.FileCopyProgress = ((decimal)counter / (decimal)manifest.Files.Count) * 100;
_cache.Set(key, progress);
}
}
string targetPath = Path.Join(tempDir1, file.Path);
List<string> knownDirectories = new List<string>();
if (manifest.IsCompressed)
{
GetFileResponse fileLookup = _indexReader.GetFile(file.Id);
if (fileLookup == null)
throw new Exception($"Failed to find expected package file {file.Id} - repository is likely corrupt");

using (var storageArchive = new ZipArchive(fileLookup.Content))
{
ZipArchiveEntry storageArchiveEntry = storageArchive.Entries[0];
using (var storageArchiveStream = storageArchiveEntry.Open())
using (FileStream writeStream = new FileStream(targetPath, FileMode.Create))
// copy async not used here because cannot get this delegate to block asParallel,
StreamsHelper.Copy(storageArchiveStream, writeStream, bufSize);
}
}
else
{
GetFileResponse fileLookup = _indexReader.GetFile(file.Id);
if (fileLookup == null)
throw new Exception($"Failed to find expected package file {file.Id}- repository is likely corrupt");

string dir = Path.GetDirectoryName(targetPath);
if (!knownDirectories.Contains(dir))
{
Directory.CreateDirectory(dir);
knownDirectories.Add(dir);
}

// is this the fastest way of copying? benchmark
using (Stream fileStream = fileLookup.Content)
using (FileStream writeStream = new FileStream(targetPath, FileMode.Create))
// copy async not used here because cannot get this delegate to block asParallel,
StreamsHelper.Copy(fileStream, writeStream, bufSize);
}

counter++;

if (cacheUpdateIncrements == 0 || counter % cacheUpdateIncrements == 0)
{
_log.LogInformation($"Gathering file {counter}/{manifest.Files.Count}, package \"{packageId}\".");
string key = this.GetArchiveProgressKey(packageId);
ArchiveProgressInfo progress = _cache.Get<ArchiveProgressInfo>(key);
if (progress != null)
{
progress.FileCopyProgress = ((decimal)counter / (decimal)manifest.Files.Count) * 100;
_cache.Set(key, progress);
}
}
});

Directory.Move(tempDir1, tempDir2);
Expand Down Expand Up @@ -282,7 +284,7 @@ private async Task Archive7Zip(string packageId, string archivePathTemp)
}
}

private async Task ArchiveDefaultMode(string packageId, string archivePathTemp)
private async Task ArchiveDotNetZip(string packageId, string archivePathTemp)
{
DateTime compressStart = DateTime.Now;

Expand Down Expand Up @@ -422,12 +424,12 @@ public async Task CreateArchive(string packageId)

try
{
if (string.IsNullOrEmpty(_settings.ExternaArchivingExecutable))
await ArchiveDefaultMode(packageId, archivePathTemp);
else
if (_settings.ArchivingMode == ArchivingModes.SevenZip)
await Archive7Zip(packageId, archivePathTemp);

// flip temp file to final path, it is ready for use only when this happens
else
await ArchiveDotNetZip(packageId, archivePathTemp);

// flip temp file to final path, it is ready for use only when this happens
_fileSystem.File.Move(archivePathTemp, archivePath);
TimeSpan totalTaken = DateTime.Now - totalStart;
_log.LogInformation($"Archive generation : package {packageId} complete, total time {Math.Round(totalTaken.TotalSeconds, 0)} seconds.");
Expand Down
2 changes: 2 additions & 0 deletions src/Tetrifact.Tests/Core/ArchiveService/GetPackageAsArchive.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ public async void GetExistingArchive()
public async void GetWithSevenZip()
{
Settings.ExternaArchivingExecutable = Path.Combine(Path.GetFullPath($"../../../../"), "lib", "7za.exe");
Settings.ArchivingMode = ArchivingModes.SevenZip;

IArchiveService archiveService = TestContext.Get<IArchiveService>();

TestPackage testPackage = PackageHelper.CreateRandomPackage();
Expand Down
10 changes: 8 additions & 2 deletions src/Tetrifact.Tests/Core/PackagePrune/Prune.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,13 @@ public Prune()

[Fact]
public void HappyPath()
{
{
Settings.PruneBrackets = new List<PruneBracket>(){
new PruneBracket{ Days=7, Amount = 3 },
new PruneBracket{ Days=31, Amount = 3 },
new PruneBracket{ Days=365, Amount = 3 }
};

// create packages :
// packages under week threshold, none of these should not be deleted
PackageHelper.CreateNewPackageFiles("under-week-1");
Expand Down Expand Up @@ -74,7 +80,7 @@ public void HappyPath()

IEnumerable<string> packages = IndexReader.GetAllPackageIds();

Assert.Equal(14, packages.Count());
Assert.Equal(14, packages.Count()); // 5 + 3 + 3 + 3

Assert.Equal(3, packages.Where(r => r.StartsWith("above-week-")).Count());
Assert.Equal(3, packages.Where(r => r.StartsWith("above-month-")).Count());
Expand Down
2 changes: 1 addition & 1 deletion src/Tetrifact.Tests/Core/RepositoryCleanService/Clean.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ public void EnsureNoLock_Coverage()
lockInstance.Lock(ProcessLockCategories.Package_Create, "some-package");
CleanResult result = repoCleaner.Clean();

Assert.Contains(result.Description, "Package locks found, clean exited before start");
Assert.Contains("Package locks found, clean exited before start", result.Description);
}
}
}
2 changes: 1 addition & 1 deletion src/Tetrifact.Tests/Helpers/PackageHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public string CreateNewPackage(ISettings settings, IEnumerable<string> filesCont
{

IFileSystem filesystem = new FileSystem();
ITetrifactMemoryCache memcache = _moqHelper.CreateInstanceWithAllMoqed<ITetrifactMemoryCache>();
ITetrifactMemoryCache memcache = _context.Get<ITetrifactMemoryCache>();
IIndexReadService indexReader = new IndexReadService(
settings,
new TestMemoryCache(),
Expand Down

0 comments on commit e0c5ac8

Please sign in to comment.