From 4052a27c6c7322aa421e0fd5dbe54cf7f4c8e841 Mon Sep 17 00:00:00 2001 From: Shukri Adams Date: Fri, 26 Jul 2024 18:01:14 +0200 Subject: [PATCH] rewrote prune logic --- src/Tetrifact.Core/ISettings.cs | 5 + src/Tetrifact.Core/PackagePruneService.cs | 134 ++++++------------ src/Tetrifact.Core/PruneBracket.cs | 14 ++ src/Tetrifact.Core/PruneBracketProcess.cs | 23 +++ src/Tetrifact.Core/PruneReport.cs | 4 +- src/Tetrifact.Core/Settings.cs | 39 +++++ .../Properties/launchSettings.json | 2 + src/Tetrifact.Web/Startup.cs | 1 + 8 files changed, 132 insertions(+), 90 deletions(-) create mode 100644 src/Tetrifact.Core/PruneBracket.cs create mode 100644 src/Tetrifact.Core/PruneBracketProcess.cs diff --git a/src/Tetrifact.Core/ISettings.cs b/src/Tetrifact.Core/ISettings.cs index f025dfc..15ebcc2 100644 --- a/src/Tetrifact.Core/ISettings.cs +++ b/src/Tetrifact.Core/ISettings.cs @@ -229,5 +229,10 @@ public interface ISettings /// Nr of threads to use for archiving process. /// int ArchiveCPUThreads { get; set; } + + /// + /// Time brackets for auto-deleting packages + /// + IEnumerable PruneBrackets {get; set; } } } diff --git a/src/Tetrifact.Core/PackagePruneService.cs b/src/Tetrifact.Core/PackagePruneService.cs index c7c9bf6..50a4fa6 100644 --- a/src/Tetrifact.Core/PackagePruneService.cs +++ b/src/Tetrifact.Core/PackagePruneService.cs @@ -36,21 +36,6 @@ public PackagePruneService(ISettings settings, IProcessLockManager processLock, #region METHODS - /// - /// - /// - /// - /// - /// - /// - /// - /// - private void RemoveKeep(ref IList keepIds, ref IList packageIds) - { - foreach (string id in keepIds) - packageIds.Remove(id); - } - public void Prune() { if (!_settings.Prune) @@ -71,7 +56,7 @@ public void Prune() foreach(string line in report.Report) _log.LogInformation(line); - foreach (string packageId in report.PackageIds) + foreach (string packageId in report.Brackets.SelectMany(b => b.Prune)) { try { @@ -91,15 +76,16 @@ public void Prune() public PruneReport Report() { - IList weeklyKeep = new List(); - IList weeklyPrune = new List(); - IList monthlyKeep = new List(); - IList monthlyPrune = new List(); - IList yearlyKeep = new List(); - IList yearlyPrune = new List(); + // sort brackets oldest to most recent + IList brackets = _settings.PruneBrackets + .Select(p => PruneBracketProcess.Clone(p)) + .OrderByDescending(p => p.Days) + .ToList(); + IList taggedKeep = new List(); IList newKeep = new List(); IList report = new List(); + int unhandled = 0; report.Add(" ******************************** Prune audit **********************************"); @@ -109,19 +95,12 @@ public PruneReport Report() // calculate periods for weekly, monthly and year pruning - weekly happens first, and after some time from NOW passes. Monthly starts at some point after weekly starts // and yearl starst some point after that and runs indefinitely. DateTime utcNow = _timeprovider.GetUtcNow(); - DateTime weeklyPruneFloor = utcNow.AddDays(-1 * _settings.PruneWeeklyThreshold); - DateTime monthlyPruneFloor = utcNow.AddDays(-1 * _settings.PruneMonthlyThreshold); - DateTime yearlyPruneFloor = utcNow.AddDays(-1 * _settings.PruneYearlyThreshold); + foreach(PruneBracketProcess pruneBracketProcess in brackets) + pruneBracketProcess.Floor = utcNow.AddDays(-1 * pruneBracketProcess.Days); - int inWeeky = 0; - int inMonthly = 0; - int inYearly = 0; int startingPackageCount = packageIds.Count; - if (_settings.DEBUG_block_prune_deletes) - report.Add($"DEBUG_block_prune_deletes is enabled, packages will not be deleted"); - - report.Add($"Found {packageIds.Count} packages :"); + report.Add($"Server currently contains {packageIds.Count} packages."); foreach (string packageId in packageIds) { @@ -133,85 +112,64 @@ public PruneReport Report() continue; } - bool isTaggedKeep = manifest.Tags.Any(tag => _settings.PruneIgnoreTags.Any(protectedTag => protectedTag.Equals(tag))); string flattenedTags = manifest.Tags.Count == 0 ? string.Empty : $"Tags : {string.Join(",", manifest.Tags)}"; int ageInDays = (int)Math.Round((utcNow - manifest.CreatedUtc).TotalDays, 0); - report.Add($"- {packageId}, added {manifest.CreatedUtc.ToIso()} ({ageInDays}) days ago). {flattenedTags}"); - - if (isTaggedKeep){ - taggedKeep.Add(packageId); - continue; - } + report.Add($"Analysing {packageId}, added {manifest.CreatedUtc.ToIso()} ({ageInDays}) days ago). Tagged with: {flattenedTags}"); - if (ageInDays > _settings.PruneYearlyThreshold) + PruneBracketProcess matchingBracket = brackets.FirstOrDefault(b => manifest.CreatedUtc < b.Floor); + if (matchingBracket == null) { - inYearly++; - if (yearlyKeep.Count < _settings.PruneYearlyKeep || isTaggedKeep) - yearlyKeep.Add(packageId); - else - yearlyPrune.Add(packageId); - - continue; + report.Add($"{packageId}, created {manifest.CreatedUtc.ToIso()}, does not land in any prune bracket, will be kept."); + unhandled ++; } - - if (ageInDays <= _settings.PruneYearlyThreshold && ageInDays > _settings.PruneMonthlyThreshold) + else { - inMonthly++; - - if (monthlyKeep.Count < _settings.PruneMonthlyKeep || isTaggedKeep) - monthlyKeep.Add(packageId); - else - monthlyPrune.Add(packageId); - - continue; - } - - if (ageInDays <= _settings.PruneMonthlyThreshold && ageInDays > _settings.PruneWeeklyThreshold) - { - inWeeky++; - - if (weeklyKeep.Count < _settings.PruneWeeklyKeep || isTaggedKeep) - weeklyKeep.Add(packageId); - else - weeklyPrune.Add(packageId); - - continue; + report.Add($"{packageId}, created {manifest.CreatedUtc.ToIso()}, lands in prune bracket {matchingBracket.Days}Days."); + bool isTaggedKeep = manifest.Tags.Any(tag => _settings.PruneIgnoreTags.Any(protectedTag => protectedTag.Equals(tag))); + + if (isTaggedKeep) + { + taggedKeep.Add(packageId); + matchingBracket.Keep.Add(packageId); + report.Add($"{packageId} marked for keep based on tag."); + } + else + { + if (matchingBracket.Keep.Count < matchingBracket.Amount) + { + report.Add($"{packageId} marked for keep, {matchingBracket.Keep.Count} packages kept so far."); + matchingBracket.Keep.Add(packageId); + } + else + { + matchingBracket.Prune.Add(packageId); + report.Add($"{packageId} marked for prune, {matchingBracket.Keep.Count} packages already kept."); + } + } } - - if (ageInDays <= _settings.PruneWeeklyThreshold) - newKeep.Add(packageId); } - RemoveKeep(ref yearlyKeep, ref packageIds); - RemoveKeep(ref monthlyKeep, ref packageIds); - RemoveKeep(ref weeklyKeep, ref packageIds); - RemoveKeep(ref newKeep, ref packageIds); - RemoveKeep(ref taggedKeep, ref packageIds); - string pruneIdList = string.Empty; if (packageIds.Count > 0) pruneIdList = $" ({string.Join(",", packageIds)})"; - string yearlyPruneFlattened = yearlyPrune.Count == 0 ? string.Empty : $" pruning {string.Join(",", yearlyPrune)}"; - string monthlyPruneFlattened = monthlyPrune.Count == 0 ? string.Empty : $" pruning {string.Join(",", monthlyPrune)}"; - string weeklyPruneFlattened = weeklyPrune.Count == 0 ? string.Empty : $" pruning {string.Join(",", weeklyPrune)}"; - - // log out audit for prune, use warning because we expect this to be logged as important report.Add(string.Empty); report.Add($"Pre-weekly ignore count is {newKeep.Count()} - {string.Join(",", newKeep)}"); + report.Add($"Unhandled: {unhandled}"); + if (taggedKeep.Count > 0) report.Add($"Kept due to tagging - {string.Join(",", taggedKeep)}."); - report.Add($"WEEKLY prune (before {weeklyPruneFloor.ToIso()}, {_settings.PruneWeeklyThreshold} days ago) count is {_settings.PruneWeeklyKeep}. Keeping {weeklyKeep.Count()} of {inWeeky} ({string.Join(",", weeklyKeep)}). {string.Join(",", weeklyKeep)}{weeklyPruneFlattened}."); - report.Add($"MONTHLY prune (before {monthlyPruneFloor.ToIso()}, {_settings.PruneMonthlyThreshold} days ago) count is {_settings.PruneMonthlyKeep}. Keeping {monthlyKeep.Count()} of {inMonthly} ({string.Join(",", monthlyKeep)}). {string.Join(",", monthlyKeep)}{monthlyPruneFlattened}."); - report.Add($"YEARLY prune (before {yearlyPruneFloor.ToIso()}, {_settings.PruneYearlyThreshold} days ago) count is {_settings.PruneYearlyKeep}. Keeping {yearlyKeep.Count()} of {inYearly} ({string.Join(",", yearlyKeep)}). {string.Join(",", yearlyKeep)}{yearlyPruneFlattened}."); + + foreach(PruneBracketProcess p in brackets) + report.Add($"Bracket {p}, keeping {p.Keep.Count} packages ({string.Join(",",p.Keep)}), pruning {p.Prune.Count} packages ({string.Join(",", p.Prune)})"); + report.Add(string.Empty); - report.Add($"Pruning {packageIds.Count} packages{pruneIdList}."); report.Add(" ******************************** Prune audit **********************************"); return new PruneReport{ Report = report, - PackageIds = packageIds + Brackets = brackets }; } diff --git a/src/Tetrifact.Core/PruneBracket.cs b/src/Tetrifact.Core/PruneBracket.cs new file mode 100644 index 0000000..99e28db --- /dev/null +++ b/src/Tetrifact.Core/PruneBracket.cs @@ -0,0 +1,14 @@ +namespace Tetrifact +{ + public class PruneBracket + { + public int Amount { get; set; } + + public int Days { get; set; } + + public override string ToString() + { + return $"{Days} days {Amount} packages"; + } + } +} diff --git a/src/Tetrifact.Core/PruneBracketProcess.cs b/src/Tetrifact.Core/PruneBracketProcess.cs new file mode 100644 index 0000000..8a177f1 --- /dev/null +++ b/src/Tetrifact.Core/PruneBracketProcess.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; + +namespace Tetrifact +{ + public class PruneBracketProcess : PruneBracket + { + public IList Keep {get ; set; } = new List(); + + public IList Prune { get; set; } = new List(); + + public DateTime Floor { get; set; } + + public static PruneBracketProcess Clone(PruneBracket pruneBracket) + { + return new PruneBracketProcess + { + Amount = pruneBracket.Amount, + Days = pruneBracket.Days + }; + } + } +} diff --git a/src/Tetrifact.Core/PruneReport.cs b/src/Tetrifact.Core/PruneReport.cs index 3ca09ba..196d90a 100644 --- a/src/Tetrifact.Core/PruneReport.cs +++ b/src/Tetrifact.Core/PruneReport.cs @@ -4,13 +4,13 @@ namespace Tetrifact.Core { public class PruneReport { - public IEnumerable PackageIds {get;set; } public IEnumerable Report {get;set; } + public IEnumerable Brackets { get;set; } public PruneReport() { this.Report = new List(); - this.PackageIds = new List(); + this.Brackets = new List(); } } } diff --git a/src/Tetrifact.Core/Settings.cs b/src/Tetrifact.Core/Settings.cs index 31c8d55..e95dc18 100644 --- a/src/Tetrifact.Core/Settings.cs +++ b/src/Tetrifact.Core/Settings.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.IO.Compression; +using System.Text.RegularExpressions; namespace Tetrifact.Core { @@ -99,6 +100,8 @@ public class Settings : ISettings public int ArchiveCPUThreads { get; set; } + public IEnumerable PruneBrackets { get; set; } + #endregion #region CTORS @@ -121,6 +124,7 @@ public Settings() this.MaxArchives = 10; this.AuthorizationLevel = AuthorizationLevel.None; this.AccessTokens = new List(); + this.PruneBrackets = new List(); this.IsStorageCompressionEnabled = false; this.DownloadArchiveCompression = CompressionLevel.Optimal; this.PruneWeeklyKeep = 7; @@ -189,6 +193,41 @@ public Settings() this.ArchiveCPUThreads = this.TryGetSetting("ARCHIVE_CPU_THREADS", this.ArchiveCPUThreads); this.SevenZipBinaryPath = this.TryGetSetting("SEVEN_ZIP_BINARY_PATH", this.SevenZipBinaryPath); + string timeBracketsAllRaw = Environment.GetEnvironmentVariable("PRUNE_BRACKETS"); + if (!string.IsNullOrEmpty(timeBracketsAllRaw)) + { + List brackets = new List(); + + string[] timeBracketsRaw = timeBracketsAllRaw.Split(","); + foreach(string timeBracketRaw in timeBracketsRaw) + { + string[] items = timeBracketRaw.Trim().Split(" "); + if (items.Length != 2) + continue; + + string intervalRaw = items[0]; + int amount; + if (!int.TryParse(items[1], out amount)) + continue; + + Regex regex = new Regex("^(\\d*)d?"); + Match match = regex.Match(intervalRaw); + if (match.Success && match.Groups.Count == 2) + { + string daysraw = match.Groups[1].Value; + + int days; + if (!int.TryParse(daysraw, out days)) + continue; + + brackets.Add(new PruneBracket { Amount = amount, Days = days }); + } + } + + this.PruneBrackets = brackets; + } + + string downloadArchiveCompressionEnvVar = Environment.GetEnvironmentVariable("DOWNLOAD_ARCHIVE_COMPRESSION"); if (downloadArchiveCompressionEnvVar == "0") DownloadArchiveCompression = CompressionLevel.NoCompression; diff --git a/src/Tetrifact.Web/Properties/launchSettings.json b/src/Tetrifact.Web/Properties/launchSettings.json index fcaacda..e5009db 100644 --- a/src/Tetrifact.Web/Properties/launchSettings.json +++ b/src/Tetrifact.Web/Properties/launchSettings.json @@ -35,6 +35,8 @@ "PRUNE_IGNORE_TAGS": "keep", "SEVEN_ZIP_BINARY_PATH": "", "ASPNETCORE_ENVIRONMENT": "Development", + "PRUNE": "true", + "PRUNE_BRACKETS" : "5d 10, 20d 2, 10d 0", "Logging__LogLevel__System": "Debug" }, "applicationUrl": "http://0.0.0.0:3000" diff --git a/src/Tetrifact.Web/Startup.cs b/src/Tetrifact.Web/Startup.cs index c5146af..4be1050 100644 --- a/src/Tetrifact.Web/Startup.cs +++ b/src/Tetrifact.Web/Startup.cs @@ -179,6 +179,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerF Console.WriteLine($"Max archives: {settings.MaxArchives}"); Console.WriteLine($"PackagePath: {settings.PackagePath}"); Console.WriteLine($"Pages per page group: {settings.PagesPerPageGroup}"); + Console.WriteLine($"Prune brackets: { string.Join(", ", settings.PruneBrackets)}"); Console.WriteLine($"Repository path: {settings.RepositoryPath}"); Console.WriteLine($"Space safety threshold: {settings.SpaceSafetyThreshold}"); Console.WriteLine($"Tags path: {settings.TagsPath}");