rewrote prune logic

src/Tetrifact.Core/ISettings.cs

@@ -229,5 +229,10 @@ public interface ISettings
         /// Nr of threads to use for archiving process.
         /// </summary>
         int ArchiveCPUThreads { get; set; }
+
+        /// <summary>
+        /// Time brackets for auto-deleting packages
+        /// </summary>
+        IEnumerable<PruneBracket> PruneBrackets {get; set; }
     }
 }

src/Tetrifact.Core/PackagePruneService.cs

@@ -36,21 +36,6 @@ public PackagePruneService(ISettings settings, IProcessLockManager processLock,
 
         #region METHODS
 
-        /// <summary>
-        /// 
-        /// </summary>
-        /// <param name="floor"></param>
-        /// <param name="keep"></param>
-        /// <param name="keepIds"></param>
-        /// <param name="packageIds"></param>
-        /// <param name="isWeekly"></param>
-        /// <returns></returns>
-        private void RemoveKeep(ref IList<string> keepIds, ref IList<string> packageIds) 
-        { 
-            foreach (string id in keepIds)
-                packageIds.Remove(id);
-        }
-
         public void Prune()
         {
             if (!_settings.Prune)
@@ -71,7 +56,7 @@ public void Prune()
             foreach(string line in report.Report)
                _log.LogInformation(line);
 
-            foreach (string packageId in report.PackageIds)
+            foreach (string packageId in report.Brackets.SelectMany(b => b.Prune))
             {
                 try
                 {
@@ -91,15 +76,16 @@ public void Prune()
 
         public PruneReport Report()
         {
-            IList<string> weeklyKeep = new List<string>();
-            IList<string> weeklyPrune = new List<string>();
-            IList<string> monthlyKeep = new List<string>();
-            IList<string> monthlyPrune = new List<string>();
-            IList<string> yearlyKeep = new List<string>();
-            IList<string> yearlyPrune = new List<string>();
+            // sort brackets oldest to most recent
+            IList<PruneBracketProcess> brackets = _settings.PruneBrackets
+                .Select(p => PruneBracketProcess.Clone(p))
+                .OrderByDescending(p => p.Days)
+                .ToList();
+
             IList<string> taggedKeep = new List<string>();
             IList<string> newKeep = new List<string>();
             IList<string> report = new List<string>();
+            int unhandled = 0;
 
             report.Add(" ******************************** Prune audit **********************************");
 
@@ -109,19 +95,12 @@ public PruneReport Report()
             // calculate periods for weekly, monthly and year pruning - weekly happens first, and after some time from NOW passes. Monthly starts at some point after weekly starts
             // and yearl starst some point after that and runs indefinitely.
             DateTime utcNow = _timeprovider.GetUtcNow();
-            DateTime weeklyPruneFloor = utcNow.AddDays(-1 * _settings.PruneWeeklyThreshold);
-            DateTime monthlyPruneFloor = utcNow.AddDays(-1 * _settings.PruneMonthlyThreshold);
-            DateTime yearlyPruneFloor = utcNow.AddDays(-1 * _settings.PruneYearlyThreshold);
+            foreach(PruneBracketProcess pruneBracketProcess in brackets)
+                pruneBracketProcess.Floor = utcNow.AddDays(-1 * pruneBracketProcess.Days);
 
-            int inWeeky = 0;
-            int inMonthly = 0;
-            int inYearly = 0;
             int startingPackageCount = packageIds.Count;
 
-            if (_settings.DEBUG_block_prune_deletes)
-                report.Add($"DEBUG_block_prune_deletes is enabled, packages will not be deleted");
-
-            report.Add($"Found {packageIds.Count} packages :");
+            report.Add($"Server currently contains {packageIds.Count} packages.");
 
             foreach (string packageId in packageIds)
             {
@@ -133,85 +112,64 @@ public PruneReport Report()
                     continue;
                 }
 
-                bool isTaggedKeep = manifest.Tags.Any(tag => _settings.PruneIgnoreTags.Any(protectedTag => protectedTag.Equals(tag)));
                 string flattenedTags = manifest.Tags.Count == 0 ? string.Empty : $"Tags : {string.Join(",", manifest.Tags)}";
                 int ageInDays = (int)Math.Round((utcNow - manifest.CreatedUtc).TotalDays, 0);
-                report.Add($"- {packageId}, added {manifest.CreatedUtc.ToIso()} ({ageInDays}) days ago). {flattenedTags}");
-
-                if (isTaggedKeep){
-                    taggedKeep.Add(packageId);
-                    continue;
-                }
+                report.Add($"Analysing {packageId}, added {manifest.CreatedUtc.ToIso()} ({ageInDays}) days ago). Tagged with: {flattenedTags}");
 
-                if (ageInDays > _settings.PruneYearlyThreshold)
+                PruneBracketProcess matchingBracket = brackets.FirstOrDefault(b => manifest.CreatedUtc < b.Floor);
+                if (matchingBracket == null)
                 {
-                    inYearly++;
-                    if (yearlyKeep.Count < _settings.PruneYearlyKeep || isTaggedKeep)
-                        yearlyKeep.Add(packageId);
-                    else
-                        yearlyPrune.Add(packageId);
-
-                    continue;
+                    report.Add($"{packageId}, created {manifest.CreatedUtc.ToIso()}, does not land in any prune bracket, will be kept.");
+                    unhandled ++;
                 }
-
-                if (ageInDays <= _settings.PruneYearlyThreshold && ageInDays > _settings.PruneMonthlyThreshold)
+                else
                 {
-                    inMonthly++;
-
-                    if (monthlyKeep.Count < _settings.PruneMonthlyKeep || isTaggedKeep)
-                        monthlyKeep.Add(packageId);
-                    else
-                        monthlyPrune.Add(packageId);
-
-                    continue;
-                }
-
-                if (ageInDays <= _settings.PruneMonthlyThreshold && ageInDays > _settings.PruneWeeklyThreshold)
-                {
-                    inWeeky++;
-
-                    if (weeklyKeep.Count < _settings.PruneWeeklyKeep || isTaggedKeep)
-                        weeklyKeep.Add(packageId);
-                    else
-                        weeklyPrune.Add(packageId);
-
-                    continue;
+                    report.Add($"{packageId}, created {manifest.CreatedUtc.ToIso()}, lands in prune bracket {matchingBracket.Days}Days.");
+                    bool isTaggedKeep = manifest.Tags.Any(tag => _settings.PruneIgnoreTags.Any(protectedTag => protectedTag.Equals(tag)));
+
+                    if (isTaggedKeep)
+                    {
+                        taggedKeep.Add(packageId);
+                        matchingBracket.Keep.Add(packageId);
+                        report.Add($"{packageId} marked for keep based on tag.");
+                    }
+                    else 
+                    { 
+                        if (matchingBracket.Keep.Count < matchingBracket.Amount)
+                        {
+                            report.Add($"{packageId} marked for keep, {matchingBracket.Keep.Count} packages kept so far.");
+                            matchingBracket.Keep.Add(packageId);
+                        }
+                        else
+                        {
+                            matchingBracket.Prune.Add(packageId);
+                            report.Add($"{packageId} marked for prune, {matchingBracket.Keep.Count} packages already kept.");
+                        }
+                    }
                 }
-
-                if (ageInDays <= _settings.PruneWeeklyThreshold)
-                    newKeep.Add(packageId);
 
             }
 
-            RemoveKeep(ref yearlyKeep, ref packageIds);
-            RemoveKeep(ref monthlyKeep, ref packageIds);
-            RemoveKeep(ref weeklyKeep, ref packageIds);
-            RemoveKeep(ref newKeep, ref packageIds);
-            RemoveKeep(ref taggedKeep, ref packageIds);
-
             string pruneIdList = string.Empty;
             if (packageIds.Count > 0)
                 pruneIdList = $" ({string.Join(",", packageIds)})";
 
-            string yearlyPruneFlattened = yearlyPrune.Count == 0 ? string.Empty : $" pruning {string.Join(",", yearlyPrune)}";
-            string monthlyPruneFlattened = monthlyPrune.Count == 0 ? string.Empty : $" pruning {string.Join(",", monthlyPrune)}";
-            string weeklyPruneFlattened = weeklyPrune.Count == 0 ? string.Empty : $" pruning {string.Join(",", weeklyPrune)}";
-
-            // log out audit for prune, use warning because we expect this to be logged as important
             report.Add(string.Empty);
             report.Add($"Pre-weekly ignore count is {newKeep.Count()} - {string.Join(",", newKeep)}");
+            report.Add($"Unhandled: {unhandled}");
+
             if (taggedKeep.Count > 0)
                 report.Add($"Kept due to tagging - {string.Join(",", taggedKeep)}.");
-            report.Add($"WEEKLY prune (before {weeklyPruneFloor.ToIso()}, {_settings.PruneWeeklyThreshold} days ago) count is {_settings.PruneWeeklyKeep}. Keeping {weeklyKeep.Count()} of {inWeeky} ({string.Join(",", weeklyKeep)}). {string.Join(",", weeklyKeep)}{weeklyPruneFlattened}.");
-            report.Add($"MONTHLY prune (before {monthlyPruneFloor.ToIso()}, {_settings.PruneMonthlyThreshold} days ago) count is {_settings.PruneMonthlyKeep}. Keeping {monthlyKeep.Count()} of {inMonthly} ({string.Join(",", monthlyKeep)}). {string.Join(",", monthlyKeep)}{monthlyPruneFlattened}.");
-            report.Add($"YEARLY prune (before {yearlyPruneFloor.ToIso()}, {_settings.PruneYearlyThreshold} days ago) count is {_settings.PruneYearlyKeep}. Keeping {yearlyKeep.Count()} of {inYearly} ({string.Join(",", yearlyKeep)}). {string.Join(",", yearlyKeep)}{yearlyPruneFlattened}.");
+
+            foreach(PruneBracketProcess p in brackets)
+                report.Add($"Bracket {p}, keeping {p.Keep.Count} packages ({string.Join(",",p.Keep)}), pruning {p.Prune.Count} packages ({string.Join(",", p.Prune)})");
+
             report.Add(string.Empty);
-            report.Add($"Pruning {packageIds.Count} packages{pruneIdList}.");
             report.Add(" ******************************** Prune audit **********************************");
 
             return new PruneReport{ 
                 Report = report, 
-                PackageIds = packageIds 
+                Brackets = brackets
             };
         }
 

src/Tetrifact.Core/PruneBracket.cs

@@ -0,0 +1,14 @@
+namespace Tetrifact
+{
+    public class PruneBracket
+    {
+        public int Amount { get; set; }
+
+        public int Days { get; set; }
+
+        public override string ToString()
+        {
+            return $"{Days} days {Amount} packages";
+        }
+    }
+}

src/Tetrifact.Core/PruneBracketProcess.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+
+namespace Tetrifact
+{
+    public class PruneBracketProcess : PruneBracket
+    {
+        public IList<string> Keep {get ; set; } = new List<string>();
+
+        public IList<string> Prune { get; set; }  = new List<string>();
+
+        public DateTime Floor { get; set; }
+
+        public static PruneBracketProcess Clone(PruneBracket pruneBracket) 
+        { 
+            return new PruneBracketProcess 
+            { 
+                Amount = pruneBracket.Amount,
+                Days = pruneBracket.Days
+            };
+        }
+    }
+}

src/Tetrifact.Core/PruneReport.cs

@@ -4,13 +4,13 @@ namespace Tetrifact.Core
 {
     public class PruneReport
     {
-        public IEnumerable<string> PackageIds {get;set; }
         public IEnumerable<string> Report {get;set; }
+        public IEnumerable<PruneBracketProcess> Brackets { get;set; }
 
         public PruneReport()
         { 
             this.Report = new List<string>();
-            this.PackageIds = new List<string>();
+            this.Brackets = new List<PruneBracketProcess>();
         }
     }
 }

src/Tetrifact.Core/Settings.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.IO.Compression;
+using System.Text.RegularExpressions;
 
 namespace Tetrifact.Core
 {
@@ -99,6 +100,8 @@ public class Settings : ISettings
 
         public int ArchiveCPUThreads { get; set; }
 
+        public IEnumerable<PruneBracket> PruneBrackets { get; set; }
+
         #endregion
 
         #region CTORS
@@ -121,6 +124,7 @@ public Settings()
             this.MaxArchives = 10;
             this.AuthorizationLevel = AuthorizationLevel.None;
             this.AccessTokens = new List<string>();
+            this.PruneBrackets = new List<PruneBracket>();
             this.IsStorageCompressionEnabled = false;
             this.DownloadArchiveCompression = CompressionLevel.Optimal;
             this.PruneWeeklyKeep = 7; 
@@ -189,6 +193,41 @@ public Settings()
             this.ArchiveCPUThreads = this.TryGetSetting("ARCHIVE_CPU_THREADS", this.ArchiveCPUThreads);
             this.SevenZipBinaryPath = this.TryGetSetting("SEVEN_ZIP_BINARY_PATH", this.SevenZipBinaryPath);
 
+            string timeBracketsAllRaw = Environment.GetEnvironmentVariable("PRUNE_BRACKETS");
+            if (!string.IsNullOrEmpty(timeBracketsAllRaw)) 
+            {
+                List<PruneBracket> brackets = new List<PruneBracket>();
+
+                string[] timeBracketsRaw = timeBracketsAllRaw.Split(",");
+                foreach(string timeBracketRaw in timeBracketsRaw) 
+                {
+                    string[] items = timeBracketRaw.Trim().Split(" ");
+                    if (items.Length != 2)
+                        continue;
+
+                    string intervalRaw = items[0];
+                    int amount;
+                    if (!int.TryParse(items[1], out amount))
+                        continue;
+
+                    Regex regex = new Regex("^(\\d*)d?");
+                    Match match = regex.Match(intervalRaw);
+                    if (match.Success && match.Groups.Count == 2) 
+                    { 
+                        string daysraw = match.Groups[1].Value;
+
+                        int days;
+                        if (!int.TryParse(daysraw, out days))
+                            continue;
+
+                        brackets.Add(new PruneBracket { Amount = amount, Days = days });
+                    }
+                }
+
+                this.PruneBrackets = brackets;
+            }
+
+
             string downloadArchiveCompressionEnvVar = Environment.GetEnvironmentVariable("DOWNLOAD_ARCHIVE_COMPRESSION");
             if (downloadArchiveCompressionEnvVar == "0")
                 DownloadArchiveCompression = CompressionLevel.NoCompression;

src/Tetrifact.Web/Properties/launchSettings.json

@@ -35,6 +35,8 @@
         "PRUNE_IGNORE_TAGS": "keep",
         "SEVEN_ZIP_BINARY_PATH": "",
         "ASPNETCORE_ENVIRONMENT": "Development",
+        "PRUNE": "true",
+        "PRUNE_BRACKETS" : "5d 10, 20d 2, 10d 0",
         "Logging__LogLevel__System": "Debug"
       },
       "applicationUrl": "http://0.0.0.0:3000"

src/Tetrifact.Web/Startup.cs

@@ -179,6 +179,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerF
             Console.WriteLine($"Max archives: {settings.MaxArchives}");
             Console.WriteLine($"PackagePath: {settings.PackagePath}");
             Console.WriteLine($"Pages per page group: {settings.PagesPerPageGroup}");
+            Console.WriteLine($"Prune brackets: { string.Join(", ", settings.PruneBrackets)}");
             Console.WriteLine($"Repository path: {settings.RepositoryPath}");
             Console.WriteLine($"Space safety threshold: {settings.SpaceSafetyThreshold}");
             Console.WriteLine($"Tags path: {settings.TagsPath}");