From 107303ce718c40e7b80c390900f3a2e74899757c Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Thu, 5 Sep 2024 16:23:22 -0500 Subject: [PATCH] Always use the same SourceForge mirror --- Core/Net/Net.cs | 50 +++++++++++-------- Netkan/Transformers/SourceForgeTransformer.cs | 33 ++++++++++-- 2 files changed, 58 insertions(+), 25 deletions(-) diff --git a/Core/Net/Net.cs b/Core/Net/Net.cs index 113cc2000..a429ff666 100644 --- a/Core/Net/Net.cs +++ b/Core/Net/Net.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Collections.Generic; using System.Linq; using System.Net; @@ -9,6 +10,7 @@ using ChinhDo.Transactions.FileManager; using log4net; +using CKAN.Extensions; using CKAN.Configuration; namespace CKAN @@ -198,33 +200,37 @@ public static string Download(string url, out string? etag, string? filename = n } public static Uri? ResolveRedirect(Uri url, - string? userAgent = "") + string? userAgent = null, + int maxRedirects = 6) { - const int maxRedirects = 6; - for (int redirects = 0; redirects <= maxRedirects; ++redirects) + var urls = url.TraverseNodes(u => new RedirectWebClient(userAgent) is RedirectWebClient rwClient + && rwClient.OpenRead(u) is Stream s && DisposeStream(s) + && rwClient.ResponseHeaders is WebHeaderCollection headers + && headers["Location"] is string location + ? Uri.IsWellFormedUriString(location, UriKind.Absolute) + ? new Uri(location) + : Uri.IsWellFormedUriString(location, UriKind.Relative) + ? new Uri(u, location) + : throw new Kraken(string.Format(Properties.Resources.NetInvalidLocation, + location)) + : null) + // The first element is the input, so e.g. if we want two redirects, that's three elements + .Take(maxRedirects + 1) + .ToArray(); + if (log.IsDebugEnabled) { - var rwClient = new RedirectWebClient(userAgent); - using (rwClient.OpenRead(url)) { } - var location = rwClient.ResponseHeaders?["Location"]; - if (location == null) + foreach ((Uri from, Uri to) in urls.Zip(urls.Skip(1))) { - return url; - } - else if (Uri.IsWellFormedUriString(location, UriKind.Absolute)) - { - url = new Uri(location); - } - else if (Uri.IsWellFormedUriString(location, UriKind.Relative)) - { - url = new Uri(url, location); - log.DebugFormat("Relative URL {0} is absolute URL {1}", location, url); - } - else - { - throw new Kraken(string.Format(Properties.Resources.NetInvalidLocation, location)); + log.DebugFormat("Redirected {0} to {1}", from, to); } } - return null; + return urls.LastOrDefault(); + } + + private static bool DisposeStream(Stream s) + { + s.Dispose(); + return true; } /// diff --git a/Netkan/Transformers/SourceForgeTransformer.cs b/Netkan/Transformers/SourceForgeTransformer.cs index 6b76c70f8..ad5b6f8e4 100644 --- a/Netkan/Transformers/SourceForgeTransformer.cs +++ b/Netkan/Transformers/SourceForgeTransformer.cs @@ -1,6 +1,8 @@ using System; using System.Linq; +using System.Web; using System.Collections.Generic; +using System.Collections.Specialized; using Newtonsoft.Json.Linq; using log4net; @@ -74,8 +76,18 @@ private static Metadata TransformOne(JObject json, { "bugtracker", mod.BugTrackerLink }, })); // SourceForge doesn't send redirects to user agents it considers browser-like - json.SafeAdd("download", Net.ResolveRedirect(version.Link, "Wget") - ?.OriginalString); + if (Net.ResolveRedirect(version.Link, "Wget", 1) is Uri firstRedir) + { + // SourceForge redirects to different mirrors for load-balancing + // (IF it considers your user agent string a non-browser, which excludes the CKAN client), + // but for us that means CKAN users constantly shifting from one server + // to another in unison as the bot changes the URL in the metadata. + // https://sourceforge.net/p/forge/documentation/Mirrors/ + // Tweak the intermediate redirect URL to use the same mirror every time. + json.SafeAdd("download", Net.ResolveRedirect(SetQueryKey(firstRedir, "use_mirror", mirror), + "Wget", 1) + ?.OriginalString); + } json.SafeAdd(Metadata.UpdatedPropertyName, version.Timestamp); json.Remove("$kref"); @@ -84,7 +96,22 @@ private static Metadata TransformOne(JObject json, return new Metadata(json); } + private static Uri SetQueryKey(Uri url, string key, string value) + { + if (HttpUtility.ParseQueryString(url.Query) is NameValueCollection newQuery) + { + newQuery.Set(key, value); + return new UriBuilder(url) + { + Query = newQuery.ToString(), + }.Uri; + + } + return url; + } + private readonly ISourceForgeApi api; - private static readonly ILog log = LogManager.GetLogger(typeof(GitlabTransformer)); + private const string mirror = "psychz"; // Brooklyn, United States + private static readonly ILog log = LogManager.GetLogger(typeof(GitlabTransformer)); } }