Merge #3557 One concurrent download per host for all hosts

CHANGELOG.md

@@ -39,6 +39,7 @@ All notable changes to this project will be documented in this file.
 - [Core] Disable tx timeouts, add tx debug logging, static DLL pattern (#3512 by: HebaruSan; reviewed: DasSkelett)
 - [Core] Only delete diversely capitalized directories once on Windows (#3528 by: HebaruSan; reviewed: DasSkelett)
 - [Core] Get licenses from embedded schema, skip bad modules in deserialize (#3526 by: HebaruSan; reviewed: DasSkelett)
+- [Core] One concurrent download per host for all hosts (#3557 by: HebaruSan; reviewed: DasSkelett)
 
 ### Internal
 

Core/Net/NetAsyncDownloader.cs

@@ -96,19 +96,6 @@ private void ResetAgent()
             }
         }
 
-        /// <summary>
-        /// Downloads from hosts in this list will be done sequentially rather
-        /// than in parallel
-        /// </summary>
-        private static readonly HashSet<string> throttledHosts = new HashSet<string>()
-        {
-            /// GitHub returns a 403-Forbidden status sometimes if you try to download
-            /// too much in parallel, see https://github.com/KSP-CKAN/CKAN/issues/2210
-            "github.com",
-            "api.github.com",
-            "raw.githubusercontent.com",
-        };
-
         private static readonly ILog log = LogManager.GetLogger(typeof (NetAsyncDownloader));
 
         public readonly IUser User;
@@ -189,17 +176,16 @@ private void DownloadModule(Net.DownloadTarget target)
 
         /// <summary>
         /// Check whether a given download should be deferred to be started later.
-        /// Decision is made based on whether the host is throttled and whether
-        /// we're already downloading something else from it.
+        /// Decision is made based on whether we're already downloading something
+        /// else from the same host.
         /// </summary>
         /// <param name="target">Info about a requested download</param>
         /// <returns>
         /// true to queue, false to start immediately
         /// </returns>
         private bool shouldQueue(Net.DownloadTarget target)
         {
-            return throttledHosts.Contains(target.url.Host)
-                && downloads.Any(dl =>
+            return downloads.Any(dl =>
                     dl.target.url.Host == target.url.Host
                     && dl.bytesLeft > 0);
         }
@@ -423,16 +409,13 @@ private void FileDownloadComplete(int index, Exception error)
                 log.InfoFormat("Finished downloading {0}", downloads[index].target.url);
             }
 
-            if (throttledHosts.Contains(downloads[index].target.url.Host))
+            var next = queuedDownloads.FirstOrDefault(dl =>
+                dl.url.Host == downloads[index].target.url.Host);
+            if (next != null)
             {
-                var next = queuedDownloads.FirstOrDefault(dl =>
-                    dl.url.Host == downloads[index].target.url.Host);
-                if (next != null)
-                {
-                    // Start this host's next queued download
-                    queuedDownloads.Remove(next);
-                    DownloadModule(next);
-                }
+                // Start this host's next queued download
+                queuedDownloads.Remove(next);
+                DownloadModule(next);
             }
 
             onOneCompleted.Invoke(downloads[index].target.url, downloads[index].path, downloads[index].error);