Pass fetch_url_allowlist from DRS to HTTP source

lib/galaxy/celery/__init__.py

@@ -115,7 +115,6 @@ def get_app_properties():
 def get_config():
     kwargs = get_app_properties() or {}
     kwargs["override_tempdir"] = False
-    kwargs["fetch_url_allowlist"] = ["127.0.0.0/24"]
     return Configuration(**kwargs)
 
 

lib/galaxy/files/sources/drs.py

@@ -42,10 +42,21 @@ def __init__(self, **kwd: Unpack[FilesSourceProperties]):
         self._force_http = props.pop("force_http", False)
         self._props = props
 
+    @property
+    def _allowlist(self):
+        return self._file_sources_config.fetch_url_allowlist
+
     def _realize_to(self, source_path, native_path, user_context=None, opts: Optional[FilesSourceOptions] = None):
         props = self._serialization_props(user_context)
         headers = props.pop("http_headers", {}) or {}
-        fetch_drs_to_file(source_path, native_path, user_context, headers=headers, force_http=self._force_http)
+        fetch_drs_to_file(
+            source_path,
+            native_path,
+            user_context,
+            fetch_url_allowlist=self._allowlist,
+            headers=headers,
+            force_http=self._force_http,
+        )
 
     def _write_from(self, target_path, native_path, user_context=None, opts: Optional[FilesSourceOptions] = None):
         raise NotImplementedError()

lib/galaxy/files/sources/http.py

@@ -4,6 +4,7 @@
 from typing import (
     cast,
     Dict,
+    List,
     Optional,
 )
 
@@ -15,6 +16,7 @@
     get_charset_from_http_headers,
     stream_to_open_named_file,
 )
+from galaxy.util.config_parsers import IpAllowedListEntryT
 from . import (
     BaseFilesSource,
     FilesSourceOptions,
@@ -27,6 +29,7 @@
 class HTTPFilesSourceProperties(FilesSourceProperties, total=False):
     url_regex: str
     http_headers: Dict[str, str]
+    fetch_url_allowlist: List[IpAllowedListEntryT]
 
 
 class HTTPFilesSource(BaseFilesSource):
@@ -61,7 +64,7 @@ def _realize_to(
 
         with urllib.request.urlopen(req, timeout=DEFAULT_SOCKET_TIMEOUT) as page:
             # Verify url post-redirects is still allowlisted
-            validate_non_local(page.geturl(), self._allowlist)
+            validate_non_local(page.geturl(), self._allowlist or extra_props.get("fetch_url_allowlist") or [])
             f = open(native_path, "wb")  # fd will be .close()ed in stream_to_open_named_file
             return stream_to_open_named_file(
                 page, f.fileno(), native_path, source_encoding=get_charset_from_http_headers(page.headers)

lib/galaxy/util/drs.py

@@ -1,6 +1,7 @@
 import time
 from os import PathLike
 from typing import (
+    List,
     Optional,
     Tuple,
     Union,
@@ -17,6 +18,7 @@
 from galaxy.files.sources.http import HTTPFilesSourceProperties
 from galaxy.files.uris import stream_url_to_file
 from galaxy.util import DEFAULT_SOCKET_TIMEOUT
+from galaxy.util.config_parsers import IpAllowedListEntryT
 
 TargetPathT = Union[str, PathLike]
 
@@ -81,6 +83,7 @@ def fetch_drs_to_file(
     force_http=False,
     retry_options: Optional[RetryOptions] = None,
     headers: Optional[dict] = None,
+    fetch_url_allowlist: Optional[List[IpAllowedListEntryT]] = None,
 ):
     """Fetch contents of drs:// URI to a target path."""
     if not drs_uri.startswith("drs://"):
@@ -107,7 +110,10 @@ def fetch_drs_to_file(
         access_url, access_headers = _get_access_info(get_url, access_method, headers=headers)
         opts = FilesSourceOptions()
         if access_method["type"] == "https":
-            extra_props: HTTPFilesSourceProperties = {"http_headers": access_headers or {}}
+            extra_props: HTTPFilesSourceProperties = {
+                "http_headers": access_headers or {},
+                "fetch_url_allowlist": fetch_url_allowlist or [],
+            }
             opts.extra_props = extra_props
         else:
             opts.extra_props = {}