Add FTP mirroring timeout (#763)

Co-authored-by: Sasha Romijn <[email protected]>
irrdnet · Mar 6, 2023 · a5c2f2b · a5c2f2b
1 parent bd3f8af
commit a5c2f2b
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/irrd/mirroring/mirror_runners_import.py b/irrd/mirroring/mirror_runners_import.py
@@ -26,6 +26,7 @@
 from .parsers import MirrorFileImportParser, NRTMStreamParser
 
 logger = logging.getLogger(__name__)
+DOWNLOAD_TIMEOUT = 10
 
 
 class RPSLMirrorImportUpdateRunner:
@@ -167,12 +168,12 @@ def _download_file(self, destination: IO[Any], url: str, url_parsed):
         """
         if url_parsed.scheme == "ftp":
             try:
-                r = request.urlopen(url)
+                r = request.urlopen(url, timeout=DOWNLOAD_TIMEOUT)
                 shutil.copyfileobj(r, destination)
             except URLError as error:
                 raise OSError(f"Failed to download {url}: {str(error)}")
         elif url_parsed.scheme in ["http", "https"]:
-            r = requests.get(url, stream=True, timeout=10)
+            r = requests.get(url, stream=True, timeout=DOWNLOAD_TIMEOUT)
             if r.status_code == 200:
                 for chunk in r.iter_content(10240):
                     destination.write(chunk)

diff --git a/irrd/mirroring/tests/test_mirror_runners_import.py b/irrd/mirroring/tests/test_mirror_runners_import.py
@@ -189,7 +189,7 @@ def test_run_import_ftp(self, monkeypatch, config_override):
             "ftp://host/source2": b"source2",
             "ftp://host/serial": b"424242",
         }
-        request.urlopen = lambda url: MockUrlopenResponse(responses[url])
+        request.urlopen = lambda url, timeout: MockUrlopenResponse(responses[url])
         RPSLMirrorFullImportRunner("TEST").run(mock_dh, serial_newest_mirror=424241)
 
         assert MockMirrorFileImportParser.rpsl_data_calls == ["source1", "source2"]
@@ -225,7 +225,7 @@ def test_failed_import_ftp(self, monkeypatch, config_override):
             "irrd.mirroring.mirror_runners_import.BulkRouteROAValidator", mock_bulk_validator_init
         )
 
-        request.urlopen = lambda url: MockUrlopenResponse(b"", fail=True)
+        request.urlopen = lambda url, timeout: MockUrlopenResponse(b"", fail=True)
         with pytest.raises(IOError):
             RPSLMirrorFullImportRunner("TEST").run(mock_dh, serial_newest_mirror=424241)
 
@@ -296,7 +296,7 @@ def test_no_serial_ftp(self, monkeypatch, config_override):
             "ftp://host/source1.gz": b64decode("H4sIAE4CfFsAAyvOLy1KTjUEAE5Fj0oHAAAA"),
             "ftp://host/source2": b"source2",
         }
-        request.urlopen = lambda url: MockUrlopenResponse(responses[url])
+        request.urlopen = lambda url, timeout: MockUrlopenResponse(responses[url])
         RPSLMirrorFullImportRunner("TEST").run(mock_dh, serial_newest_mirror=42)
 
         assert MockMirrorFileImportParser.rpsl_data_calls == ["source1", "source2"]
@@ -331,7 +331,7 @@ def test_import_cancelled_serial_too_old(self, monkeypatch, config_override, cap
             "ftp://host/source2": b"source2",
             "ftp://host/serial": b"424242",
         }
-        request.urlopen = lambda url: MockUrlopenResponse(responses[url])
+        request.urlopen = lambda url, timeout: MockUrlopenResponse(responses[url])
         RPSLMirrorFullImportRunner("TEST").run(mock_dh, serial_newest_mirror=424243)
 
         assert not MockMirrorFileImportParser.rpsl_data_calls
@@ -365,7 +365,7 @@ def test_import_force_reload_with_serial_too_old(self, monkeypatch, config_overr
             "ftp://host/source2": b"source2",
             "ftp://host/serial": b"424242",
         }
-        request.urlopen = lambda url: MockUrlopenResponse(responses[url])
+        request.urlopen = lambda url, timeout: MockUrlopenResponse(responses[url])
         RPSLMirrorFullImportRunner("TEST").run(mock_dh, serial_newest_mirror=424243, force_reload=True)
 
         assert MockMirrorFileImportParser.rpsl_data_calls == ["source1", "source2"]