From a5c2f2b0888a90808d5cc9acbe7707b0cfc0d6ad Mon Sep 17 00:00:00 2001 From: Alexander Lopintsev Date: Mon, 6 Mar 2023 12:39:09 +0300 Subject: [PATCH] Add FTP mirroring timeout (#763) Co-authored-by: Sasha Romijn --- irrd/mirroring/mirror_runners_import.py | 5 +++-- irrd/mirroring/tests/test_mirror_runners_import.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/irrd/mirroring/mirror_runners_import.py b/irrd/mirroring/mirror_runners_import.py index 97f7cf39e..5230c62ed 100644 --- a/irrd/mirroring/mirror_runners_import.py +++ b/irrd/mirroring/mirror_runners_import.py @@ -26,6 +26,7 @@ from .parsers import MirrorFileImportParser, NRTMStreamParser logger = logging.getLogger(__name__) +DOWNLOAD_TIMEOUT = 10 class RPSLMirrorImportUpdateRunner: @@ -167,12 +168,12 @@ def _download_file(self, destination: IO[Any], url: str, url_parsed): """ if url_parsed.scheme == "ftp": try: - r = request.urlopen(url) + r = request.urlopen(url, timeout=DOWNLOAD_TIMEOUT) shutil.copyfileobj(r, destination) except URLError as error: raise OSError(f"Failed to download {url}: {str(error)}") elif url_parsed.scheme in ["http", "https"]: - r = requests.get(url, stream=True, timeout=10) + r = requests.get(url, stream=True, timeout=DOWNLOAD_TIMEOUT) if r.status_code == 200: for chunk in r.iter_content(10240): destination.write(chunk) diff --git a/irrd/mirroring/tests/test_mirror_runners_import.py b/irrd/mirroring/tests/test_mirror_runners_import.py index ad8abc94e..89b3f7ae2 100644 --- a/irrd/mirroring/tests/test_mirror_runners_import.py +++ b/irrd/mirroring/tests/test_mirror_runners_import.py @@ -189,7 +189,7 @@ def test_run_import_ftp(self, monkeypatch, config_override): "ftp://host/source2": b"source2", "ftp://host/serial": b"424242", } - request.urlopen = lambda url: MockUrlopenResponse(responses[url]) + request.urlopen = lambda url, timeout: MockUrlopenResponse(responses[url]) RPSLMirrorFullImportRunner("TEST").run(mock_dh, serial_newest_mirror=424241) assert MockMirrorFileImportParser.rpsl_data_calls == ["source1", "source2"] @@ -225,7 +225,7 @@ def test_failed_import_ftp(self, monkeypatch, config_override): "irrd.mirroring.mirror_runners_import.BulkRouteROAValidator", mock_bulk_validator_init ) - request.urlopen = lambda url: MockUrlopenResponse(b"", fail=True) + request.urlopen = lambda url, timeout: MockUrlopenResponse(b"", fail=True) with pytest.raises(IOError): RPSLMirrorFullImportRunner("TEST").run(mock_dh, serial_newest_mirror=424241) @@ -296,7 +296,7 @@ def test_no_serial_ftp(self, monkeypatch, config_override): "ftp://host/source1.gz": b64decode("H4sIAE4CfFsAAyvOLy1KTjUEAE5Fj0oHAAAA"), "ftp://host/source2": b"source2", } - request.urlopen = lambda url: MockUrlopenResponse(responses[url]) + request.urlopen = lambda url, timeout: MockUrlopenResponse(responses[url]) RPSLMirrorFullImportRunner("TEST").run(mock_dh, serial_newest_mirror=42) assert MockMirrorFileImportParser.rpsl_data_calls == ["source1", "source2"] @@ -331,7 +331,7 @@ def test_import_cancelled_serial_too_old(self, monkeypatch, config_override, cap "ftp://host/source2": b"source2", "ftp://host/serial": b"424242", } - request.urlopen = lambda url: MockUrlopenResponse(responses[url]) + request.urlopen = lambda url, timeout: MockUrlopenResponse(responses[url]) RPSLMirrorFullImportRunner("TEST").run(mock_dh, serial_newest_mirror=424243) assert not MockMirrorFileImportParser.rpsl_data_calls @@ -365,7 +365,7 @@ def test_import_force_reload_with_serial_too_old(self, monkeypatch, config_overr "ftp://host/source2": b"source2", "ftp://host/serial": b"424242", } - request.urlopen = lambda url: MockUrlopenResponse(responses[url]) + request.urlopen = lambda url, timeout: MockUrlopenResponse(responses[url]) RPSLMirrorFullImportRunner("TEST").run(mock_dh, serial_newest_mirror=424243, force_reload=True) assert MockMirrorFileImportParser.rpsl_data_calls == ["source1", "source2"]