From 6093f6d4f3960a68957080b845d1d8c647aa37b1 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 09:57:40 +0100 Subject: [PATCH] [4.2.x] Fix #742 - Add setting to limit NRTM response size (backport #745) (#751) (cherry picked from commit 0e33e13c9bb03ad4245e04a74cd38d1d4fb8b0c6) Co-authored-by: Sasha Romijn --- docs/admins/configuration.rst | 10 +++++ irrd/conf/__init__.py | 3 ++ irrd/conf/test_conf.py | 3 ++ irrd/mirroring/nrtm_generator.py | 5 +++ irrd/mirroring/tests/test_nrtm_generator.py | 44 ++++++++++++++++++++- 5 files changed, 64 insertions(+), 1 deletion(-) diff --git a/docs/admins/configuration.rst b/docs/admins/configuration.rst index 12317da2c..ae25772c5 100644 --- a/docs/admins/configuration.rst +++ b/docs/admins/configuration.rst @@ -539,6 +539,16 @@ Sources |br| **Default**: not defined, all access denied. Clients in ``nrtm_access_list``, if defined, have filtered access. |br| **Change takes effect**: after SIGHUP, upon next request. +* ``sources.{name}.nrtm_query_serial_range_limit``: the maximum number of + serials a client may request in one NRTM query, if otherwise permitted. + This is intended to limit the maximum load of NRTM queries - it is checked + before IRRd runs any heavy database queries. The limit is applied to the + requested range regardless of any gaps, i.e. querying a range of ``10-20`` + is allowed if this setting to be at least 10, even if there are no entries + for some of those serials. IRRd is aware of the serial ``LAST`` refers to + and will take that into account. + |br| **Default**: not defined, no limits on NRTM query size. + |br| **Change takes effect**: after SIGHUP, upon next request. * ``sources.{name}.strict_import_keycert_objects``: a setting used when migrating authoritative data that may contain `key-cert` objects. See the :doc:`data migration guide ` diff --git a/irrd/conf/__init__.py b/irrd/conf/__init__.py index c61642ac2..7bdde382d 100644 --- a/irrd/conf/__init__.py +++ b/irrd/conf/__init__.py @@ -98,6 +98,7 @@ 'export_timer', 'nrtm_access_list', 'nrtm_access_list_unfiltered', + 'nrtm_query_serial_range_limit', 'strict_import_keycert_objects', 'rpki_excluded', 'scopefilter_excluded', @@ -398,6 +399,8 @@ def _check_staging_config(self) -> List[str]: errors.append(f'Setting import_timer for source {name} must be a number.') if not str(details.get('export_timer', '0')).isnumeric(): errors.append(f'Setting export_timer for source {name} must be a number.') + if not str(details.get('nrtm_query_serial_range_limit', '0')).isnumeric(): + errors.append(f'Setting nrtm_query_serial_range_limit for source {name} must be a number.') if details.get('nrtm_access_list'): expected_access_lists.add(details.get('nrtm_access_list')) diff --git a/irrd/conf/test_conf.py b/irrd/conf/test_conf.py index 7a98c9032..807f9add5 100644 --- a/irrd/conf/test_conf.py +++ b/irrd/conf/test_conf.py @@ -80,6 +80,7 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp 'TESTDB': { 'authoritative': True, 'keep_journal': True, + 'nrtm_query_serial_range_limit': 10, }, 'TESTDB2': { 'nrtm_host': '192.0.2.1', @@ -259,6 +260,7 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir): 'export_timer': 'bar', 'nrtm_host': '192.0.2.1', 'unknown': True, + 'nrtm_query_serial_range_limit': 'not-a-number', }, 'TESTDB2': { 'authoritative': True, @@ -316,6 +318,7 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir): assert 'Setting rpki.notify_invalid_header must be a string, if defined.' in str(ce.value) assert 'Setting import_timer for source TESTDB must be a number.' in str(ce.value) assert 'Setting export_timer for source TESTDB must be a number.' in str(ce.value) + assert 'Setting nrtm_query_serial_range_limit for source TESTDB must be a number.' in str(ce.value) assert 'Invalid source name: lowercase' in str(ce.value) assert 'Invalid source name: invalid char' in str(ce.value) assert 'but rpki.notify_invalid_enabled is not set' in str(ce.value) diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index 5572e8e65..59670a62b 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -57,7 +57,12 @@ def generate(self, source: str, version: str, serial_end_display = serial_end_available if serial_end_requested is None else serial_end_requested + range_limit = get_setting(f'sources.{source}.nrtm_query_serial_range_limit') + if range_limit and int(range_limit) < (serial_end_display - serial_start_requested): + raise NRTMGeneratorException(f'Serial range requested exceeds maximum range of {range_limit}') + q = RPSLDatabaseJournalQuery().sources([source]).serial_range(serial_start_requested, serial_end_requested) + operations = list(database_handler.execute_query(q)) output = f'%START Version: {version} {source} {serial_start_requested}-{serial_end_display}\n' diff --git a/irrd/mirroring/tests/test_nrtm_generator.py b/irrd/mirroring/tests/test_nrtm_generator.py index 300240b86..fc2af5eba 100644 --- a/irrd/mirroring/tests/test_nrtm_generator.py +++ b/irrd/mirroring/tests/test_nrtm_generator.py @@ -15,6 +15,7 @@ def prepare_generator(monkeypatch, config_override): 'sources': { 'TEST': { 'keep_journal': True, + 'nrtm_query_serial_range_limit': 200, } } }) @@ -84,7 +85,7 @@ def test_generate_serial_range_v1(self, prepare_generator): %END TEST""").strip() - def test_generate_until_last(self, prepare_generator): + def test_generate_until_last(self, prepare_generator, config_override): generator, mock_dh = prepare_generator result = generator.generate('TEST', '3', 110, None, mock_dh) @@ -169,6 +170,47 @@ def test_no_source_status_entry(self, prepare_generator, config_override): generator.generate('TEST', '3', 110, 300, mock_dh) assert 'There are no journal entries for this source.' in str(nge.value) + def test_v3_range_limit_not_set(self, prepare_generator, config_override): + generator, mock_dh = prepare_generator + config_override({ + 'sources': { + 'TEST': { + 'keep_journal': True, + } + } + }) + + result = generator.generate('TEST', '3', 110, 190, mock_dh) + + assert result == textwrap.dedent(""" + %START Version: 3 TEST 110-190 + + ADD 120 + + object 1 🦄 + auth: CRYPT-PW DummyValue # Filtered for security + + DEL 180 + + object 2 🌈 + + %END TEST""").strip() + + def test_range_limit_exceeded(self, prepare_generator, config_override): + generator, mock_dh = prepare_generator + config_override({ + 'sources': { + 'TEST': { + 'keep_journal': True, + 'nrtm_query_serial_range_limit': 50, + } + } + }) + + with pytest.raises(NRTMGeneratorException) as nge: + generator.generate('TEST', '3', 110, 190, mock_dh) + assert 'Serial range requested exceeds maximum range of 50' in str(nge.value) + def test_include_auth_hash(self, prepare_generator): generator, mock_dh = prepare_generator result = generator.generate('TEST', '3', 110, 190, mock_dh, False)