From 97cbd22fa1123aa6bac0101f45ec1382bef16588 Mon Sep 17 00:00:00 2001
From: Sasha Romijn <github@mxsasha.eu>
Date: Tue, 11 Apr 2023 19:58:31 +0200
Subject: [PATCH] Disable routepref filter_prefixes for large filter sets

When using filter_prefixes, we run one SQL query per prefix,
which has overhead for large sets. This ignored it when there
are >5000.
---
 irrd/routepref/routepref.py            |  4 ++++
 irrd/routepref/tests/test_routepref.py | 31 +++++++++++++++++++++++++-
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/irrd/routepref/routepref.py b/irrd/routepref/routepref.py
index 45ae34ef5..566139754 100644
--- a/irrd/routepref/routepref.py
+++ b/irrd/routepref/routepref.py
@@ -12,6 +12,7 @@
 from .status import RoutePreferenceStatus
 
 logger = logging.getLogger(__name__)
+MAX_FILTER_PREFIX_LEN = 5000
 
 
 class RoutePreferenceValidator:
@@ -137,6 +138,9 @@ def update_route_preference_status(
     and an optional set of prefixes to limit the evaluation.
     Results are saved to the database and counters are logged.
     """
+    if filter_prefixes and len(filter_prefixes) > MAX_FILTER_PREFIX_LEN:
+        filter_prefixes = None
+
     validator = build_validator(database_handler, filter_prefixes)
     pks_to_be_visible, pks_to_be_suppressed = validator.validate_known_routes()
     pks_to_become_visible = pks_to_be_visible + validator.excluded_currently_suppressed
diff --git a/irrd/routepref/tests/test_routepref.py b/irrd/routepref/tests/test_routepref.py
index 96a8fd49f..6ba743f82 100644
--- a/irrd/routepref/tests/test_routepref.py
+++ b/irrd/routepref/tests/test_routepref.py
@@ -1,3 +1,5 @@
+import itertools
+
 from IPy import IP
 
 from irrd.storage.queries import RPSLDatabaseQuery
@@ -99,7 +101,7 @@ def test_route_preference_validator(config_override):
     assert to_be_suppressed == []
 
 
-def test_update_route_preference_status(config_override):
+def test_update_route_preference_status(config_override, monkeypatch):
     config_override(
         {
             "sources": {
@@ -189,3 +191,30 @@ def test_update_route_preference_status(config_override):
             {"rpsl_objs_now_visible": [], "rpsl_objs_now_suppressed": []},
         )
     ]
+
+    # Finally, test with a "large" set of prefixes
+    monkeypatch.setattr("irrd.routepref.routepref.MAX_FILTER_PREFIX_LEN", 5)
+    mock_dh.reset_mock()
+    mock_dh.query_responses[RPSLDatabaseQuery] = iter(route_objects)
+    ip = IP("192.0.0.0/23")
+    filter_prefixes = list(itertools.chain.from_iterable(itertools.repeat(ip, 10)))
+    update_route_preference_status(mock_dh, filter_prefixes)
+    assert mock_dh.queries == [
+        RPSLDatabaseQuery(column_names=expected_columns, ordered_by_sources=False).object_classes(
+            object_classes
+        ),
+        RPSLDatabaseQuery(
+            enrich_columns,
+            enable_ordering=False,
+        ).pks(["route-A"]),
+        RPSLDatabaseQuery(
+            enrich_columns,
+            enable_ordering=False,
+        ).pks(["route-D"]),
+    ]
+    assert mock_dh.other_calls == [
+        (
+            "update_route_preference_status",
+            {"rpsl_objs_now_visible": [], "rpsl_objs_now_suppressed": []},
+        )
+    ]