From 8cf25a14da2f18ab91e5ad5d30fa9ef85f341b75 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Thu, 23 Feb 2023 15:11:54 +0100 Subject: [PATCH] Run isort --- irrd/conf/known_keys.py | 2 +- irrd/conf/test_conf.py | 12 +++- irrd/daemon/main.py | 17 +++--- irrd/integration_tests/run.py | 54 +++++++++++------ irrd/mirroring/mirror_runners_export.py | 5 +- irrd/mirroring/mirror_runners_import.py | 9 +-- irrd/mirroring/nrtm_generator.py | 2 +- irrd/mirroring/nrtm_operation.py | 4 +- irrd/mirroring/parsers.py | 11 ++-- irrd/mirroring/scheduler.py | 22 ++++--- .../tests/test_mirror_runners_export.py | 5 +- .../tests/test_mirror_runners_import.py | 13 ++-- irrd/mirroring/tests/test_nrtm_generator.py | 5 +- irrd/mirroring/tests/test_nrtm_operation.py | 12 +++- irrd/mirroring/tests/test_parsers.py | 36 ++++++++--- irrd/mirroring/tests/test_scheduler.py | 5 +- irrd/routepref/routepref.py | 4 +- irrd/routepref/tests/test_routepref.py | 1 + irrd/rpki/importer.py | 7 +-- irrd/rpki/tests/test_importer.py | 1 + irrd/rpki/tests/test_notifications.py | 7 ++- irrd/rpki/tests/test_validators.py | 6 +- irrd/rpki/validators.py | 11 ++-- irrd/rpsl/fields.py | 7 ++- irrd/rpsl/parser.py | 9 +-- irrd/rpsl/parser_state.py | 2 +- irrd/rpsl/passwords.py | 6 +- irrd/rpsl/rpsl_objects.py | 41 +++++++++---- irrd/rpsl/tests/test_fields.py | 27 ++++++--- irrd/rpsl/tests/test_rpsl_objects.py | 60 +++++++++++++------ irrd/scopefilter/tests/test_scopefilter.py | 3 +- irrd/scopefilter/validators.py | 3 +- irrd/scripts/database_downgrade.py | 6 +- irrd/scripts/database_upgrade.py | 6 +- irrd/scripts/expire_journal.py | 2 +- irrd/scripts/load_database.py | 6 +- irrd/scripts/load_pgp_keys.py | 8 +-- irrd/scripts/load_test.py | 3 +- irrd/scripts/mirror_force_reload.py | 5 +- irrd/scripts/query_qa_comparison.py | 12 ++-- irrd/scripts/rpsl_read.py | 3 +- irrd/scripts/set_last_modified_auth.py | 6 +- irrd/scripts/submit_changes.py | 3 +- irrd/scripts/submit_email.py | 5 +- irrd/scripts/tests/test_expire_journal.py | 1 + irrd/scripts/tests/test_irr_rpsl_submit.py | 7 ++- irrd/scripts/tests/test_load_database.py | 1 + irrd/scripts/tests/test_load_pgp_keys.py | 6 +- .../scripts/tests/test_mirror_force_reload.py | 1 + .../tests/test_set_last_modified_auth.py | 1 + irrd/scripts/tests/test_submit_update.py | 2 +- irrd/scripts/tests/test_update_database.py | 1 + irrd/scripts/update_database.py | 7 +-- irrd/server/graphql/resolvers.py | 15 ++--- irrd/server/graphql/schema_builder.py | 26 +++++--- irrd/server/graphql/schema_generator.py | 14 +++-- irrd/server/graphql/tests/test_resolvers.py | 5 +- irrd/server/http/app.py | 9 ++- irrd/server/http/endpoints.py | 5 +- irrd/server/http/event_stream.py | 18 +++--- irrd/server/http/server.py | 3 +- irrd/server/http/tests/test_endpoints.py | 10 +++- irrd/server/http/tests/test_event_stream.py | 3 +- .../http/tests/test_status_generator.py | 1 + irrd/server/query_resolver.py | 15 +++-- irrd/server/tests/test_query_resolver.py | 5 +- irrd/server/whois/query_parser.py | 19 ++++-- irrd/server/whois/server.py | 2 +- irrd/server/whois/tests/test_query_parser.py | 9 ++- .../server/whois/tests/test_query_response.py | 7 ++- irrd/server/whois/tests/test_server.py | 1 + irrd/storage/alembic/env.py | 4 +- ...0548f1aa4f10_add_rpsl_objects_suspended.py | 2 +- .../1743f98a456d_add_serial_newest_mirror.py | 3 +- .../181670a62643_add_journal_entry_origin.py | 3 +- .../versions/28dc1cd85bdc_initial_db.py | 2 +- .../versions/39e4f15ed80c_add_bogon_status.py | 3 +- .../4a514ead8fc2_bogon_to_scope_filter.py | 2 +- .../8744b4b906bb_fix_rpsl_unique_key.py | 1 - .../893d0d5363b3_add_rpsl_prefix_idx.py | 2 +- .../8b8357acd333_add_global_serial.py | 6 +- ...aa3_set_prefix_length_in_existing_rpsl_.py | 1 - .../versions/b175c262448f_set_rpsl_prefix.py | 1 - .../e07863eac52f_add_roa_object_table.py | 2 +- .../versions/f4c837d8258c_add_rpsl_prefix.py | 2 +- ...d4473bc1a10_add_route_preference_status.py | 2 +- irrd/storage/database_handler.py | 31 ++++++---- irrd/storage/preload.py | 5 +- irrd/storage/queries.py | 17 ++++-- irrd/storage/tests/test_database.py | 15 +++-- irrd/storage/tests/test_event_stream.py | 6 +- irrd/storage/tests/test_preload.py | 11 +++- irrd/updates/email.py | 1 + irrd/updates/handler.py | 9 +-- irrd/updates/parser.py | 15 ++--- irrd/updates/parser_state.py | 4 +- irrd/updates/suspension.py | 12 ++-- irrd/updates/tests/test_email.py | 3 +- irrd/updates/tests/test_handler.py | 3 +- irrd/updates/tests/test_parser.py | 22 +++++-- irrd/updates/tests/test_suspension.py | 21 ++++--- irrd/updates/tests/test_validators.py | 17 ++++-- irrd/updates/validators.py | 5 +- irrd/utils/email.py | 4 +- irrd/utils/pgp.py | 6 +- irrd/utils/test_utils.py | 6 +- irrd/utils/tests/test_email.py | 4 +- irrd/utils/tests/test_text.py | 13 +++- irrd/utils/tests/test_validators.py | 2 +- irrd/utils/tests/test_whois_client.py | 7 ++- irrd/utils/text.py | 2 +- irrd/utils/validators.py | 3 +- irrd/utils/whois_client.py | 2 +- irrd/vendor/dotted/collection.py | 2 - irrd/vendor/postgres_copy/__init__.py | 2 +- 115 files changed, 603 insertions(+), 351 deletions(-) diff --git a/irrd/conf/known_keys.py b/irrd/conf/known_keys.py index 5d9a91df9..2eb0a392a 100644 --- a/irrd/conf/known_keys.py +++ b/irrd/conf/known_keys.py @@ -1,7 +1,7 @@ from irrd.conf import AUTH_SET_CREATION_COMMON_KEY from irrd.rpsl.passwords import PASSWORD_HASHERS_ALL -from irrd.vendor.dotted.collection import DottedDict from irrd.rpsl.rpsl_objects import OBJECT_CLASS_MAPPING, RPSLSet +from irrd.vendor.dotted.collection import DottedDict # Note that sources are checked separately, # and 'access_lists' is always permitted diff --git a/irrd/conf/test_conf.py b/irrd/conf/test_conf.py index b5e0540fd..2bf379014 100644 --- a/irrd/conf/test_conf.py +++ b/irrd/conf/test_conf.py @@ -1,12 +1,18 @@ import os +import signal import textwrap +from typing import Dict import pytest -import signal import yaml -from typing import Dict -from . import get_setting, ConfigurationError, config_init, is_config_initialised, get_configuration +from . import ( + ConfigurationError, + config_init, + get_configuration, + get_setting, + is_config_initialised, +) @pytest.fixture() diff --git a/irrd/daemon/main.py b/irrd/daemon/main.py index 34bbf2f96..9749765ec 100755 --- a/irrd/daemon/main.py +++ b/irrd/daemon/main.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # flake8: noqa: E402 import argparse +import grp import logging import multiprocessing import os @@ -9,10 +10,9 @@ import sys import time from pathlib import Path -from typing import Tuple, Optional +from typing import Optional, Tuple import daemon -import grp import psutil from daemon.daemon import change_process_owner from pid import PidFile, PidFileError @@ -20,14 +20,13 @@ logger = logging.getLogger(__name__) sys.path.append(str(Path(__file__).resolve().parents[2])) -from irrd.utils.process_support import ExceptionLoggingProcess, set_traceback_handler -from irrd.storage.preload import PreloadStoreManager -from irrd.server.whois.server import start_whois_server -from irrd.server.http.server import run_http_server +from irrd import ENV_MAIN_PROCESS_PID, __version__ +from irrd.conf import CONFIG_PATH_DEFAULT, config_init, get_configuration, get_setting from irrd.mirroring.scheduler import MirrorScheduler -from irrd.conf import config_init, CONFIG_PATH_DEFAULT, get_setting, get_configuration -from irrd import __version__, ENV_MAIN_PROCESS_PID - +from irrd.server.http.server import run_http_server +from irrd.server.whois.server import start_whois_server +from irrd.storage.preload import PreloadStoreManager +from irrd.utils.process_support import ExceptionLoggingProcess, set_traceback_handler # This file does not have a unit test, but is instead tested through # the integration tests. Writing a unit test would be too complex. diff --git a/irrd/integration_tests/run.py b/irrd/integration_tests/run.py index 232ea660d..5b4ab1a69 100644 --- a/irrd/integration_tests/run.py +++ b/irrd/integration_tests/run.py @@ -1,33 +1,53 @@ -import sys -import time -import unittest - -import ujson - import base64 import email import os -import requests import signal import socket -import sqlalchemy as sa import subprocess +import sys import textwrap -import yaml -from alembic import command, config +import time +import unittest from pathlib import Path +import requests +import sqlalchemy as sa +import ujson +import yaml +from alembic import command, config from python_graphql_client import GraphqlClient -from irrd.conf import config_init, PASSWORD_HASH_DUMMY_VALUE -from irrd.utils.rpsl_samples import (SAMPLE_MNTNER, SAMPLE_PERSON, SAMPLE_KEY_CERT, SIGNED_PERSON_UPDATE_VALID, - SAMPLE_AS_SET, SAMPLE_AUT_NUM, SAMPLE_DOMAIN, SAMPLE_FILTER_SET, SAMPLE_INET_RTR, - SAMPLE_INET6NUM, SAMPLE_INETNUM, SAMPLE_PEERING_SET, SAMPLE_ROLE, SAMPLE_ROUTE, - SAMPLE_ROUTE_SET, SAMPLE_ROUTE6, SAMPLE_RTR_SET, SAMPLE_AS_BLOCK) +from irrd.conf import PASSWORD_HASH_DUMMY_VALUE, config_init +from irrd.utils.rpsl_samples import ( + SAMPLE_AS_BLOCK, + SAMPLE_AS_SET, + SAMPLE_AUT_NUM, + SAMPLE_DOMAIN, + SAMPLE_FILTER_SET, + SAMPLE_INET6NUM, + SAMPLE_INET_RTR, + SAMPLE_INETNUM, + SAMPLE_KEY_CERT, + SAMPLE_MNTNER, + SAMPLE_PEERING_SET, + SAMPLE_PERSON, + SAMPLE_ROLE, + SAMPLE_ROUTE, + SAMPLE_ROUTE6, + SAMPLE_ROUTE_SET, + SAMPLE_RTR_SET, + SIGNED_PERSON_UPDATE_VALID, +) from irrd.utils.whois_client import whois_query, whois_query_irrd -from .constants import (EMAIL_SMTP_PORT, EMAIL_DISCARD_MSGS_COMMAND, EMAIL_RETURN_MSGS_COMMAND, EMAIL_SEPARATOR, - EMAIL_END) + from ..storage import translate_url +from .constants import ( + EMAIL_DISCARD_MSGS_COMMAND, + EMAIL_END, + EMAIL_RETURN_MSGS_COMMAND, + EMAIL_SEPARATOR, + EMAIL_SMTP_PORT, +) IRRD_ROOT_PATH = str(Path(__file__).resolve().parents[2]) sys.path.append(IRRD_ROOT_PATH) diff --git a/irrd/mirroring/mirror_runners_export.py b/irrd/mirroring/mirror_runners_export.py index bdc8c0759..6efd83992 100644 --- a/irrd/mirroring/mirror_runners_export.py +++ b/irrd/mirroring/mirror_runners_export.py @@ -1,7 +1,6 @@ -import os - import gzip import logging +import os import shutil from pathlib import Path from tempfile import NamedTemporaryFile @@ -11,7 +10,7 @@ from irrd.rpki.status import RPKIStatus from irrd.scopefilter.status import ScopeFilterStatus from irrd.storage.database_handler import DatabaseHandler -from irrd.storage.queries import RPSLDatabaseQuery, DatabaseStatusQuery +from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseQuery from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func EXPORT_PERMISSIONS = 0o644 diff --git a/irrd/mirroring/mirror_runners_import.py b/irrd/mirroring/mirror_runners_import.py index b037f8ccf..a93d4dfe5 100644 --- a/irrd/mirroring/mirror_runners_import.py +++ b/irrd/mirroring/mirror_runners_import.py @@ -4,24 +4,25 @@ import shutil from io import BytesIO from tempfile import NamedTemporaryFile -from typing import Optional, Tuple, Any, IO +from typing import IO, Any, Optional, Tuple from urllib import request -from urllib.parse import urlparse from urllib.error import URLError +from urllib.parse import urlparse import requests -from irrd.conf import get_setting, RPKI_IRR_PSEUDO_SOURCE +from irrd.conf import RPKI_IRR_PSEUDO_SOURCE, get_setting from irrd.conf.defaults import DEFAULT_SOURCE_NRTM_PORT +from irrd.routepref.routepref import update_route_preference_status from irrd.rpki.importer import ROADataImporter, ROAParserException from irrd.rpki.notifications import notify_rpki_invalid_owners from irrd.rpki.validators import BulkRouteROAValidator -from irrd.routepref.routepref import update_route_preference_status from irrd.scopefilter.validators import ScopeFilterValidator from irrd.storage.database_handler import DatabaseHandler from irrd.storage.event_stream import EventStreamPublisher from irrd.storage.queries import DatabaseStatusQuery from irrd.utils.whois_client import whois_query + from .parsers import MirrorFileImportParser, NRTMStreamParser logger = logging.getLogger(__name__) diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index 81a99d30a..b6a7d1e80 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -2,7 +2,7 @@ from irrd.conf import get_setting from irrd.storage.database_handler import DatabaseHandler -from irrd.storage.queries import RPSLDatabaseJournalQuery, DatabaseStatusQuery +from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseJournalQuery from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func diff --git a/irrd/mirroring/nrtm_operation.py b/irrd/mirroring/nrtm_operation.py index 7c32780b5..565f3c71c 100644 --- a/irrd/mirroring/nrtm_operation.py +++ b/irrd/mirroring/nrtm_operation.py @@ -1,9 +1,9 @@ import logging -from typing import Optional, List +from typing import List, Optional from irrd.rpki.validators import SingleRouteROAValidator from irrd.rpsl.parser import UnknownRPSLObjectClassException -from irrd.rpsl.rpsl_objects import rpsl_object_from_text, RPSLKeyCert +from irrd.rpsl.rpsl_objects import RPSLKeyCert, rpsl_object_from_text from irrd.scopefilter.validators import ScopeFilterValidator from irrd.storage.database_handler import DatabaseHandler from irrd.storage.models import DatabaseOperation, JournalEntryOrigin diff --git a/irrd/mirroring/parsers.py b/irrd/mirroring/parsers.py index ef571bece..b16c9d71c 100644 --- a/irrd/mirroring/parsers.py +++ b/irrd/mirroring/parsers.py @@ -1,17 +1,18 @@ import logging import re -from typing import List, Set, Optional +from typing import List, Optional, Set from irrd.conf import get_setting from irrd.rpki.validators import BulkRouteROAValidator -from irrd.rpsl.parser import UnknownRPSLObjectClassException, RPSLObject -from irrd.rpsl.rpsl_objects import rpsl_object_from_text, RPSLKeyCert +from irrd.rpsl.parser import RPSLObject, UnknownRPSLObjectClassException +from irrd.rpsl.rpsl_objects import RPSLKeyCert, rpsl_object_from_text from irrd.scopefilter.validators import ScopeFilterValidator from irrd.storage.database_handler import DatabaseHandler from irrd.storage.models import DatabaseOperation, JournalEntryOrigin -from irrd.utils.text import split_paragraphs_rpsl, remove_last_modified -from .nrtm_operation import NRTMOperation +from irrd.utils.text import remove_last_modified, split_paragraphs_rpsl + from ..storage.queries import RPSLDatabaseQuery +from .nrtm_operation import NRTMOperation logger = logging.getLogger(__name__) nrtm_start_line_re = re.compile(r'^% *START *Version: *(?P\d+) +(?P[\w-]+) +(?P\d+)-(?P\d+)( FILTERED)?\n$', flags=re.MULTILINE) diff --git a/irrd/mirroring/scheduler.py b/irrd/mirroring/scheduler.py index eb29cae16..1c2ff7c7e 100644 --- a/irrd/mirroring/scheduler.py +++ b/irrd/mirroring/scheduler.py @@ -1,19 +1,23 @@ -import time -from collections import defaultdict - import gc import logging import multiprocessing - import signal -from setproctitle import setproctitle +import time +from collections import defaultdict from typing import Dict -from irrd.conf import get_setting, RPKI_IRR_PSEUDO_SOURCE -from irrd.conf.defaults import DEFAULT_SOURCE_IMPORT_TIMER, DEFAULT_SOURCE_EXPORT_TIMER +from setproctitle import setproctitle + +from irrd.conf import RPKI_IRR_PSEUDO_SOURCE, get_setting +from irrd.conf.defaults import DEFAULT_SOURCE_EXPORT_TIMER, DEFAULT_SOURCE_IMPORT_TIMER + from .mirror_runners_export import SourceExportRunner -from .mirror_runners_import import RPSLMirrorImportUpdateRunner, ROAImportRunner, \ - ScopeFilterUpdateRunner, RoutePreferenceUpdateRunner +from .mirror_runners_import import ( + ROAImportRunner, + RoutePreferenceUpdateRunner, + RPSLMirrorImportUpdateRunner, + ScopeFilterUpdateRunner, +) logger = logging.getLogger(__name__) diff --git a/irrd/mirroring/tests/test_mirror_runners_export.py b/irrd/mirroring/tests/test_mirror_runners_export.py index 2dfbabd8c..88ca06b19 100644 --- a/irrd/mirroring/tests/test_mirror_runners_export.py +++ b/irrd/mirroring/tests/test_mirror_runners_export.py @@ -4,11 +4,12 @@ from pathlib import Path from unittest.mock import Mock -from irrd.rpki.status import RPKIStatus from irrd.routepref.status import RoutePreferenceStatus +from irrd.rpki.status import RPKIStatus from irrd.scopefilter.status import ScopeFilterStatus from irrd.utils.test_utils import flatten_mock_calls -from ..mirror_runners_export import SourceExportRunner, EXPORT_PERMISSIONS + +from ..mirror_runners_export import EXPORT_PERMISSIONS, SourceExportRunner class TestSourceExportRunner: diff --git a/irrd/mirroring/tests/test_mirror_runners_import.py b/irrd/mirroring/tests/test_mirror_runners_import.py index 0c25d367b..87d7a3dd6 100644 --- a/irrd/mirroring/tests/test_mirror_runners_import.py +++ b/irrd/mirroring/tests/test_mirror_runners_import.py @@ -6,15 +6,20 @@ import pytest +from irrd.routepref.routepref import update_route_preference_status from irrd.rpki.importer import ROAParserException from irrd.rpki.validators import BulkRouteROAValidator -from irrd.routepref.routepref import update_route_preference_status -from irrd.storage.database_handler import DatabaseHandler from irrd.scopefilter.validators import ScopeFilterValidator +from irrd.storage.database_handler import DatabaseHandler from irrd.utils.test_utils import flatten_mock_calls + from ..mirror_runners_import import ( - RPSLMirrorImportUpdateRunner, RPSLMirrorFullImportRunner, NRTMImportUpdateStreamRunner, - ROAImportRunner, ScopeFilterUpdateRunner, RoutePreferenceUpdateRunner + NRTMImportUpdateStreamRunner, + ROAImportRunner, + RoutePreferenceUpdateRunner, + RPSLMirrorFullImportRunner, + RPSLMirrorImportUpdateRunner, + ScopeFilterUpdateRunner, ) diff --git a/irrd/mirroring/tests/test_nrtm_generator.py b/irrd/mirroring/tests/test_nrtm_generator.py index f9de24054..f6c0468f5 100644 --- a/irrd/mirroring/tests/test_nrtm_generator.py +++ b/irrd/mirroring/tests/test_nrtm_generator.py @@ -1,10 +1,11 @@ +import textwrap from itertools import cycle, repeat +from unittest.mock import Mock import pytest -import textwrap -from unittest.mock import Mock from irrd.storage.models import DatabaseOperation + from ..nrtm_generator import NRTMGenerator, NRTMGeneratorException diff --git a/irrd/mirroring/tests/test_nrtm_operation.py b/irrd/mirroring/tests/test_nrtm_operation.py index 59c247000..1e139116a 100644 --- a/irrd/mirroring/tests/test_nrtm_operation.py +++ b/irrd/mirroring/tests/test_nrtm_operation.py @@ -5,9 +5,15 @@ from irrd.scopefilter.status import ScopeFilterStatus from irrd.scopefilter.validators import ScopeFilterValidator from irrd.storage.models import DatabaseOperation, JournalEntryOrigin -from irrd.utils.rpsl_samples import (SAMPLE_MNTNER, SAMPLE_UNKNOWN_CLASS, - SAMPLE_MALFORMED_EMPTY_LINE, SAMPLE_KEY_CERT, - KEY_CERT_SIGNED_MESSAGE_VALID, SAMPLE_ROUTE) +from irrd.utils.rpsl_samples import ( + KEY_CERT_SIGNED_MESSAGE_VALID, + SAMPLE_KEY_CERT, + SAMPLE_MALFORMED_EMPTY_LINE, + SAMPLE_MNTNER, + SAMPLE_ROUTE, + SAMPLE_UNKNOWN_CLASS, +) + from ..nrtm_operation import NRTMOperation diff --git a/irrd/mirroring/tests/test_parsers.py b/irrd/mirroring/tests/test_parsers.py index 07e320137..c4d9a5ddb 100644 --- a/irrd/mirroring/tests/test_parsers.py +++ b/irrd/mirroring/tests/test_parsers.py @@ -10,15 +10,35 @@ from irrd.scopefilter.validators import ScopeFilterValidator from irrd.storage.models import DatabaseOperation, JournalEntryOrigin from irrd.utils.rpsl_samples import ( - SAMPLE_ROUTE, SAMPLE_UNKNOWN_CLASS, SAMPLE_UNKNOWN_ATTRIBUTE, SAMPLE_MALFORMED_PK, - SAMPLE_ROUTE6, SAMPLE_KEY_CERT, KEY_CERT_SIGNED_MESSAGE_VALID, SAMPLE_LEGACY_IRRD_ARTIFACT, - SAMPLE_ROLE, SAMPLE_RTR_SET) + KEY_CERT_SIGNED_MESSAGE_VALID, + SAMPLE_KEY_CERT, + SAMPLE_LEGACY_IRRD_ARTIFACT, + SAMPLE_MALFORMED_PK, + SAMPLE_ROLE, + SAMPLE_ROUTE, + SAMPLE_ROUTE6, + SAMPLE_RTR_SET, + SAMPLE_UNKNOWN_ATTRIBUTE, + SAMPLE_UNKNOWN_CLASS, +) from irrd.utils.test_utils import flatten_mock_calls -from .nrtm_samples import (SAMPLE_NRTM_V3, SAMPLE_NRTM_V1, SAMPLE_NRTM_V1_TOO_MANY_ITEMS, - SAMPLE_NRTM_INVALID_VERSION, SAMPLE_NRTM_V3_NO_END, - SAMPLE_NRTM_V3_SERIAL_GAP, SAMPLE_NRTM_V3_INVALID_MULTIPLE_START_LINES, - SAMPLE_NRTM_INVALID_NO_START_LINE, SAMPLE_NRTM_V3_SERIAL_OUT_OF_ORDER) -from ..parsers import NRTMStreamParser, MirrorFileImportParser, MirrorUpdateFileImportParser + +from ..parsers import ( + MirrorFileImportParser, + MirrorUpdateFileImportParser, + NRTMStreamParser, +) +from .nrtm_samples import ( + SAMPLE_NRTM_INVALID_NO_START_LINE, + SAMPLE_NRTM_INVALID_VERSION, + SAMPLE_NRTM_V1, + SAMPLE_NRTM_V1_TOO_MANY_ITEMS, + SAMPLE_NRTM_V3, + SAMPLE_NRTM_V3_INVALID_MULTIPLE_START_LINES, + SAMPLE_NRTM_V3_NO_END, + SAMPLE_NRTM_V3_SERIAL_GAP, + SAMPLE_NRTM_V3_SERIAL_OUT_OF_ORDER, +) @pytest.fixture diff --git a/irrd/mirroring/tests/test_scheduler.py b/irrd/mirroring/tests/test_scheduler.py index 4ee1793bb..f59c1326c 100644 --- a/irrd/mirroring/tests/test_scheduler.py +++ b/irrd/mirroring/tests/test_scheduler.py @@ -1,8 +1,7 @@ -import time - import threading +import time -from ..scheduler import MirrorScheduler, ScheduledTaskProcess, MAX_SIMULTANEOUS_RUNS +from ..scheduler import MAX_SIMULTANEOUS_RUNS, MirrorScheduler, ScheduledTaskProcess thread_run_count = 0 diff --git a/irrd/routepref/routepref.py b/irrd/routepref/routepref.py index a08426e00..31ddf22e4 100644 --- a/irrd/routepref/routepref.py +++ b/irrd/routepref/routepref.py @@ -1,5 +1,5 @@ import logging -from typing import List, Dict, Iterable, Tuple, Optional +from typing import Dict, Iterable, List, Optional, Tuple import radix from IPy import IP @@ -8,8 +8,8 @@ from irrd.conf import get_setting from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import RPSLDatabaseQuery -from .status import RoutePreferenceStatus +from .status import RoutePreferenceStatus logger = logging.getLogger(__name__) diff --git a/irrd/routepref/tests/test_routepref.py b/irrd/routepref/tests/test_routepref.py index 46cb97c18..e93182b0e 100644 --- a/irrd/routepref/tests/test_routepref.py +++ b/irrd/routepref/tests/test_routepref.py @@ -2,6 +2,7 @@ from irrd.storage.queries import RPSLDatabaseQuery from irrd.utils.test_utils import MockDatabaseHandler + from ..routepref import RoutePreferenceValidator, update_route_preference_status from ..status import RoutePreferenceStatus diff --git a/irrd/rpki/importer.py b/irrd/rpki/importer.py index 49d3f7351..f78645851 100644 --- a/irrd/rpki/importer.py +++ b/irrd/rpki/importer.py @@ -1,14 +1,13 @@ +import logging from collections import defaultdict +from typing import Dict, List, Optional, Set import ujson - -import logging from IPy import IP, IPSet -from typing import List, Optional, Dict, Set from irrd.conf import RPKI_IRR_PSEUDO_SOURCE, get_setting from irrd.rpki.status import RPKIStatus -from irrd.rpsl.parser import RPSLObject, RPSL_ATTRIBUTE_TEXT_WIDTH +from irrd.rpsl.parser import RPSL_ATTRIBUTE_TEXT_WIDTH, RPSLObject from irrd.rpsl.rpsl_objects import RPSL_ROUTE_OBJECT_CLASS_FOR_IP_VERSION from irrd.scopefilter.validators import ScopeFilterValidator from irrd.storage.database_handler import DatabaseHandler diff --git a/irrd/rpki/tests/test_importer.py b/irrd/rpki/tests/test_importer.py index 0364e1f08..ede31c86c 100644 --- a/irrd/rpki/tests/test_importer.py +++ b/irrd/rpki/tests/test_importer.py @@ -9,6 +9,7 @@ from irrd.scopefilter.validators import ScopeFilterValidator from irrd.storage.database_handler import DatabaseHandler from irrd.utils.test_utils import flatten_mock_calls + from ..importer import ROADataImporter, ROAParserException diff --git a/irrd/rpki/tests/test_notifications.py b/irrd/rpki/tests/test_notifications.py index 151206597..6951c738f 100644 --- a/irrd/rpki/tests/test_notifications.py +++ b/irrd/rpki/tests/test_notifications.py @@ -1,13 +1,14 @@ import textwrap from unittest.mock import Mock -from ..notifications import notify_rpki_invalid_owners -from irrd.utils.test_utils import flatten_mock_calls from irrd.storage.database_handler import DatabaseHandler from irrd.utils.rpsl_samples import SAMPLE_ROUTE, SAMPLE_ROUTE6 -from ..status import RPKIStatus +from irrd.utils.test_utils import flatten_mock_calls + from ...storage.queries import RPSLDatabaseQuery from ...utils.email import send_email +from ..notifications import notify_rpki_invalid_owners +from ..status import RPKIStatus class TestNotifyRPKIInvalidOwners: diff --git a/irrd/rpki/tests/test_validators.py b/irrd/rpki/tests/test_validators.py index 7057cb9cb..8a77cf0ef 100644 --- a/irrd/rpki/tests/test_validators.py +++ b/irrd/rpki/tests/test_validators.py @@ -1,10 +1,12 @@ -from IPy import IP from unittest.mock import Mock +from IPy import IP + from irrd.conf import RPKI_IRR_PSEUDO_SOURCE from irrd.storage.database_handler import DatabaseHandler -from irrd.storage.queries import RPSLDatabaseQuery, ROADatabaseObjectQuery +from irrd.storage.queries import ROADatabaseObjectQuery, RPSLDatabaseQuery from irrd.utils.test_utils import flatten_mock_calls + from ..importer import ROA from ..status import RPKIStatus from ..validators import BulkRouteROAValidator, SingleRouteROAValidator diff --git a/irrd/rpki/validators.py b/irrd/rpki/validators.py index 321aca387..7ae4bc366 100644 --- a/irrd/rpki/validators.py +++ b/irrd/rpki/validators.py @@ -1,14 +1,15 @@ -import datrie -from collections import defaultdict - import codecs import socket +from collections import defaultdict +from typing import Dict, List, Optional, Tuple + +import datrie from IPy import IP -from typing import Optional, List, Tuple, Dict from irrd.conf import RPKI_IRR_PSEUDO_SOURCE, get_setting from irrd.storage.database_handler import DatabaseHandler -from irrd.storage.queries import RPSLDatabaseQuery, ROADatabaseObjectQuery +from irrd.storage.queries import ROADatabaseObjectQuery, RPSLDatabaseQuery + from .importer import ROA from .status import RPKIStatus diff --git a/irrd/rpsl/fields.py b/irrd/rpsl/fields.py index 26c86d2fc..07f923ddf 100644 --- a/irrd/rpsl/fields.py +++ b/irrd/rpsl/fields.py @@ -1,14 +1,15 @@ import datetime import re -from typing import List, Type, Optional +from typing import List, Optional, Type from urllib.parse import urlparse from IPy import IP from irrd.utils.text import clean_ip_value_error -from irrd.utils.validators import parse_as_number, ValidationError +from irrd.utils.validators import ValidationError, parse_as_number + +from .parser_state import RPSLFieldParseResult, RPSLParserMessages from .passwords import get_password_hashers -from .parser_state import RPSLParserMessages, RPSLFieldParseResult # The IPv4/IPv6 regexes are for initial screening - not full validators diff --git a/irrd/rpsl/parser.py b/irrd/rpsl/parser.py index 9e0ad0e5a..fd113001a 100644 --- a/irrd/rpsl/parser.py +++ b/irrd/rpsl/parser.py @@ -2,18 +2,19 @@ import itertools import json import re -from collections import OrderedDict, Counter -from typing import Dict, List, Optional, Tuple, Any, Set +from collections import Counter, OrderedDict +from typing import Any, Dict, List, Optional, Set, Tuple from IPy import IP -from irrd.rpki.status import RPKIStatus from irrd.routepref.status import RoutePreferenceStatus +from irrd.rpki.status import RPKIStatus from irrd.rpsl.parser_state import RPSLParserMessages from irrd.scopefilter.status import ScopeFilterStatus from irrd.utils.text import splitline_unicodesafe -from .fields import RPSLTextField + from ..conf import get_setting +from .fields import RPSLTextField RPSL_ATTRIBUTE_TEXT_WIDTH = 16 TypeRPSLObjectData = List[Tuple[str, str, List[str]]] diff --git a/irrd/rpsl/parser_state.py b/irrd/rpsl/parser_state.py index be705c659..c0f877d32 100644 --- a/irrd/rpsl/parser_state.py +++ b/irrd/rpsl/parser_state.py @@ -1,4 +1,4 @@ -from typing import Optional, TypeVar, List +from typing import List, Optional, TypeVar from IPy import IP diff --git a/irrd/rpsl/passwords.py b/irrd/rpsl/passwords.py index 8c9933d42..4d18d679b 100644 --- a/irrd/rpsl/passwords.py +++ b/irrd/rpsl/passwords.py @@ -1,6 +1,8 @@ -from enum import unique, Enum +from enum import Enum, unique + +from passlib.hash import bcrypt, des_crypt, md5_crypt + from irrd.conf import get_setting -from passlib.hash import des_crypt, md5_crypt, bcrypt @unique diff --git a/irrd/rpsl/rpsl_objects.py b/irrd/rpsl/rpsl_objects.py index 8ced2e378..95c48d840 100644 --- a/irrd/rpsl/rpsl_objects.py +++ b/irrd/rpsl/rpsl_objects.py @@ -1,19 +1,36 @@ from collections import OrderedDict +from typing import List, Optional, Set, Union -from typing import Set, List, Optional, Union - -from irrd.conf import AUTH_SET_CREATION_COMMON_KEY, PASSWORD_HASH_DUMMY_VALUE, get_setting +from irrd.conf import ( + AUTH_SET_CREATION_COMMON_KEY, + PASSWORD_HASH_DUMMY_VALUE, + get_setting, +) from irrd.utils.pgp import get_gpg_instance -from .passwords import PASSWORD_REPLACEMENT_HASH, get_password_hashers -from .fields import (RPSLTextField, RPSLIPv4PrefixField, RPSLIPv4PrefixesField, RPSLIPv6PrefixField, - RPSLIPv6PrefixesField, RPSLIPv4AddressRangeField, RPSLASNumberField, - RPSLASBlockField, - RPSLSetNameField, RPSLEmailField, RPSLDNSNameField, RPSLGenericNameField, - RPSLReferenceField, - RPSLReferenceListField, RPSLAuthField, RPSLRouteSetMembersField, - RPSLChangedField, RPSLURLField) + +from ..utils.validators import ValidationError, parse_as_number +from .fields import ( + RPSLASBlockField, + RPSLASNumberField, + RPSLAuthField, + RPSLChangedField, + RPSLDNSNameField, + RPSLEmailField, + RPSLGenericNameField, + RPSLIPv4AddressRangeField, + RPSLIPv4PrefixesField, + RPSLIPv4PrefixField, + RPSLIPv6PrefixesField, + RPSLIPv6PrefixField, + RPSLReferenceField, + RPSLReferenceListField, + RPSLRouteSetMembersField, + RPSLSetNameField, + RPSLTextField, + RPSLURLField, +) from .parser import RPSLObject, UnknownRPSLObjectClassException -from ..utils.validators import parse_as_number, ValidationError +from .passwords import PASSWORD_REPLACEMENT_HASH, get_password_hashers RPSL_ROUTE_OBJECT_CLASS_FOR_IP_VERSION = { 4: 'route', diff --git a/irrd/rpsl/tests/test_fields.py b/irrd/rpsl/tests/test_fields.py index e495bb3ee..d7f7c8dae 100644 --- a/irrd/rpsl/tests/test_fields.py +++ b/irrd/rpsl/tests/test_fields.py @@ -1,13 +1,26 @@ from IPy import IP from pytest import raises -from ..fields import (RPSLIPv4PrefixField, RPSLIPv4PrefixesField, RPSLIPv6PrefixField, - RPSLIPv6PrefixesField, RPSLIPv4AddressRangeField, RPSLASNumberField, - RPSLASBlockField, - RPSLSetNameField, RPSLEmailField, RPSLDNSNameField, RPSLGenericNameField, - RPSLReferenceField, - RPSLReferenceListField, RPSLTextField, RPSLAuthField, RPSLRouteSetMemberField, - RPSLChangedField, RPSLURLField) +from ..fields import ( + RPSLASBlockField, + RPSLASNumberField, + RPSLAuthField, + RPSLChangedField, + RPSLDNSNameField, + RPSLEmailField, + RPSLGenericNameField, + RPSLIPv4AddressRangeField, + RPSLIPv4PrefixesField, + RPSLIPv4PrefixField, + RPSLIPv6PrefixesField, + RPSLIPv6PrefixField, + RPSLReferenceField, + RPSLReferenceListField, + RPSLRouteSetMemberField, + RPSLSetNameField, + RPSLTextField, + RPSLURLField, +) from ..parser_state import RPSLParserMessages diff --git a/irrd/rpsl/tests/test_rpsl_objects.py b/irrd/rpsl/tests/test_rpsl_objects.py index d47c36993..842c940b3 100644 --- a/irrd/rpsl/tests/test_rpsl_objects.py +++ b/irrd/rpsl/tests/test_rpsl_objects.py @@ -5,26 +5,50 @@ from pytest import raises from pytz import timezone -from irrd.conf import PASSWORD_HASH_DUMMY_VALUE, AUTH_SET_CREATION_COMMON_KEY -from irrd.utils.rpsl_samples import (object_sample_mapping, SAMPLE_MALFORMED_EMPTY_LINE, - SAMPLE_MALFORMED_ATTRIBUTE_NAME, - SAMPLE_UNKNOWN_CLASS, SAMPLE_MISSING_MANDATORY_ATTRIBUTE, - SAMPLE_MALFORMED_SOURCE, - SAMPLE_MALFORMED_PK, SAMPLE_UNKNOWN_ATTRIBUTE, - SAMPLE_INVALID_MULTIPLE_ATTRIBUTE, - KEY_CERT_SIGNED_MESSAGE_VALID, KEY_CERT_SIGNED_MESSAGE_INVALID, - KEY_CERT_SIGNED_MESSAGE_CORRUPT, - KEY_CERT_SIGNED_MESSAGE_WRONG_KEY, - TEMPLATE_ROUTE_OBJECT, - TEMPLATE_PERSON_OBJECT, - SAMPLE_LINE_NEITHER_CONTINUATION_NOR_ATTR, - SAMPLE_MISSING_SOURCE, SAMPLE_ROUTE) +from irrd.conf import AUTH_SET_CREATION_COMMON_KEY, PASSWORD_HASH_DUMMY_VALUE +from irrd.utils.rpsl_samples import ( + KEY_CERT_SIGNED_MESSAGE_CORRUPT, + KEY_CERT_SIGNED_MESSAGE_INVALID, + KEY_CERT_SIGNED_MESSAGE_VALID, + KEY_CERT_SIGNED_MESSAGE_WRONG_KEY, + SAMPLE_INVALID_MULTIPLE_ATTRIBUTE, + SAMPLE_LINE_NEITHER_CONTINUATION_NOR_ATTR, + SAMPLE_MALFORMED_ATTRIBUTE_NAME, + SAMPLE_MALFORMED_EMPTY_LINE, + SAMPLE_MALFORMED_PK, + SAMPLE_MALFORMED_SOURCE, + SAMPLE_MISSING_MANDATORY_ATTRIBUTE, + SAMPLE_MISSING_SOURCE, + SAMPLE_ROUTE, + SAMPLE_UNKNOWN_ATTRIBUTE, + SAMPLE_UNKNOWN_CLASS, + TEMPLATE_PERSON_OBJECT, + TEMPLATE_ROUTE_OBJECT, + object_sample_mapping, +) from ..parser import UnknownRPSLObjectClassException -from ..rpsl_objects import (RPSLAsBlock, RPSLAsSet, RPSLAutNum, RPSLDomain, RPSLFilterSet, RPSLInetRtr, - RPSLInet6Num, RPSLInetnum, RPSLKeyCert, RPSLMntner, RPSLPeeringSet, - RPSLPerson, RPSLRole, RPSLRoute, RPSLRouteSet, RPSLRoute6, RPSLRtrSet, - OBJECT_CLASS_MAPPING, rpsl_object_from_text) +from ..rpsl_objects import ( + OBJECT_CLASS_MAPPING, + RPSLAsBlock, + RPSLAsSet, + RPSLAutNum, + RPSLDomain, + RPSLFilterSet, + RPSLInet6Num, + RPSLInetnum, + RPSLInetRtr, + RPSLKeyCert, + RPSLMntner, + RPSLPeeringSet, + RPSLPerson, + RPSLRole, + RPSLRoute, + RPSLRoute6, + RPSLRouteSet, + RPSLRtrSet, + rpsl_object_from_text, +) class TestRPSLParsingGeneric: diff --git a/irrd/scopefilter/tests/test_scopefilter.py b/irrd/scopefilter/tests/test_scopefilter.py index 95181e763..21531785d 100644 --- a/irrd/scopefilter/tests/test_scopefilter.py +++ b/irrd/scopefilter/tests/test_scopefilter.py @@ -6,8 +6,9 @@ from irrd.rpsl.rpsl_objects import rpsl_object_from_text from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import RPSLDatabaseQuery -from irrd.utils.rpsl_samples import SAMPLE_AUT_NUM, SAMPLE_ROUTE, SAMPLE_INETNUM +from irrd.utils.rpsl_samples import SAMPLE_AUT_NUM, SAMPLE_INETNUM, SAMPLE_ROUTE from irrd.utils.test_utils import flatten_mock_calls + from ..status import ScopeFilterStatus from ..validators import ScopeFilterValidator diff --git a/irrd/scopefilter/validators.py b/irrd/scopefilter/validators.py index cc11a1aeb..b0c89bb80 100644 --- a/irrd/scopefilter/validators.py +++ b/irrd/scopefilter/validators.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Optional, Tuple, List, Dict +from typing import Dict, List, Optional, Tuple from IPy import IP @@ -7,6 +7,7 @@ from irrd.rpsl.parser import RPSLObject from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import RPSLDatabaseQuery + from .status import ScopeFilterStatus diff --git a/irrd/scripts/database_downgrade.py b/irrd/scripts/database_downgrade.py index 3bf11839a..46ac180e5 100755 --- a/irrd/scripts/database_downgrade.py +++ b/irrd/scripts/database_downgrade.py @@ -1,16 +1,16 @@ #!/usr/bin/env python # flake8: noqa: E402 +import argparse import sys +from pathlib import Path -import argparse from alembic import command from alembic.config import Config -from pathlib import Path irrd_root = str(Path(__file__).resolve().parents[2]) sys.path.append(irrd_root) -from irrd.conf import config_init, CONFIG_PATH_DEFAULT +from irrd.conf import CONFIG_PATH_DEFAULT, config_init def run(version): diff --git a/irrd/scripts/database_upgrade.py b/irrd/scripts/database_upgrade.py index 50d140429..f3a9247d5 100755 --- a/irrd/scripts/database_upgrade.py +++ b/irrd/scripts/database_upgrade.py @@ -1,16 +1,16 @@ #!/usr/bin/env python # flake8: noqa: E402 +import argparse import sys +from pathlib import Path -import argparse from alembic import command from alembic.config import Config -from pathlib import Path irrd_root = str(Path(__file__).resolve().parents[2]) sys.path.append(irrd_root) -from irrd.conf import config_init, CONFIG_PATH_DEFAULT +from irrd.conf import CONFIG_PATH_DEFAULT, config_init def run(version): diff --git a/irrd/scripts/expire_journal.py b/irrd/scripts/expire_journal.py index da21ecc0e..cb39183f1 100755 --- a/irrd/scripts/expire_journal.py +++ b/irrd/scripts/expire_journal.py @@ -16,9 +16,9 @@ logger = logging.getLogger(__name__) sys.path.append(str(Path(__file__).resolve().parents[2])) +from irrd.conf import CONFIG_PATH_DEFAULT, config_init, get_setting from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import RPSLDatabaseJournalQuery -from irrd.conf import config_init, CONFIG_PATH_DEFAULT, get_setting def expire_journal(skip_confirmation: bool, expire_before: datetime, source: str): diff --git a/irrd/scripts/load_database.py b/irrd/scripts/load_database.py index d7aa84386..60ea7c37f 100755 --- a/irrd/scripts/load_database.py +++ b/irrd/scripts/load_database.py @@ -3,10 +3,8 @@ import argparse import logging import sys - from pathlib import Path - """ Load an RPSL file into the database. """ @@ -14,10 +12,10 @@ logger = logging.getLogger(__name__) sys.path.append(str(Path(__file__).resolve().parents[2])) +from irrd.conf import CONFIG_PATH_DEFAULT, config_init, get_setting +from irrd.mirroring.parsers import MirrorFileImportParser from irrd.rpki.validators import BulkRouteROAValidator from irrd.storage.database_handler import DatabaseHandler -from irrd.mirroring.parsers import MirrorFileImportParser -from irrd.conf import config_init, CONFIG_PATH_DEFAULT, get_setting def load(source, filename, serial) -> int: diff --git a/irrd/scripts/load_pgp_keys.py b/irrd/scripts/load_pgp_keys.py index e34a77b09..21124f4d5 100755 --- a/irrd/scripts/load_pgp_keys.py +++ b/irrd/scripts/load_pgp_keys.py @@ -3,7 +3,6 @@ import argparse import logging import sys - from pathlib import Path """ @@ -13,10 +12,11 @@ logger = logging.getLogger(__name__) sys.path.append(str(Path(__file__).resolve().parents[2])) -from irrd.conf import config_init, CONFIG_PATH_DEFAULT -from irrd.storage.queries import RPSLDatabaseQuery -from irrd.storage.database_handler import DatabaseHandler +from irrd.conf import CONFIG_PATH_DEFAULT, config_init from irrd.rpsl.rpsl_objects import rpsl_object_from_text +from irrd.storage.database_handler import DatabaseHandler +from irrd.storage.queries import RPSLDatabaseQuery + def load_pgp_keys(source: str) -> None: dh = DatabaseHandler() diff --git a/irrd/scripts/load_test.py b/irrd/scripts/load_test.py index c60e271a7..d8444ad55 100755 --- a/irrd/scripts/load_test.py +++ b/irrd/scripts/load_test.py @@ -5,11 +5,10 @@ A simple load tester for IRRd. Sends random !g queries. """ -import time - import argparse import random import socket +import time def main(host, port, count): diff --git a/irrd/scripts/mirror_force_reload.py b/irrd/scripts/mirror_force_reload.py index 33b8e3ce1..a91f85b05 100755 --- a/irrd/scripts/mirror_force_reload.py +++ b/irrd/scripts/mirror_force_reload.py @@ -3,7 +3,6 @@ import argparse import logging import sys - from pathlib import Path """ @@ -13,10 +12,10 @@ logger = logging.getLogger(__name__) sys.path.append(str(Path(__file__).resolve().parents[2])) -from irrd.conf import config_init, CONFIG_PATH_DEFAULT, get_setting - +from irrd.conf import CONFIG_PATH_DEFAULT, config_init, get_setting from irrd.storage.database_handler import DatabaseHandler + def set_force_reload(source) -> None: dh = DatabaseHandler() dh.set_force_reload(source) diff --git a/irrd/scripts/query_qa_comparison.py b/irrd/scripts/query_qa_comparison.py index a29eff798..c41b56ab2 100755 --- a/irrd/scripts/query_qa_comparison.py +++ b/irrd/scripts/query_qa_comparison.py @@ -7,21 +7,19 @@ import argparse import difflib -import sys - import re -from IPy import IP -from ordered_set import OrderedSet +import sys from pathlib import Path from typing import Optional +from IPy import IP +from ordered_set import OrderedSet sys.path.append(str(Path(__file__).resolve().parents[2])) from irrd.rpsl.rpsl_objects import rpsl_object_from_text -from irrd.utils.text import splitline_unicodesafe, split_paragraphs_rpsl -from irrd.utils.whois_client import whois_query_irrd, whois_query, WhoisQueryError - +from irrd.utils.text import split_paragraphs_rpsl, splitline_unicodesafe +from irrd.utils.whois_client import WhoisQueryError, whois_query, whois_query_irrd SSP_QUERIES = ['!6', '!g', '!i'] ASDOT_RE = re.compile(r'as\d+\.\d*', flags=re.IGNORECASE) diff --git a/irrd/scripts/rpsl_read.py b/irrd/scripts/rpsl_read.py index 94cdeb434..93615e6d3 100755 --- a/irrd/scripts/rpsl_read.py +++ b/irrd/scripts/rpsl_read.py @@ -6,7 +6,6 @@ """ import argparse import sys - from pathlib import Path from typing import Optional, Set @@ -15,9 +14,9 @@ sys.path.append(str(Path(__file__).resolve().parents[2])) from irrd.conf import CONFIG_PATH_DEFAULT, config_init, get_setting -from irrd.storage.database_handler import DatabaseHandler from irrd.rpsl.parser import UnknownRPSLObjectClassException from irrd.rpsl.rpsl_objects import rpsl_object_from_text +from irrd.storage.database_handler import DatabaseHandler from irrd.utils.text import split_paragraphs_rpsl diff --git a/irrd/scripts/set_last_modified_auth.py b/irrd/scripts/set_last_modified_auth.py index c90d647ab..d8414c22f 100755 --- a/irrd/scripts/set_last_modified_auth.py +++ b/irrd/scripts/set_last_modified_auth.py @@ -5,7 +5,6 @@ import sys from pathlib import Path - """ Set last-modified attribute on all authoritative objects. """ @@ -13,12 +12,13 @@ logger = logging.getLogger(__name__) sys.path.append(str(Path(__file__).resolve().parents[2])) -from irrd.storage.database_handler import DatabaseHandler -from irrd.conf import config_init, CONFIG_PATH_DEFAULT, get_setting +from irrd.conf import CONFIG_PATH_DEFAULT, config_init, get_setting from irrd.rpsl.rpsl_objects import rpsl_object_from_text +from irrd.storage.database_handler import DatabaseHandler from irrd.storage.models import RPSLDatabaseObject from irrd.storage.queries import RPSLDatabaseQuery + def set_last_modified(): dh = DatabaseHandler() auth_sources = [k for k, v in get_setting('sources').items() if v.get('authoritative')] diff --git a/irrd/scripts/submit_changes.py b/irrd/scripts/submit_changes.py index 96a35d32e..8a0962ef8 100755 --- a/irrd/scripts/submit_changes.py +++ b/irrd/scripts/submit_changes.py @@ -13,12 +13,11 @@ """ import argparse import sys - from pathlib import Path sys.path.append(str(Path(__file__).resolve().parents[2])) -from irrd.conf import config_init, CONFIG_PATH_DEFAULT +from irrd.conf import CONFIG_PATH_DEFAULT, config_init from irrd.updates.handler import ChangeSubmissionHandler diff --git a/irrd/scripts/submit_email.py b/irrd/scripts/submit_email.py index 92f4d5b00..7191502b6 100755 --- a/irrd/scripts/submit_email.py +++ b/irrd/scripts/submit_email.py @@ -1,10 +1,9 @@ #!/usr/bin/env python # flake8: noqa: E402 -import sys - import argparse import logging +import sys from pathlib import Path """ @@ -17,7 +16,7 @@ logger = logging.getLogger(__name__) sys.path.append(str(Path(__file__).resolve().parents[2])) -from irrd.conf import config_init, CONFIG_PATH_DEFAULT +from irrd.conf import CONFIG_PATH_DEFAULT, config_init from irrd.updates.email import handle_email_submission diff --git a/irrd/scripts/tests/test_expire_journal.py b/irrd/scripts/tests/test_expire_journal.py index 2a22345cb..8e569d99e 100644 --- a/irrd/scripts/tests/test_expire_journal.py +++ b/irrd/scripts/tests/test_expire_journal.py @@ -5,6 +5,7 @@ from irrd.storage.queries import RPSLDatabaseJournalQuery from irrd.utils.test_utils import MockDatabaseHandler + from ..expire_journal import expire_journal EXPIRY_DATE = datetime(2022, 1, 1, tzinfo=pytz.utc) diff --git a/irrd/scripts/tests/test_irr_rpsl_submit.py b/irrd/scripts/tests/test_irr_rpsl_submit.py index 5c5aeccd7..1cd4ca7b1 100755 --- a/irrd/scripts/tests/test_irr_rpsl_submit.py +++ b/irrd/scripts/tests/test_irr_rpsl_submit.py @@ -1,14 +1,15 @@ -import json import io +import json import os -import pytest import re -from urllib import request import subprocess import sys import unittest +from urllib import request from urllib.error import HTTPError +import pytest + from .. import irr_rpsl_submit IRRD_HOST = "fake.example.com" diff --git a/irrd/scripts/tests/test_load_database.py b/irrd/scripts/tests/test_load_database.py index 3d6ebe479..bb2e29141 100644 --- a/irrd/scripts/tests/test_load_database.py +++ b/irrd/scripts/tests/test_load_database.py @@ -1,6 +1,7 @@ from unittest.mock import Mock from irrd.utils.test_utils import flatten_mock_calls + from ..load_database import load diff --git a/irrd/scripts/tests/test_load_pgp_keys.py b/irrd/scripts/tests/test_load_pgp_keys.py index a571a01e0..d9cd9ec21 100644 --- a/irrd/scripts/tests/test_load_pgp_keys.py +++ b/irrd/scripts/tests/test_load_pgp_keys.py @@ -1,8 +1,10 @@ -import pytest from unittest.mock import Mock -from irrd.utils.test_utils import flatten_mock_calls +import pytest + from irrd.utils.rpsl_samples import SAMPLE_KEY_CERT +from irrd.utils.test_utils import flatten_mock_calls + from ..load_pgp_keys import load_pgp_keys diff --git a/irrd/scripts/tests/test_mirror_force_reload.py b/irrd/scripts/tests/test_mirror_force_reload.py index 0b662b28f..e94cfc86c 100644 --- a/irrd/scripts/tests/test_mirror_force_reload.py +++ b/irrd/scripts/tests/test_mirror_force_reload.py @@ -1,6 +1,7 @@ from unittest.mock import Mock from irrd.utils.test_utils import flatten_mock_calls + from ..mirror_force_reload import set_force_reload diff --git a/irrd/scripts/tests/test_set_last_modified_auth.py b/irrd/scripts/tests/test_set_last_modified_auth.py index 53ebb9978..d1b9177c3 100644 --- a/irrd/scripts/tests/test_set_last_modified_auth.py +++ b/irrd/scripts/tests/test_set_last_modified_auth.py @@ -6,6 +6,7 @@ from irrd.utils.rpsl_samples import SAMPLE_RTR_SET from irrd.utils.test_utils import flatten_mock_calls + from ..set_last_modified_auth import set_last_modified diff --git a/irrd/scripts/tests/test_submit_update.py b/irrd/scripts/tests/test_submit_update.py index 561001e4d..9035a575a 100644 --- a/irrd/scripts/tests/test_submit_update.py +++ b/irrd/scripts/tests/test_submit_update.py @@ -1,7 +1,7 @@ from unittest.mock import Mock -from ..submit_changes import main from ...updates.handler import ChangeSubmissionHandler +from ..submit_changes import main def test_submit_changes(capsys, monkeypatch): diff --git a/irrd/scripts/tests/test_update_database.py b/irrd/scripts/tests/test_update_database.py index bff6b4330..14120f2c8 100644 --- a/irrd/scripts/tests/test_update_database.py +++ b/irrd/scripts/tests/test_update_database.py @@ -1,6 +1,7 @@ from unittest.mock import Mock from irrd.utils.test_utils import flatten_mock_calls + from ..update_database import update diff --git a/irrd/scripts/update_database.py b/irrd/scripts/update_database.py index d3366ece9..dce2e77bd 100644 --- a/irrd/scripts/update_database.py +++ b/irrd/scripts/update_database.py @@ -3,10 +3,8 @@ import argparse import logging import sys - from pathlib import Path - """ Update a database based on a RPSL file. """ @@ -14,10 +12,11 @@ logger = logging.getLogger(__name__) sys.path.append(str(Path(__file__).resolve().parents[2])) +from irrd.conf import CONFIG_PATH_DEFAULT, config_init, get_setting +from irrd.mirroring.parsers import MirrorUpdateFileImportParser from irrd.rpki.validators import BulkRouteROAValidator from irrd.storage.database_handler import DatabaseHandler -from irrd.mirroring.parsers import MirrorUpdateFileImportParser -from irrd.conf import config_init, CONFIG_PATH_DEFAULT, get_setting + def update(source, filename) -> int: if any([ diff --git a/irrd/server/graphql/resolvers.py b/irrd/server/graphql/resolvers.py index 97b877795..324862a8c 100644 --- a/irrd/server/graphql/resolvers.py +++ b/irrd/server/graphql/resolvers.py @@ -1,21 +1,22 @@ from collections import OrderedDict -from typing import Set, Dict, Optional, List +from typing import Dict, List, Optional, Set import ariadne import graphql +from graphql import GraphQLError, GraphQLResolveInfo from IPy import IP -from graphql import GraphQLResolveInfo, GraphQLError -from irrd.conf import get_setting, RPKI_IRR_PSEUDO_SOURCE -from irrd.rpki.status import RPKIStatus +from irrd.conf import RPKI_IRR_PSEUDO_SOURCE, get_setting from irrd.routepref.status import RoutePreferenceStatus +from irrd.rpki.status import RPKIStatus from irrd.rpsl.rpsl_objects import OBJECT_CLASS_MAPPING, lookup_field_names from irrd.scopefilter.status import ScopeFilterStatus from irrd.server.access_check import is_client_permitted -from irrd.storage.queries import RPSLDatabaseQuery, RPSLDatabaseJournalQuery -from irrd.utils.text import snake_to_camel_case, remove_auth_hashes -from .schema_generator import SchemaGenerator +from irrd.storage.queries import RPSLDatabaseJournalQuery, RPSLDatabaseQuery +from irrd.utils.text import remove_auth_hashes, snake_to_camel_case + from ..query_resolver import QueryResolver +from .schema_generator import SchemaGenerator """ Resolvers resolve GraphQL queries, usually by translating them diff --git a/irrd/server/graphql/schema_builder.py b/irrd/server/graphql/schema_builder.py index 87b8f58c0..6a25af6cb 100644 --- a/irrd/server/graphql/schema_builder.py +++ b/irrd/server/graphql/schema_builder.py @@ -1,17 +1,25 @@ -from IPy import IP from ariadne import make_executable_schema from asgiref.sync import sync_to_async as sta from graphql import GraphQLError +from IPy import IP -from .resolvers import (resolve_rpsl_objects, resolve_rpsl_object_type, - resolve_database_status, resolve_rpsl_object_mnt_by_objs, - resolve_rpsl_object_member_of_objs, resolve_rpsl_object_members_by_ref_objs, - resolve_rpsl_object_members_objs, resolve_rpsl_object_adminc_objs, - resolve_asn_prefixes, resolve_as_set_prefixes, - resolve_recursive_set_members, resolve_rpsl_object_techc_objs, - resolve_rpsl_object_journal) -from .schema_generator import SchemaGenerator from ...utils.text import clean_ip_value_error +from .resolvers import ( + resolve_as_set_prefixes, + resolve_asn_prefixes, + resolve_database_status, + resolve_recursive_set_members, + resolve_rpsl_object_adminc_objs, + resolve_rpsl_object_journal, + resolve_rpsl_object_member_of_objs, + resolve_rpsl_object_members_by_ref_objs, + resolve_rpsl_object_members_objs, + resolve_rpsl_object_mnt_by_objs, + resolve_rpsl_object_techc_objs, + resolve_rpsl_object_type, + resolve_rpsl_objects, +) +from .schema_generator import SchemaGenerator def build_executable_schema(): diff --git a/irrd/server/graphql/schema_generator.py b/irrd/server/graphql/schema_generator.py index 97b221bce..0c23910b1 100644 --- a/irrd/server/graphql/schema_generator.py +++ b/irrd/server/graphql/schema_generator.py @@ -1,13 +1,19 @@ from collections import OrderedDict, defaultdict -from typing import Optional, Dict, Tuple, List +from typing import Dict, List, Optional, Tuple import ariadne from irrd.routepref.status import RoutePreferenceStatus from irrd.rpki.status import RPKIStatus -from irrd.rpsl.fields import RPSLFieldListMixin, RPSLTextField, RPSLReferenceField -from irrd.rpsl.rpsl_objects import (lookup_field_names, OBJECT_CLASS_MAPPING, RPSLAutNum, - RPSLInetRtr, RPSLPerson, RPSLRole) +from irrd.rpsl.fields import RPSLFieldListMixin, RPSLReferenceField, RPSLTextField +from irrd.rpsl.rpsl_objects import ( + OBJECT_CLASS_MAPPING, + RPSLAutNum, + RPSLInetRtr, + RPSLPerson, + RPSLRole, + lookup_field_names, +) from irrd.scopefilter.status import ScopeFilterStatus from irrd.utils.text import snake_to_camel_case diff --git a/irrd/server/graphql/tests/test_resolvers.py b/irrd/server/graphql/tests/test_resolvers.py index 9373a2a6d..f4084b611 100644 --- a/irrd/server/graphql/tests/test_resolvers.py +++ b/irrd/server/graphql/tests/test_resolvers.py @@ -1,8 +1,8 @@ from unittest.mock import Mock import pytest -from IPy import IP from graphql import GraphQLError +from IPy import IP from starlette.requests import HTTPConnection from irrd.routepref.status import RoutePreferenceStatus @@ -12,8 +12,9 @@ from irrd.storage.database_handler import DatabaseHandler from irrd.storage.models import DatabaseOperation, JournalEntryOrigin from irrd.storage.preload import Preloader -from irrd.storage.queries import RPSLDatabaseQuery, RPSLDatabaseJournalQuery +from irrd.storage.queries import RPSLDatabaseJournalQuery, RPSLDatabaseQuery from irrd.utils.test_utils import flatten_mock_calls + from .. import resolvers EXPECTED_RPSL_GRAPHQL_OUTPUT = [{ diff --git a/irrd/server/http/app.py b/irrd/server/http/app.py index 6f7078cd3..7b06417d1 100644 --- a/irrd/server/http/app.py +++ b/irrd/server/http/app.py @@ -14,15 +14,18 @@ from irrd import ENV_MAIN_PROCESS_PID from irrd.conf import config_init from irrd.server.graphql import ENV_UVICORN_WORKER_CONFIG_PATH -from irrd.server.graphql.extensions import error_formatter, QueryMetadataExtension +from irrd.server.graphql.extensions import QueryMetadataExtension, error_formatter from irrd.server.graphql.schema_builder import build_executable_schema from irrd.server.http.endpoints import ( + ObjectSubmissionEndpoint, StatusEndpoint, SuspensionSubmissionEndpoint, WhoisQueryEndpoint, - ObjectSubmissionEndpoint, ) -from irrd.server.http.event_stream import EventStreamEndpoint, EventStreamInitialDownloadEndpoint +from irrd.server.http.event_stream import ( + EventStreamEndpoint, + EventStreamInitialDownloadEndpoint, +) from irrd.storage.database_handler import DatabaseHandler from irrd.storage.preload import Preloader from irrd.utils.process_support import memory_trim, set_traceback_handler diff --git a/irrd/server/http/endpoints.py b/irrd/server/http/endpoints.py index 197597147..336529aea 100644 --- a/irrd/server/http/endpoints.py +++ b/irrd/server/http/endpoints.py @@ -7,14 +7,15 @@ from asgiref.sync import sync_to_async from starlette.endpoints import HTTPEndpoint from starlette.requests import Request -from starlette.responses import PlainTextResponse, Response, JSONResponse +from starlette.responses import JSONResponse, PlainTextResponse, Response from irrd.server.access_check import is_client_permitted from irrd.updates.handler import ChangeSubmissionHandler from irrd.utils.validators import RPSLChangeSubmission, RPSLSuspensionSubmission -from .status_generator import StatusGenerator + from ..whois.query_parser import WhoisQueryParser from ..whois.query_response import WhoisQueryResponseType +from .status_generator import StatusGenerator logger = logging.getLogger(__name__) diff --git a/irrd/server/http/event_stream.py b/irrd/server/http/event_stream.py index bf612f1c4..2a6533938 100644 --- a/irrd/server/http/event_stream.py +++ b/irrd/server/http/event_stream.py @@ -5,30 +5,32 @@ import socket import sys import tempfile -from typing import Any, List, Optional, Callable +from typing import Any, Callable, List, Literal, Optional import pydantic import ujson -from starlette.endpoints import WebSocketEndpoint, HTTPEndpoint +from starlette.endpoints import HTTPEndpoint, WebSocketEndpoint from starlette.requests import Request -from starlette.responses import Response, StreamingResponse, PlainTextResponse +from starlette.responses import PlainTextResponse, Response, StreamingResponse from starlette.status import WS_1003_UNSUPPORTED_DATA, WS_1008_POLICY_VIOLATION from starlette.websockets import WebSocket -from typing import Literal from irrd.conf import get_setting +from irrd.routepref.status import RoutePreferenceStatus from irrd.rpki.status import RPKIStatus from irrd.rpsl.rpsl_objects import rpsl_object_from_text -from irrd.routepref.status import RoutePreferenceStatus from irrd.scopefilter.status import ScopeFilterStatus from irrd.server.access_check import is_client_permitted from irrd.storage.database_handler import DatabaseHandler -from irrd.storage.event_stream import AsyncEventStreamRedisClient, REDIS_STREAM_END_IDENTIFIER +from irrd.storage.event_stream import ( + REDIS_STREAM_END_IDENTIFIER, + AsyncEventStreamRedisClient, +) from irrd.storage.queries import ( DatabaseStatusQuery, - RPSLDatabaseQuery, - RPSLDatabaseJournalStatisticsQuery, RPSLDatabaseJournalQuery, + RPSLDatabaseJournalStatisticsQuery, + RPSLDatabaseQuery, ) from irrd.utils.text import remove_auth_hashes from irrd.vendor import postgres_copy diff --git a/irrd/server/http/server.py b/irrd/server/http/server.py index 9247ec043..05bf53e5c 100644 --- a/irrd/server/http/server.py +++ b/irrd/server/http/server.py @@ -13,8 +13,7 @@ sys.path.append(str(Path(__file__).resolve().parents[3])) from irrd import __version__ -from irrd.conf import get_setting, get_configuration - +from irrd.conf import get_configuration, get_setting from irrd.server.graphql import ENV_UVICORN_WORKER_CONFIG_PATH diff --git a/irrd/server/http/tests/test_endpoints.py b/irrd/server/http/tests/test_endpoints.py index 987fd79b7..bcae59d83 100644 --- a/irrd/server/http/tests/test_endpoints.py +++ b/irrd/server/http/tests/test_endpoints.py @@ -8,12 +8,16 @@ from irrd.storage.preload import Preloader from irrd.updates.handler import ChangeSubmissionHandler from irrd.utils.validators import RPSLChangeSubmission, RPSLSuspensionSubmission + +from ...whois.query_parser import WhoisQueryParser +from ...whois.query_response import ( + WhoisQueryResponse, + WhoisQueryResponseMode, + WhoisQueryResponseType, +) from ..app import app from ..endpoints import StatusEndpoint, WhoisQueryEndpoint from ..status_generator import StatusGenerator -from ...whois.query_parser import WhoisQueryParser -from ...whois.query_response import WhoisQueryResponse, WhoisQueryResponseType, \ - WhoisQueryResponseMode class TestStatusEndpoint: diff --git a/irrd/server/http/tests/test_event_stream.py b/irrd/server/http/tests/test_event_stream.py index a6d7956ea..17a7adfc9 100644 --- a/irrd/server/http/tests/test_event_stream.py +++ b/irrd/server/http/tests/test_event_stream.py @@ -18,13 +18,14 @@ from irrd.scopefilter.status import ScopeFilterStatus from irrd.storage.event_stream import OPERATION_JOURNAL_EXTENDED from irrd.storage.queries import ( - RPSLDatabaseJournalStatisticsQuery, RPSLDatabaseJournalQuery, + RPSLDatabaseJournalStatisticsQuery, RPSLDatabaseQuery, ) from irrd.utils.rpsl_samples import SAMPLE_MNTNER from irrd.utils.test_utils import MockDatabaseHandler from irrd.vendor import postgres_copy + from ..app import app from ..event_stream import AsyncEventStreamFollower diff --git a/irrd/server/http/tests/test_status_generator.py b/irrd/server/http/tests/test_status_generator.py index 08f39edb4..cb15d3e9c 100644 --- a/irrd/server/http/tests/test_status_generator.py +++ b/irrd/server/http/tests/test_status_generator.py @@ -5,6 +5,7 @@ from irrd import __version__ from irrd.conf import get_setting + from ..status_generator import StatusGenerator diff --git a/irrd/server/query_resolver.py b/irrd/server/query_resolver.py index 99fe78f40..851a1031f 100644 --- a/irrd/server/query_resolver.py +++ b/irrd/server/query_resolver.py @@ -1,20 +1,23 @@ import logging from collections import OrderedDict from enum import Enum -from typing import Optional, List, Set, Tuple, Any, Dict +from typing import Any, Dict, List, Optional, Set, Tuple from IPy import IP from pytz import timezone -from irrd.conf import get_setting, RPKI_IRR_PSEUDO_SOURCE +from irrd.conf import RPKI_IRR_PSEUDO_SOURCE, get_setting from irrd.routepref.status import RoutePreferenceStatus from irrd.rpki.status import RPKIStatus -from irrd.rpsl.rpsl_objects import (OBJECT_CLASS_MAPPING, lookup_field_names) +from irrd.rpsl.rpsl_objects import OBJECT_CLASS_MAPPING, lookup_field_names from irrd.scopefilter.status import ScopeFilterStatus -from irrd.storage.database_handler import DatabaseHandler, is_serial_synchronised, \ - RPSLDatabaseResponse +from irrd.storage.database_handler import ( + DatabaseHandler, + RPSLDatabaseResponse, + is_serial_synchronised, +) from irrd.storage.preload import Preloader -from irrd.storage.queries import RPSLDatabaseQuery, DatabaseStatusQuery +from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseQuery from irrd.utils.validators import parse_as_number logger = logging.getLogger(__name__) diff --git a/irrd/server/tests/test_query_resolver.py b/irrd/server/tests/test_query_resolver.py index 6763a5f29..ba847f5ad 100644 --- a/irrd/server/tests/test_query_resolver.py +++ b/irrd/server/tests/test_query_resolver.py @@ -7,12 +7,13 @@ from IPy import IP from pytz import timezone -from irrd.rpki.status import RPKIStatus from irrd.routepref.status import RoutePreferenceStatus +from irrd.rpki.status import RPKIStatus from irrd.scopefilter.status import ScopeFilterStatus from irrd.storage.preload import Preloader from irrd.utils.test_utils import flatten_mock_calls -from ..query_resolver import QueryResolver, RouteLookupType, InvalidQueryException + +from ..query_resolver import InvalidQueryException, QueryResolver, RouteLookupType # Note that these mock objects are not entirely valid RPSL objects, # as they are meant to test all the scenarios in the query resolver. diff --git a/irrd/server/whois/query_parser.py b/irrd/server/whois/query_parser.py index d478c7935..6e68d6b90 100644 --- a/irrd/server/whois/query_parser.py +++ b/irrd/server/whois/query_parser.py @@ -7,17 +7,26 @@ from ordered_set import OrderedSet from irrd import __version__ -from irrd.conf import get_setting, RPKI_IRR_PSEUDO_SOURCE, SOCKET_DEFAULT_TIMEOUT +from irrd.conf import RPKI_IRR_PSEUDO_SOURCE, SOCKET_DEFAULT_TIMEOUT, get_setting from irrd.mirroring.nrtm_generator import NRTMGenerator, NRTMGeneratorException from irrd.rpki.status import RPKIStatus -from irrd.rpsl.rpsl_objects import (OBJECT_CLASS_MAPPING, RPKI_RELEVANT_OBJECT_CLASSES) -from irrd.server.query_resolver import QueryResolver, RouteLookupType, InvalidQueryException +from irrd.rpsl.rpsl_objects import OBJECT_CLASS_MAPPING, RPKI_RELEVANT_OBJECT_CLASSES +from irrd.server.query_resolver import ( + InvalidQueryException, + QueryResolver, + RouteLookupType, +) from irrd.storage.database_handler import DatabaseHandler, RPSLDatabaseResponse from irrd.storage.preload import Preloader from irrd.storage.queries import DatabaseStatusQuery -from irrd.utils.validators import parse_as_number, ValidationError -from .query_response import WhoisQueryResponseType, WhoisQueryResponseMode, WhoisQueryResponse +from irrd.utils.validators import ValidationError, parse_as_number + from ..access_check import is_client_permitted +from .query_response import ( + WhoisQueryResponse, + WhoisQueryResponseMode, + WhoisQueryResponseType, +) logger = logging.getLogger(__name__) diff --git a/irrd/server/whois/server.py b/irrd/server/whois/server.py index c3ee5f5e4..1575620dc 100644 --- a/irrd/server/whois/server.py +++ b/irrd/server/whois/server.py @@ -7,8 +7,8 @@ import threading import time -from IPy import IP from daemon.daemon import change_process_owner +from IPy import IP from setproctitle import setproctitle from irrd import ENV_MAIN_PROCESS_PID diff --git a/irrd/server/whois/tests/test_query_parser.py b/irrd/server/whois/tests/test_query_parser.py index b96aba003..355b6cbcd 100644 --- a/irrd/server/whois/tests/test_query_parser.py +++ b/irrd/server/whois/tests/test_query_parser.py @@ -6,11 +6,16 @@ from irrd.mirroring.nrtm_generator import NRTMGeneratorException from irrd.rpki.status import RPKIStatus -from irrd.server.query_resolver import QueryResolver, RouteLookupType, InvalidQueryException +from irrd.server.query_resolver import ( + InvalidQueryException, + QueryResolver, + RouteLookupType, +) from irrd.storage.database_handler import DatabaseHandler from irrd.utils.test_utils import flatten_mock_calls + from ..query_parser import WhoisQueryParser -from ..query_response import WhoisQueryResponseType, WhoisQueryResponseMode +from ..query_response import WhoisQueryResponseMode, WhoisQueryResponseType # Note that these mock objects are not entirely valid RPSL objects, # as they are meant to test all the scenarios in the query parser. diff --git a/irrd/server/whois/tests/test_query_response.py b/irrd/server/whois/tests/test_query_response.py index 070f58479..723b39a4d 100644 --- a/irrd/server/whois/tests/test_query_response.py +++ b/irrd/server/whois/tests/test_query_response.py @@ -2,7 +2,12 @@ from irrd.conf import PASSWORD_HASH_DUMMY_VALUE from irrd.utils.rpsl_samples import SAMPLE_MNTNER -from ..query_response import WhoisQueryResponse, WhoisQueryResponseMode, WhoisQueryResponseType + +from ..query_response import ( + WhoisQueryResponse, + WhoisQueryResponseMode, + WhoisQueryResponseType, +) class TestWhoisQueryResponse: diff --git a/irrd/server/whois/tests/test_server.py b/irrd/server/whois/tests/test_server.py index 88455e957..c281a6616 100644 --- a/irrd/server/whois/tests/test_server.py +++ b/irrd/server/whois/tests/test_server.py @@ -7,6 +7,7 @@ import pytest from irrd.storage.preload import Preloader + from ..server import WhoisWorker diff --git a/irrd/storage/alembic/env.py b/irrd/storage/alembic/env.py index 64d119c6c..34f2d7761 100644 --- a/irrd/storage/alembic/env.py +++ b/irrd/storage/alembic/env.py @@ -1,16 +1,16 @@ # flake8: noqa: E402 import os import sys +from pathlib import Path from alembic import context -from pathlib import Path from sqlalchemy import create_engine from irrd.storage import translate_url sys.path.append(str(Path(__file__).resolve().parents[3])) -from irrd.conf import get_setting, config_init, is_config_initialised +from irrd.conf import config_init, get_setting, is_config_initialised from irrd.storage.models import Base target_metadata = Base.metadata diff --git a/irrd/storage/alembic/versions/0548f1aa4f10_add_rpsl_objects_suspended.py b/irrd/storage/alembic/versions/0548f1aa4f10_add_rpsl_objects_suspended.py index 98d910c2c..9d4a586ac 100644 --- a/irrd/storage/alembic/versions/0548f1aa4f10_add_rpsl_objects_suspended.py +++ b/irrd/storage/alembic/versions/0548f1aa4f10_add_rpsl_objects_suspended.py @@ -5,8 +5,8 @@ Create Date: 2021-12-02 14:34:10.566178 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql from sqlalchemy.exc import ProgrammingError diff --git a/irrd/storage/alembic/versions/1743f98a456d_add_serial_newest_mirror.py b/irrd/storage/alembic/versions/1743f98a456d_add_serial_newest_mirror.py index 73088a2f3..a3f9b8732 100644 --- a/irrd/storage/alembic/versions/1743f98a456d_add_serial_newest_mirror.py +++ b/irrd/storage/alembic/versions/1743f98a456d_add_serial_newest_mirror.py @@ -5,9 +5,8 @@ Create Date: 2020-04-15 20:08:59.925809 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '1743f98a456d' diff --git a/irrd/storage/alembic/versions/181670a62643_add_journal_entry_origin.py b/irrd/storage/alembic/versions/181670a62643_add_journal_entry_origin.py index 0e17bd3f3..84890d2b8 100644 --- a/irrd/storage/alembic/versions/181670a62643_add_journal_entry_origin.py +++ b/irrd/storage/alembic/versions/181670a62643_add_journal_entry_origin.py @@ -5,9 +5,8 @@ Create Date: 2020-04-15 10:19:09.146000 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '181670a62643' diff --git a/irrd/storage/alembic/versions/28dc1cd85bdc_initial_db.py b/irrd/storage/alembic/versions/28dc1cd85bdc_initial_db.py index 12d989d29..80335f2ad 100644 --- a/irrd/storage/alembic/versions/28dc1cd85bdc_initial_db.py +++ b/irrd/storage/alembic/versions/28dc1cd85bdc_initial_db.py @@ -5,8 +5,8 @@ Create Date: 2018-06-11 14:37:13.472465 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/irrd/storage/alembic/versions/39e4f15ed80c_add_bogon_status.py b/irrd/storage/alembic/versions/39e4f15ed80c_add_bogon_status.py index c4450f05d..f1f0947a5 100644 --- a/irrd/storage/alembic/versions/39e4f15ed80c_add_bogon_status.py +++ b/irrd/storage/alembic/versions/39e4f15ed80c_add_bogon_status.py @@ -5,11 +5,10 @@ Create Date: 2020-04-22 14:43:57.985437 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.engine.reflection import Inspector - # revision identifiers, used by Alembic. revision = '39e4f15ed80c' diff --git a/irrd/storage/alembic/versions/4a514ead8fc2_bogon_to_scope_filter.py b/irrd/storage/alembic/versions/4a514ead8fc2_bogon_to_scope_filter.py index 531f56165..0ad572044 100644 --- a/irrd/storage/alembic/versions/4a514ead8fc2_bogon_to_scope_filter.py +++ b/irrd/storage/alembic/versions/4a514ead8fc2_bogon_to_scope_filter.py @@ -5,8 +5,8 @@ Create Date: 2020-07-09 20:11:45.873381 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.engine.reflection import Inspector from sqlalchemy.exc import ProgrammingError diff --git a/irrd/storage/alembic/versions/8744b4b906bb_fix_rpsl_unique_key.py b/irrd/storage/alembic/versions/8744b4b906bb_fix_rpsl_unique_key.py index 6f68c8b71..8dbeb5f54 100644 --- a/irrd/storage/alembic/versions/8744b4b906bb_fix_rpsl_unique_key.py +++ b/irrd/storage/alembic/versions/8744b4b906bb_fix_rpsl_unique_key.py @@ -10,7 +10,6 @@ """ from alembic import op - # revision identifiers, used by Alembic. revision = '8744b4b906bb' down_revision = '893d0d5363b3' diff --git a/irrd/storage/alembic/versions/893d0d5363b3_add_rpsl_prefix_idx.py b/irrd/storage/alembic/versions/893d0d5363b3_add_rpsl_prefix_idx.py index 1437b5182..9375ac4d5 100644 --- a/irrd/storage/alembic/versions/893d0d5363b3_add_rpsl_prefix_idx.py +++ b/irrd/storage/alembic/versions/893d0d5363b3_add_rpsl_prefix_idx.py @@ -5,8 +5,8 @@ Create Date: 2021-03-01 16:11:28.275554 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '893d0d5363b3' diff --git a/irrd/storage/alembic/versions/8b8357acd333_add_global_serial.py b/irrd/storage/alembic/versions/8b8357acd333_add_global_serial.py index 56a3d6c2d..a61ec0e5a 100644 --- a/irrd/storage/alembic/versions/8b8357acd333_add_global_serial.py +++ b/irrd/storage/alembic/versions/8b8357acd333_add_global_serial.py @@ -5,8 +5,8 @@ Create Date: 2022-10-07 13:43:46.598866 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '8b8357acd333' @@ -16,7 +16,7 @@ def upgrade(): - from sqlalchemy.schema import Sequence, CreateSequence + from sqlalchemy.schema import CreateSequence, Sequence op.execute(CreateSequence(Sequence('rpsl_database_journal_serial_global_seq', start=1000000))) op.add_column('rpsl_database_journal', sa.Column('serial_global', sa.BigInteger(), nullable=True)) @@ -42,7 +42,7 @@ def upgrade(): def downgrade(): - from sqlalchemy.schema import Sequence, DropSequence + from sqlalchemy.schema import DropSequence, Sequence op.drop_index(op.f('ix_rpsl_database_journal_timestamp'), table_name='rpsl_database_journal') op.drop_index(op.f('ix_rpsl_database_journal_serial_global'), table_name='rpsl_database_journal') op.drop_column('rpsl_database_journal', 'serial_global') diff --git a/irrd/storage/alembic/versions/a8609af97aa3_set_prefix_length_in_existing_rpsl_.py b/irrd/storage/alembic/versions/a8609af97aa3_set_prefix_length_in_existing_rpsl_.py index 094f9fd7d..2b19fc2b1 100644 --- a/irrd/storage/alembic/versions/a8609af97aa3_set_prefix_length_in_existing_rpsl_.py +++ b/irrd/storage/alembic/versions/a8609af97aa3_set_prefix_length_in_existing_rpsl_.py @@ -10,7 +10,6 @@ from sqlalchemy.dialects import postgresql as pg from sqlalchemy.ext.declarative import declarative_base - # revision identifiers, used by Alembic. revision = 'a8609af97aa3' down_revision = '64a3d6faf6d4' diff --git a/irrd/storage/alembic/versions/b175c262448f_set_rpsl_prefix.py b/irrd/storage/alembic/versions/b175c262448f_set_rpsl_prefix.py index 6e57cee69..e92d1f06a 100644 --- a/irrd/storage/alembic/versions/b175c262448f_set_rpsl_prefix.py +++ b/irrd/storage/alembic/versions/b175c262448f_set_rpsl_prefix.py @@ -7,7 +7,6 @@ """ from alembic import op - # revision identifiers, used by Alembic. revision = 'b175c262448f' down_revision = 'f4c837d8258c' diff --git a/irrd/storage/alembic/versions/e07863eac52f_add_roa_object_table.py b/irrd/storage/alembic/versions/e07863eac52f_add_roa_object_table.py index 1fa3e92c2..19b43e950 100644 --- a/irrd/storage/alembic/versions/e07863eac52f_add_roa_object_table.py +++ b/irrd/storage/alembic/versions/e07863eac52f_add_roa_object_table.py @@ -5,8 +5,8 @@ Create Date: 2019-02-28 16:03:57.797697 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/irrd/storage/alembic/versions/f4c837d8258c_add_rpsl_prefix.py b/irrd/storage/alembic/versions/f4c837d8258c_add_rpsl_prefix.py index e321a67c7..aabaa7f8e 100644 --- a/irrd/storage/alembic/versions/f4c837d8258c_add_rpsl_prefix.py +++ b/irrd/storage/alembic/versions/f4c837d8258c_add_rpsl_prefix.py @@ -5,8 +5,8 @@ Create Date: 2021-03-01 15:38:26.513071 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/irrd/storage/alembic/versions/fd4473bc1a10_add_route_preference_status.py b/irrd/storage/alembic/versions/fd4473bc1a10_add_route_preference_status.py index 383aa0476..632dad6b2 100644 --- a/irrd/storage/alembic/versions/fd4473bc1a10_add_route_preference_status.py +++ b/irrd/storage/alembic/versions/fd4473bc1a10_add_route_preference_status.py @@ -5,8 +5,8 @@ Create Date: 2022-12-24 16:25:59.648631 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.exc import ProgrammingError # revision identifiers, used by Alembic. diff --git a/irrd/storage/database_handler.py b/irrd/storage/database_handler.py index a1266f3ee..bf5d11d6f 100644 --- a/irrd/storage/database_handler.py +++ b/irrd/storage/database_handler.py @@ -3,30 +3,41 @@ from datetime import datetime, timezone from functools import lru_cache from io import StringIO -from typing import List, Set, Dict, Any, Tuple, Iterator, Union, Optional, Iterable +from typing import Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union import sqlalchemy as sa -from IPy import IP from asgiref.sync import sync_to_async +from IPy import IP from sqlalchemy.dialects import postgresql as pg from irrd.conf import get_setting +from irrd.routepref.status import RoutePreferenceStatus from irrd.rpki.status import RPKIStatus from irrd.rpsl.parser import RPSLObject from irrd.rpsl.rpsl_objects import OBJECT_CLASS_MAPPING, RPKI_RELEVANT_OBJECT_CLASSES from irrd.scopefilter.status import ScopeFilterStatus +from irrd.utils.misc import chunked_iterable from irrd.vendor import postgres_copy + from . import get_engine from .event_stream import EventStreamPublisher -from .models import RPSLDatabaseObject, RPSLDatabaseJournal, DatabaseOperation, RPSLDatabaseStatus, \ - ROADatabaseObject, JournalEntryOrigin, RPSLDatabaseObjectSuspended +from .models import ( + DatabaseOperation, + JournalEntryOrigin, + ROADatabaseObject, + RPSLDatabaseJournal, + RPSLDatabaseObject, + RPSLDatabaseObjectSuspended, + RPSLDatabaseStatus, +) from .preload import Preloader -from .queries import (BaseRPSLObjectDatabaseQuery, DatabaseStatusQuery, - RPSLDatabaseObjectStatisticsQuery, ROADatabaseObjectQuery, - RPSLDatabaseJournalStatisticsQuery) -from irrd.routepref.status import RoutePreferenceStatus -from irrd.utils.misc import chunked_iterable - +from .queries import ( + BaseRPSLObjectDatabaseQuery, + DatabaseStatusQuery, + ROADatabaseObjectQuery, + RPSLDatabaseJournalStatisticsQuery, + RPSLDatabaseObjectStatisticsQuery, +) QueryType = Union[ BaseRPSLObjectDatabaseQuery, DatabaseStatusQuery, diff --git a/irrd/storage/preload.py b/irrd/storage/preload.py index 512ba03fc..ee3344538 100644 --- a/irrd/storage/preload.py +++ b/irrd/storage/preload.py @@ -4,16 +4,17 @@ import threading import time from collections import defaultdict -from typing import Optional, List, Set, Dict, Union +from typing import Dict, List, Optional, Set, Union import redis from setproctitle import setproctitle from irrd.conf import get_setting -from irrd.rpki.status import RPKIStatus from irrd.routepref.status import RoutePreferenceStatus +from irrd.rpki.status import RPKIStatus from irrd.scopefilter.status import ScopeFilterStatus from irrd.utils.process_support import ExceptionLoggingProcess + from .queries import RPSLDatabaseQuery SENTINEL_HASH_CREATED = b'SENTINEL_HASH_CREATED' diff --git a/irrd/storage/queries.py b/irrd/storage/queries.py index 2eb137b1b..646370f27 100644 --- a/irrd/storage/queries.py +++ b/irrd/storage/queries.py @@ -3,18 +3,23 @@ from typing import List, Optional import sqlalchemy as sa -from IPy import IP -from sqlalchemy.sql import Select, ColumnCollection import sqlalchemy.dialects.postgresql as pg +from IPy import IP +from sqlalchemy.sql import ColumnCollection, Select from irrd.conf import get_setting -from irrd.rpki.status import RPKIStatus from irrd.routepref.status import RoutePreferenceStatus +from irrd.rpki.status import RPKIStatus from irrd.rpsl.rpsl_objects import lookup_field_names from irrd.scopefilter.status import ScopeFilterStatus -from irrd.storage.models import (RPSLDatabaseObject, RPSLDatabaseJournal, RPSLDatabaseStatus, - ROADatabaseObject, RPSLDatabaseObjectSuspended) -from irrd.utils.validators import parse_as_number, ValidationError +from irrd.storage.models import ( + ROADatabaseObject, + RPSLDatabaseJournal, + RPSLDatabaseObject, + RPSLDatabaseObjectSuspended, + RPSLDatabaseStatus, +) +from irrd.utils.validators import ValidationError, parse_as_number logger = logging.getLogger(__name__) diff --git a/irrd/storage/tests/test_database.py b/irrd/storage/tests/test_database.py index ceec7964e..2fd03854a 100644 --- a/irrd/storage/tests/test_database.py +++ b/irrd/storage/tests/test_database.py @@ -7,16 +7,23 @@ from pytest import raises from sqlalchemy.exc import ProgrammingError -from irrd.rpki.status import RPKIStatus from irrd.routepref.status import RoutePreferenceStatus +from irrd.rpki.status import RPKIStatus from irrd.scopefilter.status import ScopeFilterStatus from irrd.utils.test_utils import flatten_mock_calls + from .. import get_engine from ..database_handler import DatabaseHandler -from ..models import RPSLDatabaseObject, DatabaseOperation, JournalEntryOrigin +from ..models import DatabaseOperation, JournalEntryOrigin, RPSLDatabaseObject from ..preload import Preloader -from ..queries import (RPSLDatabaseQuery, RPSLDatabaseJournalQuery, DatabaseStatusQuery, - RPSLDatabaseObjectStatisticsQuery, ROADatabaseObjectQuery, RPSLDatabaseSuspendedQuery) +from ..queries import ( + DatabaseStatusQuery, + ROADatabaseObjectQuery, + RPSLDatabaseJournalQuery, + RPSLDatabaseObjectStatisticsQuery, + RPSLDatabaseQuery, + RPSLDatabaseSuspendedQuery, +) """ These tests for the database use a live PostgreSQL database, diff --git a/irrd/storage/tests/test_event_stream.py b/irrd/storage/tests/test_event_stream.py index 11513b7e7..85b469ad6 100644 --- a/irrd/storage/tests/test_event_stream.py +++ b/irrd/storage/tests/test_event_stream.py @@ -1,4 +1,8 @@ -from ..event_stream import EventStreamPublisher, AsyncEventStreamRedisClient, REDIS_STREAM_END_IDENTIFIER +from ..event_stream import ( + REDIS_STREAM_END_IDENTIFIER, + AsyncEventStreamRedisClient, + EventStreamPublisher, +) # Use different stores in tests TEST_REDIS_STREAM_RPSL = "TEST-irrd-eventstream-rpsl-stream" diff --git a/irrd/storage/tests/test_preload.py b/irrd/storage/tests/test_preload.py index 4f48d3b07..4ae91c3ed 100644 --- a/irrd/storage/tests/test_preload.py +++ b/irrd/storage/tests/test_preload.py @@ -4,13 +4,18 @@ import pytest -from irrd.rpki.status import RPKIStatus from irrd.routepref.status import RoutePreferenceStatus +from irrd.rpki.status import RPKIStatus from irrd.scopefilter.status import ScopeFilterStatus from irrd.utils.test_utils import flatten_mock_calls + from ..database_handler import DatabaseHandler -from ..preload import (Preloader, PreloadStoreManager, PreloadUpdater, - REDIS_KEY_ORIGIN_SOURCE_SEPARATOR) +from ..preload import ( + REDIS_KEY_ORIGIN_SOURCE_SEPARATOR, + Preloader, + PreloadStoreManager, + PreloadUpdater, +) from ..queries import RPSLDatabaseQuery # Use different stores in tests diff --git a/irrd/updates/email.py b/irrd/updates/email.py index 03a67a9d8..47e9ad205 100644 --- a/irrd/updates/email.py +++ b/irrd/updates/email.py @@ -3,6 +3,7 @@ from typing import Optional from irrd.utils import email + from .handler import ChangeSubmissionHandler logger = logging.getLogger(__name__) diff --git a/irrd/updates/handler.py b/irrd/updates/handler.py index 0e2c0d8a2..b64d9d898 100644 --- a/irrd/updates/handler.py +++ b/irrd/updates/handler.py @@ -1,7 +1,7 @@ import logging import textwrap from collections import defaultdict -from typing import List, Optional, Dict, Union +from typing import Dict, List, Optional, Union from ordered_set import OrderedSet @@ -10,10 +10,11 @@ from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import RPSLDatabaseQuery from irrd.utils import email -from .parser import parse_change_requests, ChangeRequest, SuspensionRequest -from .parser_state import UpdateRequestStatus, UpdateRequestType -from .validators import ReferenceValidator, AuthValidator + from ..utils.validators import RPSLChangeSubmission, RPSLSuspensionSubmission +from .parser import ChangeRequest, SuspensionRequest, parse_change_requests +from .parser_state import UpdateRequestStatus, UpdateRequestType +from .validators import AuthValidator, ReferenceValidator logger = logging.getLogger(__name__) diff --git a/irrd/updates/parser.py b/irrd/updates/parser.py index 07a6a14c7..ac450f502 100644 --- a/irrd/updates/parser.py +++ b/irrd/updates/parser.py @@ -1,21 +1,22 @@ import difflib import logging -from typing import List, Optional, Set, Dict, Union +from typing import Dict, List, Optional, Set, Union from irrd.conf import get_setting from irrd.rpki.status import RPKIStatus from irrd.rpki.validators import SingleRouteROAValidator -from irrd.rpsl.parser import UnknownRPSLObjectClassException, RPSLObject -from irrd.rpsl.rpsl_objects import rpsl_object_from_text, RPSLMntner +from irrd.rpsl.parser import RPSLObject, UnknownRPSLObjectClassException +from irrd.rpsl.rpsl_objects import RPSLMntner, rpsl_object_from_text from irrd.scopefilter.status import ScopeFilterStatus from irrd.scopefilter.validators import ScopeFilterValidator from irrd.storage.database_handler import DatabaseHandler from irrd.storage.models import JournalEntryOrigin from irrd.storage.queries import RPSLDatabaseQuery -from irrd.utils.text import splitline_unicodesafe, remove_auth_hashes -from .parser_state import UpdateRequestType, UpdateRequestStatus, SuspensionRequestType -from .validators import ReferenceValidator, AuthValidator, RulesValidator -from .suspension import suspend_for_mntner, reactivate_for_mntner +from irrd.utils.text import remove_auth_hashes, splitline_unicodesafe + +from .parser_state import SuspensionRequestType, UpdateRequestStatus, UpdateRequestType +from .suspension import reactivate_for_mntner, suspend_for_mntner +from .validators import AuthValidator, ReferenceValidator, RulesValidator logger = logging.getLogger(__name__) diff --git a/irrd/updates/parser_state.py b/irrd/updates/parser_state.py index 39faf299c..5610bf6af 100644 --- a/irrd/updates/parser_state.py +++ b/irrd/updates/parser_state.py @@ -1,6 +1,6 @@ -from enum import unique, Enum +from enum import Enum, unique -from irrd.conf import get_setting, AUTH_SET_CREATION_COMMON_KEY +from irrd.conf import AUTH_SET_CREATION_COMMON_KEY, get_setting @unique diff --git a/irrd/updates/suspension.py b/irrd/updates/suspension.py index b9c5ff515..dfa038eb9 100644 --- a/irrd/updates/suspension.py +++ b/irrd/updates/suspension.py @@ -1,15 +1,15 @@ -import logging -from typing import List, Dict, Tuple import functools +import logging +from typing import Dict, List, Tuple -from irrd.storage.models import JournalEntryOrigin from irrd.conf import get_setting +from irrd.rpki.validators import SingleRouteROAValidator +from irrd.rpsl.parser import RPSLObject from irrd.rpsl.rpsl_objects import RPSLMntner, rpsl_object_from_text +from irrd.scopefilter.validators import ScopeFilterValidator from irrd.storage.database_handler import DatabaseHandler +from irrd.storage.models import JournalEntryOrigin from irrd.storage.queries import RPSLDatabaseQuery, RPSLDatabaseSuspendedQuery -from irrd.rpsl.parser import RPSLObject -from irrd.scopefilter.validators import ScopeFilterValidator -from irrd.rpki.validators import SingleRouteROAValidator logger = logging.getLogger(__name__) diff --git a/irrd/updates/tests/test_email.py b/irrd/updates/tests/test_email.py index dfe8eaf98..e42d26135 100644 --- a/irrd/updates/tests/test_email.py +++ b/irrd/updates/tests/test_email.py @@ -1,7 +1,8 @@ -import pytest import textwrap from unittest.mock import Mock +import pytest + from irrd.storage.database_handler import DatabaseHandler from irrd.updates.email import handle_email_submission diff --git a/irrd/updates/tests/test_handler.py b/irrd/updates/tests/test_handler.py index 85bb6e2af..024962290 100644 --- a/irrd/updates/tests/test_handler.py +++ b/irrd/updates/tests/test_handler.py @@ -8,8 +8,9 @@ from irrd.storage.models import JournalEntryOrigin from irrd.utils.rpsl_samples import SAMPLE_MNTNER from irrd.utils.test_utils import flatten_mock_calls -from ..handler import ChangeSubmissionHandler + from ...utils.validators import RPSLChangeSubmission, RPSLSuspensionSubmission +from ..handler import ChangeSubmissionHandler from ..parser_state import SuspensionRequestType diff --git a/irrd/updates/tests/test_parser.py b/irrd/updates/tests/test_parser.py index 75d67b045..577b607a2 100644 --- a/irrd/updates/tests/test_parser.py +++ b/irrd/updates/tests/test_parser.py @@ -11,14 +11,26 @@ from irrd.scopefilter.status import ScopeFilterStatus from irrd.scopefilter.validators import ScopeFilterValidator from irrd.storage.models import JournalEntryOrigin -from irrd.utils.rpsl_samples import SAMPLE_INETNUM, SAMPLE_AS_SET, SAMPLE_PERSON, SAMPLE_MNTNER, \ - SAMPLE_ROUTE, SAMPLE_ROUTE6 +from irrd.updates.suspension import suspend_for_mntner +from irrd.utils.rpsl_samples import ( + SAMPLE_AS_SET, + SAMPLE_INETNUM, + SAMPLE_MNTNER, + SAMPLE_PERSON, + SAMPLE_ROUTE, + SAMPLE_ROUTE6, +) from irrd.utils.test_utils import flatten_mock_calls from irrd.utils.text import splitline_unicodesafe -from irrd.updates.suspension import suspend_for_mntner + from ..parser import parse_change_requests -from ..parser_state import SuspensionRequestType, UpdateRequestType, UpdateRequestStatus -from ..validators import ReferenceValidator, AuthValidator, RulesValidator, ValidatorResult +from ..parser_state import SuspensionRequestType, UpdateRequestStatus, UpdateRequestType +from ..validators import ( + AuthValidator, + ReferenceValidator, + RulesValidator, + ValidatorResult, +) @pytest.fixture() diff --git a/irrd/updates/tests/test_suspension.py b/irrd/updates/tests/test_suspension.py index e805b3558..0180807e4 100644 --- a/irrd/updates/tests/test_suspension.py +++ b/irrd/updates/tests/test_suspension.py @@ -1,16 +1,23 @@ -import pytest from unittest.mock import Mock +import pytest + +from irrd.rpki.status import RPKIStatus +from irrd.rpki.validators import SingleRouteROAValidator from irrd.rpsl.rpsl_objects import RPSLMntner +from irrd.scopefilter.status import ScopeFilterStatus +from irrd.scopefilter.validators import ScopeFilterValidator from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import RPSLDatabaseQuery, RPSLDatabaseSuspendedQuery -from irrd.utils.rpsl_samples import SAMPLE_MNTNER, SAMPLE_PERSON, SAMPLE_ROLE, SAMPLE_ROUTE +from irrd.utils.rpsl_samples import ( + SAMPLE_MNTNER, + SAMPLE_PERSON, + SAMPLE_ROLE, + SAMPLE_ROUTE, +) from irrd.utils.test_utils import flatten_mock_calls -from irrd.scopefilter.validators import ScopeFilterValidator -from irrd.rpki.validators import SingleRouteROAValidator -from irrd.rpki.status import RPKIStatus -from irrd.scopefilter.status import ScopeFilterStatus -from ..suspension import suspend_for_mntner, reactivate_for_mntner + +from ..suspension import reactivate_for_mntner, suspend_for_mntner class TestSuspension: diff --git a/irrd/updates/tests/test_validators.py b/irrd/updates/tests/test_validators.py index 9d5d7e9c6..798bf18d8 100644 --- a/irrd/updates/tests/test_validators.py +++ b/irrd/updates/tests/test_validators.py @@ -6,14 +6,21 @@ from irrd.conf import AUTH_SET_CREATION_COMMON_KEY from irrd.rpsl.rpsl_objects import rpsl_object_from_text from irrd.storage.database_handler import DatabaseHandler -from irrd.utils.rpsl_samples import (SAMPLE_AS_SET, SAMPLE_FILTER_SET, SAMPLE_MNTNER, SAMPLE_MNTNER_CRYPT, - SAMPLE_MNTNER_MD5, SAMPLE_PERSON, - SAMPLE_ROUTE, SAMPLE_ROUTE6) +from irrd.storage.queries import RPSLDatabaseSuspendedQuery +from irrd.updates.parser_state import UpdateRequestType +from irrd.utils.rpsl_samples import ( + SAMPLE_AS_SET, + SAMPLE_FILTER_SET, + SAMPLE_MNTNER, + SAMPLE_MNTNER_CRYPT, + SAMPLE_MNTNER_MD5, + SAMPLE_PERSON, + SAMPLE_ROUTE, + SAMPLE_ROUTE6, +) from irrd.utils.test_utils import flatten_mock_calls from irrd.utils.text import remove_auth_hashes -from irrd.storage.queries import RPSLDatabaseSuspendedQuery -from irrd.updates.parser_state import UpdateRequestType from ..validators import AuthValidator, RulesValidator VALID_PW = 'override-password' diff --git a/irrd/updates/validators.py b/irrd/updates/validators.py index ab76e01e3..730c8e901 100644 --- a/irrd/updates/validators.py +++ b/irrd/updates/validators.py @@ -1,16 +1,17 @@ import functools import logging from dataclasses import dataclass, field -from typing import Set, Tuple, List, Optional, TYPE_CHECKING, Union +from typing import TYPE_CHECKING, List, Optional, Set, Tuple, Union from ordered_set import OrderedSet from passlib.hash import md5_crypt from irrd.conf import get_setting from irrd.rpsl.parser import RPSLObject -from irrd.rpsl.rpsl_objects import RPSLMntner, rpsl_object_from_text, RPSLSet +from irrd.rpsl.rpsl_objects import RPSLMntner, RPSLSet, rpsl_object_from_text from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import RPSLDatabaseQuery, RPSLDatabaseSuspendedQuery + from .parser_state import RPSLSetAutnumAuthenticationMode, UpdateRequestType if TYPE_CHECKING: # pragma: no cover diff --git a/irrd/utils/email.py b/irrd/utils/email.py index 391df8d72..ad273ba7f 100644 --- a/irrd/utils/email.py +++ b/irrd/utils/email.py @@ -1,7 +1,6 @@ +import email import logging import socket - -import email from email.mime.text import MIMEText from smtplib import SMTP from typing import Optional @@ -10,7 +9,6 @@ from irrd.conf import get_setting from irrd.utils.pgp import validate_pgp_signature - logger = logging.getLogger(__name__) diff --git a/irrd/utils/pgp.py b/irrd/utils/pgp.py index 08dd2b4b4..dd6b5249b 100644 --- a/irrd/utils/pgp.py +++ b/irrd/utils/pgp.py @@ -4,11 +4,13 @@ from tempfile import NamedTemporaryFile from typing import Optional, Tuple -# Mypy struggles to see gnupg.GPG -import gnupg # type: ignore +# isort: off +import gnupg from irrd.conf import get_setting +# isort: on + logger = logging.getLogger(__name__) pgp_inline_re = re.compile(r'-----BEGIN PGP SIGNED MESSAGE-----(\n.+)?\n\n((?s:.+))\n-----BEGIN PGP SIGNATURE-----\n', flags=re.MULTILINE) diff --git a/irrd/utils/test_utils.py b/irrd/utils/test_utils.py index fb14e34b7..e1ba670b4 100644 --- a/irrd/utils/test_utils.py +++ b/irrd/utils/test_utils.py @@ -3,7 +3,11 @@ from irrd.storage.database_handler import QueryType, RPSLDatabaseResponse from irrd.storage.models import DatabaseOperation, JournalEntryOrigin -from irrd.storage.queries import RPSLDatabaseJournalStatisticsQuery, RPSLDatabaseJournalQuery, DatabaseStatusQuery +from irrd.storage.queries import ( + DatabaseStatusQuery, + RPSLDatabaseJournalQuery, + RPSLDatabaseJournalStatisticsQuery, +) from irrd.utils.rpsl_samples import SAMPLE_MNTNER diff --git a/irrd/utils/tests/test_email.py b/irrd/utils/tests/test_email.py index b25b900c7..916e20e10 100644 --- a/irrd/utils/tests/test_email.py +++ b/irrd/utils/tests/test_email.py @@ -1,8 +1,10 @@ -import pytest import textwrap from unittest.mock import Mock +import pytest + from irrd.conf import get_setting + from ..email import EmailParser, send_email diff --git a/irrd/utils/tests/test_text.py b/irrd/utils/tests/test_text.py index c4c61b1df..8ecc9fa46 100644 --- a/irrd/utils/tests/test_text.py +++ b/irrd/utils/tests/test_text.py @@ -1,10 +1,17 @@ from io import StringIO +import pytest + from irrd.conf import PASSWORD_HASH_DUMMY_VALUE from irrd.utils.rpsl_samples import SAMPLE_MNTNER -import pytest -from ..text import (splitline_unicodesafe, split_paragraphs_rpsl, remove_auth_hashes, - remove_last_modified, snake_to_camel_case) + +from ..text import ( + remove_auth_hashes, + remove_last_modified, + snake_to_camel_case, + split_paragraphs_rpsl, + splitline_unicodesafe, +) def test_remove_auth_hashes(): diff --git a/irrd/utils/tests/test_validators.py b/irrd/utils/tests/test_validators.py index 3811745ff..b3223d9fb 100644 --- a/irrd/utils/tests/test_validators.py +++ b/irrd/utils/tests/test_validators.py @@ -2,7 +2,7 @@ import pytest from pytest import raises -from ..validators import parse_as_number, ValidationError, RPSLChangeSubmission +from ..validators import RPSLChangeSubmission, ValidationError, parse_as_number def test_validate_as_number(): diff --git a/irrd/utils/tests/test_whois_client.py b/irrd/utils/tests/test_whois_client.py index 9c71692ad..040eee57b 100644 --- a/irrd/utils/tests/test_whois_client.py +++ b/irrd/utils/tests/test_whois_client.py @@ -5,7 +5,12 @@ import pytest from ..test_utils import flatten_mock_calls -from ..whois_client import whois_query, whois_query_irrd, whois_query_source_status, WhoisQueryError +from ..whois_client import ( + WhoisQueryError, + whois_query, + whois_query_irrd, + whois_query_source_status, +) class TestWhoisQuery: diff --git a/irrd/utils/text.py b/irrd/utils/text.py index 7096ccd53..954b18045 100644 --- a/irrd/utils/text.py +++ b/irrd/utils/text.py @@ -1,5 +1,5 @@ import re -from typing import Iterator, Union, TextIO, Optional, List, Set +from typing import Iterator, List, Optional, Set, TextIO, Union from irrd.conf import PASSWORD_HASH_DUMMY_VALUE from irrd.rpsl.passwords import PASSWORD_HASHERS_ALL diff --git a/irrd/utils/validators.py b/irrd/utils/validators.py index af693cd32..032001083 100644 --- a/irrd/utils/validators.py +++ b/irrd/utils/validators.py @@ -1,6 +1,7 @@ -from typing import Tuple, List, Union, Optional +from typing import List, Optional, Tuple, Union import pydantic + from irrd.updates.parser_state import SuspensionRequestType diff --git a/irrd/utils/whois_client.py b/irrd/utils/whois_client.py index e37c8b4fd..517ca8fc3 100644 --- a/irrd/utils/whois_client.py +++ b/irrd/utils/whois_client.py @@ -1,6 +1,6 @@ import logging import socket -from typing import List, Tuple, Optional +from typing import List, Optional, Tuple logger = logging.getLogger(__name__) diff --git a/irrd/vendor/dotted/collection.py b/irrd/vendor/dotted/collection.py index 6a8bf1032..2eed832bc 100644 --- a/irrd/vendor/dotted/collection.py +++ b/irrd/vendor/dotted/collection.py @@ -27,10 +27,8 @@ import collections import json import re - from abc import ABCMeta, abstractmethod - SPLIT_REGEX = r"(?