Ref #617 - Add new web UI and IRRD-INTERNAL-AUTH (#671)

.circleci/config.yml

@@ -108,10 +108,10 @@ commands:
 
       - restore_cache:
           keys:
-            - v2-dependencies-{{ .Environment.CIRCLE_JOB }}-{{ checksum
+            - v3-dependencies-{{ .Environment.CIRCLE_JOB }}-{{ checksum
               "poetry.lock" }}
             # fallback to using the latest cache if no exact match is found
-            - v2-dependencies-{{ .Environment.CIRCLE_JOB }}
+            - v3-dependencies-{{ .Environment.CIRCLE_JOB }}
 
       - run:
           name: install python dependencies
@@ -123,7 +123,7 @@ commands:
       - save_cache:
           paths:
             - /mnt/ramdisk/.venv
-          key: v2-dependencies-{{ .Environment.CIRCLE_JOB }}-{{ checksum
+          key: v3-dependencies-{{ .Environment.CIRCLE_JOB }}-{{ checksum
             "poetry.lock" }}
 
   wait_for_postgres:

conftest.py

@@ -6,9 +6,17 @@
 import pytest
 from typing import Dict, Any
 
+import redis
+from sqlalchemy.exc import ProgrammingError
+
 from irrd import conf
 from irrd.rpsl.rpsl_objects import rpsl_object_from_text
-from irrd.utils.rpsl_samples import SAMPLE_KEY_CERT
+from irrd.storage import get_engine
+from irrd.storage.database_handler import DatabaseHandler
+from irrd.storage.models import RPSLDatabaseObject, JournalEntryOrigin
+from irrd.storage.orm_provider import ORMSessionProvider
+from irrd.utils.factories import set_factory_session, AuthUserFactory
+from irrd.utils.rpsl_samples import SAMPLE_KEY_CERT, SAMPLE_MNTNER, SAMPLE_PERSON, SAMPLE_ROLE
 from irrd.vendor.dotted.collection import DottedDict
 
 
@@ -52,7 +60,7 @@ def preload_gpg_key():
 
 def pytest_configure(config):
     """
-    This function is called by py.test, and will set a flag to
+    This function is called by py.test, and will set flags to
     indicate tests are running. This is used by the configuration
     checker to not require a full working config for most tests.
     Can be checked with:
@@ -61,6 +69,7 @@ def pytest_configure(config):
     """
     import sys
     sys._called_from_test = True
+    os.environ["TESTING"] = "true"
 
 
 @pytest.fixture(autouse=True)
@@ -71,3 +80,74 @@ def reset_config():
     """
     conf.config_init(None)
     conf.testing_overrides = None
+
+
+@pytest.fixture(scope="package")
+def irrd_database_create_destroy():
+    """
+    Some tests use a live PostgreSQL database, as it's rather complicated
+    to mock, and mocking would not make them less useful.
+    Using in-memory SQLite is not an option due to using specific
+    PostgreSQL features.
+
+    To improve performance, these tests do not run full migrations, but
+    the database is only created once per session, and truncated between tests.
+    """
+    if not conf.is_config_initialised():
+        conf.config_init(None)
+        conf.testing_overrides = None
+
+    engine = get_engine()
+    try:
+        engine.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto")
+    except ProgrammingError as pe:  # pragma: no cover
+        print(f"WARNING: unable to create extension pgcrypto on the database. Queries may fail: {pe}")
+
+    table_name = RPSLDatabaseObject.__tablename__
+    if engine.has_table(engine, table_name):  # pragma: no cover
+        if engine.url.database not in ["irrd_test", "circle_test"]:
+            print(
+                f"The database on URL {engine.url} already has a table named {table_name} - "
+                "delete existing database and all data in it?"
+            )
+            confirm = input(f"Type '{engine.url.database}' to confirm deletion\n> ")
+            if confirm != engine.url.database:
+                pytest.exit("Not overwriting database, terminating test run")
+        RPSLDatabaseObject.metadata.drop_all(engine)
+
+    RPSLDatabaseObject.metadata.create_all(engine)
+
+    yield engine
+
+    engine.dispose()
+    RPSLDatabaseObject.metadata.drop_all(engine)
+
+
+@pytest.fixture(scope="function")
+def irrd_db(irrd_database_create_destroy):
+    engine = irrd_database_create_destroy
+    dh = DatabaseHandler()
+    for table in engine.table_names():
+        dh.execute_statement(f"TRUNCATE {table} CASCADE")
+    dh.commit()
+    dh.close()
+
+
+@pytest.fixture(scope="function")
+def irrd_db_session_with_user(irrd_db):
+    dh = DatabaseHandler()
+    dh.upsert_rpsl_object(rpsl_object_from_text(SAMPLE_MNTNER), origin=JournalEntryOrigin.unknown)
+    dh.upsert_rpsl_object(rpsl_object_from_text(SAMPLE_PERSON), origin=JournalEntryOrigin.unknown)
+    dh.upsert_rpsl_object(rpsl_object_from_text(SAMPLE_ROLE), origin=JournalEntryOrigin.unknown)
+    dh.commit()
+    dh.close()
+
+    provider = ORMSessionProvider()
+    set_factory_session(provider.session)
+    user = AuthUserFactory()
+    provider.session.commit()
+    provider.session.connection().execute("COMMIT")
+
+    yield provider, user
+
+    provider.commit_close()

docs/admins/configuration.rst

@@ -19,6 +19,8 @@ a key ``roa_source`` under a key ``rpki``.
    :local:
    :depth: 2
 
+
+
 Example configuration file
 --------------------------
 
@@ -41,6 +43,8 @@ This sample shows most configuration options
         piddir: /var/run/
         user: irrd
         group: irrd
+        # required, but no default included for safety
+        secret_key: null
 
         access_lists:
             http_database_status:
@@ -55,6 +59,7 @@ This sample shows most configuration options
                 status_access_list: http_database_status
                 interface: '::0'
                 port: 8080
+                url: "https://irrd.example.com/"
             whois:
                 interface: '::0'
                 max_connections: 50
@@ -63,6 +68,7 @@ This sample shows most configuration options
         auth:
             gnupg_keyring: /home/irrd/gnupg-keyring/
             override_password: {hash}
+            webui_auth_failure_rate_limit: "30/hour"
             password_hashers:
                 md5-pw: legacy
 
@@ -220,6 +226,13 @@ General settings
   need for IRRd to bind to port 80 or 443.
   |br| **Default**: not defined, IRRd does not drop privileges.
   |br| **Change takes effect**: after full IRRd restart.
+* ``secret_key``: a random secret string. **The secrecy of this key protects
+  all web authentication.** If rotated, all sessions and password resets
+  are invalidated, requiring users to log in or request new password
+  reset links. Second factor authentication is *not* attached to this key.
+  Minimum 30 characters.
+  |br| **Default**: not defined, but required.
+  |br| **Change takes effect**: after full IRRd restart.
 
 
 Servers
@@ -249,6 +262,13 @@ Servers
   stream. If not defined, all access is denied.
   |br| **Default**: not defined, all access denied for event stream.
   |br| **Change takes effect**: after SIGHUP.
+* ``server.http.url``: the external URL on which users will reach the
+  IRRD instance. This is used for WebAuthn security tokens.
+  **Changing this URL after users have configured security tokens
+  will invalidate all tokens ** - an intentional anti-phishing
+  design feature of WebAuthn. The scheme must be included.
+  |br| **Default**: not defined, but required.
+  |br| **Change takes effect**: after SIGHUP.
 * ``server.whois.max_connections``: the maximum number of simultaneous whois
   connections permitted. Note that each permitted connection will result in
   one IRRd whois worker to be started, each of which use about 200 MB memory.
@@ -319,6 +339,19 @@ Authentication and validation
 * ``auth.gnupg_keyring``: the full path to the gnupg keyring.
   |br| **Default**: not defined, but required.
   |br| **Change takes effect**: after full IRRd restart.
+* ``auth.irrd_internal_migration_enabled``: whether users can initiate a migration
+  their mntners to IRRD-INTERNAL-AUTH authentication through the web interface.
+  If this is disabled after some mntners have already been migrated, those
+  will remain available and usable - this setting only affects new migrations.
+  |br| **Default**: disabled.
+  |br| **Change takes effect**: after SIGHUP, for all subsequent attempts to
+  initiate a migration.
+* ``auth.webui_auth_failure_rate_limit``: the rate limit for failed
+  authentication attempts through the web interface. This includes logins,
+  but also other cases that request passwords. This is a moving window
+  in the format of the limits_ library.
+  |br| **Default**: ``30/hour``.
+  |br| **Change takes effect**: after full IRRd restart.
 * ``auth.password_hashers``: which password hashers to allow in mntner objects.
   This is a dictionary with the hashers (``crypt-pw``, ``md5-pw``, ``bcrypt-pw``) as
   possible keys, and ``enabled``, ``legacy``, or ``disabled`` as possible values.
@@ -346,6 +379,8 @@ Authentication and validation
     However, you can use the ``irrd_load_pgp_keys`` command to refill the keyring
     in ``auth.gnupg_keyring``.
 
+.. _limits: https://limits.readthedocs.io/en/latest/index.html
+
 .. _conf-auth-set-creation:
 
 auth.set_creation

docs/admins/suspension.rst

@@ -155,3 +155,4 @@ You can use a shortened `mntner` syntax, like so::
     suspension: suspend
     mntner:     EXAMPLE-MNT
     source:     EXAMPLE
+

docs/admins/webui.rst

@@ -0,0 +1,89 @@
+Web interface and internal authentication
+=========================================
+
+Along with HTTP based API calls, GraphQL and the status page, IRRD contains
+a web interface that allows users to migrate their authoritative maintainers
+to an IRRD internal authentication method. It also offers more secure
+override access.
+
+The web interface contains a RPSL submission form, accepting
+the same format as emails, to make object changes. This form accepts
+the new internal authentication as well as passwords, and is meant
+as a more practical and more secure alternative.
+
+The submission form and internal authentication only affect
+objects in authoritative sources.
+
+IRRD internal authentication
+----------------------------
+
+Traditional maintainer objects authenticate with a list of passwords
+and/or PGP keys in the maintainer object. In IRRD internal authentication,
+the permissions are kept in a separate storage, i.e. not in RPSL
+objects. The major features of internal over traditional are:
+
+* Users can choose whether to give other users access to change
+  permissions on the maintainer, or only modify other objects.
+  In traditional authentication, anyone with maintainer access can
+  essentially take over the maintainer.
+* Users can create API keys with limited permissions, rather than include
+  a password in emails.
+* Users can submit object updates after logging in, without needing
+  to pass further authentication.
+* Internal authentication can be combined with traditional, but
+  this is not recommended.
+* Logins on the web interface can be protected with two-factor
+  authentication.
+* Hashes of (new) user passwords are no longer part of RPSL objects.
+* User passwords can not be used directly for authentication of
+  e.g. email updates.
+
+You can allow migrations with the
+``auth.irrd_internal_migration_enabled`` setting.
+By default, this is disabled.
+
+Override access
+---------------
+Independent of whether regular users can migrate their account
+(``auth.irrd_internal_migration_enabled``), you can
+use the web interface to provide override access.
+Rather than sharing a single password with your staff with traditional
+override access, you can use this feature to restrict override access
+to HTTPS and two-factor authenticated users.
+
+To enable override access for a user, the user must first create
+an account and set up two-factor authentication.
+Then, use the ``irrdctl user-change-override`` command
+to enable or disable access for the user.
+
+User registration
+-----------------
+Users can register their own account through the interface, after verifying
+their e-mail address. Users can also independently change their details or
+request a link to reset. Two-factor authentication is
+supported with WebAuthn tokens (SoloKeys, YubiKey, PassKey, etc.) or
+one time password (TOTP, through Google Authenticator, Authy, etc.)
+
+Significant changes and authentication failures are logged in IRRD's log file,
+and a notification is mailed to the user.
+Important endpoints (e.g. login attempts) have rate limiting.
+
+If a user loses access to all their two-factor authentication methods,
+an IRRD operator needs to reset this for them. You can do this with
+the ``irrdctl user-mfa-clear`` command.
+
+Maintainer migration
+--------------------
+Migrating a maintainer can be done by any registered user, and involves
+the following steps:
+
+* The user requests migration with the maintainer name and one of the
+  current valid passwords on the maintainer.
+* IRRD will mail all admin-c contacts on the maintainer with a
+  confirmation link.
+* The same user must open the confirmation link, and confirm again with
+  a current valid password.
+* The maintainer is now migrated. Existing methods are kept.
+
+A migrated maintainer must have ``IRRD-INTERNAL-AUTH`` as one of
+the ``auth`` methods. This is added as part of the migration process.

docs/conf.py

@@ -80,7 +80,7 @@
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
-# If true, `todo` and `todoList` produce output, else they produce nothing.
+# If true, `to^do` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
 

docs/index.rst

@@ -34,11 +34,6 @@ older versions lead to the IRRd v4 project.
 .. _Internetstiftelsen: https://internetstiftelsen.se/
 .. _Older versions: https://github.com/irrdnet/irrd-legacy
 
-.. warning::
-    IRRd 4.2.x versions prior to 4.2.3 had a security issue that exposed password
-    hashes in some cases. All 4.2.x users are urged to
-    update to 4.2.3 or later.
-    See the :doc:`4.2.3 release notes </releases/4.2.3>` for further details.
 
 For administrators
 ------------------
@@ -54,6 +49,7 @@ This documentation is mainly for administrators of IRRd deployments.
    admins/availability-and-migration
    admins/migrating-legacy-irrd
    admins/status_page
+   admins/webui
    admins/faq