Skip to content

Commit

Permalink
Ref #617 - Add change log (#807)
Browse files Browse the repository at this point in the history
  • Loading branch information
mxsasha authored Jun 29, 2023
1 parent abc74a1 commit 800ce07
Show file tree
Hide file tree
Showing 29 changed files with 1,231 additions and 214 deletions.
2 changes: 1 addition & 1 deletion docs/users/database-changes.rst
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ Here is an example of a JSON response::

{
"request_meta": {
"HTTP-client-IP": "127.0.0.1",
"HTTP-Client-IP": "127.0.0.1",
"HTTP-User-Agent": "user-agent"
},
"summary": {
Expand Down
1 change: 1 addition & 0 deletions irrd/__init__.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
__version__ = "4.4-dev"
ENV_MAIN_PROCESS_PID = "IRRD_MAIN_PROCESS_PID"
META_KEY_HTTP_CLIENT_IP = "HTTP-Client-IP"
2 changes: 1 addition & 1 deletion irrd/rpki/notifications.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def notify_rpki_invalid_owners(
if not get_setting("rpki.notify_invalid_enabled"):
return 0

rpsl_objs = []
rpsl_objs: List[RPSLObject] = []
for obj in rpsl_dicts_now_invalid:
source = obj["source"]
authoritative = get_setting(f"sources.{source}.authoritative")
Expand Down
13 changes: 8 additions & 5 deletions irrd/rpsl/auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,25 +44,28 @@ def get_password_hashers(permit_legacy=True):

def verify_auth_lines(
auth_lines: List[str], passwords: List[str], keycert_obj_pk: Optional[str] = None
) -> bool:
) -> Optional[str]:
"""
Verify whether one of a given list of passwords matches
any of the auth lines in the provided list, or match the
keycert object PK.
Returns None for auth failed, a scheme or PGP key PK
for success.
"""
hashers = get_password_hashers(permit_legacy=True)
for auth in auth_lines:
if keycert_obj_pk and auth.upper() == keycert_obj_pk.upper():
return True
return keycert_obj_pk.upper()
if " " not in auth:
continue
scheme, hash = auth.split(" ", 1)
hasher = hashers.get(scheme.upper())
scheme = scheme.upper()
hasher = hashers.get(scheme)
if hasher:
for password in passwords:
try:
if hasher.verify(password, hash):
return True
return scheme
except ValueError:
pass
return False
return None
2 changes: 1 addition & 1 deletion irrd/rpsl/rpsl_objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -435,7 +435,7 @@ def clean(self):
"Either all password auth hashes in a submitted mntner must be dummy objects, or none."
)

def verify_auth(self, passwords: List[str], keycert_obj_pk: Optional[str] = None) -> bool:
def verify_auth(self, passwords: List[str], keycert_obj_pk: Optional[str] = None) -> Optional[str]:
return verify_auth_lines(self.parsed_data["auth"], passwords, keycert_obj_pk)

def has_dummy_auth_value(self) -> bool:
Expand Down
14 changes: 7 additions & 7 deletions irrd/rpsl/tests/test_rpsl_objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -405,13 +405,13 @@ def test_verify(self, tmp_gpg_dir):
rpsl_text + "auth: UNKNOWN_HASH foo\nauth: MD5-PW 💩", strict_validation=False
)

assert obj.verify_auth(["crypt-password"])
assert obj.verify_auth(["md5-password"])
assert obj.verify_auth(["bcrypt-password"])
assert obj.verify_auth(["md5-password"], "PGPKey-80F238C6")
assert not obj.verify_auth(["other-password"])
assert not obj.verify_auth([KEY_CERT_SIGNED_MESSAGE_CORRUPT])
assert not obj.verify_auth([KEY_CERT_SIGNED_MESSAGE_WRONG_KEY])
assert obj.verify_auth(["crypt-password"]) == "CRYPT-PW"
assert obj.verify_auth(["md5-password"]) == "MD5-PW"
assert obj.verify_auth(["bcrypt-password"]) == "BCRYPT-PW"
assert obj.verify_auth(["md5-password"], "PGPKey-80F238C6") == "PGPKEY-80F238C6"
assert obj.verify_auth(["other-password"]) is None
assert obj.verify_auth([KEY_CERT_SIGNED_MESSAGE_CORRUPT]) is None
assert obj.verify_auth([KEY_CERT_SIGNED_MESSAGE_WRONG_KEY]) is None


class TestRPSLPeeringSet:
Expand Down
5 changes: 3 additions & 2 deletions irrd/server/http/endpoints_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from irrd.updates.handler import ChangeSubmissionHandler
from irrd.utils.validators import RPSLChangeSubmission, RPSLSuspensionSubmission

from ... import META_KEY_HTTP_CLIENT_IP
from ...storage.models import AuthoritativeChangeOrigin
from ..whois.query_parser import WhoisQueryParser
from ..whois.query_response import WhoisQueryResponseType
Expand Down Expand Up @@ -84,7 +85,7 @@ async def _handle_submission(self, request: Request, delete=False):
except (JSONDecodeError, KeyError):
request_meta = {}

request_meta["HTTP-client-IP"] = request.client.host
request_meta[META_KEY_HTTP_CLIENT_IP] = request.client.host
request_meta["HTTP-User-Agent"] = request.headers.get("User-Agent")
try:
remote_ip = IP(request.client.host)
Expand Down Expand Up @@ -113,7 +114,7 @@ async def post(self, request: Request) -> Response:
return PlainTextResponse(str(error), status_code=400)

request_meta = {
"HTTP-client-IP": request.client.host,
META_KEY_HTTP_CLIENT_IP: request.client.host,
"HTTP-User-Agent": request.headers.get("User-Agent"),
}
handler = ChangeSubmissionHandler()
Expand Down
7 changes: 4 additions & 3 deletions irrd/server/http/tests/test_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from starlette.requests import HTTPConnection
from starlette.testclient import TestClient

from irrd import META_KEY_HTTP_CLIENT_IP
from irrd.storage.database_handler import DatabaseHandler
from irrd.storage.models import AuthoritativeChangeOrigin
from irrd.storage.preload import Preloader
Expand Down Expand Up @@ -186,7 +187,7 @@ def test_endpoint(self, monkeypatch):
data=expected_data,
origin=AuthoritativeChangeOrigin.webapi,
delete=False,
request_meta={"HTTP-client-IP": "testclient", "HTTP-User-Agent": "testclient", "meta": 2},
request_meta={META_KEY_HTTP_CLIENT_IP: "testclient", "HTTP-User-Agent": "testclient", "meta": 2},
remote_ip=None,
)
mock_handler.send_notification_target_reports.assert_called_once()
Expand All @@ -199,7 +200,7 @@ def test_endpoint(self, monkeypatch):
data=expected_data,
origin=AuthoritativeChangeOrigin.webapi,
delete=True,
request_meta={"HTTP-client-IP": "testclient", "HTTP-User-Agent": "testclient"},
request_meta={META_KEY_HTTP_CLIENT_IP: "testclient", "HTTP-User-Agent": "testclient"},
remote_ip=None,
)
mock_handler.send_notification_target_reports.assert_called_once()
Expand Down Expand Up @@ -236,7 +237,7 @@ def test_endpoint(self, monkeypatch):
assert response_post.text == '{"response":true}'
mock_handler.load_suspension_submission.assert_called_once_with(
data=expected_data,
request_meta={"HTTP-client-IP": "testclient", "HTTP-User-Agent": "testclient"},
request_meta={META_KEY_HTTP_CLIENT_IP: "testclient", "HTTP-User-Agent": "testclient"},
)
mock_handler.reset_mock()

Expand Down
78 changes: 78 additions & 0 deletions irrd/storage/alembic/versions/5d942647566e_add_changelog.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
"""add_changelog
Revision ID: 5d942647566e
Revises: 500027f85a55
Create Date: 2023-06-27 23:04:08.424619
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = "5d942647566e"
down_revision = "500027f85a55"
branch_labels = None
depends_on = None


def upgrade():
updaterequesttype = postgresql.ENUM(
"CREATE", "MODIFY", "DELETE", name="updaterequesttype", create_type=False
)
updaterequesttype.create(op.get_bind(), checkfirst=True)

op.create_table(
"change_log",
sa.Column(
"pk", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False
),
sa.Column("auth_by_user_id", postgresql.UUID(), nullable=True),
sa.Column("auth_by_user_email", sa.String(), nullable=True),
sa.Column("auth_by_api_key_id", postgresql.UUID(), nullable=True),
sa.Column("auth_by_api_key_id_fixed", postgresql.UUID(), nullable=True),
sa.Column("auth_through_mntner_id", postgresql.UUID(), nullable=True),
sa.Column("auth_through_rpsl_mntner_pk", sa.String(), nullable=True),
sa.Column("auth_by_rpsl_mntner_password", sa.Boolean(), nullable=False),
sa.Column("auth_by_rpsl_mntner_pgp_key", sa.Boolean(), nullable=False),
sa.Column("auth_by_override", sa.Boolean(), nullable=True),
sa.Column("from_email", sa.String(), nullable=True),
sa.Column("from_ip", postgresql.INET(), nullable=True),
sa.Column("auth_change_descr", sa.String(), nullable=True),
sa.Column("auth_affected_user_id", postgresql.UUID(), nullable=True),
sa.Column("auth_affected_mntner_id", postgresql.UUID(), nullable=True),
sa.Column("rpsl_target_request_type", updaterequesttype, nullable=True),
sa.Column("rpsl_target_pk", sa.String(), nullable=True),
sa.Column("rpsl_target_source", sa.String(), nullable=True),
sa.Column("rpsl_target_object_class", sa.String(), nullable=True),
sa.Column("rpsl_target_object_text_old", sa.Text(), nullable=True),
sa.Column("rpsl_target_object_text_new", sa.Text(), nullable=True),
sa.Column("timestamp", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(["auth_affected_mntner_id"], ["auth_mntner.pk"], ondelete="SET NULL"),
sa.ForeignKeyConstraint(["auth_affected_user_id"], ["auth_user.pk"], ondelete="SET NULL"),
sa.ForeignKeyConstraint(["auth_by_api_key_id"], ["auth_api_token.pk"], ondelete="SET NULL"),
sa.ForeignKeyConstraint(["auth_by_user_id"], ["auth_user.pk"], ondelete="SET NULL"),
sa.ForeignKeyConstraint(["auth_through_mntner_id"], ["auth_mntner.pk"], ondelete="SET NULL"),
sa.PrimaryKeyConstraint("pk"),
)
op.create_index(
op.f("ix_change_log_auth_affected_mntner_id"), "change_log", ["auth_affected_mntner_id"], unique=False
)
op.create_index(
op.f("ix_change_log_auth_through_mntner_id"), "change_log", ["auth_through_mntner_id"], unique=False
)
op.create_index(
op.f("ix_change_log_auth_through_rpsl_mntner_pk"),
"change_log",
["auth_through_rpsl_mntner_pk"],
unique=False,
)
op.create_index(op.f("ix_change_log_rpsl_target_pk"), "change_log", ["rpsl_target_pk"], unique=False)


def downgrade():
op.drop_index(op.f("ix_change_log_rpsl_target_pk"), table_name="change_log")
op.drop_index(op.f("ix_change_log_auth_through_rpsl_mntner_pk"), table_name="change_log")
op.drop_index(op.f("ix_change_log_auth_through_mntner_id"), table_name="change_log")
op.drop_index(op.f("ix_change_log_auth_affected_mntner_id"), table_name="change_log")
op.drop_table("change_log")
70 changes: 70 additions & 0 deletions irrd/storage/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from irrd.rpki.status import RPKIStatus
from irrd.rpsl.rpsl_objects import lookup_field_names
from irrd.scopefilter.status import ScopeFilterStatus
from irrd.updates.parser_state import UpdateRequestType


class DatabaseOperation(enum.Enum):
Expand Down Expand Up @@ -478,6 +479,75 @@ def __repr__(self):
return f"AuthMntner<{self.pk}, {self.rpsl_mntner_pk}>"


class ChangeLog(Base): # type: ignore
__tablename__ = "change_log"

pk = sa.Column(pg.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), primary_key=True)
auth_by_user_id = sa.Column(pg.UUID, sa.ForeignKey("auth_user.pk", ondelete="SET NULL"), nullable=True)
auth_by_user_email = sa.Column(sa.String, nullable=True)
auth_by_api_key_id = sa.Column(
pg.UUID, sa.ForeignKey("auth_api_token.pk", ondelete="SET NULL"), nullable=True
)
auth_by_api_key = relationship(
"AuthApiToken",
foreign_keys="ChangeLog.auth_by_api_key_id",
)
auth_by_api_key_id_fixed = sa.Column(pg.UUID, nullable=True)
auth_through_mntner_id = sa.Column(
pg.UUID, sa.ForeignKey("auth_mntner.pk", ondelete="SET NULL"), index=True, nullable=True
)
auth_through_mntner = relationship(
"AuthMntner",
foreign_keys="ChangeLog.auth_through_mntner_id",
)
auth_through_rpsl_mntner_pk = sa.Column(sa.String, index=True, nullable=True)
auth_by_rpsl_mntner_password = sa.Column(sa.Boolean, nullable=False, default=False)
auth_by_rpsl_mntner_pgp_key = sa.Column(sa.Boolean, nullable=False, default=False)
auth_by_override = sa.Column(sa.Boolean, default=False)

from_email = sa.Column(sa.String, nullable=True)
from_ip = sa.Column(pg.INET, nullable=True)

auth_change_descr = sa.Column(sa.String, nullable=True)
auth_affected_user_id = sa.Column(
pg.UUID, sa.ForeignKey("auth_user.pk", ondelete="SET NULL"), nullable=True
)
auth_affected_mntner_id = sa.Column(
pg.UUID, sa.ForeignKey("auth_mntner.pk", ondelete="SET NULL"), index=True, nullable=True
)
auth_affected_mntner = relationship(
"AuthMntner",
foreign_keys="ChangeLog.auth_affected_mntner_id",
)
auth_affected_user = relationship(
"AuthUser",
foreign_keys="ChangeLog.auth_affected_user_id",
)

rpsl_target_request_type = sa.Column(sa.Enum(UpdateRequestType), nullable=True)
rpsl_target_pk = sa.Column(sa.String, index=True, nullable=True)
rpsl_target_source = sa.Column(sa.String, nullable=True)
rpsl_target_object_class = sa.Column(sa.String, nullable=True)
rpsl_target_object_text_old = sa.Column(sa.Text, nullable=True)
rpsl_target_object_text_new = sa.Column(sa.Text, nullable=True)

timestamp = sa.Column(sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False)

def __repr__(self):
return f"<{self.pk}/{self.description()}>"

def description(self) -> str:
if self.rpsl_target_pk:
return (
f"{self.rpsl_target_request_type.value} of"
f" {self.rpsl_target_object_class} {self.rpsl_target_pk} in {self.rpsl_target_source}"
)
elif self.auth_change_descr:
return self.auth_change_descr
else: # pragma: no cover
return "<unknown>"


# Before you update this, please check the storage documentation for changing lookup fields.
expected_lookup_field_names = {
"admin-c",
Expand Down
13 changes: 11 additions & 2 deletions irrd/updates/handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,11 @@ def load_text_blob(
self.database_handler, origin, self._pgp_key_id, internal_authenticated_user
)
change_requests = parse_change_requests(
object_texts_blob, self.database_handler, auth_validator, reference_validator
object_texts_blob,
self.database_handler,
auth_validator,
reference_validator,
self.request_meta,
)

self._handle_change_requests(change_requests, reference_validator, auth_validator)
Expand Down Expand Up @@ -90,7 +94,12 @@ def load_change_submission(
assert object_text # enforced by pydantic
change_requests.append(
ChangeRequest(
object_text, self.database_handler, auth_validator, reference_validator, delete_reason
object_text,
self.database_handler,
auth_validator,
reference_validator,
delete_reason,
self.request_meta,
)
)

Expand Down
Loading

0 comments on commit 800ce07

Please sign in to comment.