Migrate discovery services to integration config (PP-94) (ThePalacePr…

…oject#1321)

Convert the library registry to use IntegrationConfiguration rather then ExternalIntegration. The library registration data wasn't a great fit for LibraryIntegrationConfiguration, so that data is stored in its own table discovery_service_registrations.

As part of the conversion, I moved the files for the registry into a discovery module and fully type hinted them, so it was a bit easier to sort out what was happening. As part of the type hinting process, I added hints to core/util/problem_detail.py, which necessitated some changes in files unrelated directly to this PR.

See JIRA ticket: PP-94.
This is part of the larger CM Config Settings epic: PP-4.

alembic/versions/20230810_0df58829fc1a_add_discovery_service_tables.py

@@ -0,0 +1,151 @@
+"""Add discovery service tables
+
+Revision ID: 0df58829fc1a
+Revises: 2f1a51aa0ee8
+Create Date: 2023-08-10 15:49:36.784169+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+from api.discovery.opds_registration import OpdsRegistrationService
+from core.migration.migrate_external_integration import (
+    _migrate_external_integration,
+    get_configuration_settings,
+    get_integrations,
+    get_library_for_integration,
+)
+from core.migration.util import drop_enum, pg_update_enum
+
+# revision identifiers, used by Alembic.
+revision = "0df58829fc1a"
+down_revision = "2f1a51aa0ee8"
+branch_labels = None
+depends_on = None
+
+old_goals_enum = [
+    "PATRON_AUTH_GOAL",
+    "LICENSE_GOAL",
+]
+
+new_goals_enum = old_goals_enum + ["DISCOVERY_GOAL"]
+
+
+def upgrade() -> None:
+    op.create_table(
+        "discovery_service_registrations",
+        sa.Column(
+            "status",
+            sa.Enum("SUCCESS", "FAILURE", name="registrationstatus"),
+            nullable=False,
+        ),
+        sa.Column(
+            "stage",
+            sa.Enum("TESTING", "PRODUCTION", name="registrationstage"),
+            nullable=False,
+        ),
+        sa.Column("web_client", sa.Unicode(), nullable=True),
+        sa.Column("short_name", sa.Unicode(), nullable=True),
+        sa.Column("shared_secret", sa.Unicode(), nullable=True),
+        sa.Column("integration_id", sa.Integer(), nullable=False),
+        sa.Column("library_id", sa.Integer(), nullable=False),
+        sa.Column("vendor_id", sa.Unicode(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["integration_id"], ["integration_configurations.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(["library_id"], ["libraries.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("integration_id", "library_id"),
+    )
+    pg_update_enum(
+        op,
+        "integration_configurations",
+        "goal",
+        "goals",
+        old_goals_enum,
+        new_goals_enum,
+    )
+
+    # Migrate data
+    connection = op.get_bind()
+    external_integrations = get_integrations(connection, "discovery")
+    for external_integration in external_integrations:
+        # This should always be the case, but we want to make sure
+        assert external_integration.protocol == "OPDS Registration"
+
+        # Create the settings and library settings dicts from the configurationsettings
+        settings_dict, library_settings, self_test_result = get_configuration_settings(
+            connection, external_integration
+        )
+
+        # Write the configurationsettings into the integration_configurations table
+        integration_configuration_id = _migrate_external_integration(
+            connection,
+            external_integration,
+            OpdsRegistrationService,
+            "DISCOVERY_GOAL",
+            settings_dict,
+            self_test_result,
+        )
+
+        # Get the libraries that are associated with this external integration
+        interation_libraries = get_library_for_integration(
+            connection, external_integration.id
+        )
+
+        vendor_id = settings_dict.get("vendor_id")
+
+        # Write the library settings into the discovery_service_registrations table
+        for library in interation_libraries:
+            library_id = library.library_id
+            library_settings_dict = library_settings[library_id]
+
+            status = library_settings_dict.get("library-registration-status")
+            if status is None:
+                status = "FAILURE"
+            else:
+                status = status.upper()
+
+            stage = library_settings_dict.get("library-registration-stage")
+            if stage is None:
+                stage = "TESTING"
+            else:
+                stage = stage.upper()
+
+            web_client = library_settings_dict.get("library-registration-web-client")
+            short_name = library_settings_dict.get("username")
+            shared_secret = library_settings_dict.get("password")
+
+            connection.execute(
+                "insert into discovery_service_registrations "
+                "(status, stage, web_client, short_name, shared_secret, integration_id, library_id, vendor_id) "
+                "values (%s, %s, %s, %s, %s, %s, %s, %s)",
+                (
+                    status,
+                    stage,
+                    web_client,
+                    short_name,
+                    shared_secret,
+                    integration_configuration_id,
+                    library_id,
+                    vendor_id,
+                ),
+            )
+
+
+def downgrade() -> None:
+    connection = op.get_bind()
+    connection.execute(
+        "DELETE from integration_configurations where goal = %s", "DISCOVERY_GOAL"
+    )
+
+    op.drop_table("discovery_service_registrations")
+    drop_enum(op, "registrationstatus")
+    drop_enum(op, "registrationstage")
+    pg_update_enum(
+        op,
+        "integration_configurations",
+        "goal",
+        "goals",
+        new_goals_enum,
+        old_goals_enum,
+    )

api/admin/controller/discovery_service_library_registrations.py

@@ -1,69 +1,84 @@
+from __future__ import annotations
+
 import json
+from typing import Any, Dict
 
 import flask
 from flask import Response, url_for
 from flask_babel import lazy_gettext as _
+from sqlalchemy import select
+from sqlalchemy.orm import Session
 
-from api.admin.controller.settings import SettingsController
+from api.admin.controller.base import AdminPermissionsControllerMixin
 from api.admin.problem_details import MISSING_SERVICE, NO_SUCH_LIBRARY
-from api.registration.registry import Registration, RemoteRegistry
-from core.model import ExternalIntegration, Library, get_one
-from core.util.http import HTTP
-from core.util.problem_detail import ProblemDetail
+from api.controller import CirculationManager
+from api.discovery.opds_registration import OpdsRegistrationService
+from api.integration.registry.discovery import DiscoveryRegistry
+from core.integration.goals import Goals
+from core.model import IntegrationConfiguration, Library, get_one
+from core.model.discovery_service_registration import (
+    DiscoveryServiceRegistration,
+    RegistrationStage,
+)
+from core.problem_details import INVALID_INPUT
+from core.util.problem_detail import ProblemDetail, ProblemError
 
 
-class DiscoveryServiceLibraryRegistrationsController(SettingsController):
+class DiscoveryServiceLibraryRegistrationsController(AdminPermissionsControllerMixin):
 
     """List the libraries that have been registered with a specific
-    RemoteRegistry, and allow the admin to register a library with
-    a RemoteRegistry.
-
-    :param registration_class: Mock class to use instead of Registration.
+    OpdsRegistrationService, and allow the admin to register a library with
+    a OpdsRegistrationService.
     """
 
-    def __init__(self, manager):
-        super().__init__(manager)
-        self.goal = ExternalIntegration.DISCOVERY_GOAL
+    def __init__(self, manager: CirculationManager):
+        self._db: Session = manager._db
+        self.goal = Goals.DISCOVERY_GOAL
+        self.registry = DiscoveryRegistry()
 
     def process_discovery_service_library_registrations(
         self,
-        registration_class=None,
-        do_get=HTTP.debuggable_get,
-        do_post=HTTP.debuggable_post,
-    ):
-        registration_class = registration_class or Registration
+    ) -> Response | Dict[str, Any] | ProblemDetail:
         self.require_system_admin()
-        if flask.request.method == "GET":
-            return self.process_get(do_get)
-        else:
-            return self.process_post(registration_class, do_get, do_post)
+        try:
+            if flask.request.method == "GET":
+                return self.process_get()
+            else:
+                return self.process_post()
+        except ProblemError as e:
+            self._db.rollback()
+            return e.problem_detail
 
-    def process_get(self, do_get=HTTP.debuggable_get):
+    def process_get(self) -> Dict[str, Any]:
         """Make a list of all discovery services, each with the
         list of libraries registered with that service and the
         status of the registration."""
 
         services = []
-        for registry in RemoteRegistry.for_protocol_and_goal(
-            self._db, ExternalIntegration.OPDS_REGISTRATION, self.goal
-        ):
-            result = registry.fetch_registration_document(do_get=do_get)
-            if isinstance(result, ProblemDetail):
-                # Unlike most cases like this, a ProblemDetail doesn't
+        integration_query = select(IntegrationConfiguration).where(
+            IntegrationConfiguration.goal == self.goal,
+            IntegrationConfiguration.protocol
+            == self.registry.get_protocol(OpdsRegistrationService),
+        )
+        integrations = self._db.scalars(integration_query).all()
+        for integration in integrations:
+            registry = OpdsRegistrationService.for_integration(self._db, integration)
+            try:
+                access_problem = None
+                (
+                    terms_of_service_link,
+                    terms_of_service_html,
+                ) = registry.fetch_registration_document()
+            except ProblemError as e:
+                # Unlike most cases like this, a ProblemError doesn't
                 # mean the whole request is ruined -- just that one of
                 # the discovery services isn't working. Turn the
                 # ProblemDetail into a JSON object and return it for
                 # handling on the client side.
-                access_problem = json.loads(result.response[0])
+                access_problem = json.loads(e.problem_detail.response[0])
                 terms_of_service_link = terms_of_service_html = None
-            else:
-                access_problem = None
-                terms_of_service_link, terms_of_service_html = result
-            libraries = []
-            for registration in registry.registrations:
-                library_info = self.get_library_info(registration)
-                if library_info:
-                    libraries.append(library_info)
+
+            libraries = [self.get_library_info(r) for r in registry.registrations]
 
             services.append(
                 dict(
@@ -77,59 +92,77 @@ def process_get(self, do_get=HTTP.debuggable_get):
 
         return dict(library_registrations=services)
 
-    def get_library_info(self, registration):
+    def get_library_info(
+        self, registration: DiscoveryServiceRegistration
+    ) -> Dict[str, str]:
         """Find the relevant information about the library which the user
         is trying to register"""
 
-        library = registration.library
-        library_info = dict(short_name=library.short_name)
-        status = registration.status_field.value
-        stage_field = registration.stage_field.value
-        if stage_field:
-            library_info["stage"] = stage_field
+        library_info = {"short_name": str(registration.library.short_name)}
+        status = registration.status
+        stage = registration.stage
+        if stage:
+            library_info["stage"] = stage.value
         if status:
-            library_info["status"] = status
-            return library_info
+            library_info["status"] = status.value
 
-    def look_up_registry(self, integration_id):
-        """Find the RemoteRegistry that the user is trying to register the library with,
+        return library_info
+
+    def look_up_registry(self, integration_id: int) -> OpdsRegistrationService:
+        """Find the OpdsRegistrationService that the user is trying to register the library with,
         and check that it actually exists."""
 
-        registry = RemoteRegistry.for_integration_id(
-            self._db, integration_id, self.goal
-        )
+        registry = OpdsRegistrationService.for_integration(self._db, integration_id)
         if not registry:
-            return MISSING_SERVICE
+            raise ProblemError(problem_detail=MISSING_SERVICE)
         return registry
 
-    def look_up_library(self, library_short_name):
+    def look_up_library(self, library_short_name: str) -> Library:
         """Find the library the user is trying to register, and check that it actually exists."""
 
         library = get_one(self._db, Library, short_name=library_short_name)
         if not library:
-            return NO_SUCH_LIBRARY
+            raise ProblemError(problem_detail=NO_SUCH_LIBRARY)
         return library
 
-    def process_post(self, registration_class, do_get, do_post):
-        """Attempt to register a library with a RemoteRegistry."""
+    def process_post(self) -> Response:
+        """Attempt to register a library with a OpdsRegistrationService."""
 
-        integration_id = flask.request.form.get("integration_id")
+        integration_id = flask.request.form.get("integration_id", type=int)
         library_short_name = flask.request.form.get("library_short_name")
-        stage = (
-            flask.request.form.get("registration_stage") or Registration.TESTING_STAGE
-        )
+        stage_string = flask.request.form.get("registration_stage")
 
+        if integration_id is None:
+            raise ProblemError(
+                problem_detail=INVALID_INPUT.detailed(
+                    "Missing required parameter 'integration_id'"
+                )
+            )
         registry = self.look_up_registry(integration_id)
-        if isinstance(registry, ProblemDetail):
-            return registry
 
+        if library_short_name is None:
+            raise ProblemError(
+                problem_detail=INVALID_INPUT.detailed(
+                    "Missing required parameter 'library_short_name'"
+                )
+            )
         library = self.look_up_library(library_short_name)
-        if isinstance(library, ProblemDetail):
-            return library
 
-        registration = registration_class(registry, library)
-        registered = registration.push(stage, url_for, do_get=do_get, do_post=do_post)
-        if isinstance(registered, ProblemDetail):
-            return registered
+        if stage_string is None:
+            raise ProblemError(
+                problem_detail=INVALID_INPUT.detailed(
+                    "Missing required parameter 'registration_stage'"
+                )
+            )
+        try:
+            stage = RegistrationStage(stage_string)
+        except ValueError:
+            raise ProblemError(
+                problem_detail=INVALID_INPUT.detailed(
+                    f"'{stage_string}' is not a valid registration stage"
+                )
+            )
+
+        registry.register_library(library, stage, url_for)
 
         return Response(str(_("Success")), 200)