Merge pull request #13 from nationalarchives/AYR-421/flask-app-search…

…-integration

Ayr 421/flask app search integration

.github/workflows/unit_tests.yml

@@ -28,4 +28,4 @@ jobs:
       run: poetry install
 
     - name: Run Pytest tests
-      run: poetry run pytest --cov=app/main  --cov-report term-missing  -vvv app/tests/
+      run: AWS_DEFAULT_REGION=eu-west-2 poetry run pytest --cov=app/main  --cov-report term-missing  -vvv app/tests/

.pre-commit-config.yaml

@@ -26,4 +26,4 @@ repos:
     rev: 23.9.1
     hooks:
     - id: black
-      language_version: python3.11
+      language_version: python3

app/__init__.py

@@ -1,3 +1,4 @@
+import boto3
 from flask import Flask
 from flask_assets import Bundle, Environment
 from flask_compress import Compress
@@ -25,8 +26,16 @@ def null_to_dash(value):
 def create_app(config_class=Config):
     app = Flask(__name__, static_url_path="/assets")
     app.config.from_object(config_class)
+
+    force_https = False if app.config["TESTING"] else True
+
+    # use only for local development
+    if app.config["DEFAULT_AWS_PROFILE"]:
+        boto3.setup_default_session(profile_name=app.config["DEFAULT_AWS_PROFILE"])
+
     app.jinja_env.lstrip_blocks = True
     app.jinja_env.trim_blocks = True
+    app.jinja_env.filters["null_to_dash"] = null_to_dash
     app.jinja_loader = ChoiceLoader(
         [
             PackageLoader("app"),
@@ -45,14 +54,12 @@ def create_app(config_class=Config):
         "script-src": ["'self'"],
     }
 
-    app.jinja_env.filters["null_to_dash"] = null_to_dash
-
     # Initialise app extensions
     assets.init_app(app)
     compress.init_app(app)
     csrf.init_app(app)
     limiter.init_app(app)
-    talisman.init_app(app, content_security_policy=csp)
+    talisman.init_app(app, content_security_policy=csp, force_https=force_https)
     WTFormsHelpers(app)
 
     # Create static asset bundles

app/main/aws/open_search.py

@@ -0,0 +1,61 @@
+from typing import Tuple
+
+import boto3
+from opensearchpy import AWSV4SignerAuth, OpenSearch
+
+from app.main.aws.parameter import (
+    get_aws_environment_prefix,
+    get_parameter_store_key_value,
+)
+
+
+def get_open_search_index_from_aws_params() -> str:
+    return get_parameter_store_key_value(
+        get_aws_environment_prefix() + "AWS_OPEN_SEARCH_INDEX"
+    )
+
+
+def generate_open_search_client_from_aws_params() -> OpenSearch:
+    host = get_parameter_store_key_value(
+        get_aws_environment_prefix() + "AWS_OPEN_SEARCH_HOST"
+    )
+    http_auth = _get_open_search_http_auth()
+
+    open_search_client = OpenSearch(
+        hosts=[{"host": host, "port": 443}],
+        http_auth=http_auth,
+        use_ssl=True,
+        verify_certs=True,
+        http_compress=True,
+        ssl_assert_hostname=False,
+        ssl_show_warn=True,
+    )
+    return open_search_client
+
+
+def _get_open_search_http_auth(
+    auth_method: str = "username_password",
+) -> Tuple[str, str] | AWSV4SignerAuth:
+    if auth_method == "username_password":
+        return _get_open_search_username_password_auth()
+    return _get_open_search_iam_auth()
+
+
+def _get_open_search_username_password_auth() -> Tuple[str, str]:
+    username = get_parameter_store_key_value(
+        get_aws_environment_prefix() + "AWS_OPEN_SEARCH_USERNAME"
+    )
+    password = get_parameter_store_key_value(
+        get_aws_environment_prefix() + "AWS_OPEN_SEARCH_PASSWORD"
+    )
+    return (username, password)
+
+
+def _get_open_search_iam_auth() -> AWSV4SignerAuth:
+    credentials = boto3.Session().get_credentials()
+    aws_region = get_parameter_store_key_value(
+        get_aws_environment_prefix() + "AWS_REGION"
+    )
+    service = "es"
+    aws_auth = AWSV4SignerAuth(credentials, aws_region, service)
+    return aws_auth

app/main/aws/parameter.py

@@ -0,0 +1,27 @@
+import boto3
+from botocore.exceptions import ClientError
+import logging
+
+
+def get_aws_environment_prefix() -> str:
+    environment_name = get_parameter_store_key_value("ENVIRONMENT_NAME")
+    print(environment_name)
+    return "/" + environment_name + "/"
+
+
+def get_parameter_store_key_value(key: str) -> str:
+    """
+    Get string value of `key` in Parameter Store.
+    :param key: Name of key whose value will be returned.
+    :return: String value of requested Parameter Store key.
+    """
+    ssm_client = boto3.client("ssm")
+    parameter_value = ""
+    try:
+        parameter_value = ssm_client.get_parameter(Name=key)["Parameter"]["Value"]
+        logging.info("Parameter value retrieved successfully")
+    except ClientError as e:
+        logging.error(
+            "Failed to get parameter value, Error : %s", e.response["Error"]["Code"]
+        )
+    return parameter_value

app/main/routes.py

@@ -8,15 +8,15 @@
     session,
 )
 from flask_wtf.csrf import CSRFError
+
+from app.main.search import search_logic
 from .forms import SearchForm
 from werkzeug.exceptions import HTTPException
 import os
 
 from app.main import bp
 from app.main.forms import CookiesForm
 
-# from app.data.data import consignment_response, consignment_files_response
-
 from keycloak import KeycloakOpenID
 
 KEYCLOAK_BASE_URI = os.getenv("KEYCLOAK_BASE_URI")
@@ -33,31 +33,6 @@
 )
 
 
-sample_records = [
-    {
-        "title": "1.2_record1.pdf",
-        "description": "⚊",
-        "last_modified": "2023-01-15",
-        "status": "Open",
-        "closure_period_years": "⚊",
-    },
-    {
-        "title": "1.1_record2.doc",
-        "description": "⚊",
-        "last_modified": "2023-02-20",
-        "status": "Closed",
-        "closure_period_years": 50,
-    },
-    {
-        "title": "record_3.jpg",
-        "description": "⚊",
-        "last_modified": "2023-09-23",
-        "status": "Closed",
-        "closure_period_years": 20,
-    },
-]
-
-
 @bp.route("/", methods=["GET"])
 def index():
     return render_template("index.html")
@@ -118,19 +93,15 @@ def poc_search():
     form = SearchForm()
     results = []
     query = request.form.get("query", "").lower()
+
     if query:
-        search_terms = query.split()
-        for term in search_terms:
-            results.extend(
-                [
-                    record
-                    for record in sample_records
-                    if term in record["title"].lower()
-                    or term in record["description"].lower()
-                    or term in record["status"].lower()
-                ]
-            )
+        open_search_response = (
+            search_logic.generate_open_search_client_and_make_poc_search(query)
+        )
+        results = open_search_response["hits"]["hits"]
+
     num_records_found = len(results)
+
     return render_template(
         "poc-search.html",
         form=form,
@@ -149,15 +120,6 @@ def quick_access():
     return render_template("quick-access.html")
 
 
-# @bp.route("/record", methods=["GET"])
-# def record():
-#     return render_template(
-#         "record.html",
-#         consignment=consignment_response,
-#         consignment_files=consignment_files_response,
-#     )
-
-
 @bp.route("/all-departments", methods=["GET"])
 def departments():
     return render_template("departments.html")

app/main/search/search_logic.py

@@ -0,0 +1,48 @@
+import logging
+from typing import Any
+
+from opensearchpy import ImproperlyConfigured
+from app.main.aws.open_search import (
+    generate_open_search_client_from_aws_params,
+    get_open_search_index_from_aws_params,
+)
+
+
+def generate_open_search_client_and_make_poc_search(query: str) -> Any:
+    fields = [
+        "legal_status",
+        "description",
+        "closure_type",
+        "Internal-Sender_Identifier",
+        "id",
+        "Contact_Email",
+        "Source_Organization",
+        "Consignment_Series.keyword",
+        "Consignment_Series",
+        "Contact_Name",
+    ]
+    open_search_client = generate_open_search_client_from_aws_params()
+
+    try:
+        open_search_client.ping()
+    except ImproperlyConfigured as e:
+        logging.error("OpenSearch client improperly configured: " + str(e))
+        raise e
+
+    logging.info("OpenSearch client has been connected successfully")
+
+    open_search_index = get_open_search_index_from_aws_params()
+    open_search_query = {
+        "query": {
+            "multi_match": {
+                "query": query,
+                "fields": fields,
+                "fuzziness": "AUTO",
+                "type": "best_fields",
+            }
+        }
+    }
+    search_results = open_search_client.search(
+        body=open_search_query, index=open_search_index
+    )
+    return search_results

app/templates/main/poc-search.html

@@ -32,17 +32,9 @@ <h3 class="govuk-heading-s govuk-!-font-weight-bold">Search for digital records<
         {% if num_records_found > 0 %}
         <div class="govuk-grid-row">
           <div class="govuk-grid-column-full">
-            <h2 class="govuk-heading-m">{{ num_records_found}} records found</h2>
+            <h2 class="govuk-heading-m">{{ num_records_found }} records found</h2>
             <table class="govuk-table">
               <thead class="govuk-table__head">
-                <tr class="govuk-table__row">
-                  <th scope="row" class="govuk-table__header govuk-!-width-one-third"></th>
-                  <td class="govuk-table__cell"></td>
-                  <td class="govuk-table__cell"></td>
-                  <td class="govuk-table__cell"></td>
-                  <td class="govuk-table__cell"></td>
-                  <td class="govuk-table__cell"></td>
-                </tr>
                 <tr class="govuk-table__row govuk-!-font-size-14">
                   <th scope="col" class="govuk-table__header govuk-!-font-weight-bold">Title</th>
                   <th scope="col" class="govuk-table__header govuk-!-font-weight-bold">Description</th>
@@ -54,12 +46,11 @@ <h2 class="govuk-heading-m">{{ num_records_found}} records found</h2>
               <tbody class="govuk-table__body">
                 {% for record in results %}
                 <tr class="govuk-table__row">
-                  <td class="govuk-table__cell govuk-body govuk-!-font-size-14"><a href="{{'#'}}">{{ record.title }}</a></td>
-                  <td class="govuk-table__cell govuk-body govuk-!-font-size-14">{{ record.description }}</td>
-                  <td class="govuk-table__cell govuk-body govuk-!-font-size-14">{{ record.last_modified }}</td>
-                  <td class="govuk-table__cell govuk-body govuk-!-font-size-14">{{ record.status }}</td>
-                  <td class="govuk-table__cell govuk-body govuk-!-font-size-14">{{ record.closure_period_years }}</td>
-                  <td class="govuk-table__cell govuk-body govuk-!-font-size-14"></td>
+                  <td class="govuk-table__cell govuk-body govuk-!-font-size-14"><a href="{{'#'}}">{{ record._source.file_name }}</a></td>
+                  <td class="govuk-table__cell govuk-body govuk-!-font-size-14">{{ record._source.description }}</td>
+                  <td class="govuk-table__cell govuk-body govuk-!-font-size-14">{{ record._source.date_last_modified }}</td>
+                  <td class="govuk-table__cell govuk-body govuk-!-font-size-14">{{ record._source.legal_status }}</td>
+                  <td class="govuk-table__cell govuk-body govuk-!-font-size-14">{{ record._source.closure_period }}</td>
                 </tr>
                 {% endfor %}
               </tbody>

app/tests/conftest.py

@@ -1,12 +1,11 @@
 import pytest
 from app import create_app
+from testing_config import TestingConfig
 
 
 @pytest.fixture
 def app():
-    app = create_app()
-    app.config["TESTING"] = True
-    app.config["WTF_CSRF_ENABLED"] = False
+    app = create_app(config_class=TestingConfig)
     yield app