Skip to content

Commit

Permalink
More config fixup, refactors and tests
Browse files Browse the repository at this point in the history
  • Loading branch information
anthonyhashemi committed Nov 3, 2023
1 parent 4f83a59 commit 4152a31
Show file tree
Hide file tree
Showing 8 changed files with 240 additions and 126 deletions.
62 changes: 26 additions & 36 deletions app/main/aws/open_search.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,22 @@
from typing import Tuple

import boto3
from flask import current_app
from opensearchpy import AWSV4SignerAuth, OpenSearch

from app.main.aws.parameter import (
get_aws_environment_prefix,
get_parameter_store_key_value,
)

def generate_open_search_client_from_current_app_config() -> OpenSearch:
"""
Generate an OpenSearch client with the specified configuration for the AYR application.
def get_open_search_index_from_aws_params() -> str:
return get_parameter_store_key_value(
get_aws_environment_prefix() + "AWS_OPEN_SEARCH_INDEX"
)


def generate_open_search_client_from_aws_params() -> OpenSearch:
host = get_parameter_store_key_value(
get_aws_environment_prefix() + "AWS_OPEN_SEARCH_HOST"
)
http_auth = _get_open_search_http_auth()

Returns:
OpenSearch: An OpenSearch client configured with settings obtained from the current app's configuration.
"""
open_search_client = OpenSearch(
hosts=[{"host": host, "port": 443}],
http_auth=http_auth,
hosts=[
{"host": current_app.config["AWS_OPEN_SEARCH_HOST"], "port": 443}
],
http_auth=get_open_search_http_auth(),
use_ssl=True,
verify_certs=True,
http_compress=True,
Expand All @@ -33,29 +26,26 @@ def generate_open_search_client_from_aws_params() -> OpenSearch:
return open_search_client


def _get_open_search_http_auth(
auth_method: str = "username_password",
) -> Tuple[str, str] | AWSV4SignerAuth:
if auth_method == "username_password":
return _get_open_search_username_password_auth()
return _get_open_search_iam_auth()
def get_open_search_http_auth(iam=False) -> Tuple[str, str] | AWSV4SignerAuth:
"""
Get the authentication method for OpenSearch.
Args:
iam (bool): A boolean indicating whether IAM authentication should be used.
def _get_open_search_username_password_auth() -> Tuple[str, str]:
username = get_parameter_store_key_value(
get_aws_environment_prefix() + "AWS_OPEN_SEARCH_USERNAME"
)
password = get_parameter_store_key_value(
get_aws_environment_prefix() + "AWS_OPEN_SEARCH_PASSWORD"
)
return (username, password)
Returns:
Tuple[str, str] | AWSV4SignerAuth: Depending on the IAM parameter
"""
if not iam:
return (
current_app.config["AWS_OPEN_SEARCH_USERNAME"],
current_app.config["AWS_OPEN_SEARCH_PASSWORD"],
)
return _get_open_search_iam_auth(current_app.config["AWS_REGION"])


def _get_open_search_iam_auth() -> AWSV4SignerAuth:
def _get_open_search_iam_auth(aws_region) -> AWSV4SignerAuth:
credentials = boto3.Session().get_credentials()
aws_region = get_parameter_store_key_value(
get_aws_environment_prefix() + "AWS_REGION"
)
service = "es"
aws_auth = AWSV4SignerAuth(credentials, aws_region, service)
return aws_auth
4 changes: 3 additions & 1 deletion app/main/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,9 @@ def poc_search():

if query:
open_search_response = (
search_logic.generate_open_search_client_and_make_poc_search(query)
search_logic.generate_open_search_client_and_make_poc_search(
query, current_app.config["AWS_OPEN_SEARCH_INDEX"]
)
)
results = open_search_response["hits"]["hits"]
session["search_results"] = results
Expand Down
27 changes: 13 additions & 14 deletions app/main/search/search_logic.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,20 @@
from opensearchpy import ImproperlyConfigured

from app.main.aws.open_search import (
generate_open_search_client_from_aws_params,
get_open_search_index_from_aws_params,
generate_open_search_client_from_current_app_config,
)


def generate_open_search_client_and_make_poc_search(query: str) -> Any:
def generate_open_search_client_and_make_poc_search(query: str, index) -> Any:
open_search_client = generate_open_search_client_from_current_app_config()
try:
open_search_client.ping()
except ImproperlyConfigured as e:
logging.error("OpenSearch client improperly configured: " + str(e))
raise e

logging.info("OpenSearch client has been connected successfully")

fields = [
"legal_status",
"description",
Expand All @@ -22,17 +30,7 @@ def generate_open_search_client_and_make_poc_search(query: str) -> Any:
"Consignment_Series",
"Contact_Name",
]
open_search_client = generate_open_search_client_from_aws_params()

try:
open_search_client.ping()
except ImproperlyConfigured as e:
logging.error("OpenSearch client improperly configured: " + str(e))
raise e

logging.info("OpenSearch client has been connected successfully")

open_search_index = get_open_search_index_from_aws_params()
open_search_query = {
"query": {
"multi_match": {
Expand All @@ -43,7 +41,8 @@ def generate_open_search_client_and_make_poc_search(query: str) -> Any:
}
}
}

search_results = open_search_client.search(
body=open_search_query, index=open_search_index
body=open_search_query, index=index
)
return search_results
129 changes: 68 additions & 61 deletions app/tests/test_aws_open_search.py
Original file line number Diff line number Diff line change
@@ -1,73 +1,80 @@
from unittest.mock import patch

import boto3
from moto import mock_ssm
from flask import Flask
from opensearchpy import AWSV4SignerAuth

from app.main.aws.open_search import (
generate_open_search_client_from_aws_params,
get_open_search_index_from_aws_params,
generate_open_search_client_from_current_app_config,
get_open_search_http_auth,
)


@mock_ssm
def test_get_open_search_index_from_aws_params():
ssm_client = boto3.client("ssm")
ssm_client.put_parameter(
Name="ENVIRONMENT_NAME",
Value="test_env",
Type="String",
Overwrite=True,
)
ssm_client.put_parameter(
Name="/test_env/AWS_OPEN_SEARCH_INDEX",
Value="test_index",
Type="String",
Overwrite=True,
)
@patch("app.main.aws.open_search.get_open_search_http_auth")
@patch("app.main.aws.open_search.OpenSearch")
def test_generate_open_search_client_from_current_app_config(
mock_opensearch, mock_get_open_search_http_auth
):
"""
Given a Flask application and the necessary configuration set for OpenSearch
When generating an OpenSearch client using the current application's configuration
Then an OpenSearch client should be created with the expected configuration
"""
app = Flask("test")
app.config["AWS_OPEN_SEARCH_HOST"] = "mock_host"

assert get_open_search_index_from_aws_params() == "test_index"
with app.app_context():
assert (
generate_open_search_client_from_current_app_config()
== mock_opensearch.return_value
)
mock_opensearch.assert_called_once_with(
hosts=[{"host": "mock_host", "port": 443}],
http_auth=mock_get_open_search_http_auth.return_value,
use_ssl=True,
verify_certs=True,
http_compress=True,
ssl_assert_hostname=False,
ssl_show_warn=True,
)


@mock_ssm
@patch("app.main.aws.open_search.OpenSearch")
def test_generate_open_search_client_from_aws_params(mock_open_search):
ssm_client = boto3.client("ssm")
ssm_client.put_parameter(
Name="ENVIRONMENT_NAME",
Value="test_env",
Type="String",
Overwrite=True,
)
ssm_client.put_parameter(
Name="/test_env/AWS_OPEN_SEARCH_HOST",
Value="mock_opensearch_host",
Type="String",
Overwrite=True,
)
ssm_client.put_parameter(
Name="/test_env/AWS_OPEN_SEARCH_USERNAME",
Value="mock_username",
Type="String",
Overwrite=True,
)
ssm_client.put_parameter(
Name="/test_env/AWS_OPEN_SEARCH_PASSWORD",
Value="mock_password",
Type="String",
Overwrite=True,
)
def test_get_open_search_http_auth():
"""
Given a Flask application with AWS_OPEN_SEARCH_USERNAME, AWS_OPEN_SEARCH_PASSWORD in the config
When calling get_open_search_http_auth with iam as `False`
Then a tuple of the username and password is returned
"""
app = Flask("test")
app.config["AWS_OPEN_SEARCH_USERNAME"] = "test_username"
app.config[
"AWS_OPEN_SEARCH_PASSWORD"
] = "test_password" # pragma: allowlist secret

with app.app_context():
assert get_open_search_http_auth(iam=False) == (
"test_username",
"test_password",
)


@patch("app.main.aws.open_search.boto3.Session")
def test_get_open_search_http_auth_iam(mock_boto3_session):
"""
Given a mock Boto3 Session and the current app's configuration set for OpenSearch
When calling get_open_search_http_auth with iam as "True"
Then it should return an AWSV4SignerAuth object created using the mock AWS
credentials and AWS region from the current app's configuration.
"""
app = Flask("test")
app.config["AWS_REGION"] = "us-east-1"

mock_session_instance = mock_boto3_session.return_value
mock_session_instance.get_credentials.return_value = "mock_credentials"

assert (
generate_open_search_client_from_aws_params()
== mock_open_search.return_value
)
with app.app_context():
auth = get_open_search_http_auth(iam=True)

mock_open_search.assert_called_once_with(
hosts=[{"host": "mock_opensearch_host", "port": 443}],
http_auth=("mock_username", "mock_password"),
use_ssl=True,
verify_certs=True,
http_compress=True,
ssl_assert_hostname=False,
ssl_show_warn=True,
)
assert isinstance(auth, AWSV4SignerAuth)
assert auth.credentials == "mock_credentials"
assert auth.region == "us-east-1"
assert auth.service == "es"
32 changes: 32 additions & 0 deletions app/tests/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,43 @@ def test_aws_params_config_initialized():
Type="String",
Overwrite=True,
)
ssm_client.put_parameter(
Name="/test_env/AWS_OPEN_SEARCH_INDEX",
Value="test_open_search_index",
Type="String",
Overwrite=True,
)
ssm_client.put_parameter(
Name="/test_env/AWS_OPEN_SEARCH_HOST",
Value="test_open_search_host",
Type="String",
Overwrite=True,
)
ssm_client.put_parameter(
Name="/test_env/AWS_OPEN_SEARCH_USERNAME",
Value="test_open_search_username",
Type="String",
Overwrite=True,
)
ssm_client.put_parameter(
Name="/test_env/AWS_OPEN_SEARCH_PASSWORD",
Value="test_open_search_password",
Type="String",
Overwrite=True,
)

config = Config()

assert config.AWS_ENVIRONMENT_PREFIX == "/test_env/"

assert config.AWS_OPEN_SEARCH_INDEX == "test_open_search_index"
assert config.AWS_OPEN_SEARCH_HOST == "test_open_search_host"
assert config.AWS_OPEN_SEARCH_USERNAME == "test_open_search_username"
assert (
config.AWS_OPEN_SEARCH_PASSWORD
== "test_open_search_password" # pragma: allowlist secret
)

assert config.KEYCLOAK_BASE_URI == "a"
assert config.KEYCLOAK_CLIENT_ID == "b"
assert config.KEYCLOAK_REALM_NAME == "c"
Expand Down
Loading

0 comments on commit 4152a31

Please sign in to comment.