From 31ab2acb21a83ea63987f51b0404560c80f7be69 Mon Sep 17 00:00:00 2001 From: aaronfriedman Date: Thu, 25 Jul 2024 15:18:42 -0400 Subject: [PATCH 1/2] Add SecretsManager client --- CHANGELOG.md | 3 + README.md | 3 +- pyproject.toml | 8 ++- .../classes/secrets_manager_client.py | 67 +++++++++++++++++++ tests/test_kms_client.py | 4 +- tests/test_mysql_client.py | 2 +- tests/test_secrets_manager_client.py | 55 +++++++++++++++ 7 files changed, 136 insertions(+), 6 deletions(-) create mode 100644 src/nypl_py_utils/classes/secrets_manager_client.py create mode 100644 tests/test_secrets_manager_client.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d080e37..ecf7553 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## v1.3.0 7/25/24 +- Added SecretsManager client + ## v1.2.0 7/17/24 - Generalized Avro functions and separated encoding/decoding behavior. diff --git a/README.md b/README.md index 8f260da..3b4fad8 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ This package contains common Python utility classes and functions. * Setting and retrieving a resource in S3 * Decrypting values with KMS * Encoding and decoding records using a given Avro schema +* Retrieving secrets from AWS Secrets Manager * Connecting to and querying a MySQL database * Connecting to and querying a PostgreSQL database * Connecting to and querying a PostgreSQL database using a connection pool @@ -35,7 +36,7 @@ kinesis_client = KinesisClient(...) # Do not use any version below 1.0.0 # All available optional dependencies can be found in pyproject.toml. # See the "Managing dependencies" section below for more details. -nypl-py-utils[kinesis-client,config-helper]==1.1.2 +nypl-py-utils[kinesis-client,config-helper]==1.3.0 ``` ## Developing locally diff --git a/pyproject.toml b/pyproject.toml index 0672e8b..2e7c83c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "nypl_py_utils" -version = "1.2.0" +version = "1.3.0" authors = [ { name="Aaron Friedman", email="aaronfriedman@nypl.org" }, ] @@ -56,6 +56,10 @@ s3-client = [ "boto3>=1.26.5", "botocore>=1.29.5" ] +secrets-manager-client = [ + "boto3>=1.26.5", + "botocore>=1.29.5" +] config-helper = [ "nypl_py_utils[kms-client]", "PyYAML>=6.0" @@ -67,7 +71,7 @@ research-catalog-identifier-helper = [ "requests>=2.28.1" ] development = [ - "nypl_py_utils[avro-client,kinesis-client,kms-client,mysql-client,oauth2-api-client,postgresql-client,postgresql-pool-client,redshift-client,s3-client,config-helper,obfuscation-helper,research-catalog-identifier-helper]", + "nypl_py_utils[avro-client,kinesis-client,kms-client,mysql-client,oauth2-api-client,postgresql-client,postgresql-pool-client,redshift-client,s3-client,secrets-manager-client,config-helper,obfuscation-helper,research-catalog-identifier-helper]", "flake8>=6.0.0", "freezegun>=1.2.2", "mock>=4.0.3", diff --git a/src/nypl_py_utils/classes/secrets_manager_client.py b/src/nypl_py_utils/classes/secrets_manager_client.py new file mode 100644 index 0000000..a25f405 --- /dev/null +++ b/src/nypl_py_utils/classes/secrets_manager_client.py @@ -0,0 +1,67 @@ +import boto3 +import json +import os + +from botocore.exceptions import ClientError +from nypl_py_utils.functions.log_helper import create_log + + +class SecretsManagerClient: + """Client for interacting with AWS Secrets Manager""" + + def __init__(self): + self.logger = create_log('secrets_manager_client') + + try: + self.secrets_manager_client = boto3.client( + 'secretsmanager', region_name=os.environ.get('AWS_REGION', + 'us-east-1')) + except ClientError as e: + self.logger.error( + 'Could not create Secrets Manager client: {err}'.format( + err=e)) + raise SecretsManagerClientError( + 'Could not create Secrets Manager client: {err}'.format( + err=e)) from None + + def close(self): + self.secrets_manager_client.close() + + def get_secret(self, secret_name, is_json=True): + """ + Retrieves secret with the given name from the Secrets Manager. + + Parameters + ---------- + secret_name: str + The name of the secret to retrieve + is_json: bool, optional + Whether the value of the secret is a JSON string that should be + returned as a dictionary + + Returns + ------- + dict or str + Dictionary if `is_json` is True; string if `is_json` is False + """ + self.logger.debug('Retrieving \'{}\' from Secrets Manager'.format( + secret_name)) + try: + response = self.secrets_manager_client.get_secret_value( + SecretId=secret_name) + if is_json: + return json.loads(response['SecretString']) + else: + return response['SecretString'] + except ClientError as e: + self.logger.error( + ('Could not retrieve \'{secret}\' from Secrets Manager: {err}') + .format(secret=secret_name, err=e)) + raise SecretsManagerClientError( + ('Could not retrieve \'{secret}\' from Secrets Manager: {err}') + .format(secret=secret_name, err=e)) from None + + +class SecretsManagerClientError(Exception): + def __init__(self, message=None): + self.message = message diff --git a/tests/test_kms_client.py b/tests/test_kms_client.py index e500b03..6bb31d5 100644 --- a/tests/test_kms_client.py +++ b/tests/test_kms_client.py @@ -21,10 +21,10 @@ def test_instance(self, mocker): def test_decrypt(self, test_instance): test_instance.kms_client.decrypt.return_value = _TEST_DECRYPTION - assert test_instance.kms_client.decrypt.called_once_with( - CiphertextBlob=b'test-encrypted-value') assert test_instance.decrypt( _TEST_ENCRYPTED_VALUE) == 'test-decrypted-value' + test_instance.kms_client.decrypt.assert_called_once_with( + CiphertextBlob=b'test-encrypted-value') def test_base64_error(self, test_instance): with pytest.raises(KmsClientError): diff --git a/tests/test_mysql_client.py b/tests/test_mysql_client.py index 11bb94f..a1f8a87 100644 --- a/tests/test_mysql_client.py +++ b/tests/test_mysql_client.py @@ -60,7 +60,7 @@ def test_execute_write_query_with_params(self, mock_mysql_conn, 'test query %s %s', query_params=('a', 1)) is None mock_cursor.execute.assert_called_once_with('test query %s %s', ('a', 1)) - test_instance.conn.commit.called_once() + test_instance.conn.commit.assert_called_once() mock_cursor.close.assert_called_once() def test_execute_query_with_exception( diff --git a/tests/test_secrets_manager_client.py b/tests/test_secrets_manager_client.py new file mode 100644 index 0000000..3d069be --- /dev/null +++ b/tests/test_secrets_manager_client.py @@ -0,0 +1,55 @@ +import pytest + +from botocore.exceptions import ClientError +from datetime import datetime +from nypl_py_utils.classes.secrets_manager_client import ( + SecretsManagerClient, SecretsManagerClientError) + +_TEST_RESPONSE = { + 'ARN': 'test_arn', + 'Name': 'test_secret', + 'VersionId': 'test_version', + 'SecretString': '{\n "key1": "value1",\n "key2": "value2"\n}', + 'VersionStages': ['AWSCURRENT'], + 'CreatedDate': datetime(2024, 1, 1, 1, 1, 1, 1), + 'ResponseMetadata': { + 'RequestId': 'test-request-id', + 'HTTPStatusCode': 200, + 'HTTPHeaders': { + 'x-amzn-requestid': 'test-request-id', + 'content-type': 'application/x-amz-json-1.1', + 'content-length': '155', + 'date': 'Mon, 1 Jan 2024 07:01:01 GMT' + }, + 'RetryAttempts': 0} +} + + +class TestSecretsManagerClient: + + @pytest.fixture + def test_instance(self, mocker): + mocker.patch('boto3.client') + return SecretsManagerClient() + + def test_get_secret(self, test_instance): + test_instance.secrets_manager_client.get_secret_value.return_value = \ + _TEST_RESPONSE + assert test_instance.get_secret('test_secret') == { + 'key1': 'value1', 'key2': 'value2'} + test_instance.secrets_manager_client.get_secret_value\ + .assert_called_once_with(SecretId='test_secret') + + def test_get_secret_non_json(self, test_instance): + test_instance.secrets_manager_client.get_secret_value.return_value = \ + _TEST_RESPONSE + assert test_instance.get_secret('test_secret', is_json=False) == ( + '{\n "key1": "value1",\n "key2": "value2"\n}') + test_instance.secrets_manager_client.get_secret_value\ + .assert_called_once_with(SecretId='test_secret') + + def test_get_secret_error(self, test_instance): + test_instance.secrets_manager_client.get_secret_value.side_effect = \ + ClientError({}, 'GetSecretValue') + with pytest.raises(SecretsManagerClientError): + test_instance.get_secret('test_secret') From c79753b0a687623a6ec1ab5ca3f084afb1b7fbbe Mon Sep 17 00:00:00 2001 From: aaronfriedman Date: Tue, 30 Jul 2024 14:55:42 -0400 Subject: [PATCH 2/2] Update CHANGELOG date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08ce218..b7ce843 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # Changelog -## v1.3.0 7/25/24 +## v1.3.0 7/30/24 - Added SecretsManager client ## v1.2.1 7/25/24