From a5a6dada6807fa537123b112474086997ccc058f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABl=20Perras?= Date: Thu, 22 Jun 2023 14:08:59 -0400 Subject: [PATCH 1/2] feat: Optionally load default environment keys The `build_env_list()` function has been rewritten to allow the optional inclusion of the default environment name, set by `DEFAULT_ENV_FOR_DYNACONF`. The use of this default environment key can be disabled completely by setting `SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF` to `False`. The README has been updated to reflect this. --- README.rst | 4 ++- dynaconf_aws_loader/loader.py | 29 ++++++++++++++++-- tests/conftest.py | 57 +++++++++++++++++++++++++++++++++++ tests/test_loader.py | 43 ++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 0218b2e..07de357 100644 --- a/README.rst +++ b/README.rst @@ -42,6 +42,7 @@ The following optional variables should be set in your ``settings.toml`` (or equ - ``SSM_ENDPOINT_URL_FOR_DYNACONF``: If your AWS SSM uses a different endpoint than the AWS default. This can be useful for local development when you are running something like `LocalStack `_. - ``SSM_SESSION_FOR_DYNACONF``: If you require custom `boto3.session.Session `_ arguments, you can specify then as a dictionary here. Note that this will override the default ``boto3`` credential configuration. +- ``SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF``: Boolean, defaults to ``True``. If you want the SSM loader to load keys under the ``default`` environment name. The key name itself can be set via the Dynaconf setting of ``DEFAULT_ENV_FOR_DYNACONF`` if you want it to be something other than ``default``. Parameter Store Details @@ -63,7 +64,8 @@ An optional ``namespace`` can be specified as a sub-project grouping for paramet Note that if you choose to use a ``namespace`` identifier, it must not conflict with existing ``environment`` identifiers. -The ``environment`` identifiers of: ``default``, ``main``, ``global``, and ``dynaconf`` indicate that any parameters nested below said identifiers will be inherited by *all* environments; these names are Dynaconf defaults. This can be useful for setting a default value that can then be overridden on a per-environment basis as necessary. +If ``SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF`` is set to ``True`` (which is the default value), the loader will add whatever the value of ``DEFAULT_ENV_FOR_DYNACONF`` as an ``environment`` key to load from SSM. The typical use case here is to have a default value for all environments that can be overriden on a per-environment basis as necessary. + Security Considerations ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/dynaconf_aws_loader/loader.py b/dynaconf_aws_loader/loader.py index 4047232..3024e68 100644 --- a/dynaconf_aws_loader/loader.py +++ b/dynaconf_aws_loader/loader.py @@ -18,6 +18,7 @@ if t.TYPE_CHECKING: from mypy_boto3_ssm.client import SSMClient + from dynaconf.base import LazySettings, Settings logger = logging.getLogger("dynaconf.aws_loader") @@ -32,6 +33,30 @@ def get_client(obj) -> SSMClient: return client +def build_env_list(obj: Settings | LazySettings, env: t.Optional[str]) -> list[str]: + """ + Build env list for loader to iterate. + + Ensure we are building the env list in such a way that we are not going to + be hitting the remote AWS SSM Parameter Store endpoints needlessly. Let's only + hit `DEFAULT_ENV_FOR_DYNACONF`, the currently set environment on the settings object, + and the manually set `env` (if any). + + Some of this logic is similar to `dynaconf.utils.build_env_list`. + """ + + env_list = [] + if obj.get("SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF", True) is True: + env_list.append((obj.get("DEFAULT_ENV_FOR_DYNACONF") or "default").lower()) + + # add a manually set env, if specified + if env and env.lower() not in env_list: + env_list.append(env.lower()) + + # Ensure any leading/trailing whitespace is removed + return map(str.strip, env_list) + + def load( obj, env: str | None = None, @@ -97,8 +122,6 @@ def load( return raise - env_list = build_env_list(obj, env or obj.current_env) - project_prefix = pull_from_env_or_obj(prefix_key_name, os.environ, obj) namespace_prefix = pull_from_env_or_obj(namespace_key_name, os.environ, obj) @@ -108,6 +131,8 @@ def load( " or environment for AWS SSM loader to work." ) + env_list = build_env_list(obj, env or obj.current_env) + for env_name in env_list: env_name = env_name.lower() path = f"/{project_prefix}/{env_name}" diff --git a/tests/conftest.py b/tests/conftest.py index 766172f..535e5e9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -98,6 +98,63 @@ def basic_settings( ) +@pytest.fixture +def basic_settings_disable_default_env( + tmp_path: Path, ssm_client: SSMClient +) -> t.Generator[Path, None, None]: + """ + Basic settings with environment layers, with default environment disabled + """ + + project_name = "basic" + + data = f""" + [default] + SSM_PARAMETER_PROJECT_PREFIX_FOR_DYNACONF = '{project_name}' + SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF = false + PRODUCT_NAME = "foobar" + """ + + # We'll set some default values, just to ensure that our loader does not + # access them. + ssm_client.put_parameter( + Name=f"/{project_name}/default/products", + # Use @json cast for Dynaconf + # https://www.dynaconf.com/configuration/#auto_cast + Value="@json %s" % json.dumps({"plans": ["monthly", "yearly"]}), + Type="String", + ) + ssm_client.put_parameter( + Name=f"/{project_name}/development/my_config_value", + Value="test123", + Type="String", + ) + ssm_client.put_parameter( + Name=f"/{project_name}/production/database/password", + Value="password", + Type="SecureString", + ) + ssm_client.put_parameter( + Name=f"/{project_name}/production/database/host", + Value="db.example.com", + Type="String", + ) + + settings_file = tmp_path / "settings.toml" + settings_file.write_text(data) + + yield settings_file + + ssm_client.delete_parameters( + Names=[ + f"/{project_name}/production/database/host", + f"/{project_name}/production/database/password", + f"/{project_name}/development/my_config_value", + f"/{project_name}/default/products", + ] + ) + + @pytest.fixture def settings_without_environments( tmp_path: Path, ssm_client: SSMClient diff --git a/tests/test_loader.py b/tests/test_loader.py index 8f7de08..a187ced 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -75,6 +75,49 @@ def test_basic_environment_based_settings( assert prod_settings.MY_CONFIG_VALUE == "test123" +def test_basic_environment_based_settings_without_default_env( + basic_settings_disable_default_env: pathlib.Path, +): + """ + Test simple loading of configuration from AWS SSM on a per-environment + basis, omitting the default environment + + """ + + dev_settings = Dynaconf( + environments=True, + FORCE_ENV_FOR_DYNACONF="development", + settings_file=str(basic_settings_disable_default_env.resolve()), + LOADERS_FOR_DYNACONF=[ + "dynaconf_aws_loader.loader", + ], + ) + + assert dev_settings.current_env == "development" + assert dev_settings.SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF is False + + # This should not be set, since we are avoiding defaults + with pytest.raises(AttributeError): + dev_settings.PRODUCTS + + prod_settings = Dynaconf( + environments=True, + FORCE_ENV_FOR_DYNACONF="production", + settings_file=str(basic_settings_disable_default_env.resolve()), + LOADERS_FOR_DYNACONF=[ + "dynaconf_aws_loader.loader", + ], + ) + + assert prod_settings.current_env == "production" + assert prod_settings.SSM_PARAMETER_PROJECT_PREFIX_FOR_DYNACONF == "basic" + + # Loaded from default env + # This should not be set, since we are avoiding defaults + with pytest.raises(AttributeError): + prod_settings.PRODUCTS + + def test_settings_with_no_environments( settings_without_environments: pathlib.Path, ): From 1e6b281cd8a1d6c754adae33a806e60575e01794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABl=20Perras?= Date: Thu, 22 Jun 2023 17:03:08 -0400 Subject: [PATCH 2/2] chore: Guard against non-string type for DEFAULT_ENV_FOR_DYNACONF This is very unlikely to occur, but Python is Python, *shrugs*. --- dynaconf_aws_loader/loader.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dynaconf_aws_loader/loader.py b/dynaconf_aws_loader/loader.py index 3024e68..cf82684 100644 --- a/dynaconf_aws_loader/loader.py +++ b/dynaconf_aws_loader/loader.py @@ -10,7 +10,6 @@ import boto3 from botocore.exceptions import ClientError, BotoCoreError, NoRegionError -from dynaconf.utils import build_env_list from dynaconf.utils.parse_conf import parse_conf_data from . import IDENTIFIER @@ -18,7 +17,7 @@ if t.TYPE_CHECKING: from mypy_boto3_ssm.client import SSMClient - from dynaconf.base import LazySettings, Settings + from dynaconf.base import Settings logger = logging.getLogger("dynaconf.aws_loader") @@ -33,7 +32,7 @@ def get_client(obj) -> SSMClient: return client -def build_env_list(obj: Settings | LazySettings, env: t.Optional[str]) -> list[str]: +def build_env_list(obj: Settings, env: t.Optional[str]) -> t.Iterable[str]: """ Build env list for loader to iterate. @@ -47,11 +46,13 @@ def build_env_list(obj: Settings | LazySettings, env: t.Optional[str]) -> list[s env_list = [] if obj.get("SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF", True) is True: - env_list.append((obj.get("DEFAULT_ENV_FOR_DYNACONF") or "default").lower()) + default_env_name: str = obj.get("DEFAULT_ENV_FOR_DYNACONF", "default") + if default_env_name and (default_env_name := default_env_name.lower()): + env_list.append(default_env_name) # add a manually set env, if specified - if env and env.lower() not in env_list: - env_list.append(env.lower()) + if env and (env_identifier := env.lower()) not in env_list: + env_list.append(env_identifier) # Ensure any leading/trailing whitespace is removed return map(str.strip, env_list)