Merge pull request #10 from fictivekin/configurable-default-dynaconf-…

…paths

Rework how environment list is built for environment path key prefix generation

README.rst

@@ -42,6 +42,7 @@ The following optional variables should be set in your ``settings.toml`` (or equ
 
 - ``SSM_ENDPOINT_URL_FOR_DYNACONF``: If your AWS SSM uses a different endpoint than the AWS default. This can be useful for local development when you are running something like `LocalStack <https://localstack.cloud/>`_.
 - ``SSM_SESSION_FOR_DYNACONF``: If you require custom `boto3.session.Session <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html>`_ arguments, you can specify then as a dictionary here. Note that this will override the default ``boto3`` credential configuration.
+- ``SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF``: Boolean, defaults to ``True``. If you want the SSM loader to load keys under the ``default`` environment name. The key name itself can be set via the Dynaconf setting of ``DEFAULT_ENV_FOR_DYNACONF`` if you want it to be something other than ``default``.
 
 
 Parameter Store Details
@@ -63,7 +64,8 @@ An optional ``namespace`` can be specified as a sub-project grouping for paramet
 
 Note that if you choose to use a ``namespace`` identifier, it must not conflict with existing ``environment`` identifiers.
 
-The ``environment`` identifiers of: ``default``, ``main``, ``global``, and ``dynaconf`` indicate that any parameters nested below said identifiers will be inherited by *all* environments; these names are Dynaconf defaults. This can be useful for setting a default value that can then be overridden on a per-environment basis as necessary.
+If ``SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF`` is set to ``True`` (which is the default value), the loader will add whatever the value of ``DEFAULT_ENV_FOR_DYNACONF`` as an ``environment`` key to load from SSM. The typical use case here is to have a default value for all environments that can be overriden on a per-environment basis as necessary.
+
 
 Security Considerations
 ~~~~~~~~~~~~~~~~~~~~~~~

dynaconf_aws_loader/loader.py

@@ -10,14 +10,14 @@
 import boto3
 from botocore.exceptions import ClientError, BotoCoreError, NoRegionError
 
-from dynaconf.utils import build_env_list
 from dynaconf.utils.parse_conf import parse_conf_data
 
 from . import IDENTIFIER
 from .util import slashes_to_dict, pull_from_env_or_obj
 
 if t.TYPE_CHECKING:
     from mypy_boto3_ssm.client import SSMClient
+    from dynaconf.base import Settings
 
 
 logger = logging.getLogger("dynaconf.aws_loader")
@@ -32,6 +32,32 @@ def get_client(obj) -> SSMClient:
     return client
 
 
+def build_env_list(obj: Settings, env: t.Optional[str]) -> t.Iterable[str]:
+    """
+    Build env list for loader to iterate.
+
+    Ensure we are building the env list in such a way that we are not going to
+    be hitting the remote AWS SSM Parameter Store endpoints needlessly. Let's only
+    hit `DEFAULT_ENV_FOR_DYNACONF`, the currently set environment on the settings object,
+    and the manually set `env` (if any).
+
+    Some of this logic is similar to `dynaconf.utils.build_env_list`.
+    """
+
+    env_list = []
+    if obj.get("SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF", True) is True:
+        default_env_name: str = obj.get("DEFAULT_ENV_FOR_DYNACONF", "default")
+        if default_env_name and (default_env_name := default_env_name.lower()):
+            env_list.append(default_env_name)
+
+    # add a manually set env, if specified
+    if env and (env_identifier := env.lower()) not in env_list:
+        env_list.append(env_identifier)
+
+    # Ensure any leading/trailing whitespace is removed
+    return map(str.strip, env_list)
+
+
 def load(
     obj,
     env: str | None = None,
@@ -97,8 +123,6 @@ def load(
             return
         raise
 
-    env_list = build_env_list(obj, env or obj.current_env)
-
     project_prefix = pull_from_env_or_obj(prefix_key_name, os.environ, obj)
     namespace_prefix = pull_from_env_or_obj(namespace_key_name, os.environ, obj)
 
@@ -108,6 +132,8 @@ def load(
             " or environment for AWS SSM loader to work."
         )
 
+    env_list = build_env_list(obj, env or obj.current_env)
+
     for env_name in env_list:
         env_name = env_name.lower()
         path = f"/{project_prefix}/{env_name}"

tests/conftest.py

@@ -98,6 +98,63 @@ def basic_settings(
     )
 
 
+@pytest.fixture
+def basic_settings_disable_default_env(
+    tmp_path: Path, ssm_client: SSMClient
+) -> t.Generator[Path, None, None]:
+    """
+    Basic settings with environment layers, with default environment disabled
+    """
+
+    project_name = "basic"
+
+    data = f"""
+    [default]
+    SSM_PARAMETER_PROJECT_PREFIX_FOR_DYNACONF = '{project_name}'
+    SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF = false
+    PRODUCT_NAME = "foobar"
+    """
+
+    # We'll set some default values, just to ensure that our loader does not
+    # access them.
+    ssm_client.put_parameter(
+        Name=f"/{project_name}/default/products",
+        # Use @json cast for Dynaconf
+        # https://www.dynaconf.com/configuration/#auto_cast
+        Value="@json %s" % json.dumps({"plans": ["monthly", "yearly"]}),
+        Type="String",
+    )
+    ssm_client.put_parameter(
+        Name=f"/{project_name}/development/my_config_value",
+        Value="test123",
+        Type="String",
+    )
+    ssm_client.put_parameter(
+        Name=f"/{project_name}/production/database/password",
+        Value="password",
+        Type="SecureString",
+    )
+    ssm_client.put_parameter(
+        Name=f"/{project_name}/production/database/host",
+        Value="db.example.com",
+        Type="String",
+    )
+
+    settings_file = tmp_path / "settings.toml"
+    settings_file.write_text(data)
+
+    yield settings_file
+
+    ssm_client.delete_parameters(
+        Names=[
+            f"/{project_name}/production/database/host",
+            f"/{project_name}/production/database/password",
+            f"/{project_name}/development/my_config_value",
+            f"/{project_name}/default/products",
+        ]
+    )
+
+
 @pytest.fixture
 def settings_without_environments(
     tmp_path: Path, ssm_client: SSMClient

tests/test_loader.py

@@ -75,6 +75,49 @@ def test_basic_environment_based_settings(
         assert prod_settings.MY_CONFIG_VALUE == "test123"
 
 
+def test_basic_environment_based_settings_without_default_env(
+    basic_settings_disable_default_env: pathlib.Path,
+):
+    """
+    Test simple loading of configuration from AWS SSM on a per-environment
+    basis, omitting the default environment
+
+    """
+
+    dev_settings = Dynaconf(
+        environments=True,
+        FORCE_ENV_FOR_DYNACONF="development",
+        settings_file=str(basic_settings_disable_default_env.resolve()),
+        LOADERS_FOR_DYNACONF=[
+            "dynaconf_aws_loader.loader",
+        ],
+    )
+
+    assert dev_settings.current_env == "development"
+    assert dev_settings.SSM_LOAD_DEFAULT_ENV_FOR_DYNACONF is False
+
+    # This should not be set, since we are avoiding defaults
+    with pytest.raises(AttributeError):
+        dev_settings.PRODUCTS
+
+    prod_settings = Dynaconf(
+        environments=True,
+        FORCE_ENV_FOR_DYNACONF="production",
+        settings_file=str(basic_settings_disable_default_env.resolve()),
+        LOADERS_FOR_DYNACONF=[
+            "dynaconf_aws_loader.loader",
+        ],
+    )
+
+    assert prod_settings.current_env == "production"
+    assert prod_settings.SSM_PARAMETER_PROJECT_PREFIX_FOR_DYNACONF == "basic"
+
+    # Loaded from default env
+    # This should not be set, since we are avoiding defaults
+    with pytest.raises(AttributeError):
+        prod_settings.PRODUCTS
+
+
 def test_settings_with_no_environments(
     settings_without_environments: pathlib.Path,
 ):