From 414796edbe4dc936a99202c11bd5768a0655f3f5 Mon Sep 17 00:00:00 2001
From: James Nesbitt <jnesbitt@mirantis.com>
Date: Tue, 16 Mar 2021 13:00:36 -0400
Subject: [PATCH] refactored formatting

- pattern now uses double braces {{ }} for pattern identification
- patterns now use "plugin::" to pass arguments to a formatter plugin
- plugins now receive "plugin::target"" target strings instead of full data
- single pass instead of a pass per plugin (centralized patterns)
- unlimited default ? options
- "special::None" , [], {} "null" options

Signed-off-by: James Nesbitt <jnesbitt@mirantis.com>
---
 configerus/__init__.py               |   1 -
 configerus/config.py                 |  20 +-
 configerus/contrib/deep/README.md    |   0
 configerus/contrib/deep/__init__.py  |  21 --
 configerus/contrib/deep/format.py    |  41 ----
 configerus/contrib/files/__init__.py |   2 +-
 configerus/contrib/files/format.py   |  97 ++++------
 configerus/contrib/get/format.py     |  84 +++-----
 configerus/format.py                 | 277 +++++++++++++++++++++++++++
 configerus/plugin.py                 |   4 +-
 configerus/test/test_5_templating.py | 122 ++++++++----
 setup.cfg                            |   4 +-
 12 files changed, 445 insertions(+), 228 deletions(-)
 delete mode 100644 configerus/contrib/deep/README.md
 delete mode 100644 configerus/contrib/deep/__init__.py
 delete mode 100644 configerus/contrib/deep/format.py
 create mode 100644 configerus/format.py

diff --git a/configerus/__init__.py b/configerus/__init__.py
index f9f89da..fb91b20 100644
--- a/configerus/__init__.py
+++ b/configerus/__init__.py
@@ -6,7 +6,6 @@
 
 
 CONFIGERUS_BOOSTRAP_DEFAULT = [
-    'deep',
     'get',
     'files'
 ]
diff --git a/configerus/config.py b/configerus/config.py
index 2474c35..ae0858a 100644
--- a/configerus/config.py
+++ b/configerus/config.py
@@ -1,13 +1,14 @@
 import logging
 from importlib import metadata
 import copy
-from typing import Any
+from typing import Any, Dict, List
 
 from .plugin import Factory, Type
 from .instances import PluginInstances
 from .shared import tree_merge
 from .loaded import Loaded
 from .validator import ValidationError
+from .format import Formatter
 
 logger = logging.getLogger('configerus:config')
 
@@ -271,6 +272,9 @@ def add_formatter(self, plugin_id: str, instance_id: str = '', priority: int = P
     def format(self, data, default_label: Any, validator: str = ""):
         """ Format some data using the config object formatters
 
+        Parameters:
+        -----------
+
         data (Any): primitive data that should be formatted. The data will be
             passed to the formatter plugins in descending priority order.
 
@@ -286,12 +290,20 @@ def format(self, data, default_label: Any, validator: str = ""):
 
             if empty/None then no validation is performed.
 
+        Raises:
+        -------
+
+        Will throw a ValueError if your data contains a format tag that cannot
+        be interpreted.
+        Will raise a KeyError if your data contains a format tag with a bad key
+        for action, or for a value as interpreted by the plugin.
         """
-        for formatter in self.plugins.get_plugins(type=Type.FORMATTER, exception_if_missing=False):
-            data = formatter.format(data, default_label)
+
+        formatter = Formatter(self)
+        data = formatter.format(data=data, default_label=default_label)
 
         if validator:
-            self.validate(self.loaded[label].data, validator)
+            self.validate(data, validator)
 
         return data
 
diff --git a/configerus/contrib/deep/README.md b/configerus/contrib/deep/README.md
deleted file mode 100644
index e69de29..0000000
diff --git a/configerus/contrib/deep/__init__.py b/configerus/contrib/deep/__init__.py
deleted file mode 100644
index d945c47..0000000
--- a/configerus/contrib/deep/__init__.py
+++ /dev/null
@@ -1,21 +0,0 @@
-
-from configerus.config import Config
-from configerus.plugin import FormatFactory
-
-from .format import ConfigFormatDeepPlugin
-
-PLUGIN_ID_FORMAT_DEEP = 'deep'
-PLUGIN_PRIORITY_FORMAT_DEEP = 90
-
-""" Format plugin_id for the configerus deep format plugin """
-
-
-@FormatFactory(plugin_id=PLUGIN_ID_FORMAT_DEEP)
-def plugin_factory_format_deep(config: Config, instance_id: str = ''):
-    """ create an format plugin which recursiveley formats data """
-    return ConfigFormatDeepPlugin(config, instance_id)
-
-
-def configerus_bootstrap(config: Config):
-    """ Bootstrap a config object by adding our formatter """
-    config.add_formatter(PLUGIN_ID_FORMAT_DEEP, priority=PLUGIN_PRIORITY_FORMAT_DEEP)
diff --git a/configerus/contrib/deep/format.py b/configerus/contrib/deep/format.py
deleted file mode 100644
index 11b35d1..0000000
--- a/configerus/contrib/deep/format.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from typing import List, Dict, Any
-
-from configerus.config import Config
-
-
-class ConfigFormatDeepPlugin():
-    """   """
-
-    def __init__(self, config: Config, instance_id: str):
-        """  """
-        self.config = config
-        self.instance_id = instance_id
-
-    def format(self, target, default_source: str):
-        """ Deep formatter for formatting that reruns formatting across iterables
-
-        this formatter just reruns formatting on elements of primitive iterables.
-
-        """
-        # try to iterate across the target
-        # There is probably a more `python` approach that would cover more
-        # iterables, as long as we can re-assign
-        if isinstance(target, Dict):
-            return self.format_dict(target, default_source)
-        elif isinstance(target, List):
-            return self.format_list(target, default_source)
-
-        # anything we can't iterate through we just return
-        return target
-
-    def format_list(self, target: List[Any], default_source: str):
-        """ Search a List for strings to format @see format_string """
-        for index, value in enumerate(target):
-            target[index] = self.config.format(value, default_source)
-        return target
-
-    def format_dict(self, target: Dict[str, Any], default_source: str):
-        """ Search a Dict for strings to format @see format_string """
-        for key, value in target.items():
-            target[key] = self.config.format(value, default_source)
-        return target
diff --git a/configerus/contrib/files/__init__.py b/configerus/contrib/files/__init__.py
index f671035..60d1870 100644
--- a/configerus/contrib/files/__init__.py
+++ b/configerus/contrib/files/__init__.py
@@ -19,7 +19,7 @@ def plugin_factory_configsource_path(config: Config, instance_id: str = ''):
     return ConfigSourcePathPlugin(config, instance_id)
 
 
-PLUGIN_ID_FORMAT_FILE = 'configerus.plugin.formatter.file'
+PLUGIN_ID_FORMAT_FILE = 'file'
 """ Format plugin_id for the configerus filepath format plugin """
 
 
diff --git a/configerus/contrib/files/format.py b/configerus/contrib/files/format.py
index 3bb5196..7440de5 100644
--- a/configerus/contrib/files/format.py
+++ b/configerus/contrib/files/format.py
@@ -7,14 +7,14 @@
 
 from configerus.config import Config
 
-FILES_FORMAT_MATCH_PATTERN = r'(\[(file\:)(?P<file>(\~?\/?\w+\/)*\w*(\.\w+)?)\])'
+FILES_FORMAT_MATCH_PATTERN = r'(?P<file>(\~?\/?\w+\/)*\w*(\.\w+)?)'
 """ A regex pattern to identify files that should be embedded """
 
 logger = logging.getLogger('configerus.contrib.files:format')
 
 
 class ConfigFormatFilePlugin:
-    """   """
+    """ Format a key by returning the contents of a file """
 
     def __init__(self, config: Config, instance_id: str):
         """  """
@@ -23,51 +23,18 @@ def __init__(self, config: Config, instance_id: str):
 
         self.pattern = re.compile(FILES_FORMAT_MATCH_PATTERN)
 
-    def format(self, target, default_source: str):
+    def format(self, key, default_label: str):
         """ Format a string by substituting config values
 
         Parameters
         ----------
 
-        target: a string that should be formatted. If not a string then no
-            formatting is performed
+        key: a string that should gies instructions to the formatter on how to
+            create a format replacements
 
-        default_source : if format/replace patterns don't have a source defined
+        default_label : if format/replace patterns don't have a source defined
             then this is used as a source.
 
-        """
-        if not isinstance(target, str):
-            return target
-
-        # if the entire target is the match, then replace whatever type we get
-        # out of the config .get() call
-        match = self.pattern.fullmatch(target)
-        if match:
-            return self._get(match, return_only_string=False)
-
-        # search through the target replacing any found matches with
-        start = 0
-        match = self.pattern.search(target, start)
-        while match:
-            rep = str(self._get(match))
-            target = target[:match.start()] + rep + target[match.end()]
-            start = start + len(rep)
-            match = self.pattern.search(target, start)
-
-        return target
-
-    def _get(self, match, return_only_string: bool = True):
-        """ find a file match and return the file contents
-
-        Parameters
-        ----------
-
-        match (re Match) a regex match which we use to determine file path
-
-        return_only_string (bool) optional indictor that the function is
-            expecting only a string return.  Knowing this means that we can
-            skip unmarshalling file contents which could be expensive.
-
         Raises
         ------
 
@@ -84,40 +51,42 @@ def _get(self, match, return_only_string: bool = True):
         unmarshalled json/yml file or string contents of the file
 
         """
+
+        match = self.pattern.fullmatch(key.strip())
+        if not match:
+            raise KeyError("Could not interpret Format action target '{}'".format(key))
+
         file = match.group('file')
-        extension = ''
+        """ path to the file to return as a replacement """
 
+        extension = ''
+        """ file extension, used to make decisions about parsing/unmarshalling """
         file_split = os.path.splitext(file)
         if len(file_split) > 0:
             extension = file_split[1].lower()
 
         try:
-            with open(file) as file_o:
-                if not return_only_string:
-                    # try to parse/unmarshall a file instead of just returing it
-
-                    if extension == ".json":
-                        try:
-                            data = json.load(file_o)
-                        except json.decoder.JSONDecodeError as e:
-                            raise ValueError(
-                                "Failed to parse one of the config files '{}': {}".format(
-                                    os.path.join(
-                                        self.path, file), e))
-                        return data
-
-                    elif extension == ".yml" or extension == ".yaml":
-                        try:
-                            data = yaml.load(file_o, Loader=yaml.FullLoader)
-                        except yaml.YAMLError as e:
-                            raise ValueError(
-                                "Failed to parse one of the config files '{}': {}".format(
-                                    os.path.join(
-                                        self.path, file), e))
-                        return data
+            with open(file) as fo:
+                if extension == ".json":
+                    try:
+                        return json.load(fo)
+                    except json.decoder.JSONDecodeError as e:
+                        raise ValueError(
+                            "Failed to parse one of the config files '{}': {}".format(
+                                os.path.join(
+                                    self.path, file), e))
+
+                elif extension == ".yml" or extension == ".yaml":
+                    try:
+                        return yaml.load(fo, Loader=yaml.FullLoader)
+                    except yaml.YAMLError as e:
+                        raise ValueError(
+                            "Failed to parse one of the config files '{}': {}".format(
+                                os.path.join(
+                                    self.path, file), e))
 
                 # return file contents as a string (above parsing didn't happen)
-                return file_o.read()
+                return fo.read()
 
         except FileNotFoundError as e:
             raise KeyError("Could not embed file as config as file could not be found: {}".format(file))
diff --git a/configerus/contrib/get/format.py b/configerus/contrib/get/format.py
index 2ea8412..ee27172 100644
--- a/configerus/contrib/get/format.py
+++ b/configerus/contrib/get/format.py
@@ -5,27 +5,12 @@
 
 logger = logging.getLogger('configerus.contrib.get:formatter')
 
-CONFIG_DEFAULT_MATCH_PATTERN = r'\{((?P<source>\w+):)?(?P<key>[-\w.]+)(\?(?P<default>[^\}]+))?\}'
-""" Default regex pattern used to string template, which needs to identify
-keys that can be used for replacement. Needs to have a named group for
-key, and can have named group for source and default
-
-The above regex pattern resolves for three named groups: source, key and default:
-
-the {first?check} value          : key=first, default=check
-a {common.dot.notation} test     : key=commond.dot.notation
-a {label:dot.notation.test} here : source=label, key=dot.notation.test
-
-so:
-1. all replacements are wrapped in "{}"
-2. an optional "source:" group tells config to look in a specifig label
-3. a "?default" group allows a default (untemplated) value to be used if the
-   key cannot be found
-"""
+CONFIG_DEFAULT_MATCH_PATTERN = r'((?P<label>\w+)\:)?(?P<key>[-\w.]+)'
+""" Default regex pattern used detect format targets for config retrieval """
 
 
 class ConfigFormatGetPlugin():
-    """   """
+    """ return a format replacement by retriving data from config """
 
     def __init__(self, config: Config, instance_id: str):
         """  """
@@ -34,51 +19,38 @@ def __init__(self, config: Config, instance_id: str):
 
         self.pattern = re.compile(CONFIG_DEFAULT_MATCH_PATTERN)
 
-    def format(self, target, default_source: str):
-        """ Format a string by substituting config values
+    def format(self, key, default_label: str):
+        """ Format a key by returning config values
 
-        target: a string that should be formatted. If not a string then no
-            formatting is performed
+        Parameters:
+        -----------
 
-        default_source : if format/replace patterns don't have a source defined
-            then this is used as a source.
+        key: a string that should gies instructions to the formatter on how to
+            create a format replacements
 
-        """
+        default_label : if format/replace patterns don't have a label defined
+            then this is used as a label.
 
-        if not isinstance(target, str):
-            return target
+        Raises:
+        -------
 
-        # if the entire target is the match, then replace whatever type we get
-        # out of the config .get() call
-        match = self.pattern.fullmatch(target)
-        if match:
-            return self._get(match, default_source)
+        KeyError if the source or the key could not be found.
 
-        # search through the target replacing any found matches with
-        start = 0
-        match = self.pattern.search(target, start)
-        while match:
-            rep = str(self._get(match, default_source))
-            target = target[:match.start()] + rep + (target[match.end():] if len(target) > match.end() else '')
-            start = start + len(rep)
-            match = self.pattern.search(target, start)
+        """
 
-        return target
+        match = self.pattern.fullmatch(key.strip())
+        if not match:
+            raise KeyError("Could not interpret Format action key '{}'".format(key))
 
-    def _get(self, match, default_source: str):
         """ from an re.match get data from config """
+        label = match.group('label')
         key = match.group('key')
-        source = match.group('source')
-        default = match.group('default')
-
-        if source is None:
-            source = default_source
-        source_config = self.config.load(source)
-        try:
-            return source_config.get(key, exception_if_missing=True)
-        except KeyError as e:
-            if default is not None:
-                return default
-            else:
-                # if a template string wasn't found then exception
-                raise e
+
+        if label is None:
+            label = default_label
+
+        if label is None:
+            label = default_label
+
+        loaded = self.config.load(label)
+        return loaded.get(key, exception_if_missing=True)
diff --git a/configerus/format.py b/configerus/format.py
new file mode 100644
index 0000000..81d0193
--- /dev/null
+++ b/configerus/format.py
@@ -0,0 +1,277 @@
+"""
+
+Config formatting code
+
+@NOTE this code has gone through three major iterations, starting with a single
+  function, expanded to be module, expanded further with regex, expanded again
+  with centralized processing using regex, and finally this parser.  Some
+  of the comments may be out of date.
+
+"""
+import json
+import logging
+from typing import Any, Dict, List
+from enum import Enum
+
+from .plugin import Type
+
+logger = logging.getLogger('configerus.format')
+
+DEFAULT_FORMATTER_PLUGIN_ID = 'get'
+""" this plugin is used for formatting when a target doesn't specify a plugin """
+FORMATTER_PLUGIN_ID_PASSTHROUGH = 'value'
+""" this config formatter plugin id just passes the target through """
+FORMATTER_PLUGIN_ID_INTEPRET = 'special'
+""" this config formatter plugin id to intepret the value """
+
+FORMATTER_SPECIAL_VALUES = {
+    'Null': None,
+    'None': None,
+    '{}': {},
+    '[]': [],
+    'empty': ''
+}
+""" A map of interpation values for the interpet target """
+
+
+class ParserState(Enum):
+    """ what the parser currently thinks it is looking at """
+    OUTSIDE = 0  # we are outside of a matching pattern
+    START = 1    # we are inside a matching pattern, either a plugin or a target
+    KEY = 2      # we know we are in a target key name
+    IGNORE = 3   # we are ignoring the current context
+
+
+class ParserStackNode:
+    """ We parse a string using a language stack.  This is a node in the stack """
+
+    def __init__(self, parent, state=ParserState.OUTSIDE):
+        """
+
+        Parameters:
+        -----------
+
+        parent (node) : a stack needs to know its parent for popping
+
+        state (State) : every node corresponds to a single state
+
+        """
+        self.parent = parent
+        self.state = state
+
+        self.plugin = None
+        self.key = None
+        """ some nodes track information collected for interpretation """
+
+        self.parts = []
+        """ list of subject parts that are a part of this node """
+
+    def push(self, state):
+        """ push a state child and return it """
+        return ParserStackNode(parent=self, state=state)
+
+    def pop(self):
+        """ get the node parent """
+        return self.parent
+
+    def __str__(self):
+        """ write to a string for debugging """
+        return "{}:{}[{}][p:{},k:{}]".format(self.parent, self.state.value, self.join(), self.plugin, self.key)
+
+    def join(self):
+        """ join the parts back together
+
+        Returns:
+
+        If only 1 item is in the node, then it is returned raw, otherwise we return
+        a string joing of all of the parts.
+
+        @NOTE this can result in false identities when comparing dicts as strings
+          to things like json, as Python uses single quotes, but JSON uses double.
+
+        """
+        if len(self.parts) == 1:
+            return self.parts[0]
+
+        return ''.join(["{}".format(part) for part in self.parts])
+
+    def append(self, part):
+        """ append a part to the node """
+        self.parts.append(part)
+
+
+class Formatter:
+
+    START_MATCH = '{{'
+    STOP_MATCH = '}}'
+    PLUGINEND_MATCH = '::'
+    DEFAULT_MATCH = '?'
+
+    def __init__(self, config, default_plugin_for_target: str = DEFAULT_FORMATTER_PLUGIN_ID,
+                 default_plugin_for_default: str = FORMATTER_PLUGIN_ID_PASSTHROUGH, fail_on: List[str] = []):
+        self.config = config
+        self.default_plugin_for_target = default_plugin_for_target
+        self.default_plugin_for_default = default_plugin_for_default
+
+        self.fail_on = fail_on
+        """ For debugging, always fail on these values """
+
+    def format(self, data: Any, default_label: str):
+        """ perform a deep format """
+        return self.recursive_format(data=data, default_label=default_label)
+
+    def recursive_format(self, data: Any, default_label: str):
+        """ recursive deep formatting """
+        # try to iterate across the target
+        # There is probably a more `python` approach that would cover more
+        # iterables, as long as we can re-assign
+        if isinstance(data, list):
+            for index, value in enumerate(data):
+                data[index] = self.recursive_format(data=value, default_label=default_label)
+        if isinstance(data, dict):
+            for index, value in data.items():
+                data[index] = self.recursive_format(data=value, default_label=default_label)
+
+        # strings get some searching for format actions
+        elif isinstance(data, str):
+            # if the entire target is the match, then replace whatever type we get
+            # out of the config .get() call
+            data = self.format_string(subject=data, default_label=default_label)
+
+        else:
+            # all sorts of primitives and custom objects are just ignored.
+            pass
+
+        return data
+
+    def format_string(self, subject: str, default_label: str):
+        """ format subject string, looking for and processing any formatting tags in the subject """
+
+        subject_tokenized = subject
+        for token in [
+            self.START_MATCH,
+            self.STOP_MATCH,
+            self.PLUGINEND_MATCH,
+            self.DEFAULT_MATCH
+        ]:
+            subject_tokenized = subject_tokenized.replace(token, '@@@{}@@@'.format(token))
+        subject_parts = subject_tokenized.split('@@@')
+        """ subject string split/exploded into words """
+
+        stack = ParserStackNode(parent=None, state=ParserState.OUTSIDE)
+        # print ("PRE: {} => {}".format(subject, subject_parts))
+
+        for part in subject_parts:
+
+            if part == '':
+                # print("SKIP EMPTY -> {}".format(part))
+                continue
+
+            if part in [self.START_MATCH]:
+                stack = stack.push(ParserState.START)
+                stack.plugin = self.default_plugin_for_target
+                # print("START: '{}' :: {}".format(part, stack))
+
+            elif part in [self.STOP_MATCH]:
+
+                if stack.state is ParserState.START:
+                    # we had started a tag, we had collected only a tag key
+                    # and now we see s stop tag, so assume that is all we get
+                    key = stack.join()
+                    stack = stack.pop()
+                    plugin = stack.plugin if stack.plugin is not None else self.default_plugin_for_target
+
+                    replace = self.find_replacement(plugin, key, default_label=default_label)
+                    stack.append(replace)
+                    # print("STOP: (start) '{}' :: {}".format(part, stack))
+
+                elif stack.state is ParserState.KEY:
+                    # we had started a tag, and had collected
+                    key = stack.join()
+                    stack = stack.pop()
+                    plugin = stack.plugin if stack.plugin is not None else self.default_plugin_for_target
+
+                    replace = self.find_replacement(plugin, key, default_label=default_label)
+                    stack.append(replace)
+                    # print("STOP: (key) '{}' :: {}".format(part, stack))
+
+                elif stack.state is ParserState.IGNORE:
+                    # we are ignoring this stuff
+                    stack = stack.pop()
+                    # print("STOP: (ignore) '{}' :: {}".format(part, stack))
+
+                else:
+                    # print("STOP: (what is this) '{}' :: {}".format(part, stack))
+                    raise ValueError("Unexpected tag end : {}".format(stack))
+
+            elif stack.state in [ParserState.IGNORE]:
+                # If we are in an ignore state, only track tag start and stop
+                # so skip everything below
+                # print("IGNORING: {}".format(part))
+                pass
+
+            elif part in [self.PLUGINEND_MATCH]:
+                plugin = stack.join()
+                stack = stack.pop()
+
+                stack.plugin = plugin
+                stack = stack.push(ParserState.KEY)
+                # print("PLUGIN: '{}' :: {}".format(part, stack))
+
+            elif part in [self.DEFAULT_MATCH]:
+                key = stack.join()
+                stack = stack.pop()
+
+                stack.key = key
+                plugin = stack.plugin if stack.plugin is not None else self.default_plugin_for_target
+
+                try:
+                    replace = self.find_replacement(plugin, key, default_label=default_label)
+                    stack.append(replace)
+
+                    # if the replacement retrieval did not throw an exception then ignore the rest of the tag
+                    stack = stack.push(ParserState.IGNORE)
+                    # print("DEFAULT: IGNORING DEFAULT '{}' :: {}".format(part, stack))
+                except KeyError:
+                    # if retrieving a replacement raised an exception then we switch into DEFAULT mode
+                    stack.plugin = self.default_plugin_for_default
+                    stack = stack.push(ParserState.START)
+                    # print("DEFAULT: PROCESSING DEFAULT '{}' :: {}".format(part, stack))
+
+            else:
+                stack.append(part)
+                # print("INC: {}".format(part))
+
+        # print("DONE:  {}".format(stack))
+        return stack.join()
+
+    def find_replacement(self, plugin: str, key: str, default_label: str):
+        """ use a plugin and key to find a replacement """
+
+        if plugin == FORMATTER_PLUGIN_ID_PASSTHROUGH:
+            # In this case we are just passing through.  This is used primarily by the
+            # default processor as it has little value from real source.
+            return key
+
+        if plugin == FORMATTER_PLUGIN_ID_INTEPRET:
+            # In this case interpret the value as something special
+            return self.interpret(key)
+
+        elif self.config.plugins.has_plugin(instance_id=plugin, type=Type.FORMATTER):
+            plugin = self.config.plugins.get_plugin(instance_id=plugin, type=Type.FORMATTER)
+        elif self.config.plugins.has_plugin(plugin_id=plugin, type=Type.FORMATTER):
+            plugin = self.config.plugins.get_plugin(plugin_id=plugin, type=Type.FORMATTER)
+        else:
+            logger.error([instance.instance_id for instance in self.config.plugins.instances])
+            raise RuntimeError(
+                "Unknown format plugin '{}::{}', no such plugin has been added.".format(
+                    Type.FORMATTER.value, plugin))
+
+        return plugin.format(key=key, default_label=default_label)
+
+    def interpret(self, key: str):
+        """ interpret that value to mean something special """
+        try:
+            return FORMATTER_SPECIAL_VALUES[key.strip()]
+        except Exception:
+            raise ValueError("Could not interpret special key '{}'".format(key))
diff --git a/configerus/plugin.py b/configerus/plugin.py
index 542527f..29dce70 100644
--- a/configerus/plugin.py
+++ b/configerus/plugin.py
@@ -71,7 +71,9 @@ def create(self, config, instance_id: str):
                 "Configerus Plugin instance '{}:{}' has not been registered.".format(
                     self.type.value, self.plugin_id))
         except Exception as e:
-            raise Exception("Could not create Plugin instance '{}:{}' as the plugin factory produced an exception".format(self.type.value, self.plugin_id)) from e
+            raise Exception(
+                "Could not create Plugin instance '{}:{}' as the plugin factory produced an exception".format(
+                    self.type.value, self.plugin_id)) from e
 
         return factory(config=config, instance_id=instance_id)
 
diff --git a/configerus/test/test_5_templating.py b/configerus/test/test_5_templating.py
index daf18bb..49cc7f5 100644
--- a/configerus/test/test_5_templating.py
+++ b/configerus/test/test_5_templating.py
@@ -61,7 +61,9 @@
         'data': {
             'config': {
                 '10': "{2}",
-                '11': "__{2}__"
+                '11': "__{{2}}__",
+                '12': "__{{2}}",
+                '13': "{{2}}__"
             }
         }
     },
@@ -71,10 +73,10 @@
         'type': PLUGIN_ID_SOURCE_DICT,
         'data': {
             'config': {
-                '20': "{does.not.exist?default}",
-                '21': "{does.not.exist}",
-                '22': "{variables:does.not.exist?megadefault}",
-                '23': "{variables:three?default}",
+                '20': "{{does.not.exist?default}}",
+                '21': "{{does.not.exist}}",
+                '22': "{{variables:does.not.exist?megadefault}}",
+                '23': "{{variables:three?default}}",
             }
         }
     },
@@ -95,11 +97,11 @@
         'data': {
             'config': {
                 '40': "deep 40",
-                '41': "{40}",
+                '41': "{{40}}",
                 '42': {
-                    '1': "{40}",
+                    '1': "{{40}}",
                     '2': {
-                        '1': "{40}"
+                        '1': "{{40}}"
                     }
                 }
             }
@@ -150,47 +152,60 @@ def tearDownClass(cls):
     def test_format_sane(self):
         """ simple template replacing using format """
 
-        self.assertEqual(self.config.format('{1}', 'config'), "first 1")
-        self.assertEqual(self.loaded_config.format('{1}'), "first 1")
+        self.assertEqual(self.config.format('{{get::1}}', 'config'), "first 1")
+        self.assertEqual(self.loaded_config.format('{{get::1}}'), "first 1")
+        """ Simple formatting at the config and loadedconfig level """
+
+    def test_format_passthrough(self):
+        """ simple template replacing using passthrough """
+
+        self.assertEqual(self.config.format('{{value::myvalue}}', 'config'), "myvalue")
+        """ passthrough is a mechanism used internally, as a defult plugin for embedded """
+
+    def test_format_sane(self):
+        """ simple template replacing using format relying on default plugin """
+
+        self.assertEqual(self.config.format('{{1}}', 'config'), "first 1")
+        self.assertEqual(self.loaded_config.format('{{1}}'), "first 1")
         """ Simple formatting at the config and loadedconfig level """
 
     def test_format_dot_notation(self):
         """ test that format catches dot notation """
 
-        self.assertEqual(self.config.format('{2.1}', 'config'), "first 2.1")
-        self.assertEqual(self.config.format('{3.2.1}', 'config'), "first 3.2.1")
+        self.assertEqual(self.config.format('{{2.1}}', 'config'), "first 2.1")
+        self.assertEqual(self.config.format('{{3.2.1}}', 'config'), "first 3.2.1")
         """ Simple dot notation descending format """
 
     def test_format_embedding_and_casting(self):
         """ simple template replacing using format """
 
-        self.assertEqual(self.config.format('12345{1}67890', 'config'), "12345first 167890")
-        self.assertEqual(self.config.format('1 23 45 {1}6 78 90', 'config'), "1 23 45 first 16 78 90")
+        self.assertEqual(self.config.format('12345{{1}}67890', 'config'), "12345first 167890")
+        self.assertEqual(self.config.format('1 23 45 {{1}}6 78 90', 'config'), "1 23 45 first 16 78 90")
         """ format replacements in the middle of a string """
 
-        self.assertEqual(self.config.format('{2}', 'config'), {'1': "first 2.1"})
-        self.assertEqual(self.config.format(' {2} ', 'config'), " {'1': 'first 2.1'} ")
+        self.assertEqual(self.config.format('{{2}}', 'config'), {'1': 'first 2.1'})
+        self.assertEqual(self.config.format(' {{2}} ', 'config'), " {'1': 'first 2.1'} ")
         """ format should string cast only not full matches """
 
     def test_format_with_sources(self):
         """ test that sources work with format """
 
-        variables_two = self.config.format("{two}", 'variables')
+        variables_two = self.config.format("{{two}}", 'variables')
         self.assertEqual(variables_two, "first two")
-        self.assertEqual(self.config.format("{variables:two}", 'config'), variables_two)
-        self.assertEqual(self.config.format("{variables:two}", 'variables'), variables_two)
+        self.assertEqual(self.config.format("{{variables:two}}", 'config'), variables_two)
+        self.assertEqual(self.config.format("{{variables:two}}", 'variables'), variables_two)
         """ specifying a source should work """
 
     def test_format_with_defaults(self):
         """ test the defaulting with formatting """
 
-        self.assertEqual(self.config.format("{does.not.exist?default1}", 'config'), 'default1')
-        self.assertEqual(self.config.format("{variables:does.not.exist?default2}", 'config'), 'default2')
+        self.assertEqual(self.config.format("{{does.not.exist?default1}}", 'config'), 'default1')
+        self.assertEqual(self.config.format("{{variables:does.not.exist?default2}}", 'config'), 'default2')
         """ defaults should apply """
 
-        variables_two = self.config.format("{two}", 'variables')
+        variables_two = self.config.format("{{two}}", 'variables')
         self.assertEqual(variables_two, "first two")
-        self.assertEqual(self.config.format("{variables:two?default}", 'config'), variables_two)
+        self.assertEqual(self.config.format('{{variables:two?default}}', 'config'), variables_two)
         """ defaults should not overwrite values that can be found """
 
     def test_get_formatted(self):
@@ -198,29 +213,29 @@ def test_get_formatted(self):
 
         config_one = self.loaded_config.get('1')
         self.assertEqual(config_one, "first 1")
-        self.assertEqual(self.config.format('{1}', 'config'), config_one)
+        self.assertEqual(self.config.format('{{1}}', 'config'), config_one)
         """ get should mimic format behaviour, which we tested above """
 
         variables_three = self.loaded_variables.get('three')
         self.assertEqual(variables_three, 3)
-        self.assertEqual(self.loaded_config.format("{variables:three}"), variables_three)
-        self.assertEqual(self.loaded_config.format(" {variables:three} "), " {} ".format(variables_three))
+        self.assertEqual(self.loaded_config.format("{{variables:three}}"), variables_three)
+        self.assertEqual(self.loaded_config.format(" {{variables:three}} "), " {} ".format(variables_three))
         """ test templating for different sources """
 
         self.assertEqual(self.loaded_config.get('20'), "default")
         self.assertEqual(self.loaded_config.get('22'), "megadefault")
         self.assertEqual(self.loaded_config.get('23'), variables_three)
-        """ test the templateing on get() """
+        """ test the templating on get() """
 
     def test_missing_raises_exceptions(self):
         """ missing values should raise exceptions """
 
         with self.assertRaises(KeyError):
-            self.config.format("{does.not.exist}", 'config')
+            self.config.format("{{does.not.exist}}", 'config')
             """ test template fails on missing keys in config format """
 
         with self.assertRaises(Exception):
-            self.loaded_config.format("{does.not.exist}")
+            self.loaded_config.format("{{does.not.exist}}")
             """ test template fails on missing keys in loaded format """
 
         with self.assertRaises(Exception):
@@ -245,7 +260,7 @@ def test_file_templating(self):
 
         self.assertIsNotNone(replace_json)
 
-        replace_path = replace_config.format('{paths:replace}')
+        replace_path = replace_config.format('{{paths:replace}}')
         self.assertIsNotNone(replace_path)
 
         replace_full_path = os.path.join(replace_path, 'replace.json')
@@ -254,10 +269,45 @@ def test_file_templating(self):
         self.assertIsNotNone(replace_json)
 
         # confirm that a file replace full match works
-        self.assertEqual(replace_json, replace_config.format('[file:' + replace_full_path + ']'))
-        self.assertEqual(
-            ' {} '.format(
-                json.dumps(replace_json)), replace_config.format(
-                ' [file:' + replace_full_path + '] '))
+        self.assertEqual(replace_json, replace_config.format('{{file::' + replace_full_path + '}}'))
+
+        # This test works but it writes using single quotes which don't match the direct json dump
+        # self.assertEqual(
+        #     ' {} '.format(json.dumps(replace_json)),
+        #     replace_config.format(' {{file::' + replace_full_path + '}} ')
+        # )
         # confirm mixed templating
-        self.assertEqual(replace_json, replace_config.format('[file:{paths:replace}/replace.json]'))
+        self.assertEqual(replace_json, replace_config.format('{{file::{{paths:replace}}/replace.json}}'))
+        self.assertEqual('default', replace_config.format('{{file::{{paths:replace}}/no_such_file.json?default}}'))
+
+    def test_crazy_stuff(self):
+        """ test some crazy templating ideas """
+        config_fourty = self.loaded_config.get('40')
+
+        self.assertEqual(self.loaded_config.format(' A{{40}} Z'), ' A{} Z'.format(config_fourty))
+
+        self.assertEqual(self.loaded_config.format('A{{value::target}}Z'), 'AtargetZ')
+
+        self.assertEqual(self.loaded_config.format('A{{get::40?default}}Z'), 'A{}Z'.format(config_fourty))
+
+        self.assertEqual(self.loaded_config.format('A{{fail?default}}Z'), 'AdefaultZ')
+
+        self.assertEqual(
+            self.loaded_config.format('A{{get::{{value::40}}?value::custom}}Z'),
+            'A{}Z'.format(config_fourty))
+
+        self.assertEqual(self.loaded_config.format('A{{value::{{value::{{value::test}}}}?value::custom}}Z'), 'AtestZ')
+
+        self.assertEqual(self.loaded_config.format('A{{get::fail?default}}Z'), 'AdefaultZ')
+
+        self.assertEqual(self.loaded_config.format('A{{fail?get::fail?get::fail?default}}Z'), 'AdefaultZ')
+
+    def test_special_formats(self):
+        """ test some of the special format options """
+
+        self.assertIsNone(self.loaded_config.format('{{special::None}}'))
+        self.assertIsNone(self.loaded_config.format('{{fail?special::None}}'))
+        self.assertIsNone(self.loaded_config.format('{{fail?get::fail?special::None}}'))
+
+        self.assertEqual(self.loaded_config.format('{{special::[]}}'), [])
+        self.assertEqual(self.loaded_config.format('{{special::{} }}'), {})
diff --git a/setup.cfg b/setup.cfg
index 79e7994..41a9826 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = configerus
-version = 1.0.0
+version = 2.0.0
 description = Plugin-Based configuration manager
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -30,7 +30,6 @@ project_urls =
 packages =
     configerus
     configerus.contrib.env
-    configerus.contrib.deep
     configerus.contrib.dict
     configerus.contrib.files
     configerus.contrib.get
@@ -43,7 +42,6 @@ install_requires =
 
 [options.entry_points]
 configerus.bootstrap =
-    deep        = configerus.contrib.deep:configerus_bootstrap
     env         = configerus.contrib.env:configerus_bootstrap
     dict        = configerus.contrib.dict:configerus_bootstrap
     files       = configerus.contrib.files:configerus_bootstrap