diff --git a/Rake/Gemfile/molecule/config.py b/Rake/Gemfile/molecule/config.py index ac3f9c0b..366c8c1f 100644 --- a/Rake/Gemfile/molecule/config.py +++ b/Rake/Gemfile/molecule/config.py @@ -21,6 +21,7 @@ import os import anyconfig +import six from molecule import interpolation from molecule import logger @@ -53,6 +54,15 @@ MERGE_STRATEGY = anyconfig.MS_DICTS +# https://stackoverflow.com/questions/16017397/injecting-function-call-after-init-with-decorator # noqa +class NewInitCaller(type): + def __call__(cls, *args, **kwargs): + obj = type.__call__(cls, *args, **kwargs) + obj.after_init() + return obj + + +@six.add_metaclass(NewInitCaller) class Config(object): """ Molecule searches the current directory for `molecule.yml` files by @@ -90,9 +100,11 @@ def __init__(self, self.args = args self.command_args = command_args self.ansible_args = ansible_args - self.config = self._combine() + self.config = self._get_config() self._action = None + def after_init(self): + self.config = self._reget_config() self._validate() @property @@ -245,7 +257,29 @@ def _get_driver_name(self): return driver_name - def _combine(self): + def _get_config(self): + """ + Perform a prioritized recursive merge of config files, and returns + a new dict. Prior to merging the config files are interpolated with + environment variables. + + :return: dict + """ + return self._combine(keep_string='MOLECULE_') + + def _reget_config(self): + """ + Perform the same prioritized recursive merge from `get_config`, this + time, interpolating the `keep_string` left behind in the original + `get_config` call. This is probably __very__ bad. + + :return: dict + """ + env = util.merge_dicts(os.environ.copy(), self.env) + + return self._combine(env=env) + + def _combine(self, env=os.environ, keep_string=None): """ Perform a prioritized recursive merge of config files, and returns a new dict. Prior to merging the config files are interpolated with @@ -263,23 +297,24 @@ def _combine(self): if base_config: if os.path.exists(base_config): with util.open_file(base_config) as stream: - interpolated_config = self._interpolate(stream.read()) + interpolated_config = self._interpolate( + stream.read(), env, keep_string) defaults = util.merge_dicts( defaults, util.safe_load(interpolated_config)) with util.open_file(self.molecule_file) as stream: - interpolated_config = self._interpolate(stream.read()) + interpolated_config = self._interpolate(stream.read(), env, + keep_string) defaults = util.merge_dicts(defaults, util.safe_load(interpolated_config)) return defaults - def _interpolate(self, stream): - i = interpolation.Interpolator(interpolation.TemplateWithDefaults, - os.environ) + def _interpolate(self, stream, env, keep_string): + i = interpolation.Interpolator(interpolation.TemplateWithDefaults, env) try: - return i.interpolate(stream) + return i.interpolate(stream, keep_string) except interpolation.InvalidInterpolation as e: msg = ("parsing config file '{}'.\n\n" '{}\n{}'.format(self.molecule_file, e.place, e.string)) @@ -398,11 +433,6 @@ def _validate(self): msg = 'Validating schema {}.'.format(self.molecule_file) LOG.info(msg) - # Prior to validation, we must set values. This allows us to perform - # validation in one place. This feels gross. - self.config['dependency']['command'] = self.dependency.command - self.config['driver']['name'] = self.driver.name - errors = schema_v2.validate(self.config) if errors: msg = "Failed to validate.\n\n{}".format(errors) diff --git a/Rake/Gemfile/molecule/dependency/shell.py b/Rake/Gemfile/molecule/dependency/shell.py index 96d47751..66027715 100644 --- a/Rake/Gemfile/molecule/dependency/shell.py +++ b/Rake/Gemfile/molecule/dependency/shell.py @@ -73,7 +73,11 @@ def __init__(self, config): super(Shell, self).__init__(config) self._sh_command = None - self.command = config.config['dependency']['command'] + # self.command = config..config['dependency']['command'] + + @property + def command(self): + return self._config.config['dependency']['command'] @property def default_options(self): diff --git a/Rake/Gemfile/molecule/interpolation.py b/Rake/Gemfile/molecule/interpolation.py index e791210f..d6f964d8 100644 --- a/Rake/Gemfile/molecule/interpolation.py +++ b/Rake/Gemfile/molecule/interpolation.py @@ -49,15 +49,19 @@ class Interpolator(object): If a literal dollar sign is needed in a configuration, use a double dollar sign (`$$`). + + Molecule will substitute special `MOLECULE_` environment variables defined + in `molecule.yml`. However, use at your own risk, this should be used + sparingly. """ def __init__(self, templater, mapping): self.templater = templater self.mapping = mapping - def interpolate(self, string): + def interpolate(self, string, keep_string=None): try: - return self.templater(string).substitute(self.mapping) + return self.templater(string).substitute(self.mapping, keep_string) except ValueError as e: raise InvalidInterpolation(string, e) @@ -66,12 +70,14 @@ class TemplateWithDefaults(string.Template): idpattern = r'[_a-z][_a-z0-9]*(?::?-[^}]+)?' # Modified from python2.7/string.py - def substitute(self, mapping): + def substitute(self, mapping, keep_string): # Helper function for .sub() def convert(mo): # Check the most common path first. named = mo.group('named') or mo.group('braced') if named is not None: + if keep_string and named.startswith(keep_string): + return '$%s' % named if ':-' in named: var, _, default = named.partition(':-') return mapping.get(var) or default diff --git a/Rake/Gemfile/molecule/model/schema_v2.py b/Rake/Gemfile/molecule/model/schema_v2.py index e6118ec0..62bbdf28 100644 --- a/Rake/Gemfile/molecule/model/schema_v2.py +++ b/Rake/Gemfile/molecule/model/schema_v2.py @@ -52,6 +52,7 @@ }, 'command': { 'type': 'string', + 'nullable': True, }, } }, @@ -571,6 +572,18 @@ }, } +dependency_command_nullable_schema = { + 'dependency': { + 'type': 'dict', + 'schema': { + 'command': { + 'type': 'string', + 'nullable': False, + }, + } + }, +} + verifier_options_readonly_schema = { 'verifier': { 'type': 'dict', @@ -677,6 +690,10 @@ def _validate_disallowed(self, disallowed, field, value): def validate(c): schema = copy.deepcopy(base_schema) + # Dependency + if c['dependency']['name'] == 'shell': + util.merge_dicts(schema, dependency_command_nullable_schema) + # Driver util.merge_dicts(schema, platforms_base_schema) if c['driver']['name'] == 'docker': diff --git a/Rake/Gemfile/molecule/provisioner/ansible/plugins/libraries/molecule_vagrant.py b/Rake/Gemfile/molecule/provisioner/ansible/plugins/libraries/molecule_vagrant.py index b6b0d185..81d725eb 100644 --- a/Rake/Gemfile/molecule/provisioner/ansible/plugins/libraries/molecule_vagrant.py +++ b/Rake/Gemfile/molecule/provisioner/ansible/plugins/libraries/molecule_vagrant.py @@ -24,6 +24,7 @@ import contextlib import datetime import os +import subprocess import sys import molecule diff --git a/Rake/Gemfile/test/scenarios/dependency/molecule/shell/molecule.yml b/Rake/Gemfile/test/scenarios/dependency/molecule/shell/molecule.yml index 44cbe7c1..6b66e003 100644 --- a/Rake/Gemfile/test/scenarios/dependency/molecule/shell/molecule.yml +++ b/Rake/Gemfile/test/scenarios/dependency/molecule/shell/molecule.yml @@ -1,10 +1,7 @@ --- dependency: name: shell - # TODO(retr0h): Setup the MOLECULE_ env vars prior to the - # interpolator. - # command: $MOLECULE_SCENARIO_DIRECTORY/run.bash - command: ./molecule/shell/run.bash + command: $MOLECULE_SCENARIO_DIRECTORY/run.bash driver: name: docker # NOTE(retr0h): Required for functional tests, since diff --git a/Rake/Gemfile/test/scenarios/overrride_driver/molecule/default/molecule.yml b/Rake/Gemfile/test/scenarios/overrride_driver/molecule/default/molecule.yml index c2a3630a..9563b1ac 100644 --- a/Rake/Gemfile/test/scenarios/overrride_driver/molecule/default/molecule.yml +++ b/Rake/Gemfile/test/scenarios/overrride_driver/molecule/default/molecule.yml @@ -4,6 +4,8 @@ dependency: driver: # NOTE(retr0h): Functional test overrides this on command line. name: vagrant + provider: + name: virtualbox lint: name: yamllint options: diff --git a/Rake/Gemfile/test/unit/conftest.py b/Rake/Gemfile/test/unit/conftest.py index 3b22ecf9..b1cd47a1 100644 --- a/Rake/Gemfile/test/unit/conftest.py +++ b/Rake/Gemfile/test/unit/conftest.py @@ -270,6 +270,8 @@ def patched_testinfra(mocker): @pytest.fixture def patched_scenario_setup(mocker): + mocker.patch('molecule.config.Config.env') + return mocker.patch('molecule.scenario.Scenario._setup') diff --git a/Rake/Gemfile/test/unit/model/v2/test_dependency_section.py b/Rake/Gemfile/test/unit/model/v2/test_dependency_section.py index f5cda8cc..27a6f6a7 100644 --- a/Rake/Gemfile/test/unit/model/v2/test_dependency_section.py +++ b/Rake/Gemfile/test/unit/model/v2/test_dependency_section.py @@ -69,7 +69,6 @@ def test_dependency_has_errors(_config): 'dependency': [{ 'name': ['must be of string type'], 'enabled': ['must be of boolean type'], - 'command': ['null value not allowed'], 'options': ['must be of dict type'], 'env': [{ 'foo': ["value does not match regex '^[A-Z0-9_-]+$'"], @@ -135,3 +134,21 @@ def test_dependency_invalid_dependency_name_has_errors(_config): x = {'dependency': [{'name': ['unallowed value ']}]} assert x == schema_v2.validate(_config) + + +@pytest.fixture +def _model_dependency_shell_errors_section_data(): + return { + 'dependency': { + 'name': 'shell', + 'command': None, + } + } + + +@pytest.mark.parametrize( + '_config', ['_model_dependency_shell_errors_section_data'], indirect=True) +def test_dependency_shell_has_errors(_config): + x = {'dependency': [{'command': ['null value not allowed']}]} + + assert x == schema_v2.validate(_config) diff --git a/Rake/Gemfile/test/unit/test_config.py b/Rake/Gemfile/test/unit/test_config.py index 6ca51746..248b1385 100644 --- a/Rake/Gemfile/test/unit/test_config.py +++ b/Rake/Gemfile/test/unit/test_config.py @@ -406,25 +406,29 @@ def test_get_driver_name_raises_when_different_driver_used( patched_logger_critical.assert_called_once_with(msg) -def test_combine(config_instance): - assert isinstance(config_instance._combine(), dict) +def test_get_config(config_instance): + assert isinstance(config_instance._get_config(), dict) -def test_combine_with_base_config(config_instance): +def test_get_config_with_base_config(config_instance): config_instance.args = {'base_config': './foo.yml'} contents = {'foo': 'bar'} util.write_file(config_instance.args['base_config'], util.safe_dump(contents)) - result = config_instance._combine() + result = config_instance._get_config() assert result['foo'] == 'bar' +def test_reget_config(config_instance): + assert isinstance(config_instance._reget_config(), dict) + + def test_interpolate(patched_logger_critical, config_instance): string = 'foo: $HOME' x = 'foo: {}'.format(os.environ['HOME']) - assert x == config_instance._interpolate(string) + assert x == config_instance._interpolate(string, os.environ, None) def test_interpolate_raises_on_failed_interpolation(patched_logger_critical, @@ -432,7 +436,7 @@ def test_interpolate_raises_on_failed_interpolation(patched_logger_critical, string = '$6$8I5Cfmpr$kGZB' with pytest.raises(SystemExit) as e: - config_instance._interpolate(string) + config_instance._interpolate(string, os.environ, None) assert 1 == e.value.code @@ -453,8 +457,6 @@ def test_validate(mocker, config_instance, patched_logger_info, patched_logger_info.assert_called_once_with(msg) m.assert_called_once_with(config_instance.config) - assert 'ansible-galaxy' == config_instance.config['dependency']['command'] - assert 'docker' == config_instance.config['driver']['name'] msg = 'Validation completed successfully.' patched_logger_success.assert_called_once_with(msg) diff --git a/Rake/Gemfile/test/unit/test_interpolation.py b/Rake/Gemfile/test/unit/test_interpolation.py index a3d0bc25..735e7825 100644 --- a/Rake/Gemfile/test/unit/test_interpolation.py +++ b/Rake/Gemfile/test/unit/test_interpolation.py @@ -23,57 +23,67 @@ def _mock_env(): 'FOO': 'foo', 'BAR': '', 'DEPENDENCY_NAME': 'galaxy', - 'VERIFIER_NAME': 'testinfra' + 'VERIFIER_NAME': 'testinfra', + 'MOLECULE_SCENARIO_NAME': 'default', } @pytest.fixture def _instance(_mock_env): return interpolation.Interpolator(interpolation.TemplateWithDefaults, - _mock_env).interpolate + _mock_env) def test_escaped_interpolation(_instance): - assert '${foo}' == _instance('$${foo}') + assert '${foo}' == _instance.interpolate('$${foo}') def test_invalid_interpolation(_instance): with pytest.raises(interpolation.InvalidInterpolation): - _instance('${') + _instance.interpolate('${') with pytest.raises(interpolation.InvalidInterpolation): - _instance('$}') + _instance.interpolate('$}') with pytest.raises(interpolation.InvalidInterpolation): - _instance('${}') + _instance.interpolate('${}') with pytest.raises(interpolation.InvalidInterpolation): - _instance('${ }') + _instance.interpolate('${ }') with pytest.raises(interpolation.InvalidInterpolation): - _instance('${ foo}') + _instance.interpolate('${ foo}') with pytest.raises(interpolation.InvalidInterpolation): - _instance('${foo }') + _instance.interpolate('${foo }') with pytest.raises(interpolation.InvalidInterpolation): - _instance('${foo!}') + _instance.interpolate('${foo!}') def test_interpolate_missing_no_default(_instance): - assert 'This var' == _instance('This ${missing} var') - assert 'This var' == _instance('This ${BAR} var') + assert 'This var' == _instance.interpolate('This ${missing} var') + assert 'This var' == _instance.interpolate('This ${BAR} var') def test_interpolate_with_value(_instance): - assert 'This foo var' == _instance('This $FOO var') - assert 'This foo var' == _instance('This ${FOO} var') + assert 'This foo var' == _instance.interpolate('This $FOO var') + assert 'This foo var' == _instance.interpolate('This ${FOO} var') def test_interpolate_missing_with_default(_instance): - assert 'ok def' == _instance('ok ${missing:-def}') - assert 'ok def' == _instance('ok ${missing-def}') - assert 'ok /non:-alphanumeric' == _instance( + assert 'ok def' == _instance.interpolate('ok ${missing:-def}') + assert 'ok def' == _instance.interpolate('ok ${missing-def}') + assert 'ok /non:-alphanumeric' == _instance.interpolate( 'ok ${BAR:-/non:-alphanumeric}') def test_interpolate_with_empty_and_default_value(_instance): - assert 'ok def' == _instance('ok ${BAR:-def}') - assert 'ok ' == _instance('ok ${BAR-def}') + assert 'ok def' == _instance.interpolate('ok ${BAR:-def}') + assert 'ok ' == _instance.interpolate('ok ${BAR-def}') + + +def test_interpolate_interpolates_MOLECULE_strings(_instance): + assert 'default' == _instance.interpolate('$MOLECULE_SCENARIO_NAME') + + +def test_interpolate_does_not_interpolate_MOLECULE_strings(_instance): + assert 'foo $MOLECULE_SCENARIO_NAME' == _instance.interpolate( + 'foo $MOLECULE_SCENARIO_NAME', keep_string='MOLECULE_') def test_interpolate_with_molecule_yaml(_instance): @@ -90,7 +100,7 @@ def test_interpolate_with_molecule_yaml(_instance): provisioner: name: ansible scenario: - name: default + name: $MOLECULE_SCENARIO_NAME verifier: name: ${VERIFIER_NAME} options: @@ -117,4 +127,4 @@ def test_interpolate_with_molecule_yaml(_instance): foo: bar """.strip() - assert x == _instance(data) + assert x == _instance.interpolate(data) diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 00000000..2f7efbea --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-minimal \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..a02d6ecb --- /dev/null +++ b/docs/index.md @@ -0,0 +1,37 @@ +## Welcome to GitHub Pages + +You can use the [editor on GitHub](https://github.com/corserp/yaml/edit/yaml/docs/index.md) to maintain and preview the content for your website in Markdown files. + +Whenever you commit to this repository, GitHub Pages will run [Jekyll](https://jekyllrb.com/) to rebuild the pages in your site, from the content in your Markdown files. + +### Markdown + +Markdown is a lightweight and easy-to-use syntax for styling your writing. It includes conventions for + +```markdown +Syntax highlighted code block + +# app.io 1 +## back 2 +### api-ly.yml 3 + +- Bulleted +- List + +1. Numbered +2. List + +**Bold** and _Italic_ and `Code` text + +[Link](url) and ![Image](src) +``` + +For more details see [GitHub Flavored Markdown](https://guides.github.com/features/mastering-markdown/). + +### Jekyll Themes + +Your Pages site will use the layout and styles from the Jekyll theme you have selected in your [repository settings](https://github.com/corserp/yaml/settings). The name of this theme is saved in the io `theme_config.yml` configuration file. + +### Support or Contact + +Having trouble with Pages? Check out our [documentation](https://docs.github.com/categories/github-pages-basics/) or [contact support](https://github.com/contact) and we’ll help you sort it out. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..ef12c3d1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,20 @@ +ansible-lint==3.4.21 +anyconfig==0.9.4 +cerberus==1.1 +click==6.7 +click-completion==0.3.1 +colorama==0.3.9 +cookiecutter==1.6.0 +flake8==3.5.0 +python-gilt==1.2.1 +Jinja2==2.10 +pbr==3.0.1 +pexpect==4.2.1 +psutil==5.2.2; sys_platform!="win32" and sys_platform!="cygwin" +PyYAML==3.12 +sh==1.12.14 +six==1.11.0 +tabulate==0.8.2 +testinfra==1.12.0 +tree-format==0.1.2 +yamllint==1.11.1