diff --git a/tests/commands/test_root.py b/tests/commands/test_root.py index bcf2193..d4bbcff 100644 --- a/tests/commands/test_root.py +++ b/tests/commands/test_root.py @@ -16,13 +16,12 @@ import os import platform from tempfile import TemporaryDirectory -from unittest import mock import pytest from deepdiff import DeepDiff import tfworker.commands.root -from tfworker.commands.root import get_platform, ordered_config_load +from tfworker.commands.root import ordered_config_load class TestMain: @@ -220,32 +219,6 @@ def test_config_formats(self, yaml_base_rootc, json_base_rootc, hcl_base_rootc): diff = DeepDiff(json_config, hcl_config) assert len(diff) == 0 - @pytest.mark.parametrize( - "opsys, machine, mock_platform_opsys, mock_platform_machine", - [ - ("linux", "i386", ["linux2"], ["i386"]), - ("linux", "arm", ["Linux"], ["arm"]), - ("linux", "amd64", ["linux"], ["x86_64"]), - ("linux", "amd64", ["linux"], ["amd64"]), - ("darwin", "amd64", ["darwin"], ["x86_64"]), - ("darwin", "amd64", ["darwin"], ["amd64"]), - ("darwin", "arm", ["darwin"], ["arm"]), - ("darwin", "arm64", ["darwin"], ["aarch64"]), - ], - ) - def test_get_platform( - self, opsys, machine, mock_platform_opsys, mock_platform_machine - ): - with mock.patch("platform.system", side_effect=mock_platform_opsys) as mock1: - with mock.patch( - "platform.machine", side_effect=mock_platform_machine - ) as mock2: - actual_opsys, actual_machine = get_platform() - assert opsys == actual_opsys - assert machine == actual_machine - mock1.assert_called_once() - mock2.assert_called_once() - class TestOrderedConfigLoad: def test_ordered_config_load(self): diff --git a/tests/commands/test_terraform.py b/tests/commands/test_terraform.py index 169c370..7539880 100644 --- a/tests/commands/test_terraform.py +++ b/tests/commands/test_terraform.py @@ -22,7 +22,7 @@ from google.cloud.exceptions import NotFound import tfworker -from tfworker.commands.terraform import BaseCommand, TerraformCommand, TerraformError +from tfworker.commands.terraform import TerraformCommand, TerraformError from tfworker.definitions import Definition from tfworker.handlers import HandlerError @@ -185,27 +185,6 @@ def test_run(self, tf_cmd: str, method: callable, args: list, request): for arg in args: assert arg in call_as_string - @pytest.mark.parametrize( - "stdout, major, minor, expected_exception", - [ - ("Terraform v0.12.29", 0, 12, does_not_raise()), - ("Terraform v1.3.5", 1, 3, does_not_raise()), - ("TF 14", "", "", pytest.raises(SystemExit)), - ], - ) - def test_get_tf_version( - self, stdout: str, major: int, minor: int, expected_exception: callable - ): - with mock.patch( - "tfworker.commands.base.pipe_exec", - side_effect=mock_tf_version, - ) as mocked: - with expected_exception: - (actual_major, actual_minor) = BaseCommand.get_terraform_version(stdout) - assert actual_major == major - assert actual_minor == minor - mocked.assert_called_once() - def test_worker_options(self, tf_13cmd_options): # Verify that the options from the CLI override the options from the config assert tf_13cmd_options._rootc.worker_options_odict.get("backend") == "s3" @@ -236,7 +215,7 @@ def test_worker_options(self, tf_13cmd_options): def test_no_create_backend_bucket_fails_gcs(self, grootc_no_create_backend_bucket): with pytest.raises(SystemExit): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.commands.base.get_terraform_version", side_effect=lambda x: (13, 3), ): with mock.patch( @@ -367,7 +346,7 @@ def test_exec_valid_flow(self, terraform_command, definition): mock_prep_modules.assert_called_once_with( terraform_command._terraform_modules_dir, terraform_command._temp_dir, - required=False, + required=True, ) mock_prep_and_init.assert_called_once_with(def_iter) mock_check_plan.assert_called_once_with(definition) @@ -414,7 +393,7 @@ def test_exec_without_plan(self, terraform_command, definition): mock_prep_modules.assert_called_once_with( terraform_command._terraform_modules_dir, terraform_command._temp_dir, - required=False, + required=True, ) mock_prep_and_init.assert_called_once_with(def_iter) mock_check_plan.assert_called_once_with(definition) @@ -449,7 +428,7 @@ def test_exec_with_no_apply_or_destroy(self, terraform_command, definition): mock_prep_modules.assert_called_once_with( terraform_command._terraform_modules_dir, terraform_command._temp_dir, - required=False, + required=True, ) mock_prep_and_init.assert_called_once_with(def_iter) mock_check_plan.assert_called_once_with(definition) diff --git a/tests/conftest.py b/tests/conftest.py index 78a84b4..f390aba 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -304,7 +304,7 @@ def rootc_options(s3_client, dynamodb_client, sts_client): @pytest.fixture def basec(rootc, s3_client): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.commands.base.get_terraform_version", side_effect=lambda x: (13, 3), ): with mock.patch( @@ -319,7 +319,7 @@ def basec(rootc, s3_client): @pytest.fixture def gbasec(grootc): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.commands.base.get_terraform_version", side_effect=lambda x: (13, 3), ): with mock.patch( @@ -342,7 +342,7 @@ def tf_Xcmd(rootc): @pytest.fixture def tf_15cmd(rootc): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.util.terraform.get_terraform_version", side_effect=lambda x: (15, 0), ): with mock.patch( @@ -357,7 +357,7 @@ def tf_15cmd(rootc): @pytest.fixture def tf_14cmd(rootc): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.util.terraform.get_terraform_version", side_effect=lambda x: (14, 5), ): with mock.patch( @@ -372,7 +372,7 @@ def tf_14cmd(rootc): @pytest.fixture def tf_13cmd(rootc): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.util.terraform.get_terraform_version", side_effect=lambda x: (13, 5), ): with mock.patch( @@ -387,7 +387,7 @@ def tf_13cmd(rootc): @pytest.fixture def tf_12cmd(rootc): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.util.terraform.get_terraform_version", side_effect=lambda x: (12, 27), ): with mock.patch( @@ -402,7 +402,7 @@ def tf_12cmd(rootc): @pytest.fixture def tf_13cmd_options(rootc_options): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.util.terraform.get_terraform_version", side_effect=lambda x: (13, 5), ): with mock.patch( diff --git a/tests/test_plugins.py b/tests/test_plugins.py index f2208b4..4398045 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -20,9 +20,10 @@ import tfworker.commands.root import tfworker.plugins +from tfworker.util.system import get_platform # values needed by multiple tests -opsys, machine = tfworker.commands.root.get_platform() +opsys, machine = get_platform() _platform = f"{opsys}_{machine}" diff --git a/tests/util/test_hooks.py b/tests/util/test_hooks.py new file mode 100644 index 0000000..265cf23 --- /dev/null +++ b/tests/util/test_hooks.py @@ -0,0 +1,219 @@ +import pytest +from unittest import mock +from tfworker.util.system import pipe_exec +from tfworker.exceptions import HookError + +import tfworker.util.hooks as hooks +from tfworker.types import TerraformAction, TerraformStage + +# Fixture for a mock Terraform state file +@pytest.fixture +def mock_terraform_state(): + return """ + { + "version": 4, + "terraform_version": "0.13.5", + "serial": 1, + "lineage": "8a2b56d2-4e16-48de-9c5b-c640d6b3a52d", + "outputs": {}, + "resources": [ + { + "module": "module.remote_state", + "mode": "data", + "type": "terraform_remote_state", + "name": "example", + "provider": "provider[\"registry.terraform.io/hashicorp/terraform\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "backend": "gcs", + "config": {}, + "outputs": { + "key": "value", + "another_key": "another_value" + } + } + } + ] + } + ] + } + """ + +@pytest.fixture +def mock_terraform_locals(): + return """locals { + local_key = data.terraform_remote_state.example.outputs.key + local_another_key = data.terraform_remote_state.example.outputs.another_key +} +""" + +# Test for `get_state_item` +class TestGetStateItem: + @mock.patch('tfworker.util.hooks._get_state_item_from_output') + @mock.patch('tfworker.util.hooks._get_state_item_from_remote') + def test_get_state_item_from_output_success(self, mock_remote, mock_output): + mock_output.return_value = '{"key": "value"}' + result = hooks.get_state_item("working_dir", {}, "terraform_bin", "state", "item") + assert result == '{"key": "value"}' + mock_output.assert_called_once() + mock_remote.assert_not_called() + + @mock.patch('tfworker.util.hooks._get_state_item_from_output', side_effect=FileNotFoundError) + @mock.patch('tfworker.util.hooks._get_state_item_from_remote') + def test_get_state_item_from_remote_success(self, mock_remote, mock_output): + mock_remote.return_value = '{"key": "value"}' + result = hooks.get_state_item("working_dir", {}, "terraform_bin", "state", "item") + assert result == '{"key": "value"}' + mock_output.assert_called_once() + mock_remote.assert_called_once() + +# Test for `_get_state_item_from_output` +class TestGetStateItemFromOutput: + @mock.patch('tfworker.util.hooks.pipe_exec') + def test_get_state_item_from_output_success(self, mock_pipe_exec): + mock_pipe_exec.return_value = (0, '{"key":"value"}', '') + result = hooks._get_state_item_from_output("working_dir", {}, "terraform_bin", "state", "item") + assert result == '{"key":"value"}' + mock_pipe_exec.assert_called_once() + + @mock.patch('tfworker.util.hooks.pipe_exec', side_effect=FileNotFoundError) + def test_get_state_item_from_output_file_not_found(self, mock_pipe_exec): + with pytest.raises(FileNotFoundError): + hooks._get_state_item_from_output("working_dir", {}, "terraform_bin", "state", "item") + + @mock.patch('tfworker.util.hooks.pipe_exec') + def test_get_state_item_from_output_error(self, mock_pipe_exec): + mock_pipe_exec.return_value = (1, '', 'error') + with pytest.raises(HookError): + hooks._get_state_item_from_output("working_dir", {}, "terraform_bin", "state", "item") + + @mock.patch('tfworker.util.hooks.pipe_exec') + def test_get_state_item_from_output_empty_output(self, mock_pipe_exec): + mock_pipe_exec.return_value = (0, None, '') + with pytest.raises(HookError) as e: + hooks._get_state_item_from_output("working_dir", {}, "terraform_bin", "state", "item") + assert "Remote state item state.item is empty" in str(e.value) + + @mock.patch('tfworker.util.hooks.pipe_exec') + def test_get_state_item_from_output_invalid_json(self, mock_pipe_exec): + mock_pipe_exec.return_value = (0, 'invalid_json', '') + with pytest.raises(HookError) as e: + hooks._get_state_item_from_output("working_dir", {}, "terraform_bin", "state", "item") + assert "output is not in JSON format" in str(e.value) + +# Test for `check_hooks` +class TestCheckHooks: + @mock.patch('tfworker.util.hooks.os.path.isdir', return_value=True) + @mock.patch('tfworker.util.hooks.os.listdir', return_value=[f"{TerraformStage.PRE}_{TerraformAction.PLAN}"]) + @mock.patch('tfworker.util.hooks.os.access', return_value=True) + def test_check_hooks_exists(self, mock_access, mock_listdir, mock_isdir): + result = hooks.check_hooks(TerraformStage.PRE, 'working_dir', TerraformAction.PLAN) + assert result is True + mock_isdir.assert_called_once() + mock_listdir.assert_called_once() + mock_access.assert_called_once() + + @mock.patch('tfworker.util.hooks.os.path.isdir', return_value=False) + def test_check_hooks_no_dir(self, mock_isdir): + result = hooks.check_hooks('phase', 'working_dir', 'command') + assert result is False + mock_isdir.assert_called_once() + + @mock.patch('tfworker.util.hooks.os.path.isdir', return_value=True) + @mock.patch('tfworker.util.hooks.os.listdir', return_value=[f"{TerraformStage.PRE}_{TerraformAction.PLAN}"]) + @mock.patch('tfworker.util.hooks.os.access', return_value=False) + def test_check_hooks_not_executable(self, mock_listdir, mock_isdir, mock_access): + with pytest.raises(HookError) as e: + hooks.check_hooks(TerraformStage.PRE, 'working_dir', TerraformAction.PLAN) + assert "working_dir/hooks/pre_plan exists, but is not executable!" in str(e.value) + + @mock.patch('tfworker.util.hooks.os.path.isdir', return_value=True) + @mock.patch('tfworker.util.hooks.os.listdir', return_value=[]) + def test_check_hooks_no_hooks(self, mock_listdir, mock_isdir): + result = hooks.check_hooks('phase', 'working_dir', 'command') + assert result is False + +# Test for `hook_exec` +class TestHookExec: + @mock.patch('tfworker.util.hooks._prepare_environment') + @mock.patch('tfworker.util.hooks._find_hook_script') + @mock.patch('tfworker.util.hooks._populate_environment_with_terraform_variables') + @mock.patch('tfworker.util.hooks._populate_environment_with_terraform_remote_vars') + @mock.patch('tfworker.util.hooks._populate_environment_with_extra_vars') + @mock.patch('tfworker.util.hooks._execute_hook_script') + def test_hook_exec_success(self, mock_execute, mock_extra_vars, mock_remote_vars, mock_terraform_vars, mock_find_script, mock_prepare_env): + mock_find_script.return_value = "hook_script" + hooks.hook_exec('phase', 'command', 'working_dir', {}, 'terraform_path', debug=True, b64_encode=True) + mock_prepare_env.assert_called_once() + mock_find_script.assert_called_once() + mock_terraform_vars.assert_called_once() + mock_remote_vars.assert_called_once() + mock_extra_vars.assert_called_once() + mock_execute.assert_called_once() + +# Helper function tests +class TestHelperFunctions: + @mock.patch('tfworker.util.hooks.os.listdir', return_value=[f"{TerraformStage.PRE}_{TerraformAction.PLAN}"]) + def test_find_hook_script(self, mock_listdir): + result = hooks._find_hook_script('working_dir', TerraformStage.PRE, TerraformAction.PLAN) + assert result == f'working_dir/hooks/pre_plan' + + @mock.patch('tfworker.util.hooks.os.listdir', return_value=[]) + def test_find_hook_script_no_dir(self, mock_listdir): + with pytest.raises(HookError) as e: + hooks._find_hook_script('working_dir', TerraformStage.PRE, TerraformAction.PLAN) + assert "Hook script missing from" in str(e.value) + + def test_prepare_environment(self): + env = {'KEY': 'value'} + result = hooks._prepare_environment(env, 'terraform_path') + assert result['KEY'] == 'value' + assert result['TF_PATH'] == 'terraform_path' + + def test_set_hook_env_var(self): + local_env = {} + hooks._set_hook_env_var(local_env, hooks.TFHookVarType.VAR, 'key', 'value', False) + assert local_env['TF_VAR_KEY'] == 'value' + + @mock.patch('tfworker.util.hooks.pipe_exec') + def test_execute_hook_script(self, mock_pipe_exec, capsys): + mock_pipe_exec.return_value = (0, b'stdout', b'stderr') + hooks._execute_hook_script('hook_script', TerraformStage.PRE, TerraformAction.PLAN, 'working_dir', {}, True) + mock_pipe_exec.assert_called_once_with('hook_script pre plan', cwd='working_dir/hooks', env={}) + captured = capsys.readouterr() + captured_lines = captured.out.splitlines() + assert len(captured_lines) == 4 + assert "Results from hook script: hook_script" in captured_lines + assert "exit code: 0" in captured_lines + assert "stdout: stdout" in captured_lines + assert "stderr: stderr" in captured_lines + + @mock.patch('tfworker.util.hooks.os.path.isfile', return_value=True) + @mock.patch('builtins.open', new_callable=mock.mock_open, read_data='key=value\nanother_key=another_value') + def test_populate_environment_with_terraform_variables(self, mock_isfile, mock_open): + local_env = {} + hooks._populate_environment_with_terraform_variables(local_env, 'working_dir', 'terraform_path', False) + assert 'TF_VAR_KEY' in local_env + assert 'TF_VAR_ANOTHER_KEY' in local_env + + @mock.patch('builtins.open', new_callable=mock.mock_open) + @mock.patch('tfworker.util.hooks.os.path.isfile', return_value=True) + @mock.patch('tfworker.util.hooks.get_state_item', side_effect=["value", "another_value"]) + def test_populate_environment_with_terraform_remote_vars(self, mock_get_state_item, mock_isfile, mock_open, mock_terraform_locals): + mock_open.return_value.read.return_value = mock_terraform_locals + + local_env = {} + hooks._populate_environment_with_terraform_remote_vars(local_env, 'working_dir', 'terraform_path', False) + assert 'TF_REMOTE_LOCAL_KEY' in local_env.keys() + assert 'TF_REMOTE_LOCAL_ANOTHER_KEY' in local_env.keys() + assert local_env['TF_REMOTE_LOCAL_KEY'] == 'value' + assert local_env['TF_REMOTE_LOCAL_ANOTHER_KEY'] == 'another_value' + + + def test_populate_environment_with_extra_vars(self): + local_env = {} + extra_vars = {'extra_key': 'extra_value'} + hooks._populate_environment_with_extra_vars(local_env, extra_vars, False) + assert 'TF_EXTRA_EXTRA_KEY' in local_env diff --git a/tests/util/test_system.py b/tests/util/test_system.py index 3758e94..4fb0935 100644 --- a/tests/util/test_system.py +++ b/tests/util/test_system.py @@ -17,7 +17,7 @@ import pytest -from tfworker.util.system import get_version, pipe_exec, strip_ansi, which +from tfworker.util.system import get_platform, get_version, pipe_exec, strip_ansi, which # context manager to allow testing exceptions in parameterized tests @@ -153,3 +153,29 @@ def test_strip_ansi(self): assert strip_ansi("\x1B[32mWorld\x1B[0m") == "World" assert strip_ansi("\x1B[33mFoo\x1B[0m") == "Foo" assert strip_ansi("\x1B[34mBar\x1B[0m") == "Bar" + + @pytest.mark.parametrize( + "opsys, machine, mock_platform_opsys, mock_platform_machine", + [ + ("linux", "i386", ["linux2"], ["i386"]), + ("linux", "arm", ["Linux"], ["arm"]), + ("linux", "amd64", ["linux"], ["x86_64"]), + ("linux", "amd64", ["linux"], ["amd64"]), + ("darwin", "amd64", ["darwin"], ["x86_64"]), + ("darwin", "amd64", ["darwin"], ["amd64"]), + ("darwin", "arm", ["darwin"], ["arm"]), + ("darwin", "arm64", ["darwin"], ["aarch64"]), + ], + ) + def test_get_platform( + self, opsys, machine, mock_platform_opsys, mock_platform_machine + ): + with mock.patch("platform.system", side_effect=mock_platform_opsys) as mock1: + with mock.patch( + "platform.machine", side_effect=mock_platform_machine + ) as mock2: + actual_opsys, actual_machine = get_platform() + assert opsys == actual_opsys + assert machine == actual_machine + mock1.assert_called_once() + mock2.assert_called_once() diff --git a/tests/util/test_terraform_util.py b/tests/util/test_terraform_util.py index 53db826..566e34b 100644 --- a/tests/util/test_terraform_util.py +++ b/tests/util/test_terraform_util.py @@ -75,3 +75,24 @@ def test_prep_modules_required(tmp_path): # check the target path does not exist assert not target_path.exists() + + # @pytest.mark.parametrize( + # "stdout, major, minor, expected_exception", + # [ + # ("Terraform v0.12.29", 0, 12, does_not_raise()), + # ("Terraform v1.3.5", 1, 3, does_not_raise()), + # ("TF 14", "", "", pytest.raises(SystemExit)), + # ], + # ) + # def test_get_tf_version( + # self, stdout: str, major: int, minor: int, expected_exception: callable + # ): + # with mock.patch( + # "tfworker.commands.base.pipe_exec", + # side_effect=mock_tf_version, + # ) as mocked: + # with expected_exception: + # (actual_major, actual_minor) = BaseCommand.get_terraform_version(stdout) + # assert actual_major == major + # assert actual_minor == minor + # mocked.assert_called_once() diff --git a/tfworker/backends/base.py b/tfworker/backends/base.py index 2d4186c..8744b52 100644 --- a/tfworker/backends/base.py +++ b/tfworker/backends/base.py @@ -14,7 +14,7 @@ from abc import ABCMeta, abstractmethod -from tfworker import JSONType +from tfworker.types import JSONType class BackendError(Exception): diff --git a/tfworker/cli.py b/tfworker/cli.py index 6a1afd4..77d0424 100644 --- a/tfworker/cli.py +++ b/tfworker/cli.py @@ -21,10 +21,14 @@ import click from tfworker import constants as const -from tfworker.commands import CleanCommand, RootCommand, TerraformCommand -from tfworker.commands.env import EnvCommand -from tfworker.commands.root import get_platform -from tfworker.commands.version import VersionCommand +from tfworker.commands import ( + CleanCommand, + EnvCommand, + RootCommand, + TerraformCommand, + VersionCommand, +) +from tfworker.util.system import get_platform def validate_deployment(ctx, deployment, name): @@ -189,7 +193,7 @@ def __repr__(self): ) @click.option( "--backend-use-all-remotes/--no-backend-use-all-remotes", - default=False, + default=True, envvar="WORKER_BACKEND_USE_ALL_REMOTES", help="Generate remote data sources based on all definition paths present in the backend", ) diff --git a/tfworker/commands/__init__.py b/tfworker/commands/__init__.py index ba6e711..0b790e5 100644 --- a/tfworker/commands/__init__.py +++ b/tfworker/commands/__init__.py @@ -14,6 +14,7 @@ from .base import BaseCommand # noqa from .clean import CleanCommand # noqa +from .env import EnvCommand # noqa from .root import RootCommand # noqa from .terraform import TerraformCommand # noqa from .version import VersionCommand # noqa diff --git a/tfworker/commands/base.py b/tfworker/commands/base.py index 8f72f0c..4283dc5 100644 --- a/tfworker/commands/base.py +++ b/tfworker/commands/base.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re - import click from tfworker.authenticators import AuthenticatorsCollection @@ -23,7 +21,8 @@ from tfworker.handlers.exceptions import HandlerError, UnknownHandler from tfworker.plugins import PluginsCollection from tfworker.providers import ProvidersCollection -from tfworker.util.system import get_version, pipe_exec, which +from tfworker.util.system import get_version, which +from tfworker.util.terraform import get_terraform_version class MissingDependencyException(Exception): @@ -64,7 +63,7 @@ def __init__(self, rootc, deployment="undefined", limit=tuple(), **kwargs): ( self._tf_version_major, self._tf_version_minor, - ) = self.get_terraform_version(self._terraform_bin) + ) = get_terraform_version(self._terraform_bin) self._authenticators = AuthenticatorsCollection( rootc.args, deployment=deployment, **kwargs @@ -189,21 +188,3 @@ def _resolve_arg(self, name): if name in self._rootc.worker_options_odict: return self._rootc.worker_options_odict[name] return None - - @staticmethod - def get_terraform_version(terraform_bin): - (return_code, stdout, stderr) = pipe_exec(f"{terraform_bin} version") - if return_code != 0: - click.secho(f"unable to get terraform version\n{stderr}", fg="red") - raise SystemExit(1) - version = stdout.decode("UTF-8").split("\n")[0] - version_search = re.search(r".*\s+v(\d+)\.(\d+)\.(\d+)", version) - if version_search: - click.secho( - f"Terraform Version Result: {version}, using major:{version_search.group(1)}, minor:{version_search.group(2)}", - fg="yellow", - ) - return (int(version_search.group(1)), int(version_search.group(2))) - else: - click.secho(f"unable to get terraform version\n{stderr}", fg="red") - raise SystemExit(1) diff --git a/tfworker/commands/root.py b/tfworker/commands/root.py index f597968..5ead715 100644 --- a/tfworker/commands/root.py +++ b/tfworker/commands/root.py @@ -15,7 +15,6 @@ import io import os import pathlib -import platform import tempfile from pathlib import Path from typing import Union @@ -263,30 +262,6 @@ def ordered_config_load(config: str) -> dict: raise SystemExit(1) -def get_platform(): - """ - Returns a formatted operating system / architecture tuple that is consistent with common distribution creation tools. - - Returns: - tuple: A tuple containing the operating system and architecture. - """ - - # strip off "2" which only appears on old linux kernels - opsys = platform.system().rstrip("2").lower() - - # make sure machine uses consistent format - machine = platform.machine() - if machine == "x86_64": - machine = "amd64" - - # some 64 bit arm extensions will report aarch64, this is functionaly - # equivalent to arm64 which is recognized and the pattern used by the TF - # community - if machine == "aarch64": - machine = "arm64" - return (opsys, machine) - - def rm_tree(base_path: Union[str, Path], inner: bool = False) -> None: """ Recursively removes all files and directories. diff --git a/tfworker/commands/terraform.py b/tfworker/commands/terraform.py index e1dbcf3..cf5789b 100644 --- a/tfworker/commands/terraform.py +++ b/tfworker/commands/terraform.py @@ -11,35 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import base64 -import json import os import pathlib -import re import click +import tfworker.util.hooks as hooks import tfworker.util.terraform as tf_util from tfworker.commands.base import BaseCommand from tfworker.definitions import Definition +from tfworker.exceptions import HookError, PlanChange, TerraformError from tfworker.handlers.exceptions import HandlerError from tfworker.util.system import pipe_exec, strip_ansi -TF_STATE_CACHE_NAME = "worker_state_cache.json" - - -class HookError(Exception): - pass - - -class PlanChange(Exception): - pass - - -class TerraformError(Exception): - pass - class TerraformCommand(BaseCommand): def __init__(self, rootc, **kwargs): @@ -105,7 +89,7 @@ def exec(self) -> None: tf_util.prep_modules( self._terraform_modules_dir, self._temp_dir, - required=(self._terraform_modules_dir is not None), + required=(self._terraform_modules_dir != ""), ) # prepare the definitions and run terraform init @@ -124,6 +108,10 @@ def exec(self) -> None: ################### # Private methods # ################### + + ########################################### + # Methods for dealing with terraform init # + ########################################### def _prep_and_init(self, def_iter: list[Definition]) -> None: """Prepares the definition and runs terraform init @@ -143,6 +131,9 @@ def _prep_and_init(self, def_iter: list[Definition]) -> None: click.secho("error running terraform init", fg="red") raise SystemExit(1) + ########################################### + # Methods for dealing with terraform plan # + ########################################### def _check_plan(self, definition: Definition) -> bool: """ Determines if a plan is needed for the provided definition @@ -208,60 +199,6 @@ def _validate_plan_path(self, plan_path: pathlib.Path) -> None: ) raise SystemExit(1) - def _run_handlers( - self, definition, action, stage, plan_file=None, **kwargs - ) -> None: - """Runs the handlers for the given action and stage - - Args: - definition: the definition to run the handlers for - action: the action to run the handlers for - stage: the stage to run the handlers for - plan_file: the plan file to pass to the handlers - kwargs: additional keyword arguments to pass to the handlers - - Returns: - None - """ - try: - self._execute_handlers( - action=action, - stage=stage, - deployment=self._deployment, - definition=definition.tag, - definition_path=definition.fs_path, - planfile=plan_file, - **kwargs, - ) - except HandlerError as e: - if e.terminate: - click.secho(f"terminating due to fatal handler error {e}", fg="red") - raise SystemExit(1) - click.secho(f"handler error: {e}", fg="red") - - def _should_plan(self, definition: Definition, plan_file: pathlib.Path) -> bool: - if not self._tf_plan: - definition._ready_to_apply = True - return False - - if plan_file.exists(): - if plan_file.stat().st_size == 0: - click.secho( - f"exiting plan file {plan_file} exists but is empty; planning again", - fg="green", - ) - definition._ready_to_apply = False - return True - click.secho( - f"existing plan file {plan_file} is suitable for apply; not planning again; remove plan file to allow planning", - fg="green", - ) - definition._ready_to_apply = True - return False - - definition._ready_to_apply = False - return True - def _exec_plan(self, definition) -> bool: """_exec_plan executes a terraform plan, returns true if a plan has changes""" changes = False @@ -328,6 +265,32 @@ def _exec_plan(self, definition) -> bool: return changes + def _should_plan(self, definition: Definition, plan_file: pathlib.Path) -> bool: + if not self._tf_plan: + definition._ready_to_apply = True + return False + + if plan_file.exists(): + if plan_file.stat().st_size == 0: + click.secho( + f"exiting plan file {plan_file} exists but is empty; planning again", + fg="green", + ) + definition._ready_to_apply = False + return True + click.secho( + f"existing plan file {plan_file} is suitable for apply; not planning again; remove plan file to allow planning", + fg="green", + ) + definition._ready_to_apply = True + return False + + definition._ready_to_apply = False + return True + + #################################################### + # Methods for dealing with terraform apply/destroy # + #################################################### def _check_apply_or_destroy(self, changes, definition) -> bool: """_check_apply_or_destroy determines if a terraform execution is needed""" # never apply if --no-apply is used @@ -421,6 +384,43 @@ def _exec_apply_or_destroy(self, definition) -> None: fg="green", ) + ##################################### + # Methods for dealing with handlers # + ##################################### + def _run_handlers( + self, definition, action, stage, plan_file=None, **kwargs + ) -> None: + """Runs the handlers for the given action and stage + + Args: + definition: the definition to run the handlers for + action: the action to run the handlers for + stage: the stage to run the handlers for + plan_file: the plan file to pass to the handlers + kwargs: additional keyword arguments to pass to the handlers + + Returns: + None + """ + try: + self._execute_handlers( + action=action, + stage=stage, + deployment=self._deployment, + definition=definition.tag, + definition_path=definition.fs_path, + planfile=plan_file, + **kwargs, + ) + except HandlerError as e: + if e.terminate: + click.secho(f"terminating due to fatal handler error {e}", fg="red") + raise SystemExit(1) + click.secho(f"handler error: {e}", fg="red") + + ######################################## + # Common methods for running terraform # + ######################################## def _run( self, definition, command, debug=False, plan_action="init", plan_file=None ): @@ -460,9 +460,11 @@ def _run( # only execute hooks for plan/apply/destroy try: - if TerraformCommand.check_hooks( - "pre", working_dir, command - ) and command in ["apply", "destroy", "plan"]: + if hooks.check_hooks("pre", working_dir, command) and command in [ + "apply", + "destroy", + "plan", + ]: # pre exec hooks # want to pass remotes # want to pass tf_vars @@ -471,7 +473,7 @@ def _run( " executing ", fg="yellow", ) - TerraformCommand.hook_exec( + hooks.hook_exec( "pre", command, working_dir, @@ -536,15 +538,17 @@ def _run( # only execute hooks for plan/destroy try: - if TerraformCommand.check_hooks( - "post", working_dir, command - ) and command in ["apply", "destroy", "plan"]: + if hooks.check_hooks("post", working_dir, command) and command in [ + "apply", + "destroy", + "plan", + ]: click.secho( f"found post-{command} hook script for definition {definition.tag}," " executing ", fg="yellow", ) - TerraformCommand.hook_exec( + hooks.hook_exec( "post", command, working_dir, @@ -560,304 +564,3 @@ def _run( ) raise SystemExit(2) return True - - ################## - # Static methods # - ################## - @staticmethod - def hook_exec( - phase, - command, - working_dir, - env, - terraform_path, - debug=False, - b64_encode=False, - extra_vars={}, - ): - """ - hook_exec executes a hook script. - - Before execution it sets up the environment to make all terraform and remote - state variables available to the hook via environment vars - """ - - key_replace_items = { - " ": "", - '"': "", - "-": "_", - ".": "_", - } - val_replace_items = { - " ": "", - '"': "", - "\n": "", - } - local_env = env.copy() - local_env["TF_PATH"] = terraform_path - hook_dir = f"{working_dir}/hooks" - hook_script = None - - for f in os.listdir(hook_dir): - # this file format is specifically structured by the prep_def function - if os.path.splitext(f)[0] == f"{phase}_{command}": - hook_script = f"{hook_dir}/{f}" - # this should never have been called if the hook script didn't exist... - if hook_script is None: - raise HookError(f"hook script missing from {hook_dir}") - - # populate environment with terraform remotes - if os.path.isfile(f"{working_dir}/worker-locals.tf"): - # I'm sorry. :-) - r = re.compile( - r"\s*(?P\w+)\s*\=.+data\.terraform_remote_state\.(?P\w+)\.outputs\.(?P\w+)\s*" - ) - - with open(f"{working_dir}/worker-locals.tf") as f: - for line in f: - m = r.match(line) - if m: - item = m.group("item") - state = m.group("state") - state_item = m.group("state_item") - else: - continue - - state_value = TerraformCommand.get_state_item( - working_dir, env, terraform_path, state, state_item - ) - - if state_value is not None: - if b64_encode: - state_value = base64.b64encode(state_value.encode("utf-8")) - local_env[f"TF_REMOTE_{state}_{item}".upper()] = state_value - - # populate environment with terraform variables - if os.path.isfile(f"{working_dir}/worker.auto.tfvars"): - with open(f"{working_dir}/worker.auto.tfvars") as f: - for line in f: - tf_var = line.split("=") - - # strip bad names out for env var settings - for k, v in key_replace_items.items(): - tf_var[0] = tf_var[0].replace(k, v) - - for k, v in val_replace_items.items(): - tf_var[1] = tf_var[1].replace(k, v) - - if b64_encode: - tf_var[1] = base64.b64encode(tf_var[1].encode("utf-8")) - - local_env[f"TF_VAR_{tf_var[0].upper()}"] = tf_var[1] - else: - click.secho( - f"{working_dir}/worker.auto.tfvars not found!", - fg="red", - ) - - for k, v in extra_vars.items(): - if b64_encode: - v = base64.b64encode(v.encode("utf-8")) - local_env[f"TF_EXTRA_{k.upper()}"] = v - - # execute the hook - (exit_code, stdout, stderr) = pipe_exec( - f"{hook_script} {phase} {command}", - cwd=hook_dir, - env=local_env, - ) - - # handle output from hook_script - if debug: - click.secho(f"exit code: {exit_code}", fg="blue") - for line in stdout.decode().splitlines(): - click.secho(f"stdout: {line}", fg="blue") - for line in stderr.decode().splitlines(): - click.secho(f"stderr: {line}", fg="red") - - if exit_code != 0: - raise HookError("hook script {}") - - @staticmethod - def check_hooks(phase, working_dir, command): - """ - check_hooks determines if a hook exists for a given operation/definition - """ - hook_dir = f"{working_dir}/hooks" - if not os.path.isdir(hook_dir): - # there is no hooks dir - return False - for f in os.listdir(hook_dir): - if os.path.splitext(f)[0] == f"{phase}_{command}": - if os.access(f"{hook_dir}/{f}", os.X_OK): - return True - else: - raise HookError(f"{hook_dir}/{f} exists, but is not executable!") - return False - - @staticmethod - def get_state_item(working_dir, env, terraform_bin, state, item): - """ - The general handler function for getting a state item, it will first - try to get the item from another definitions output, but if the other - definition is not setup, it will fallback to getting the item from the - remote state. - - @param working_dir: The working directory of the terraform definition - @param env: The environment variables to pass to the terraform command - @param terraform_bin: The path to the terraform binary - @param state: The state name to get the item from - @param item: The item to get from the state - """ - try: - return TerraformCommand._get_state_item_from_output( - working_dir, env, terraform_bin, state, item - ) - except FileNotFoundError: - return TerraformCommand._get_state_item_from_remote( - working_dir, env, terraform_bin, state, item - ) - - @staticmethod - def _get_state_item_from_remote(working_dir, env, terraform_bin, state, item): - """ - get_state_item returns json encoded output from a terraform remote state - - @param working_dir: The working directory of the terraform definition - @param env: The environment variables to pass to the terraform command - @param terraform_bin: The path to the terraform binary - @param state: The state name to get the item from - @param item: The item to get from the state - """ - - remote_state = None - - # setup the state cache - cache_file = TerraformCommand._get_cache_name(working_dir) - TerraformCommand._make_state_cache(working_dir, env, terraform_bin) - - # read the cache - with open(cache_file, "r") as f: - state_cache = json.load(f) - - # Get the remote state we are looking for, and raise an error if it's not there - resources = state_cache["values"]["root_module"]["resources"] - for resource in resources: - if ( - resource["type"] == "terraform_remote_state" - and resource["name"] == state - ): - remote_state = resource - if remote_state is None: - raise HookError(f"Remote state item {state} not found") - - if item in remote_state["values"]["outputs"]: - return json.dumps( - remote_state["values"]["outputs"][item], - indent=None, - separators=(",", ":"), - ) - - raise HookError(f"Remote state item {state}.{item} not found in state cache") - - @staticmethod - def _get_state_item_from_output(working_dir, env, terraform_bin, state, item): - """ - Get a single item from the terraform output, this is the preferred - mechanism as items will be more guaranteed to be up to date, but it - creates problems when the remote state is not setup, like when using - --limit - - @param work_dir: The working directory of the terraform definition - @param env: The environment variables to pass to the terraform command - @param terraform_bin: The path to the terraform binary - @param state: The state name to get the item from - @param item: The item to get from the state - """ - - base_dir, _ = os.path.split(working_dir) - try: - (exit_code, stdout, stderr) = pipe_exec( - f"{terraform_bin} output -json -no-color {item}", - cwd=f"{base_dir}/{state}", - env=env, - ) - except FileNotFoundError: - # the remote state is not setup, likely do to use of --limit - # this is acceptable, and is the responsibility of the hook - # to ensure it has all values needed for safe execution - raise - - if exit_code != 0: - raise HookError( - f"Error reading remote state item {state}.{item}, details: {stderr}" - ) - - if stdout is None: - raise HookError( - f"Remote state item {state}.{item} is empty; This is completely" - " unexpected, failing..." - ) - json_output = json.loads(stdout) - return json.dumps(json_output, indent=None, separators=(",", ":")) - - @staticmethod - def _make_state_cache( - working_dir: str, env: dict, terraform_bin: str, refresh: bool = False - ): - """ - Using `terraform show -json` make a cache of the state file - - @param working_dir: The working directory of the terraform definition - @param env: The environment variables to pass to the terraform command - @param terraform_bin: The path to the terraform binary - @param refresh: If true, the cache will be refreshed - """ - - # check if the cache exists - state_cache = TerraformCommand._get_cache_name(working_dir) - if not refresh and os.path.exists(state_cache): - return - - # ensure the state is refreshed; but no changes to resources are made - (exit_code, stdout, stderr) = pipe_exec( - f"{terraform_bin} apply -auto-approve -refresh-only", - cwd=working_dir, - env=env, - ) - - # get the json from terraform to generate the cache - (exit_code, stdout, stderr) = pipe_exec( - f"{terraform_bin} show -json", - cwd=working_dir, - env=env, - ) - - # validate the output, check exit code and ensure output is json - if exit_code != 0: - raise HookError(f"Error reading terraform state, details: {stderr}") - try: - json.loads(stdout) - except json.JSONDecodeError: - raise HookError( - "Error parsing terraform state; output is not in json format" - ) - - # write the cache to disk - try: - with open(state_cache, "w") as f: - f.write(stdout.decode()) - except Exception as e: - raise HookError(f"Error writing state cache to {state_cache}, details: {e}") - return - - @staticmethod - def _get_cache_name(working_dir: str) -> str: - """ - Get the cache directory for the state cache - - @param working_dir: The working directory of the terraform definition - - @return: The cache directory path - """ - return f"{working_dir}/{TF_STATE_CACHE_NAME}" diff --git a/tfworker/constants.py b/tfworker/constants.py index cb7b91e..31ab4ec 100644 --- a/tfworker/constants.py +++ b/tfworker/constants.py @@ -23,4 +23,13 @@ DEFAULT_AWS_REGION = "us-east-1" DEFAULT_GCP_REGION = "us-east-1a" -RESERVED_FILES = ["terraform.tf", "worker-locals.tf", "worker.auto.tfvars"] +TF_STATE_CACHE_NAME = "worker_state_cache.json" +WORKER_LOCALS_FILENAME = "worker-locals.tf" +WORKER_TF_FILENAME = "worker.tf" +WORKER_TFVARS_FILENAME = "worker.auto.tfvars" +RESERVED_FILES = [ + WORKER_LOCALS_FILENAME, + WORKER_TF_FILENAME, + WORKER_TFVARS_FILENAME, + TF_STATE_CACHE_NAME, +] diff --git a/tfworker/definitions.py b/tfworker/definitions.py index db86f52..4eacc84 100644 --- a/tfworker/definitions.py +++ b/tfworker/definitions.py @@ -23,6 +23,7 @@ from mergedeep import merge from tfworker import constants as const +from tfworker.exceptions import ReservedFileError from tfworker.util.copier import CopyFactory TERRAFORM_TPL = """\ @@ -33,10 +34,6 @@ """ -class ReservedFileError(Exception): - pass - - class Definition: _plan_file = None _ready_to_apply = False diff --git a/tfworker/exceptions.py b/tfworker/exceptions.py new file mode 100644 index 0000000..0b33b8d --- /dev/null +++ b/tfworker/exceptions.py @@ -0,0 +1,41 @@ +class HookError(Exception): + """ + Exception is raised when a hook fails, or has execution issues. + """ + + pass + + +class PlanChange(Exception): + """ + Exception is raised when a terraform plan has changes. + """ + + pass + + +class PluginSourceParseException(Exception): + """ + Exception is raised when a plugin source cannot be parsed. + """ + + pass + + +class ReservedFileError(Exception): + """ + Exception is raised when a reserved file is found in the repository; + + Reserved files are files that are used by tfworker, and should not be + present in the repository. + """ + + pass + + +class TerraformError(Exception): + """ + Exception is raised when a terraform command fails. + """ + + pass diff --git a/tfworker/plugins.py b/tfworker/plugins.py index cc30ad6..f66f0b9 100644 --- a/tfworker/plugins.py +++ b/tfworker/plugins.py @@ -22,11 +22,8 @@ import click from tenacity import retry, stop_after_attempt, wait_chain, wait_fixed -from tfworker.commands.root import get_platform - - -class PluginSourceParseException(Exception): - pass +from tfworker.exceptions import PluginSourceParseException +from tfworker.util.system import get_platform class PluginsCollection(collections.abc.Mapping): diff --git a/tfworker/types.py b/tfworker/types.py new file mode 100644 index 0000000..d5d95cb --- /dev/null +++ b/tfworker/types.py @@ -0,0 +1,32 @@ +from enum import Enum +from typing import Any, Dict, List, Union + + +class TerraformAction(Enum): + """ + Terraform actions + """ + + PLAN = "plan" + APPLY = "apply" + DESTROY = "destroy" + INIT = "init" + + def __str__(self): + return self.value + + +class TerraformStage(Enum): + """ + Stages around a terraform action, pre and post + """ + + PRE = "pre" + POST = "post" + + def __str__(self): + return self.value + + +# https://github.com/python/typing/issues/182 +JSONType = Union[str, int, float, bool, None, Dict[str, Any], List[Any]] diff --git a/tfworker/util/hooks.py b/tfworker/util/hooks.py new file mode 100644 index 0000000..03683c2 --- /dev/null +++ b/tfworker/util/hooks.py @@ -0,0 +1,586 @@ +# This contains utility functions for dealing with hooks +# in terraform definitions, it's primary purpose is to be +# used by the TerraformCommand class, while reducing the +# responsibility of the class itself. +import base64 +import json +import os +import re +from enum import Enum +from typing import Any, Dict + +import click + +from tfworker.backends import BaseBackend +from tfworker.constants import ( + TF_STATE_CACHE_NAME, + WORKER_LOCALS_FILENAME, + WORKER_TFVARS_FILENAME, +) +from tfworker.exceptions import HookError +from tfworker.types import TerraformAction, TerraformStage +from tfworker.util.system import pipe_exec + + +class TFHookVarType(Enum): + """ + Enum for the types of hook variables. + """ + + VAR = "TF_VAR" + REMOTE = "TF_REMOTE" + EXTRA = "TF_EXTRA" + + def __str__(self): + return self.value.upper() + + +def get_state_item( + working_dir: str, + env: Dict[str, str], + terraform_bin: str, + state: str, + item: str, + backend: BaseBackend = None, +) -> str: + """ + General handler function for getting a state item. First tries to get the item from another definition's output, + and if the other definition is not set up, falls back to getting the item from the remote state. + + Args: + working_dir (str): The working directory of the terraform definition. + env (dict[str, str]): The environment variables to pass to the terraform command. + terraform_bin (str): The path to the terraform binary. + state (str): The state name to get the item from. + item (str): The item to get from the state. + + Returns: + str: The state item, key: value as a JSON string. + + Raises: + HookError: If the state item is not found in the remote state. + """ + + try: + click.secho(f"Getting state item {state}.{item} from output", fg="blue") + return _get_state_item_from_output(working_dir, env, terraform_bin, state, item) + except FileNotFoundError: + click.secho( + f"Remote state not setup, falling back to getting state item {state}.{item} from remote", + fg="blue", + ) + return _get_state_item_from_remote(working_dir, env, terraform_bin, state, item) + + +def _get_state_item_from_output( + working_dir: str, env: Dict[str, str], terraform_bin: str, state: str, item: str +) -> str: + """ + Get a single item from the terraform output. This is the preferred + mechanism as items will be more guaranteed to be up to date, but it + creates problems when the remote state is not set up, like when using + --limit. + + Args: + working_dir (str): The working directory of the terraform definition. + env (Dict[str, str]): The environment variables to pass to the terraform command. + terraform_bin (str): The path to the terraform binary. + state (str): The state name to get the item from. + item (str): The item to get from the state. + + Returns: + str: The item from the terraform output in JSON format. + + Raises: + HookError: If there is an error reading the remote state item or if the output is empty or not in JSON format. + """ + base_dir, _ = os.path.split(working_dir) + try: + (exit_code, stdout, stderr) = pipe_exec( + f"{terraform_bin} output -json -no-color {item}", + cwd=f"{base_dir}/{state}", + env=env, + ) + except FileNotFoundError: + # the remote state is not setup, likely do to use of --limit + # this is acceptable, and is the responsibility of the hook + # to ensure it has all values needed for safe execution + raise + + if exit_code != 0: + raise HookError( + f"Error reading remote state item {state}.{item}, details: {stderr}" + ) + + if stdout is None: + raise HookError( + f"Remote state item {state}.{item} is empty; This is completely" + " unexpected, failing..." + ) + + try: + json_output = json.loads(stdout) + except json.JSONDecodeError: + raise HookError( + f"Error parsing remote state item {state}.{item}; output is not in JSON format" + ) + + return json.dumps(json_output, indent=None, separators=(",", ":")) + + +def check_hooks( + phase: TerraformStage, working_dir: str, command: TerraformAction +) -> bool: + """ + Check if a hook script exists for the given phase and command. + + Args: + phase (TerraformStage): The phase of the terraform command. + working_dir (str): The working directory of the terraform definition. + command (TerraformAction): The terraform command to run. + + Returns: + bool: True if the hook script exists and is executable, False otherwise. + """ + hook_dir = f"{working_dir}/hooks" + if not os.path.isdir(hook_dir): + # there is no hooks dir + return False + for f in os.listdir(hook_dir): + if os.path.splitext(f)[0] == f"{phase}_{command}": + if os.access(f"{hook_dir}/{f}", os.X_OK): + return True + else: + raise HookError(f"{hook_dir}/{f} exists, but is not executable!") + return False + + +def hook_exec( + phase: TerraformStage, + command: TerraformAction, + working_dir: str, + env: Dict[str, str], + terraform_path: str, + debug: bool = False, + b64_encode: bool = False, + extra_vars: Dict[str, str] = None, +) -> None: + """ + Coordinates the execution of a hook script. This function is responsible for finding and executing + the script, as well as setting up the proper environment variables with all of the terraform vars, + and vars from remote data sources. + + Args: + phase (TerraformPhase): The phase of the hook. + command (TerraformAction): A poorly named variable, that is the terraform action being executed + working_dir (str): The working directory of the Terraform definition. + env (Dict[str, str]): The environment variables to pass to the hook. + terraform_path (str): The path to the Terraform binary. + debug (bool, optional): If True, debug information will be printed. Defaults to False. + b64_encode (bool, optional): If True, variables will be base64 encoded. Defaults to False. + extra_vars (Dict[str, str], optional): Additional environment variables to set. Defaults to None. + + Raises: + HookError: If the hook script is missing or if execution fails. + """ + if extra_vars is None: + extra_vars = {} + + local_env = _prepare_environment(env, terraform_path) + hook_script = _find_hook_script(working_dir, phase, command) + _populate_environment_with_terraform_variables( + local_env, working_dir, terraform_path, b64_encode + ) + _populate_environment_with_terraform_remote_vars( + local_env, working_dir, terraform_path, b64_encode + ) + _populate_environment_with_extra_vars(local_env, extra_vars, b64_encode) + _execute_hook_script(hook_script, phase, command, working_dir, local_env, debug) + + +def _find_hook_script(working_dir: str, phase: str, command: str) -> str: + """ + Finds the hook script to execute. + + Args: + working_dir (str): The working directory of the Terraform definition. + phase (str): The phase of the hook. + command (str): The command to execute. + + Returns: + str: The path to the hook script. + + Raises: + HookError: If the hook script is missing. + """ + hook_dir = os.path.join(working_dir, "hooks") + for f in os.listdir(hook_dir): + if os.path.splitext(f)[0] == f"{phase}_{command}": + return os.path.join(hook_dir, f) + raise HookError(f"Hook script missing from {hook_dir}") + + +def _prepare_environment(env: Dict[str, str], terraform_path: str) -> Dict[str, str]: + """ + Prepares the environment variables for the hook script execution. + + Args: + env (Dict[str, str]): The initial environment variables. + terraform_path (str): The path to the Terraform binary. + + Returns: + Dict[str, str]: The prepared environment variables. + """ + local_env = env.copy() + local_env["TF_PATH"] = terraform_path + return local_env + + +def _populate_environment_with_terraform_variables( + local_env: Dict[str, str], working_dir: str, terraform_path: str, b64_encode: bool +) -> None: + """ + Populates the environment with Terraform variables. + + Args: + local_env (Dict[str, str]): The environment variables. + working_dir (str): The working directory of the Terraform definition. + terraform_path (str): The path to the Terraform binary. + b64_encode (bool): If True, variables will be base64 encoded. + """ + if not os.path.isfile(os.path.join(working_dir, WORKER_TFVARS_FILENAME)): + return + + with open(os.path.join(working_dir, WORKER_TFVARS_FILENAME)) as f: + contents = f.read() + + for line in contents.splitlines(): + tf_var = line.split("=") + _set_hook_env_var( + local_env, TFHookVarType.VAR, tf_var[0], tf_var[1], b64_encode + ) + + +def _populate_environment_with_terraform_remote_vars( + local_env: Dict[str, str], working_dir: str, terraform_path: str, b64_encode: bool +) -> None: + """ + Populates the environment with Terraform variables. + + Args: + local_env (Dict[str, str]): The environment variables. + working_dir (str): The working directory of the Terraform definition. + terraform_path (str): The path to the Terraform binary. + b64_encode (bool): If True, variables will be base64 encoded. + """ + if not os.path.isfile(os.path.join(working_dir, WORKER_LOCALS_FILENAME)): + return + + with open(os.path.join(working_dir, WORKER_LOCALS_FILENAME)) as f: + contents = f.read() + + # I'm sorry. :-) + # this regex looks for variables in the form of: + # = data.terraform_remote_state..outputs. + r = re.compile( + r"\s*(?P\w+)\s*\=.+data\.terraform_remote_state\.(?P\w+)\.outputs\.(?P\w+)\s*" + ) + + for line in contents.splitlines(): + m = r.match(line) + if m: + item = m.group("item") + state = m.group("state") + state_item = m.group("state_item") + state_value = get_state_item( + working_dir, local_env, terraform_path, state, state_item + ) + _set_hook_env_var( + local_env, TFHookVarType.REMOTE, item, state_value, b64_encode + ) + + +def _populate_environment_with_extra_vars( + local_env: Dict[str, str], extra_vars: Dict[str, Any], b64_encode: bool +) -> None: + """ + Populates the environment with extra variables. + + Args: + local_env (Dict[str, str]): The environment variables. + extra_vars (Dict[str, Any]): The extra variables to set. + b64_encode (bool): If True, variables will be base64 encoded. + """ + for k, v in extra_vars.items(): + _set_hook_env_var(local_env, TFHookVarType.EXTRA, k, v, b64_encode) + + +def _set_hook_env_var( + local_env: Dict[str, str], + var_type: TFHookVarType, + key: str, + value: str, + b64_encode: bool = False, +) -> None: + """ + Sets a hook environment variable. + + Args: + local_env (Dict[str, str]): The environment variables. + var_type (TFHookVarType): The type of the variable. + key (str): The key of the variable. + value (str): The value of the variable. + b64_encode (bool, optional): If True, the value will be base64 encoded. Defaults to False. + """ + key_replace_items = {" ": "", '"': "", "-": "_", ".": "_"} + val_replace_items = {" ": "", '"': "", "\n": ""} + + for k, v in key_replace_items.items(): + key = key.replace(k, v) + + for k, v in val_replace_items.items(): + value = value.replace(k, v) + + if b64_encode: + value = base64.b64encode(value.encode()) + + local_env[f"{var_type}_{key.upper()}"] = value + + +def _execute_hook_script( + hook_script: str, + phase: str, + command: str, + working_dir: str, + local_env: Dict[str, str], + debug: bool, +) -> None: + """ + Executes the hook script and handles its output. + + Args: + hook_script (str): The path to the hook script. + phase (str): The phase of the hook. + command (str): The command to execute. + working_dir (str): The working directory of the Terraform definition. + local_env (Dict[str, str]): The environment variables. + debug (bool): If True, debug information will be printed. + + Raises: + HookError: If the hook script execution fails. + """ + hook_dir = os.path.join(working_dir, "hooks") + exit_code, stdout, stderr = pipe_exec( + f"{hook_script} {phase} {command}", cwd=hook_dir, env=local_env + ) + + if debug: + click.secho(f"Results from hook script: {hook_script}", fg="blue") + click.secho(f"exit code: {exit_code}", fg="blue") + for line in stdout.decode().splitlines(): + click.secho(f"stdout: {line}", fg="blue") + for line in stderr.decode().splitlines(): + click.secho(f"stderr: {line}", fg="red") + + if exit_code != 0: + raise HookError( + f"Hook script {hook_script} execution failed with exit code {exit_code}" + ) + + +def _get_state_item_from_remote( + working_dir: str, env: Dict[str, str], terraform_bin: str, state: str, item: str +) -> str: + """ + Retrieve a state item from terraform remote state. + + Args: + working_dir: The working directory of the terraform definition. + env: The environment variables to pass to the terraform command. + terraform_bin: The path to the terraform binary. + state: The state name to get the item from. + item: The item to get from the state. + + Returns: + A JSON string of the state item. + + Raises: + HookError: If the state item cannot be found or read. + """ + cache_file = _get_state_cache_name(working_dir) + _make_state_cache(working_dir, env, terraform_bin) + + state_cache = _read_state_cache(cache_file) + remote_state = _find_remote_state(state_cache, state) + + return _get_item_from_remote_state(remote_state, state, item) + + +def _get_state_cache_name(working_dir: str) -> str: + """ + Get the name of the state cache file. + + Args: + working_dir (str): The working directory of the terraform definition. + + Returns: + str: The name of the state cache file. + """ + return f"{working_dir}/{TF_STATE_CACHE_NAME}" + + +def _make_state_cache( + working_dir: str, env: Dict[str, str], terraform_bin: str, refresh: bool = False +) -> None: + """ + Create a cache of the terraform state file. + + Args: + working_dir (str): The working directory of the terraform definition. + env ({str, str}): The environment variables to pass to the terraform command. + terraform_bin (str): The path to the terraform binary. + refresh (bool, optional): If true, the cache will be refreshed. Defaults to False. + + Raises: + HookError: If there is an error reading or writing the state cache. + """ + state_cache = _get_state_cache_name(working_dir) + if not refresh and os.path.exists(state_cache): + return + + _run_terraform_refresh(terraform_bin, working_dir, env) + state_json = _run_terraform_show(terraform_bin, working_dir, env) + _write_state_cache(state_cache, state_json) + + +def _read_state_cache(cache_file: str) -> Dict[str, Any]: + """ + Read the state cache from a file. + + Args: + cache_file (str): The path to the state cache file. + + Returns: + Dict[str, Any]: The state cache JSON. + """ + with open(cache_file, "r") as f: + return json.load(f) + + +def _find_remote_state(state_cache: Dict[str, Any], state: str) -> Dict[str, Any]: + """ + Find the remote state in the state cache. + + Args: + state_cache (Dict[str, Any]): The state cache JSON. + state (str): The state name to find. + + Returns: + Dict[str, Any]: The remote state JSON. + + Raises: + HookError: If the remote state is not found in the state cache. + """ + resources = state_cache["values"]["root_module"]["resources"] + for resource in resources: + if resource["type"] == "terraform_remote_state" and resource["name"] == state: + return resource + raise HookError(f"Remote state item {state} not found") + + +def _get_item_from_remote_state( + remote_state: Dict[str, Any], state: str, item: str +) -> str: + """ + Get an item from the remote state JSON + + Args: + remote_state (Dict[str, Any]): The remote state JSON. + state (str): The state name. + item (str): The item to get from the state. + + Returns: + str: The item from the remote state in JSON format. + + Raises: + HookError: If the item is not found in the remote state. + """ + if item in remote_state["values"]["outputs"]: + return json.dumps( + remote_state["values"]["outputs"][item], + indent=None, + separators=(",", ":"), + ) + raise HookError(f"Remote state item {state}.{item} not found in state cache") + + +def _run_terraform_refresh( + terraform_bin: str, working_dir: str, env: Dict[str, str] +) -> None: + """ + Run `terraform apply -refresh-only` to ensure the state is refreshed. + + Args: + terraform_bin (str): The path to the terraform binary. + working_dir (str): The working directory of the terraform definition. + env (Dict[str, str]): The environment variables to pass to the terraform command. + + Raises: + HookError: If there is an error refreshing the terraform state. + """ + exit_code, _, stderr = pipe_exec( + f"{terraform_bin} apply -auto-approve -refresh-only", + cwd=working_dir, + env=env, + ) + if exit_code != 0: + raise HookError(f"Error applying terraform state, details: {stderr}") + + +def _run_terraform_show( + terraform_bin: str, working_dir: str, env: Dict[str, str] +) -> str: + """ + Run `terraform show -json` to get the state in JSON format. + + Args: + terraform_bin (str): The path to the terraform binary. + working_dir (str): The working directory of the terraform definition. + env (Dict[str, str]): The environment variables to pass to the terraform command. + + Returns: + str: The state in JSON format. + + Raises: + HookError: If there is an error reading the terraform state. + """ + exit_code, stdout, stderr = pipe_exec( + f"{terraform_bin} show -json", + cwd=working_dir, + env=env, + ) + if exit_code != 0: + raise HookError(f"Error reading terraform state, details: {stderr}") + try: + json.loads(stdout) + except json.JSONDecodeError: + raise HookError("Error parsing terraform state; output is not in JSON format") + return stdout.decode() + + +def _write_state_cache(state_cache: str, state_json: str) -> None: + """ + Write the state JSON to the cache file. + + Args: + state_cache (str): The path to the state cache file. + state_json (str): The state JSON to write. + + Raises: + HookError: If there is an error writing the state cache. + """ + try: + with open(state_cache, "w") as f: + f.write(state_json) + except Exception as e: + raise HookError(f"Error writing state cache to {state_cache}, details: {e}") diff --git a/tfworker/util/system.py b/tfworker/util/system.py index 4cff798..67bd282 100644 --- a/tfworker/util/system.py +++ b/tfworker/util/system.py @@ -12,27 +12,49 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +import platform import re import shlex import subprocess +from typing import Dict, List, Tuple, Union import click import importlib_metadata -def strip_ansi(line): +def strip_ansi(line: str) -> str: """ Strips ANSI escape sequences from a string. + + Args: + line (str): The string to strip ANSI escape sequences from. + + Returns: + str: The string with ANSI escape sequences stripped. """ ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") return ansi_escape.sub("", line) -def pipe_exec(args, stdin=None, cwd=None, env=None, stream_output=False): +def pipe_exec( + args: Union[str, List[str]], + stdin: str = None, + cwd: str = None, + env: Dict[str, str] = None, + stream_output: bool = False, +) -> Tuple[int, Union[bytes, None], Union[bytes, None]]: """ - A function to accept a list of commands and pipe them together. + A function to take one or more commands and execute them in a pipeline, returning the output of the last command. - Takes optional stdin to give to the first item in the pipe chain. + Args: + args (str or list): A string or list of strings representing the command(s) to execute. + stdin (str, optional): A string to pass as stdin to the first command + cwd (str, optional): The working directory to execute the command in. + env (dict, optional): A dictionary of environment variables to set for the command. + stream_output (bool, optional): A boolean indicating if the output should be streamed back to the caller. + + Returns: + tuple: A tuple containing the return code, stdout, and stderr of the last command in the pipeline. """ commands = [] # listed used to hold all the popen objects # use the default environment if one is not specified @@ -132,8 +154,16 @@ def pipe_exec(args, stdin=None, cwd=None, env=None, stream_output=False): return (returncode, stdout, stderr) -def which(program): - """From stack overflow""" +def which(program: str) -> Union[str, None]: + """ + A function to mimic the behavior of the `which` command in Unix-like systems. + + Args: + program (str): The program to search for in the PATH. + + Returns: + str: The full path to the program if found, otherwise None. + """ def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) @@ -153,8 +183,35 @@ def is_exe(fpath): def get_version() -> str: """ Get the version of the current package + + Returns: + str: The version of the package """ try: return importlib_metadata.version("terraform-worker") except importlib_metadata.PackageNotFoundError: return "unknown" + + +def get_platform() -> Tuple[str, str]: + """ + Returns a formatted operating system / architecture tuple that is consistent with common distribution creation tools. + + Returns: + tuple: A tuple containing the operating system and architecture. + """ + + # strip off "2" which only appears on old linux kernels + opsys = platform.system().rstrip("2").lower() + + # make sure machine uses consistent format + machine = platform.machine() + if machine == "x86_64": + machine = "amd64" + + # some 64 bit arm extensions will `report aarch64, this is functionaly + # equivalent to arm64 which is recognized and the pattern used by the TF + # community + if machine == "aarch64": + machine = "arm64" + return (opsys, machine) diff --git a/tfworker/util/terraform.py b/tfworker/util/terraform.py index b4d162d..9ff47f5 100644 --- a/tfworker/util/terraform.py +++ b/tfworker/util/terraform.py @@ -2,10 +2,14 @@ # the goal of moving these functions here is to reduce the responsibility of # the TerraformCommand class, making it easier to test and maintain import pathlib +import re import shutil import click +from tfworker.constants import DEFAULT_REPOSITORY_PATH +from tfworker.util.system import pipe_exec + def prep_modules( module_path: str, @@ -23,6 +27,11 @@ def prep_modules( ignore_patterns (list(str)): A list of patterns to ignore required (bool): If the terraform modules directory is required """ + module_path = ( + module_path + if module_path != "" + else f"{DEFAULT_REPOSITORY_PATH}/terraform-modules" + ) module_path = pathlib.Path(module_path) target_path = pathlib.Path(f"{target_path}/terraform-modules".replace("//", "/")) @@ -46,3 +55,21 @@ def prep_modules( symlinks=True, ignore=shutil.ignore_patterns(*ignore_patterns), ) + + +def get_terraform_version(terraform_bin): + (return_code, stdout, stderr) = pipe_exec(f"{terraform_bin} version") + if return_code != 0: + click.secho(f"unable to get terraform version\n{stderr}", fg="red") + raise SystemExit(1) + version = stdout.decode("UTF-8").split("\n")[0] + version_search = re.search(r".*\s+v(\d+)\.(\d+)\.(\d+)", version) + if version_search: + click.secho( + f"Terraform Version Result: {version}, using major:{version_search.group(1)}, minor:{version_search.group(2)}", + fg="yellow", + ) + return (int(version_search.group(1)), int(version_search.group(2))) + else: + click.secho(f"unable to get terraform version\n{stderr}", fg="red") + raise SystemExit(1)