From 592e7fa9482b0acd8fab089d12b2c8b740b4483d Mon Sep 17 00:00:00 2001 From: Richard Maynard Date: Fri, 14 Jun 2024 19:24:59 -0500 Subject: [PATCH] Refactoring focused on TerraformCommand The work to continue refactoring this code continues; the primary motiviation is to improve unit tests, and lower the friction to adding new functionality. The TerraformCommand is a focus for this effort due to both the number of largely untestable methods, and its criticality to the project as a whole. - centralize many exceptions into the tfworker.exceptions module - create tfworker.types to start using enums for common strings - JSON type moved out of an __init__ into types - create TerraformAction type (plan/apply/destroy) - create TerraformStage type (pre/post, relation to Action) - create a tfworker.util.terraform for terraform utility methods - move get_terraform_version to tfworker.util.terraform - move get_platform to tfworker.util.system - create tfworker.util.hooks to handle all operations related to executing hook scripts. These all forced onto the TerraformCommand class previously - hooks has high test coverage - move - clean up and create constants for references to files that tfworker creates - In TerraformCommand: - break down the logic that determines how to plan into smaller methods - reorder and group remaining methods - removed all the hook related methods There are a lot of changes overall to the tests which are not enumerated, the complexity of testing is going down, and coverage is going up. Any areas of touched code are standardizing on using the Google Python Docstring comments format, and are having type annotations added --- tests/commands/test_root.py | 29 +- tests/commands/test_terraform.py | 31 +- tests/conftest.py | 14 +- tests/test_plugins.py | 3 +- tests/util/test_hooks.py | 219 +++++++++++ tests/util/test_system.py | 28 +- tests/util/test_terraform_util.py | 21 ++ tfworker/backends/base.py | 2 +- tfworker/cli.py | 14 +- tfworker/commands/__init__.py | 1 + tfworker/commands/base.py | 25 +- tfworker/commands/root.py | 25 -- tfworker/commands/terraform.py | 467 +++++------------------- tfworker/constants.py | 11 +- tfworker/definitions.py | 5 +- tfworker/exceptions.py | 41 +++ tfworker/plugins.py | 7 +- tfworker/types.py | 32 ++ tfworker/util/hooks.py | 586 ++++++++++++++++++++++++++++++ tfworker/util/system.py | 69 +++- tfworker/util/terraform.py | 27 ++ 21 files changed, 1143 insertions(+), 514 deletions(-) create mode 100644 tests/util/test_hooks.py create mode 100644 tfworker/exceptions.py create mode 100644 tfworker/types.py create mode 100644 tfworker/util/hooks.py diff --git a/tests/commands/test_root.py b/tests/commands/test_root.py index bcf2193..d4bbcff 100644 --- a/tests/commands/test_root.py +++ b/tests/commands/test_root.py @@ -16,13 +16,12 @@ import os import platform from tempfile import TemporaryDirectory -from unittest import mock import pytest from deepdiff import DeepDiff import tfworker.commands.root -from tfworker.commands.root import get_platform, ordered_config_load +from tfworker.commands.root import ordered_config_load class TestMain: @@ -220,32 +219,6 @@ def test_config_formats(self, yaml_base_rootc, json_base_rootc, hcl_base_rootc): diff = DeepDiff(json_config, hcl_config) assert len(diff) == 0 - @pytest.mark.parametrize( - "opsys, machine, mock_platform_opsys, mock_platform_machine", - [ - ("linux", "i386", ["linux2"], ["i386"]), - ("linux", "arm", ["Linux"], ["arm"]), - ("linux", "amd64", ["linux"], ["x86_64"]), - ("linux", "amd64", ["linux"], ["amd64"]), - ("darwin", "amd64", ["darwin"], ["x86_64"]), - ("darwin", "amd64", ["darwin"], ["amd64"]), - ("darwin", "arm", ["darwin"], ["arm"]), - ("darwin", "arm64", ["darwin"], ["aarch64"]), - ], - ) - def test_get_platform( - self, opsys, machine, mock_platform_opsys, mock_platform_machine - ): - with mock.patch("platform.system", side_effect=mock_platform_opsys) as mock1: - with mock.patch( - "platform.machine", side_effect=mock_platform_machine - ) as mock2: - actual_opsys, actual_machine = get_platform() - assert opsys == actual_opsys - assert machine == actual_machine - mock1.assert_called_once() - mock2.assert_called_once() - class TestOrderedConfigLoad: def test_ordered_config_load(self): diff --git a/tests/commands/test_terraform.py b/tests/commands/test_terraform.py index 169c370..7539880 100644 --- a/tests/commands/test_terraform.py +++ b/tests/commands/test_terraform.py @@ -22,7 +22,7 @@ from google.cloud.exceptions import NotFound import tfworker -from tfworker.commands.terraform import BaseCommand, TerraformCommand, TerraformError +from tfworker.commands.terraform import TerraformCommand, TerraformError from tfworker.definitions import Definition from tfworker.handlers import HandlerError @@ -185,27 +185,6 @@ def test_run(self, tf_cmd: str, method: callable, args: list, request): for arg in args: assert arg in call_as_string - @pytest.mark.parametrize( - "stdout, major, minor, expected_exception", - [ - ("Terraform v0.12.29", 0, 12, does_not_raise()), - ("Terraform v1.3.5", 1, 3, does_not_raise()), - ("TF 14", "", "", pytest.raises(SystemExit)), - ], - ) - def test_get_tf_version( - self, stdout: str, major: int, minor: int, expected_exception: callable - ): - with mock.patch( - "tfworker.commands.base.pipe_exec", - side_effect=mock_tf_version, - ) as mocked: - with expected_exception: - (actual_major, actual_minor) = BaseCommand.get_terraform_version(stdout) - assert actual_major == major - assert actual_minor == minor - mocked.assert_called_once() - def test_worker_options(self, tf_13cmd_options): # Verify that the options from the CLI override the options from the config assert tf_13cmd_options._rootc.worker_options_odict.get("backend") == "s3" @@ -236,7 +215,7 @@ def test_worker_options(self, tf_13cmd_options): def test_no_create_backend_bucket_fails_gcs(self, grootc_no_create_backend_bucket): with pytest.raises(SystemExit): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.commands.base.get_terraform_version", side_effect=lambda x: (13, 3), ): with mock.patch( @@ -367,7 +346,7 @@ def test_exec_valid_flow(self, terraform_command, definition): mock_prep_modules.assert_called_once_with( terraform_command._terraform_modules_dir, terraform_command._temp_dir, - required=False, + required=True, ) mock_prep_and_init.assert_called_once_with(def_iter) mock_check_plan.assert_called_once_with(definition) @@ -414,7 +393,7 @@ def test_exec_without_plan(self, terraform_command, definition): mock_prep_modules.assert_called_once_with( terraform_command._terraform_modules_dir, terraform_command._temp_dir, - required=False, + required=True, ) mock_prep_and_init.assert_called_once_with(def_iter) mock_check_plan.assert_called_once_with(definition) @@ -449,7 +428,7 @@ def test_exec_with_no_apply_or_destroy(self, terraform_command, definition): mock_prep_modules.assert_called_once_with( terraform_command._terraform_modules_dir, terraform_command._temp_dir, - required=False, + required=True, ) mock_prep_and_init.assert_called_once_with(def_iter) mock_check_plan.assert_called_once_with(definition) diff --git a/tests/conftest.py b/tests/conftest.py index 78a84b4..f390aba 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -304,7 +304,7 @@ def rootc_options(s3_client, dynamodb_client, sts_client): @pytest.fixture def basec(rootc, s3_client): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.commands.base.get_terraform_version", side_effect=lambda x: (13, 3), ): with mock.patch( @@ -319,7 +319,7 @@ def basec(rootc, s3_client): @pytest.fixture def gbasec(grootc): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.commands.base.get_terraform_version", side_effect=lambda x: (13, 3), ): with mock.patch( @@ -342,7 +342,7 @@ def tf_Xcmd(rootc): @pytest.fixture def tf_15cmd(rootc): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.util.terraform.get_terraform_version", side_effect=lambda x: (15, 0), ): with mock.patch( @@ -357,7 +357,7 @@ def tf_15cmd(rootc): @pytest.fixture def tf_14cmd(rootc): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.util.terraform.get_terraform_version", side_effect=lambda x: (14, 5), ): with mock.patch( @@ -372,7 +372,7 @@ def tf_14cmd(rootc): @pytest.fixture def tf_13cmd(rootc): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.util.terraform.get_terraform_version", side_effect=lambda x: (13, 5), ): with mock.patch( @@ -387,7 +387,7 @@ def tf_13cmd(rootc): @pytest.fixture def tf_12cmd(rootc): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.util.terraform.get_terraform_version", side_effect=lambda x: (12, 27), ): with mock.patch( @@ -402,7 +402,7 @@ def tf_12cmd(rootc): @pytest.fixture def tf_13cmd_options(rootc_options): with mock.patch( - "tfworker.commands.base.BaseCommand.get_terraform_version", + "tfworker.util.terraform.get_terraform_version", side_effect=lambda x: (13, 5), ): with mock.patch( diff --git a/tests/test_plugins.py b/tests/test_plugins.py index f2208b4..4398045 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -20,9 +20,10 @@ import tfworker.commands.root import tfworker.plugins +from tfworker.util.system import get_platform # values needed by multiple tests -opsys, machine = tfworker.commands.root.get_platform() +opsys, machine = get_platform() _platform = f"{opsys}_{machine}" diff --git a/tests/util/test_hooks.py b/tests/util/test_hooks.py new file mode 100644 index 0000000..265cf23 --- /dev/null +++ b/tests/util/test_hooks.py @@ -0,0 +1,219 @@ +import pytest +from unittest import mock +from tfworker.util.system import pipe_exec +from tfworker.exceptions import HookError + +import tfworker.util.hooks as hooks +from tfworker.types import TerraformAction, TerraformStage + +# Fixture for a mock Terraform state file +@pytest.fixture +def mock_terraform_state(): + return """ + { + "version": 4, + "terraform_version": "0.13.5", + "serial": 1, + "lineage": "8a2b56d2-4e16-48de-9c5b-c640d6b3a52d", + "outputs": {}, + "resources": [ + { + "module": "module.remote_state", + "mode": "data", + "type": "terraform_remote_state", + "name": "example", + "provider": "provider[\"registry.terraform.io/hashicorp/terraform\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "backend": "gcs", + "config": {}, + "outputs": { + "key": "value", + "another_key": "another_value" + } + } + } + ] + } + ] + } + """ + +@pytest.fixture +def mock_terraform_locals(): + return """locals { + local_key = data.terraform_remote_state.example.outputs.key + local_another_key = data.terraform_remote_state.example.outputs.another_key +} +""" + +# Test for `get_state_item` +class TestGetStateItem: + @mock.patch('tfworker.util.hooks._get_state_item_from_output') + @mock.patch('tfworker.util.hooks._get_state_item_from_remote') + def test_get_state_item_from_output_success(self, mock_remote, mock_output): + mock_output.return_value = '{"key": "value"}' + result = hooks.get_state_item("working_dir", {}, "terraform_bin", "state", "item") + assert result == '{"key": "value"}' + mock_output.assert_called_once() + mock_remote.assert_not_called() + + @mock.patch('tfworker.util.hooks._get_state_item_from_output', side_effect=FileNotFoundError) + @mock.patch('tfworker.util.hooks._get_state_item_from_remote') + def test_get_state_item_from_remote_success(self, mock_remote, mock_output): + mock_remote.return_value = '{"key": "value"}' + result = hooks.get_state_item("working_dir", {}, "terraform_bin", "state", "item") + assert result == '{"key": "value"}' + mock_output.assert_called_once() + mock_remote.assert_called_once() + +# Test for `_get_state_item_from_output` +class TestGetStateItemFromOutput: + @mock.patch('tfworker.util.hooks.pipe_exec') + def test_get_state_item_from_output_success(self, mock_pipe_exec): + mock_pipe_exec.return_value = (0, '{"key":"value"}', '') + result = hooks._get_state_item_from_output("working_dir", {}, "terraform_bin", "state", "item") + assert result == '{"key":"value"}' + mock_pipe_exec.assert_called_once() + + @mock.patch('tfworker.util.hooks.pipe_exec', side_effect=FileNotFoundError) + def test_get_state_item_from_output_file_not_found(self, mock_pipe_exec): + with pytest.raises(FileNotFoundError): + hooks._get_state_item_from_output("working_dir", {}, "terraform_bin", "state", "item") + + @mock.patch('tfworker.util.hooks.pipe_exec') + def test_get_state_item_from_output_error(self, mock_pipe_exec): + mock_pipe_exec.return_value = (1, '', 'error') + with pytest.raises(HookError): + hooks._get_state_item_from_output("working_dir", {}, "terraform_bin", "state", "item") + + @mock.patch('tfworker.util.hooks.pipe_exec') + def test_get_state_item_from_output_empty_output(self, mock_pipe_exec): + mock_pipe_exec.return_value = (0, None, '') + with pytest.raises(HookError) as e: + hooks._get_state_item_from_output("working_dir", {}, "terraform_bin", "state", "item") + assert "Remote state item state.item is empty" in str(e.value) + + @mock.patch('tfworker.util.hooks.pipe_exec') + def test_get_state_item_from_output_invalid_json(self, mock_pipe_exec): + mock_pipe_exec.return_value = (0, 'invalid_json', '') + with pytest.raises(HookError) as e: + hooks._get_state_item_from_output("working_dir", {}, "terraform_bin", "state", "item") + assert "output is not in JSON format" in str(e.value) + +# Test for `check_hooks` +class TestCheckHooks: + @mock.patch('tfworker.util.hooks.os.path.isdir', return_value=True) + @mock.patch('tfworker.util.hooks.os.listdir', return_value=[f"{TerraformStage.PRE}_{TerraformAction.PLAN}"]) + @mock.patch('tfworker.util.hooks.os.access', return_value=True) + def test_check_hooks_exists(self, mock_access, mock_listdir, mock_isdir): + result = hooks.check_hooks(TerraformStage.PRE, 'working_dir', TerraformAction.PLAN) + assert result is True + mock_isdir.assert_called_once() + mock_listdir.assert_called_once() + mock_access.assert_called_once() + + @mock.patch('tfworker.util.hooks.os.path.isdir', return_value=False) + def test_check_hooks_no_dir(self, mock_isdir): + result = hooks.check_hooks('phase', 'working_dir', 'command') + assert result is False + mock_isdir.assert_called_once() + + @mock.patch('tfworker.util.hooks.os.path.isdir', return_value=True) + @mock.patch('tfworker.util.hooks.os.listdir', return_value=[f"{TerraformStage.PRE}_{TerraformAction.PLAN}"]) + @mock.patch('tfworker.util.hooks.os.access', return_value=False) + def test_check_hooks_not_executable(self, mock_listdir, mock_isdir, mock_access): + with pytest.raises(HookError) as e: + hooks.check_hooks(TerraformStage.PRE, 'working_dir', TerraformAction.PLAN) + assert "working_dir/hooks/pre_plan exists, but is not executable!" in str(e.value) + + @mock.patch('tfworker.util.hooks.os.path.isdir', return_value=True) + @mock.patch('tfworker.util.hooks.os.listdir', return_value=[]) + def test_check_hooks_no_hooks(self, mock_listdir, mock_isdir): + result = hooks.check_hooks('phase', 'working_dir', 'command') + assert result is False + +# Test for `hook_exec` +class TestHookExec: + @mock.patch('tfworker.util.hooks._prepare_environment') + @mock.patch('tfworker.util.hooks._find_hook_script') + @mock.patch('tfworker.util.hooks._populate_environment_with_terraform_variables') + @mock.patch('tfworker.util.hooks._populate_environment_with_terraform_remote_vars') + @mock.patch('tfworker.util.hooks._populate_environment_with_extra_vars') + @mock.patch('tfworker.util.hooks._execute_hook_script') + def test_hook_exec_success(self, mock_execute, mock_extra_vars, mock_remote_vars, mock_terraform_vars, mock_find_script, mock_prepare_env): + mock_find_script.return_value = "hook_script" + hooks.hook_exec('phase', 'command', 'working_dir', {}, 'terraform_path', debug=True, b64_encode=True) + mock_prepare_env.assert_called_once() + mock_find_script.assert_called_once() + mock_terraform_vars.assert_called_once() + mock_remote_vars.assert_called_once() + mock_extra_vars.assert_called_once() + mock_execute.assert_called_once() + +# Helper function tests +class TestHelperFunctions: + @mock.patch('tfworker.util.hooks.os.listdir', return_value=[f"{TerraformStage.PRE}_{TerraformAction.PLAN}"]) + def test_find_hook_script(self, mock_listdir): + result = hooks._find_hook_script('working_dir', TerraformStage.PRE, TerraformAction.PLAN) + assert result == f'working_dir/hooks/pre_plan' + + @mock.patch('tfworker.util.hooks.os.listdir', return_value=[]) + def test_find_hook_script_no_dir(self, mock_listdir): + with pytest.raises(HookError) as e: + hooks._find_hook_script('working_dir', TerraformStage.PRE, TerraformAction.PLAN) + assert "Hook script missing from" in str(e.value) + + def test_prepare_environment(self): + env = {'KEY': 'value'} + result = hooks._prepare_environment(env, 'terraform_path') + assert result['KEY'] == 'value' + assert result['TF_PATH'] == 'terraform_path' + + def test_set_hook_env_var(self): + local_env = {} + hooks._set_hook_env_var(local_env, hooks.TFHookVarType.VAR, 'key', 'value', False) + assert local_env['TF_VAR_KEY'] == 'value' + + @mock.patch('tfworker.util.hooks.pipe_exec') + def test_execute_hook_script(self, mock_pipe_exec, capsys): + mock_pipe_exec.return_value = (0, b'stdout', b'stderr') + hooks._execute_hook_script('hook_script', TerraformStage.PRE, TerraformAction.PLAN, 'working_dir', {}, True) + mock_pipe_exec.assert_called_once_with('hook_script pre plan', cwd='working_dir/hooks', env={}) + captured = capsys.readouterr() + captured_lines = captured.out.splitlines() + assert len(captured_lines) == 4 + assert "Results from hook script: hook_script" in captured_lines + assert "exit code: 0" in captured_lines + assert "stdout: stdout" in captured_lines + assert "stderr: stderr" in captured_lines + + @mock.patch('tfworker.util.hooks.os.path.isfile', return_value=True) + @mock.patch('builtins.open', new_callable=mock.mock_open, read_data='key=value\nanother_key=another_value') + def test_populate_environment_with_terraform_variables(self, mock_isfile, mock_open): + local_env = {} + hooks._populate_environment_with_terraform_variables(local_env, 'working_dir', 'terraform_path', False) + assert 'TF_VAR_KEY' in local_env + assert 'TF_VAR_ANOTHER_KEY' in local_env + + @mock.patch('builtins.open', new_callable=mock.mock_open) + @mock.patch('tfworker.util.hooks.os.path.isfile', return_value=True) + @mock.patch('tfworker.util.hooks.get_state_item', side_effect=["value", "another_value"]) + def test_populate_environment_with_terraform_remote_vars(self, mock_get_state_item, mock_isfile, mock_open, mock_terraform_locals): + mock_open.return_value.read.return_value = mock_terraform_locals + + local_env = {} + hooks._populate_environment_with_terraform_remote_vars(local_env, 'working_dir', 'terraform_path', False) + assert 'TF_REMOTE_LOCAL_KEY' in local_env.keys() + assert 'TF_REMOTE_LOCAL_ANOTHER_KEY' in local_env.keys() + assert local_env['TF_REMOTE_LOCAL_KEY'] == 'value' + assert local_env['TF_REMOTE_LOCAL_ANOTHER_KEY'] == 'another_value' + + + def test_populate_environment_with_extra_vars(self): + local_env = {} + extra_vars = {'extra_key': 'extra_value'} + hooks._populate_environment_with_extra_vars(local_env, extra_vars, False) + assert 'TF_EXTRA_EXTRA_KEY' in local_env diff --git a/tests/util/test_system.py b/tests/util/test_system.py index 3758e94..4fb0935 100644 --- a/tests/util/test_system.py +++ b/tests/util/test_system.py @@ -17,7 +17,7 @@ import pytest -from tfworker.util.system import get_version, pipe_exec, strip_ansi, which +from tfworker.util.system import get_platform, get_version, pipe_exec, strip_ansi, which # context manager to allow testing exceptions in parameterized tests @@ -153,3 +153,29 @@ def test_strip_ansi(self): assert strip_ansi("\x1B[32mWorld\x1B[0m") == "World" assert strip_ansi("\x1B[33mFoo\x1B[0m") == "Foo" assert strip_ansi("\x1B[34mBar\x1B[0m") == "Bar" + + @pytest.mark.parametrize( + "opsys, machine, mock_platform_opsys, mock_platform_machine", + [ + ("linux", "i386", ["linux2"], ["i386"]), + ("linux", "arm", ["Linux"], ["arm"]), + ("linux", "amd64", ["linux"], ["x86_64"]), + ("linux", "amd64", ["linux"], ["amd64"]), + ("darwin", "amd64", ["darwin"], ["x86_64"]), + ("darwin", "amd64", ["darwin"], ["amd64"]), + ("darwin", "arm", ["darwin"], ["arm"]), + ("darwin", "arm64", ["darwin"], ["aarch64"]), + ], + ) + def test_get_platform( + self, opsys, machine, mock_platform_opsys, mock_platform_machine + ): + with mock.patch("platform.system", side_effect=mock_platform_opsys) as mock1: + with mock.patch( + "platform.machine", side_effect=mock_platform_machine + ) as mock2: + actual_opsys, actual_machine = get_platform() + assert opsys == actual_opsys + assert machine == actual_machine + mock1.assert_called_once() + mock2.assert_called_once() diff --git a/tests/util/test_terraform_util.py b/tests/util/test_terraform_util.py index 53db826..566e34b 100644 --- a/tests/util/test_terraform_util.py +++ b/tests/util/test_terraform_util.py @@ -75,3 +75,24 @@ def test_prep_modules_required(tmp_path): # check the target path does not exist assert not target_path.exists() + + # @pytest.mark.parametrize( + # "stdout, major, minor, expected_exception", + # [ + # ("Terraform v0.12.29", 0, 12, does_not_raise()), + # ("Terraform v1.3.5", 1, 3, does_not_raise()), + # ("TF 14", "", "", pytest.raises(SystemExit)), + # ], + # ) + # def test_get_tf_version( + # self, stdout: str, major: int, minor: int, expected_exception: callable + # ): + # with mock.patch( + # "tfworker.commands.base.pipe_exec", + # side_effect=mock_tf_version, + # ) as mocked: + # with expected_exception: + # (actual_major, actual_minor) = BaseCommand.get_terraform_version(stdout) + # assert actual_major == major + # assert actual_minor == minor + # mocked.assert_called_once() diff --git a/tfworker/backends/base.py b/tfworker/backends/base.py index 2d4186c..8744b52 100644 --- a/tfworker/backends/base.py +++ b/tfworker/backends/base.py @@ -14,7 +14,7 @@ from abc import ABCMeta, abstractmethod -from tfworker import JSONType +from tfworker.types import JSONType class BackendError(Exception): diff --git a/tfworker/cli.py b/tfworker/cli.py index 6a1afd4..77d0424 100644 --- a/tfworker/cli.py +++ b/tfworker/cli.py @@ -21,10 +21,14 @@ import click from tfworker import constants as const -from tfworker.commands import CleanCommand, RootCommand, TerraformCommand -from tfworker.commands.env import EnvCommand -from tfworker.commands.root import get_platform -from tfworker.commands.version import VersionCommand +from tfworker.commands import ( + CleanCommand, + EnvCommand, + RootCommand, + TerraformCommand, + VersionCommand, +) +from tfworker.util.system import get_platform def validate_deployment(ctx, deployment, name): @@ -189,7 +193,7 @@ def __repr__(self): ) @click.option( "--backend-use-all-remotes/--no-backend-use-all-remotes", - default=False, + default=True, envvar="WORKER_BACKEND_USE_ALL_REMOTES", help="Generate remote data sources based on all definition paths present in the backend", ) diff --git a/tfworker/commands/__init__.py b/tfworker/commands/__init__.py index ba6e711..0b790e5 100644 --- a/tfworker/commands/__init__.py +++ b/tfworker/commands/__init__.py @@ -14,6 +14,7 @@ from .base import BaseCommand # noqa from .clean import CleanCommand # noqa +from .env import EnvCommand # noqa from .root import RootCommand # noqa from .terraform import TerraformCommand # noqa from .version import VersionCommand # noqa diff --git a/tfworker/commands/base.py b/tfworker/commands/base.py index 8f72f0c..4283dc5 100644 --- a/tfworker/commands/base.py +++ b/tfworker/commands/base.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re - import click from tfworker.authenticators import AuthenticatorsCollection @@ -23,7 +21,8 @@ from tfworker.handlers.exceptions import HandlerError, UnknownHandler from tfworker.plugins import PluginsCollection from tfworker.providers import ProvidersCollection -from tfworker.util.system import get_version, pipe_exec, which +from tfworker.util.system import get_version, which +from tfworker.util.terraform import get_terraform_version class MissingDependencyException(Exception): @@ -64,7 +63,7 @@ def __init__(self, rootc, deployment="undefined", limit=tuple(), **kwargs): ( self._tf_version_major, self._tf_version_minor, - ) = self.get_terraform_version(self._terraform_bin) + ) = get_terraform_version(self._terraform_bin) self._authenticators = AuthenticatorsCollection( rootc.args, deployment=deployment, **kwargs @@ -189,21 +188,3 @@ def _resolve_arg(self, name): if name in self._rootc.worker_options_odict: return self._rootc.worker_options_odict[name] return None - - @staticmethod - def get_terraform_version(terraform_bin): - (return_code, stdout, stderr) = pipe_exec(f"{terraform_bin} version") - if return_code != 0: - click.secho(f"unable to get terraform version\n{stderr}", fg="red") - raise SystemExit(1) - version = stdout.decode("UTF-8").split("\n")[0] - version_search = re.search(r".*\s+v(\d+)\.(\d+)\.(\d+)", version) - if version_search: - click.secho( - f"Terraform Version Result: {version}, using major:{version_search.group(1)}, minor:{version_search.group(2)}", - fg="yellow", - ) - return (int(version_search.group(1)), int(version_search.group(2))) - else: - click.secho(f"unable to get terraform version\n{stderr}", fg="red") - raise SystemExit(1) diff --git a/tfworker/commands/root.py b/tfworker/commands/root.py index f597968..5ead715 100644 --- a/tfworker/commands/root.py +++ b/tfworker/commands/root.py @@ -15,7 +15,6 @@ import io import os import pathlib -import platform import tempfile from pathlib import Path from typing import Union @@ -263,30 +262,6 @@ def ordered_config_load(config: str) -> dict: raise SystemExit(1) -def get_platform(): - """ - Returns a formatted operating system / architecture tuple that is consistent with common distribution creation tools. - - Returns: - tuple: A tuple containing the operating system and architecture. - """ - - # strip off "2" which only appears on old linux kernels - opsys = platform.system().rstrip("2").lower() - - # make sure machine uses consistent format - machine = platform.machine() - if machine == "x86_64": - machine = "amd64" - - # some 64 bit arm extensions will report aarch64, this is functionaly - # equivalent to arm64 which is recognized and the pattern used by the TF - # community - if machine == "aarch64": - machine = "arm64" - return (opsys, machine) - - def rm_tree(base_path: Union[str, Path], inner: bool = False) -> None: """ Recursively removes all files and directories. diff --git a/tfworker/commands/terraform.py b/tfworker/commands/terraform.py index e1dbcf3..cf5789b 100644 --- a/tfworker/commands/terraform.py +++ b/tfworker/commands/terraform.py @@ -11,35 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import base64 -import json import os import pathlib -import re import click +import tfworker.util.hooks as hooks import tfworker.util.terraform as tf_util from tfworker.commands.base import BaseCommand from tfworker.definitions import Definition +from tfworker.exceptions import HookError, PlanChange, TerraformError from tfworker.handlers.exceptions import HandlerError from tfworker.util.system import pipe_exec, strip_ansi -TF_STATE_CACHE_NAME = "worker_state_cache.json" - - -class HookError(Exception): - pass - - -class PlanChange(Exception): - pass - - -class TerraformError(Exception): - pass - class TerraformCommand(BaseCommand): def __init__(self, rootc, **kwargs): @@ -105,7 +89,7 @@ def exec(self) -> None: tf_util.prep_modules( self._terraform_modules_dir, self._temp_dir, - required=(self._terraform_modules_dir is not None), + required=(self._terraform_modules_dir != ""), ) # prepare the definitions and run terraform init @@ -124,6 +108,10 @@ def exec(self) -> None: ################### # Private methods # ################### + + ########################################### + # Methods for dealing with terraform init # + ########################################### def _prep_and_init(self, def_iter: list[Definition]) -> None: """Prepares the definition and runs terraform init @@ -143,6 +131,9 @@ def _prep_and_init(self, def_iter: list[Definition]) -> None: click.secho("error running terraform init", fg="red") raise SystemExit(1) + ########################################### + # Methods for dealing with terraform plan # + ########################################### def _check_plan(self, definition: Definition) -> bool: """ Determines if a plan is needed for the provided definition @@ -208,60 +199,6 @@ def _validate_plan_path(self, plan_path: pathlib.Path) -> None: ) raise SystemExit(1) - def _run_handlers( - self, definition, action, stage, plan_file=None, **kwargs - ) -> None: - """Runs the handlers for the given action and stage - - Args: - definition: the definition to run the handlers for - action: the action to run the handlers for - stage: the stage to run the handlers for - plan_file: the plan file to pass to the handlers - kwargs: additional keyword arguments to pass to the handlers - - Returns: - None - """ - try: - self._execute_handlers( - action=action, - stage=stage, - deployment=self._deployment, - definition=definition.tag, - definition_path=definition.fs_path, - planfile=plan_file, - **kwargs, - ) - except HandlerError as e: - if e.terminate: - click.secho(f"terminating due to fatal handler error {e}", fg="red") - raise SystemExit(1) - click.secho(f"handler error: {e}", fg="red") - - def _should_plan(self, definition: Definition, plan_file: pathlib.Path) -> bool: - if not self._tf_plan: - definition._ready_to_apply = True - return False - - if plan_file.exists(): - if plan_file.stat().st_size == 0: - click.secho( - f"exiting plan file {plan_file} exists but is empty; planning again", - fg="green", - ) - definition._ready_to_apply = False - return True - click.secho( - f"existing plan file {plan_file} is suitable for apply; not planning again; remove plan file to allow planning", - fg="green", - ) - definition._ready_to_apply = True - return False - - definition._ready_to_apply = False - return True - def _exec_plan(self, definition) -> bool: """_exec_plan executes a terraform plan, returns true if a plan has changes""" changes = False @@ -328,6 +265,32 @@ def _exec_plan(self, definition) -> bool: return changes + def _should_plan(self, definition: Definition, plan_file: pathlib.Path) -> bool: + if not self._tf_plan: + definition._ready_to_apply = True + return False + + if plan_file.exists(): + if plan_file.stat().st_size == 0: + click.secho( + f"exiting plan file {plan_file} exists but is empty; planning again", + fg="green", + ) + definition._ready_to_apply = False + return True + click.secho( + f"existing plan file {plan_file} is suitable for apply; not planning again; remove plan file to allow planning", + fg="green", + ) + definition._ready_to_apply = True + return False + + definition._ready_to_apply = False + return True + + #################################################### + # Methods for dealing with terraform apply/destroy # + #################################################### def _check_apply_or_destroy(self, changes, definition) -> bool: """_check_apply_or_destroy determines if a terraform execution is needed""" # never apply if --no-apply is used @@ -421,6 +384,43 @@ def _exec_apply_or_destroy(self, definition) -> None: fg="green", ) + ##################################### + # Methods for dealing with handlers # + ##################################### + def _run_handlers( + self, definition, action, stage, plan_file=None, **kwargs + ) -> None: + """Runs the handlers for the given action and stage + + Args: + definition: the definition to run the handlers for + action: the action to run the handlers for + stage: the stage to run the handlers for + plan_file: the plan file to pass to the handlers + kwargs: additional keyword arguments to pass to the handlers + + Returns: + None + """ + try: + self._execute_handlers( + action=action, + stage=stage, + deployment=self._deployment, + definition=definition.tag, + definition_path=definition.fs_path, + planfile=plan_file, + **kwargs, + ) + except HandlerError as e: + if e.terminate: + click.secho(f"terminating due to fatal handler error {e}", fg="red") + raise SystemExit(1) + click.secho(f"handler error: {e}", fg="red") + + ######################################## + # Common methods for running terraform # + ######################################## def _run( self, definition, command, debug=False, plan_action="init", plan_file=None ): @@ -460,9 +460,11 @@ def _run( # only execute hooks for plan/apply/destroy try: - if TerraformCommand.check_hooks( - "pre", working_dir, command - ) and command in ["apply", "destroy", "plan"]: + if hooks.check_hooks("pre", working_dir, command) and command in [ + "apply", + "destroy", + "plan", + ]: # pre exec hooks # want to pass remotes # want to pass tf_vars @@ -471,7 +473,7 @@ def _run( " executing ", fg="yellow", ) - TerraformCommand.hook_exec( + hooks.hook_exec( "pre", command, working_dir, @@ -536,15 +538,17 @@ def _run( # only execute hooks for plan/destroy try: - if TerraformCommand.check_hooks( - "post", working_dir, command - ) and command in ["apply", "destroy", "plan"]: + if hooks.check_hooks("post", working_dir, command) and command in [ + "apply", + "destroy", + "plan", + ]: click.secho( f"found post-{command} hook script for definition {definition.tag}," " executing ", fg="yellow", ) - TerraformCommand.hook_exec( + hooks.hook_exec( "post", command, working_dir, @@ -560,304 +564,3 @@ def _run( ) raise SystemExit(2) return True - - ################## - # Static methods # - ################## - @staticmethod - def hook_exec( - phase, - command, - working_dir, - env, - terraform_path, - debug=False, - b64_encode=False, - extra_vars={}, - ): - """ - hook_exec executes a hook script. - - Before execution it sets up the environment to make all terraform and remote - state variables available to the hook via environment vars - """ - - key_replace_items = { - " ": "", - '"': "", - "-": "_", - ".": "_", - } - val_replace_items = { - " ": "", - '"': "", - "\n": "", - } - local_env = env.copy() - local_env["TF_PATH"] = terraform_path - hook_dir = f"{working_dir}/hooks" - hook_script = None - - for f in os.listdir(hook_dir): - # this file format is specifically structured by the prep_def function - if os.path.splitext(f)[0] == f"{phase}_{command}": - hook_script = f"{hook_dir}/{f}" - # this should never have been called if the hook script didn't exist... - if hook_script is None: - raise HookError(f"hook script missing from {hook_dir}") - - # populate environment with terraform remotes - if os.path.isfile(f"{working_dir}/worker-locals.tf"): - # I'm sorry. :-) - r = re.compile( - r"\s*(?P\w+)\s*\=.+data\.terraform_remote_state\.(?P\w+)\.outputs\.(?P\w+)\s*" - ) - - with open(f"{working_dir}/worker-locals.tf") as f: - for line in f: - m = r.match(line) - if m: - item = m.group("item") - state = m.group("state") - state_item = m.group("state_item") - else: - continue - - state_value = TerraformCommand.get_state_item( - working_dir, env, terraform_path, state, state_item - ) - - if state_value is not None: - if b64_encode: - state_value = base64.b64encode(state_value.encode("utf-8")) - local_env[f"TF_REMOTE_{state}_{item}".upper()] = state_value - - # populate environment with terraform variables - if os.path.isfile(f"{working_dir}/worker.auto.tfvars"): - with open(f"{working_dir}/worker.auto.tfvars") as f: - for line in f: - tf_var = line.split("=") - - # strip bad names out for env var settings - for k, v in key_replace_items.items(): - tf_var[0] = tf_var[0].replace(k, v) - - for k, v in val_replace_items.items(): - tf_var[1] = tf_var[1].replace(k, v) - - if b64_encode: - tf_var[1] = base64.b64encode(tf_var[1].encode("utf-8")) - - local_env[f"TF_VAR_{tf_var[0].upper()}"] = tf_var[1] - else: - click.secho( - f"{working_dir}/worker.auto.tfvars not found!", - fg="red", - ) - - for k, v in extra_vars.items(): - if b64_encode: - v = base64.b64encode(v.encode("utf-8")) - local_env[f"TF_EXTRA_{k.upper()}"] = v - - # execute the hook - (exit_code, stdout, stderr) = pipe_exec( - f"{hook_script} {phase} {command}", - cwd=hook_dir, - env=local_env, - ) - - # handle output from hook_script - if debug: - click.secho(f"exit code: {exit_code}", fg="blue") - for line in stdout.decode().splitlines(): - click.secho(f"stdout: {line}", fg="blue") - for line in stderr.decode().splitlines(): - click.secho(f"stderr: {line}", fg="red") - - if exit_code != 0: - raise HookError("hook script {}") - - @staticmethod - def check_hooks(phase, working_dir, command): - """ - check_hooks determines if a hook exists for a given operation/definition - """ - hook_dir = f"{working_dir}/hooks" - if not os.path.isdir(hook_dir): - # there is no hooks dir - return False - for f in os.listdir(hook_dir): - if os.path.splitext(f)[0] == f"{phase}_{command}": - if os.access(f"{hook_dir}/{f}", os.X_OK): - return True - else: - raise HookError(f"{hook_dir}/{f} exists, but is not executable!") - return False - - @staticmethod - def get_state_item(working_dir, env, terraform_bin, state, item): - """ - The general handler function for getting a state item, it will first - try to get the item from another definitions output, but if the other - definition is not setup, it will fallback to getting the item from the - remote state. - - @param working_dir: The working directory of the terraform definition - @param env: The environment variables to pass to the terraform command - @param terraform_bin: The path to the terraform binary - @param state: The state name to get the item from - @param item: The item to get from the state - """ - try: - return TerraformCommand._get_state_item_from_output( - working_dir, env, terraform_bin, state, item - ) - except FileNotFoundError: - return TerraformCommand._get_state_item_from_remote( - working_dir, env, terraform_bin, state, item - ) - - @staticmethod - def _get_state_item_from_remote(working_dir, env, terraform_bin, state, item): - """ - get_state_item returns json encoded output from a terraform remote state - - @param working_dir: The working directory of the terraform definition - @param env: The environment variables to pass to the terraform command - @param terraform_bin: The path to the terraform binary - @param state: The state name to get the item from - @param item: The item to get from the state - """ - - remote_state = None - - # setup the state cache - cache_file = TerraformCommand._get_cache_name(working_dir) - TerraformCommand._make_state_cache(working_dir, env, terraform_bin) - - # read the cache - with open(cache_file, "r") as f: - state_cache = json.load(f) - - # Get the remote state we are looking for, and raise an error if it's not there - resources = state_cache["values"]["root_module"]["resources"] - for resource in resources: - if ( - resource["type"] == "terraform_remote_state" - and resource["name"] == state - ): - remote_state = resource - if remote_state is None: - raise HookError(f"Remote state item {state} not found") - - if item in remote_state["values"]["outputs"]: - return json.dumps( - remote_state["values"]["outputs"][item], - indent=None, - separators=(",", ":"), - ) - - raise HookError(f"Remote state item {state}.{item} not found in state cache") - - @staticmethod - def _get_state_item_from_output(working_dir, env, terraform_bin, state, item): - """ - Get a single item from the terraform output, this is the preferred - mechanism as items will be more guaranteed to be up to date, but it - creates problems when the remote state is not setup, like when using - --limit - - @param work_dir: The working directory of the terraform definition - @param env: The environment variables to pass to the terraform command - @param terraform_bin: The path to the terraform binary - @param state: The state name to get the item from - @param item: The item to get from the state - """ - - base_dir, _ = os.path.split(working_dir) - try: - (exit_code, stdout, stderr) = pipe_exec( - f"{terraform_bin} output -json -no-color {item}", - cwd=f"{base_dir}/{state}", - env=env, - ) - except FileNotFoundError: - # the remote state is not setup, likely do to use of --limit - # this is acceptable, and is the responsibility of the hook - # to ensure it has all values needed for safe execution - raise - - if exit_code != 0: - raise HookError( - f"Error reading remote state item {state}.{item}, details: {stderr}" - ) - - if stdout is None: - raise HookError( - f"Remote state item {state}.{item} is empty; This is completely" - " unexpected, failing..." - ) - json_output = json.loads(stdout) - return json.dumps(json_output, indent=None, separators=(",", ":")) - - @staticmethod - def _make_state_cache( - working_dir: str, env: dict, terraform_bin: str, refresh: bool = False - ): - """ - Using `terraform show -json` make a cache of the state file - - @param working_dir: The working directory of the terraform definition - @param env: The environment variables to pass to the terraform command - @param terraform_bin: The path to the terraform binary - @param refresh: If true, the cache will be refreshed - """ - - # check if the cache exists - state_cache = TerraformCommand._get_cache_name(working_dir) - if not refresh and os.path.exists(state_cache): - return - - # ensure the state is refreshed; but no changes to resources are made - (exit_code, stdout, stderr) = pipe_exec( - f"{terraform_bin} apply -auto-approve -refresh-only", - cwd=working_dir, - env=env, - ) - - # get the json from terraform to generate the cache - (exit_code, stdout, stderr) = pipe_exec( - f"{terraform_bin} show -json", - cwd=working_dir, - env=env, - ) - - # validate the output, check exit code and ensure output is json - if exit_code != 0: - raise HookError(f"Error reading terraform state, details: {stderr}") - try: - json.loads(stdout) - except json.JSONDecodeError: - raise HookError( - "Error parsing terraform state; output is not in json format" - ) - - # write the cache to disk - try: - with open(state_cache, "w") as f: - f.write(stdout.decode()) - except Exception as e: - raise HookError(f"Error writing state cache to {state_cache}, details: {e}") - return - - @staticmethod - def _get_cache_name(working_dir: str) -> str: - """ - Get the cache directory for the state cache - - @param working_dir: The working directory of the terraform definition - - @return: The cache directory path - """ - return f"{working_dir}/{TF_STATE_CACHE_NAME}" diff --git a/tfworker/constants.py b/tfworker/constants.py index cb7b91e..31ab4ec 100644 --- a/tfworker/constants.py +++ b/tfworker/constants.py @@ -23,4 +23,13 @@ DEFAULT_AWS_REGION = "us-east-1" DEFAULT_GCP_REGION = "us-east-1a" -RESERVED_FILES = ["terraform.tf", "worker-locals.tf", "worker.auto.tfvars"] +TF_STATE_CACHE_NAME = "worker_state_cache.json" +WORKER_LOCALS_FILENAME = "worker-locals.tf" +WORKER_TF_FILENAME = "worker.tf" +WORKER_TFVARS_FILENAME = "worker.auto.tfvars" +RESERVED_FILES = [ + WORKER_LOCALS_FILENAME, + WORKER_TF_FILENAME, + WORKER_TFVARS_FILENAME, + TF_STATE_CACHE_NAME, +] diff --git a/tfworker/definitions.py b/tfworker/definitions.py index db86f52..4eacc84 100644 --- a/tfworker/definitions.py +++ b/tfworker/definitions.py @@ -23,6 +23,7 @@ from mergedeep import merge from tfworker import constants as const +from tfworker.exceptions import ReservedFileError from tfworker.util.copier import CopyFactory TERRAFORM_TPL = """\ @@ -33,10 +34,6 @@ """ -class ReservedFileError(Exception): - pass - - class Definition: _plan_file = None _ready_to_apply = False diff --git a/tfworker/exceptions.py b/tfworker/exceptions.py new file mode 100644 index 0000000..0b33b8d --- /dev/null +++ b/tfworker/exceptions.py @@ -0,0 +1,41 @@ +class HookError(Exception): + """ + Exception is raised when a hook fails, or has execution issues. + """ + + pass + + +class PlanChange(Exception): + """ + Exception is raised when a terraform plan has changes. + """ + + pass + + +class PluginSourceParseException(Exception): + """ + Exception is raised when a plugin source cannot be parsed. + """ + + pass + + +class ReservedFileError(Exception): + """ + Exception is raised when a reserved file is found in the repository; + + Reserved files are files that are used by tfworker, and should not be + present in the repository. + """ + + pass + + +class TerraformError(Exception): + """ + Exception is raised when a terraform command fails. + """ + + pass diff --git a/tfworker/plugins.py b/tfworker/plugins.py index cc30ad6..f66f0b9 100644 --- a/tfworker/plugins.py +++ b/tfworker/plugins.py @@ -22,11 +22,8 @@ import click from tenacity import retry, stop_after_attempt, wait_chain, wait_fixed -from tfworker.commands.root import get_platform - - -class PluginSourceParseException(Exception): - pass +from tfworker.exceptions import PluginSourceParseException +from tfworker.util.system import get_platform class PluginsCollection(collections.abc.Mapping): diff --git a/tfworker/types.py b/tfworker/types.py new file mode 100644 index 0000000..d5d95cb --- /dev/null +++ b/tfworker/types.py @@ -0,0 +1,32 @@ +from enum import Enum +from typing import Any, Dict, List, Union + + +class TerraformAction(Enum): + """ + Terraform actions + """ + + PLAN = "plan" + APPLY = "apply" + DESTROY = "destroy" + INIT = "init" + + def __str__(self): + return self.value + + +class TerraformStage(Enum): + """ + Stages around a terraform action, pre and post + """ + + PRE = "pre" + POST = "post" + + def __str__(self): + return self.value + + +# https://github.com/python/typing/issues/182 +JSONType = Union[str, int, float, bool, None, Dict[str, Any], List[Any]] diff --git a/tfworker/util/hooks.py b/tfworker/util/hooks.py new file mode 100644 index 0000000..03683c2 --- /dev/null +++ b/tfworker/util/hooks.py @@ -0,0 +1,586 @@ +# This contains utility functions for dealing with hooks +# in terraform definitions, it's primary purpose is to be +# used by the TerraformCommand class, while reducing the +# responsibility of the class itself. +import base64 +import json +import os +import re +from enum import Enum +from typing import Any, Dict + +import click + +from tfworker.backends import BaseBackend +from tfworker.constants import ( + TF_STATE_CACHE_NAME, + WORKER_LOCALS_FILENAME, + WORKER_TFVARS_FILENAME, +) +from tfworker.exceptions import HookError +from tfworker.types import TerraformAction, TerraformStage +from tfworker.util.system import pipe_exec + + +class TFHookVarType(Enum): + """ + Enum for the types of hook variables. + """ + + VAR = "TF_VAR" + REMOTE = "TF_REMOTE" + EXTRA = "TF_EXTRA" + + def __str__(self): + return self.value.upper() + + +def get_state_item( + working_dir: str, + env: Dict[str, str], + terraform_bin: str, + state: str, + item: str, + backend: BaseBackend = None, +) -> str: + """ + General handler function for getting a state item. First tries to get the item from another definition's output, + and if the other definition is not set up, falls back to getting the item from the remote state. + + Args: + working_dir (str): The working directory of the terraform definition. + env (dict[str, str]): The environment variables to pass to the terraform command. + terraform_bin (str): The path to the terraform binary. + state (str): The state name to get the item from. + item (str): The item to get from the state. + + Returns: + str: The state item, key: value as a JSON string. + + Raises: + HookError: If the state item is not found in the remote state. + """ + + try: + click.secho(f"Getting state item {state}.{item} from output", fg="blue") + return _get_state_item_from_output(working_dir, env, terraform_bin, state, item) + except FileNotFoundError: + click.secho( + f"Remote state not setup, falling back to getting state item {state}.{item} from remote", + fg="blue", + ) + return _get_state_item_from_remote(working_dir, env, terraform_bin, state, item) + + +def _get_state_item_from_output( + working_dir: str, env: Dict[str, str], terraform_bin: str, state: str, item: str +) -> str: + """ + Get a single item from the terraform output. This is the preferred + mechanism as items will be more guaranteed to be up to date, but it + creates problems when the remote state is not set up, like when using + --limit. + + Args: + working_dir (str): The working directory of the terraform definition. + env (Dict[str, str]): The environment variables to pass to the terraform command. + terraform_bin (str): The path to the terraform binary. + state (str): The state name to get the item from. + item (str): The item to get from the state. + + Returns: + str: The item from the terraform output in JSON format. + + Raises: + HookError: If there is an error reading the remote state item or if the output is empty or not in JSON format. + """ + base_dir, _ = os.path.split(working_dir) + try: + (exit_code, stdout, stderr) = pipe_exec( + f"{terraform_bin} output -json -no-color {item}", + cwd=f"{base_dir}/{state}", + env=env, + ) + except FileNotFoundError: + # the remote state is not setup, likely do to use of --limit + # this is acceptable, and is the responsibility of the hook + # to ensure it has all values needed for safe execution + raise + + if exit_code != 0: + raise HookError( + f"Error reading remote state item {state}.{item}, details: {stderr}" + ) + + if stdout is None: + raise HookError( + f"Remote state item {state}.{item} is empty; This is completely" + " unexpected, failing..." + ) + + try: + json_output = json.loads(stdout) + except json.JSONDecodeError: + raise HookError( + f"Error parsing remote state item {state}.{item}; output is not in JSON format" + ) + + return json.dumps(json_output, indent=None, separators=(",", ":")) + + +def check_hooks( + phase: TerraformStage, working_dir: str, command: TerraformAction +) -> bool: + """ + Check if a hook script exists for the given phase and command. + + Args: + phase (TerraformStage): The phase of the terraform command. + working_dir (str): The working directory of the terraform definition. + command (TerraformAction): The terraform command to run. + + Returns: + bool: True if the hook script exists and is executable, False otherwise. + """ + hook_dir = f"{working_dir}/hooks" + if not os.path.isdir(hook_dir): + # there is no hooks dir + return False + for f in os.listdir(hook_dir): + if os.path.splitext(f)[0] == f"{phase}_{command}": + if os.access(f"{hook_dir}/{f}", os.X_OK): + return True + else: + raise HookError(f"{hook_dir}/{f} exists, but is not executable!") + return False + + +def hook_exec( + phase: TerraformStage, + command: TerraformAction, + working_dir: str, + env: Dict[str, str], + terraform_path: str, + debug: bool = False, + b64_encode: bool = False, + extra_vars: Dict[str, str] = None, +) -> None: + """ + Coordinates the execution of a hook script. This function is responsible for finding and executing + the script, as well as setting up the proper environment variables with all of the terraform vars, + and vars from remote data sources. + + Args: + phase (TerraformPhase): The phase of the hook. + command (TerraformAction): A poorly named variable, that is the terraform action being executed + working_dir (str): The working directory of the Terraform definition. + env (Dict[str, str]): The environment variables to pass to the hook. + terraform_path (str): The path to the Terraform binary. + debug (bool, optional): If True, debug information will be printed. Defaults to False. + b64_encode (bool, optional): If True, variables will be base64 encoded. Defaults to False. + extra_vars (Dict[str, str], optional): Additional environment variables to set. Defaults to None. + + Raises: + HookError: If the hook script is missing or if execution fails. + """ + if extra_vars is None: + extra_vars = {} + + local_env = _prepare_environment(env, terraform_path) + hook_script = _find_hook_script(working_dir, phase, command) + _populate_environment_with_terraform_variables( + local_env, working_dir, terraform_path, b64_encode + ) + _populate_environment_with_terraform_remote_vars( + local_env, working_dir, terraform_path, b64_encode + ) + _populate_environment_with_extra_vars(local_env, extra_vars, b64_encode) + _execute_hook_script(hook_script, phase, command, working_dir, local_env, debug) + + +def _find_hook_script(working_dir: str, phase: str, command: str) -> str: + """ + Finds the hook script to execute. + + Args: + working_dir (str): The working directory of the Terraform definition. + phase (str): The phase of the hook. + command (str): The command to execute. + + Returns: + str: The path to the hook script. + + Raises: + HookError: If the hook script is missing. + """ + hook_dir = os.path.join(working_dir, "hooks") + for f in os.listdir(hook_dir): + if os.path.splitext(f)[0] == f"{phase}_{command}": + return os.path.join(hook_dir, f) + raise HookError(f"Hook script missing from {hook_dir}") + + +def _prepare_environment(env: Dict[str, str], terraform_path: str) -> Dict[str, str]: + """ + Prepares the environment variables for the hook script execution. + + Args: + env (Dict[str, str]): The initial environment variables. + terraform_path (str): The path to the Terraform binary. + + Returns: + Dict[str, str]: The prepared environment variables. + """ + local_env = env.copy() + local_env["TF_PATH"] = terraform_path + return local_env + + +def _populate_environment_with_terraform_variables( + local_env: Dict[str, str], working_dir: str, terraform_path: str, b64_encode: bool +) -> None: + """ + Populates the environment with Terraform variables. + + Args: + local_env (Dict[str, str]): The environment variables. + working_dir (str): The working directory of the Terraform definition. + terraform_path (str): The path to the Terraform binary. + b64_encode (bool): If True, variables will be base64 encoded. + """ + if not os.path.isfile(os.path.join(working_dir, WORKER_TFVARS_FILENAME)): + return + + with open(os.path.join(working_dir, WORKER_TFVARS_FILENAME)) as f: + contents = f.read() + + for line in contents.splitlines(): + tf_var = line.split("=") + _set_hook_env_var( + local_env, TFHookVarType.VAR, tf_var[0], tf_var[1], b64_encode + ) + + +def _populate_environment_with_terraform_remote_vars( + local_env: Dict[str, str], working_dir: str, terraform_path: str, b64_encode: bool +) -> None: + """ + Populates the environment with Terraform variables. + + Args: + local_env (Dict[str, str]): The environment variables. + working_dir (str): The working directory of the Terraform definition. + terraform_path (str): The path to the Terraform binary. + b64_encode (bool): If True, variables will be base64 encoded. + """ + if not os.path.isfile(os.path.join(working_dir, WORKER_LOCALS_FILENAME)): + return + + with open(os.path.join(working_dir, WORKER_LOCALS_FILENAME)) as f: + contents = f.read() + + # I'm sorry. :-) + # this regex looks for variables in the form of: + # = data.terraform_remote_state..outputs. + r = re.compile( + r"\s*(?P\w+)\s*\=.+data\.terraform_remote_state\.(?P\w+)\.outputs\.(?P\w+)\s*" + ) + + for line in contents.splitlines(): + m = r.match(line) + if m: + item = m.group("item") + state = m.group("state") + state_item = m.group("state_item") + state_value = get_state_item( + working_dir, local_env, terraform_path, state, state_item + ) + _set_hook_env_var( + local_env, TFHookVarType.REMOTE, item, state_value, b64_encode + ) + + +def _populate_environment_with_extra_vars( + local_env: Dict[str, str], extra_vars: Dict[str, Any], b64_encode: bool +) -> None: + """ + Populates the environment with extra variables. + + Args: + local_env (Dict[str, str]): The environment variables. + extra_vars (Dict[str, Any]): The extra variables to set. + b64_encode (bool): If True, variables will be base64 encoded. + """ + for k, v in extra_vars.items(): + _set_hook_env_var(local_env, TFHookVarType.EXTRA, k, v, b64_encode) + + +def _set_hook_env_var( + local_env: Dict[str, str], + var_type: TFHookVarType, + key: str, + value: str, + b64_encode: bool = False, +) -> None: + """ + Sets a hook environment variable. + + Args: + local_env (Dict[str, str]): The environment variables. + var_type (TFHookVarType): The type of the variable. + key (str): The key of the variable. + value (str): The value of the variable. + b64_encode (bool, optional): If True, the value will be base64 encoded. Defaults to False. + """ + key_replace_items = {" ": "", '"': "", "-": "_", ".": "_"} + val_replace_items = {" ": "", '"': "", "\n": ""} + + for k, v in key_replace_items.items(): + key = key.replace(k, v) + + for k, v in val_replace_items.items(): + value = value.replace(k, v) + + if b64_encode: + value = base64.b64encode(value.encode()) + + local_env[f"{var_type}_{key.upper()}"] = value + + +def _execute_hook_script( + hook_script: str, + phase: str, + command: str, + working_dir: str, + local_env: Dict[str, str], + debug: bool, +) -> None: + """ + Executes the hook script and handles its output. + + Args: + hook_script (str): The path to the hook script. + phase (str): The phase of the hook. + command (str): The command to execute. + working_dir (str): The working directory of the Terraform definition. + local_env (Dict[str, str]): The environment variables. + debug (bool): If True, debug information will be printed. + + Raises: + HookError: If the hook script execution fails. + """ + hook_dir = os.path.join(working_dir, "hooks") + exit_code, stdout, stderr = pipe_exec( + f"{hook_script} {phase} {command}", cwd=hook_dir, env=local_env + ) + + if debug: + click.secho(f"Results from hook script: {hook_script}", fg="blue") + click.secho(f"exit code: {exit_code}", fg="blue") + for line in stdout.decode().splitlines(): + click.secho(f"stdout: {line}", fg="blue") + for line in stderr.decode().splitlines(): + click.secho(f"stderr: {line}", fg="red") + + if exit_code != 0: + raise HookError( + f"Hook script {hook_script} execution failed with exit code {exit_code}" + ) + + +def _get_state_item_from_remote( + working_dir: str, env: Dict[str, str], terraform_bin: str, state: str, item: str +) -> str: + """ + Retrieve a state item from terraform remote state. + + Args: + working_dir: The working directory of the terraform definition. + env: The environment variables to pass to the terraform command. + terraform_bin: The path to the terraform binary. + state: The state name to get the item from. + item: The item to get from the state. + + Returns: + A JSON string of the state item. + + Raises: + HookError: If the state item cannot be found or read. + """ + cache_file = _get_state_cache_name(working_dir) + _make_state_cache(working_dir, env, terraform_bin) + + state_cache = _read_state_cache(cache_file) + remote_state = _find_remote_state(state_cache, state) + + return _get_item_from_remote_state(remote_state, state, item) + + +def _get_state_cache_name(working_dir: str) -> str: + """ + Get the name of the state cache file. + + Args: + working_dir (str): The working directory of the terraform definition. + + Returns: + str: The name of the state cache file. + """ + return f"{working_dir}/{TF_STATE_CACHE_NAME}" + + +def _make_state_cache( + working_dir: str, env: Dict[str, str], terraform_bin: str, refresh: bool = False +) -> None: + """ + Create a cache of the terraform state file. + + Args: + working_dir (str): The working directory of the terraform definition. + env ({str, str}): The environment variables to pass to the terraform command. + terraform_bin (str): The path to the terraform binary. + refresh (bool, optional): If true, the cache will be refreshed. Defaults to False. + + Raises: + HookError: If there is an error reading or writing the state cache. + """ + state_cache = _get_state_cache_name(working_dir) + if not refresh and os.path.exists(state_cache): + return + + _run_terraform_refresh(terraform_bin, working_dir, env) + state_json = _run_terraform_show(terraform_bin, working_dir, env) + _write_state_cache(state_cache, state_json) + + +def _read_state_cache(cache_file: str) -> Dict[str, Any]: + """ + Read the state cache from a file. + + Args: + cache_file (str): The path to the state cache file. + + Returns: + Dict[str, Any]: The state cache JSON. + """ + with open(cache_file, "r") as f: + return json.load(f) + + +def _find_remote_state(state_cache: Dict[str, Any], state: str) -> Dict[str, Any]: + """ + Find the remote state in the state cache. + + Args: + state_cache (Dict[str, Any]): The state cache JSON. + state (str): The state name to find. + + Returns: + Dict[str, Any]: The remote state JSON. + + Raises: + HookError: If the remote state is not found in the state cache. + """ + resources = state_cache["values"]["root_module"]["resources"] + for resource in resources: + if resource["type"] == "terraform_remote_state" and resource["name"] == state: + return resource + raise HookError(f"Remote state item {state} not found") + + +def _get_item_from_remote_state( + remote_state: Dict[str, Any], state: str, item: str +) -> str: + """ + Get an item from the remote state JSON + + Args: + remote_state (Dict[str, Any]): The remote state JSON. + state (str): The state name. + item (str): The item to get from the state. + + Returns: + str: The item from the remote state in JSON format. + + Raises: + HookError: If the item is not found in the remote state. + """ + if item in remote_state["values"]["outputs"]: + return json.dumps( + remote_state["values"]["outputs"][item], + indent=None, + separators=(",", ":"), + ) + raise HookError(f"Remote state item {state}.{item} not found in state cache") + + +def _run_terraform_refresh( + terraform_bin: str, working_dir: str, env: Dict[str, str] +) -> None: + """ + Run `terraform apply -refresh-only` to ensure the state is refreshed. + + Args: + terraform_bin (str): The path to the terraform binary. + working_dir (str): The working directory of the terraform definition. + env (Dict[str, str]): The environment variables to pass to the terraform command. + + Raises: + HookError: If there is an error refreshing the terraform state. + """ + exit_code, _, stderr = pipe_exec( + f"{terraform_bin} apply -auto-approve -refresh-only", + cwd=working_dir, + env=env, + ) + if exit_code != 0: + raise HookError(f"Error applying terraform state, details: {stderr}") + + +def _run_terraform_show( + terraform_bin: str, working_dir: str, env: Dict[str, str] +) -> str: + """ + Run `terraform show -json` to get the state in JSON format. + + Args: + terraform_bin (str): The path to the terraform binary. + working_dir (str): The working directory of the terraform definition. + env (Dict[str, str]): The environment variables to pass to the terraform command. + + Returns: + str: The state in JSON format. + + Raises: + HookError: If there is an error reading the terraform state. + """ + exit_code, stdout, stderr = pipe_exec( + f"{terraform_bin} show -json", + cwd=working_dir, + env=env, + ) + if exit_code != 0: + raise HookError(f"Error reading terraform state, details: {stderr}") + try: + json.loads(stdout) + except json.JSONDecodeError: + raise HookError("Error parsing terraform state; output is not in JSON format") + return stdout.decode() + + +def _write_state_cache(state_cache: str, state_json: str) -> None: + """ + Write the state JSON to the cache file. + + Args: + state_cache (str): The path to the state cache file. + state_json (str): The state JSON to write. + + Raises: + HookError: If there is an error writing the state cache. + """ + try: + with open(state_cache, "w") as f: + f.write(state_json) + except Exception as e: + raise HookError(f"Error writing state cache to {state_cache}, details: {e}") diff --git a/tfworker/util/system.py b/tfworker/util/system.py index 4cff798..67bd282 100644 --- a/tfworker/util/system.py +++ b/tfworker/util/system.py @@ -12,27 +12,49 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +import platform import re import shlex import subprocess +from typing import Dict, List, Tuple, Union import click import importlib_metadata -def strip_ansi(line): +def strip_ansi(line: str) -> str: """ Strips ANSI escape sequences from a string. + + Args: + line (str): The string to strip ANSI escape sequences from. + + Returns: + str: The string with ANSI escape sequences stripped. """ ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") return ansi_escape.sub("", line) -def pipe_exec(args, stdin=None, cwd=None, env=None, stream_output=False): +def pipe_exec( + args: Union[str, List[str]], + stdin: str = None, + cwd: str = None, + env: Dict[str, str] = None, + stream_output: bool = False, +) -> Tuple[int, Union[bytes, None], Union[bytes, None]]: """ - A function to accept a list of commands and pipe them together. + A function to take one or more commands and execute them in a pipeline, returning the output of the last command. - Takes optional stdin to give to the first item in the pipe chain. + Args: + args (str or list): A string or list of strings representing the command(s) to execute. + stdin (str, optional): A string to pass as stdin to the first command + cwd (str, optional): The working directory to execute the command in. + env (dict, optional): A dictionary of environment variables to set for the command. + stream_output (bool, optional): A boolean indicating if the output should be streamed back to the caller. + + Returns: + tuple: A tuple containing the return code, stdout, and stderr of the last command in the pipeline. """ commands = [] # listed used to hold all the popen objects # use the default environment if one is not specified @@ -132,8 +154,16 @@ def pipe_exec(args, stdin=None, cwd=None, env=None, stream_output=False): return (returncode, stdout, stderr) -def which(program): - """From stack overflow""" +def which(program: str) -> Union[str, None]: + """ + A function to mimic the behavior of the `which` command in Unix-like systems. + + Args: + program (str): The program to search for in the PATH. + + Returns: + str: The full path to the program if found, otherwise None. + """ def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) @@ -153,8 +183,35 @@ def is_exe(fpath): def get_version() -> str: """ Get the version of the current package + + Returns: + str: The version of the package """ try: return importlib_metadata.version("terraform-worker") except importlib_metadata.PackageNotFoundError: return "unknown" + + +def get_platform() -> Tuple[str, str]: + """ + Returns a formatted operating system / architecture tuple that is consistent with common distribution creation tools. + + Returns: + tuple: A tuple containing the operating system and architecture. + """ + + # strip off "2" which only appears on old linux kernels + opsys = platform.system().rstrip("2").lower() + + # make sure machine uses consistent format + machine = platform.machine() + if machine == "x86_64": + machine = "amd64" + + # some 64 bit arm extensions will `report aarch64, this is functionaly + # equivalent to arm64 which is recognized and the pattern used by the TF + # community + if machine == "aarch64": + machine = "arm64" + return (opsys, machine) diff --git a/tfworker/util/terraform.py b/tfworker/util/terraform.py index b4d162d..9ff47f5 100644 --- a/tfworker/util/terraform.py +++ b/tfworker/util/terraform.py @@ -2,10 +2,14 @@ # the goal of moving these functions here is to reduce the responsibility of # the TerraformCommand class, making it easier to test and maintain import pathlib +import re import shutil import click +from tfworker.constants import DEFAULT_REPOSITORY_PATH +from tfworker.util.system import pipe_exec + def prep_modules( module_path: str, @@ -23,6 +27,11 @@ def prep_modules( ignore_patterns (list(str)): A list of patterns to ignore required (bool): If the terraform modules directory is required """ + module_path = ( + module_path + if module_path != "" + else f"{DEFAULT_REPOSITORY_PATH}/terraform-modules" + ) module_path = pathlib.Path(module_path) target_path = pathlib.Path(f"{target_path}/terraform-modules".replace("//", "/")) @@ -46,3 +55,21 @@ def prep_modules( symlinks=True, ignore=shutil.ignore_patterns(*ignore_patterns), ) + + +def get_terraform_version(terraform_bin): + (return_code, stdout, stderr) = pipe_exec(f"{terraform_bin} version") + if return_code != 0: + click.secho(f"unable to get terraform version\n{stderr}", fg="red") + raise SystemExit(1) + version = stdout.decode("UTF-8").split("\n")[0] + version_search = re.search(r".*\s+v(\d+)\.(\d+)\.(\d+)", version) + if version_search: + click.secho( + f"Terraform Version Result: {version}, using major:{version_search.group(1)}, minor:{version_search.group(2)}", + fg="yellow", + ) + return (int(version_search.group(1)), int(version_search.group(2))) + else: + click.secho(f"unable to get terraform version\n{stderr}", fg="red") + raise SystemExit(1)