From 9ca851a040740b40c3142ed93d26ff0ed474a580 Mon Sep 17 00:00:00 2001 From: Richard Maynard Date: Sun, 9 Jun 2024 16:50:06 -0500 Subject: [PATCH] Contiue refactoring efforts - fix linting errors that have accumulated over time - refactor TerraformCommand.exec and add tests to cover all cases - refactor TerraformCommand._check_plan and add tests - add TerraformCommand._handle_no_plan_path with full tests - add TerraformCommand._prepare_plan_file with full tests - add TerraformCommand._validate_plan_path with full tests - add TerraformCommand._run_handlers with full tests - add TerraformCommand._should_plan with full tests - create terraform.py for items that do not belong in TerraformCommand - add prep_modules method function with full tests - replace deprecated use of pkg_resources by importlib_metadata --- .isort.cfg | 2 +- .../test_authenticatorscollection.py | 2 +- tests/authenticators/test_base_auth.py | 2 - tests/backends/test_s3.py | 15 +- tests/commands/test_root.py | 11 +- tests/commands/test_terraform.py | 480 ++++++++++++++++-- tests/commands/test_version.py | 16 +- tests/conftest.py | 1 - tests/handlers/test_exceptions.py | 2 - tests/handlers/test_trivy.py | 2 - tests/test_cli.py | 14 +- tests/test_definitions.py | 4 - tests/util/test_system.py | 8 +- tests/util/test_terraform_util.py | 77 +++ tfworker/authenticators/__init__.py | 2 +- tfworker/backends/base.py | 2 +- tfworker/backends/s3.py | 4 +- tfworker/cli.py | 8 - tfworker/commands/root.py | 4 +- tfworker/commands/terraform.py | 213 +++++--- tfworker/definitions.py | 2 +- tfworker/handlers/__init__.py | 11 +- tfworker/handlers/bitbucket.py | 1 - tfworker/handlers/trivy.py | 3 +- tfworker/plugins.py | 18 +- tfworker/providers/__init__.py | 2 +- tfworker/providers/base.py | 2 - tfworker/util/system.py | 7 +- tfworker/util/terraform.py | 48 ++ 29 files changed, 747 insertions(+), 216 deletions(-) create mode 100644 tests/util/test_terraform_util.py create mode 100644 tfworker/util/terraform.py diff --git a/.isort.cfg b/.isort.cfg index f21b05a..927a14b 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] -known_third_party = atlassian,boto3,botocore,click,deepdiff,google,hcl2,jinja2,mergedeep,moto,pkg_resources,pytest,pytest_lazyfixture,tenacity,yaml +known_third_party = atlassian,boto3,botocore,click,deepdiff,google,hcl2,importlib_metadata,jinja2,mergedeep,moto,pytest,tenacity,yaml profile = black diff --git a/tests/authenticators/test_authenticatorscollection.py b/tests/authenticators/test_authenticatorscollection.py index c0a6396..feb9a64 100644 --- a/tests/authenticators/test_authenticatorscollection.py +++ b/tests/authenticators/test_authenticatorscollection.py @@ -47,5 +47,5 @@ def test_collection(self, state_args): def test_unknown_authenticator(self, state_args): ac = tfworker.authenticators.AuthenticatorsCollection(state_args=state_args) assert ac.get("aws") is not None - with pytest.raises(tfworker.authenticators.UnknownAuthenticator) as e: + with pytest.raises(tfworker.authenticators.UnknownAuthenticator): ac.get("unknown") diff --git a/tests/authenticators/test_base_auth.py b/tests/authenticators/test_base_auth.py index 00fb519..525bb66 100644 --- a/tests/authenticators/test_base_auth.py +++ b/tests/authenticators/test_base_auth.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import pytest - from tfworker.authenticators.base import BaseAuthenticator diff --git a/tests/backends/test_s3.py b/tests/backends/test_s3.py index 31ae6c2..3c31bdf 100644 --- a/tests/backends/test_s3.py +++ b/tests/backends/test_s3.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os import random import string from unittest.mock import MagicMock, patch @@ -175,12 +174,12 @@ def setup_method(self, method): def test_no_session(self): self.authenticators["aws"]._session = None with pytest.raises(BackendError): - result = S3Backend(self.authenticators, self.definitions) + S3Backend(self.authenticators, self.definitions) def test_no_backend_session(self): self.authenticators["aws"]._backend_session = None with pytest.raises(BackendError): - result = S3Backend(self.authenticators, self.definitions) + S3Backend(self.authenticators, self.definitions) @patch("tfworker.backends.S3Backend._ensure_locking_table", return_value=None) @patch("tfworker.backends.S3Backend._ensure_backend_bucket", return_value=None) @@ -210,7 +209,7 @@ def test_handler_error( mock_handler, ): with pytest.raises(SystemExit): - result = S3Backend(self.authenticators, self.definitions) + S3Backend(self.authenticators, self.definitions) class TestS3BackendEnsureBackendBucket: @@ -262,7 +261,7 @@ def test_check_bucket_exists_error(self): ) with pytest.raises(ClientError): - result = self.backend._check_bucket_exists(STATE_BUCKET) + self.backend._check_bucket_exists(STATE_BUCKET) assert self.backend._s3_client.head_bucket.called @mock_aws @@ -270,7 +269,7 @@ def test_bucket_not_exist_no_create(self, capfd): self.backend._authenticator.create_backend_bucket = False self.backend._authenticator.bucket = NO_SUCH_BUCKET with pytest.raises(BackendError): - result = self.backend._ensure_backend_bucket() + self.backend._ensure_backend_bucket() assert ( "Backend bucket not found and --no-create-backend-bucket specified." in capfd.readouterr().out @@ -313,7 +312,7 @@ def test_create_bucket_invalid_location_constraint(self, capsys): ) with pytest.raises(SystemExit): - result = self.backend._create_bucket(NO_SUCH_BUCKET) + self.backend._create_bucket(NO_SUCH_BUCKET) assert "InvalidLocationConstraint" in capsys.readouterr().out assert NO_SUCH_BUCKET not in [ @@ -341,7 +340,7 @@ def test_create_bucket_error(self): ) with pytest.raises(ClientError): - result = self.backend._create_bucket(NO_SUCH_BUCKET) + self.backend._create_bucket(NO_SUCH_BUCKET) assert self.backend._s3_client.create_bucket.called diff --git a/tests/commands/test_root.py b/tests/commands/test_root.py index 11fb8ad..bcf2193 100644 --- a/tests/commands/test_root.py +++ b/tests/commands/test_root.py @@ -22,7 +22,7 @@ from deepdiff import DeepDiff import tfworker.commands.root -from tfworker.commands.root import get_platform +from tfworker.commands.root import get_platform, ordered_config_load class TestMain: @@ -203,7 +203,7 @@ def test_stateargs_template_items_invalid(self, capfd): # check templating of config_var, no environment with pytest.raises(SystemExit) as e: - result = rc.template_items(return_as_dict=True) + rc.template_items(return_as_dict=True) out, err = capfd.readouterr() assert e.value.code == 1 assert "Invalid config-var" in out @@ -247,13 +247,6 @@ def test_get_platform( mock2.assert_called_once() -import io - -import pytest - -from tfworker.commands.root import ordered_config_load - - class TestOrderedConfigLoad: def test_ordered_config_load(self): config = """ diff --git a/tests/commands/test_terraform.py b/tests/commands/test_terraform.py index ef56a1d..169c370 100644 --- a/tests/commands/test_terraform.py +++ b/tests/commands/test_terraform.py @@ -12,19 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import filecmp -import tempfile +import pathlib from contextlib import contextmanager -from pathlib import Path from typing import Tuple from unittest import mock +from unittest.mock import MagicMock, patch import pytest from google.cloud.exceptions import NotFound import tfworker -from tfworker.backends.base import BackendError -from tfworker.commands.terraform import BaseCommand +from tfworker.commands.terraform import BaseCommand, TerraformCommand, TerraformError +from tfworker.definitions import Definition +from tfworker.handlers import HandlerError # context manager to allow testing exceptions in parameterized tests @@ -47,40 +47,42 @@ def mock_tf_version(args: str) -> Tuple[int, str, str]: return (0, args.encode(), "".encode()) +@pytest.fixture(scope="function") +def definition(): + mock_definition = MagicMock(spec=Definition) + mock_definition.tag = "test_tag" + mock_definition.path = "/path/to/definition" + mock_definition.fs_path = pathlib.Path("/path/to/definition") + # mock_definition._plan_file = None + # mock_definition._ready_to_apply = False + return mock_definition + + +@pytest.fixture(scope="function") +def terraform_command(rootc): + return TerraformCommand( + rootc, + plan_file_path="/path/to/plan", + tf_plan=True, + deployment="deployment", + show_output=True, + ) + + +@pytest.fixture(scope="function") +def terraform_destroy_command(rootc): + return TerraformCommand( + rootc, + plan_file_path="/path/to/plan", + tf_plan=True, + deployment="deployment", + show_output=True, + destroy=True, + ) + + class TestTerraformCommand: - @pytest.mark.parametrize("tf_cmd", ["tf_12cmd", "tf_13cmd", "tf_14cmd", "tf_15cmd"]) - def test_prep_modules(self, tf_cmd, rootc, request): - tf_cmd = request.getfixturevalue(tf_cmd) - tf_cmd.prep_modules() - for test_file in [ - "/terraform-modules/test_a/test.tf", - "/terraform-modules/test_b/test.tf", - ]: - src = rootc.args.repository_path + test_file - dst = rootc.temp_dir + test_file - assert filecmp.cmp(src, dst, shallow=False) - - @pytest.mark.parametrize("tf_cmd", ["tf_12cmd", "tf_13cmd", "tf_14cmd", "tf_15cmd"]) - def test_terraform_modules_dir(self, tf_cmd, rootc, request): - tf_cmd = request.getfixturevalue(tf_cmd) - with tempfile.TemporaryDirectory() as d: - test_files = [Path("test_a/test.tf"), Path("test_b/test.tf")] - for f in test_files: - testfile = Path(f"{d}/{f}") - sourcefile = Path(f"tests/fixtures/terraform-modules/{f}") - parent = testfile.parent - parent.mkdir(parents=True) - testfile.write_bytes(sourcefile.read_bytes()) - - tf_cmd._terraform_modules_dir = d - tf_cmd.prep_modules() - for test_file in [ - "/terraform-modules/test_a/test.tf", - "/terraform-modules/test_b/test.tf", - ]: - src = rootc.args.repository_path + test_file - dst = rootc.temp_dir + test_file - assert filecmp.cmp(src, dst, shallow=False) + """These are legacy tests, and will be refactored away as work on the TerraformCommand class progresses.""" @pytest.mark.parametrize( "method, tf_cmd, args", @@ -251,3 +253,405 @@ def test_no_create_backend_bucket_fails_gcs(self, grootc_no_create_backend_bucke "test-0001", tf_version_major=13, ) + + +#### +class TestTerraformCommandInit: + base_kwargs = { + "backend": "s3", + "backend_plans": False, + "b64_encode": False, + "color": True, + "deployment": "test_deployment", + "destroy": False, + "force": False, + "plan_file_path": None, + "provider_cache": "/path/to/cache", + "show_output": True, + "stream_output": False, + "terraform_bin": "/path/to/terraform", + "terraform_modules_dir": "/path/to/modules", + "tf_apply": True, + "tf_plan": True, + "tf_version": (1, 8), + } + + @pytest.fixture + def terraform_command_class(self): + return TerraformCommand + + def test_constructor_with_valid_arguments(self, rootc, terraform_command_class): + kwargs = self.base_kwargs.copy() + + with patch.object( + terraform_command_class, "_resolve_arg", side_effect=lambda arg: kwargs[arg] + ): + command = terraform_command_class(rootc, **kwargs) + assert command._destroy == kwargs["destroy"] + assert command._tf_apply == kwargs["tf_apply"] + assert command._tf_plan == kwargs["tf_plan"] + assert command._plan_file_path == kwargs["plan_file_path"] + assert command._b64_encode == kwargs["b64_encode"] + assert command._deployment == kwargs["deployment"] + assert command._force == kwargs["force"] + assert command._show_output == kwargs["show_output"] + assert command._stream_output == kwargs["stream_output"] + assert command._use_colors is True + assert command._terraform_modules_dir == kwargs["terraform_modules_dir"] + assert command._terraform_output == {} + + def test_constructor_with_apply_and_destroy(self, rootc, terraform_command_class): + kwargs = self.base_kwargs.copy() + kwargs["tf_apply"] = True + kwargs["destroy"] = True + + with patch.object( + terraform_command_class, "_resolve_arg", side_effect=lambda arg: kwargs[arg] + ): + with patch("click.secho") as mock_secho, pytest.raises(SystemExit): + terraform_command_class(rootc, **kwargs) + mock_secho.assert_called_with( + "Cannot apply and destroy in the same run", fg="red" + ) + + def test_constructor_with_backend_plans(self, rootc, terraform_command_class): + kwargs = self.base_kwargs.copy() + kwargs["backend_plans"] = True + + with patch.object( + terraform_command_class, "_resolve_arg", side_effect=lambda arg: kwargs[arg] + ): + with patch("pathlib.Path.mkdir") as mock_mkdir: + command = terraform_command_class(rootc, **kwargs) + assert command._plan_file_path == f"{command._temp_dir}/plans" + mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) + + +class TestTerraformCommandProperties: + def test_plan_for_apply(self, terraform_command): + assert terraform_command.plan_for == "apply" + + def test_plan_for_destroy(self, terraform_destroy_command): + assert terraform_destroy_command.plan_for == "destroy" + + def test_tf_version_major(self, terraform_command): + assert terraform_command.tf_version_major == 1 + + +class TestTerraformCommandExec: + + def test_exec_valid_flow(self, terraform_command, definition): + def_iter = [definition] + + with patch.object( + terraform_command.definitions, "limited", return_value=def_iter + ), patch.object( + terraform_command._plugins, "download" + ) as mock_download_plugins, patch( + "tfworker.commands.terraform.tf_util.prep_modules" + ) as mock_prep_modules, patch.object( + terraform_command, "_prep_and_init" + ) as mock_prep_and_init, patch.object( + terraform_command, "_check_plan", return_value=True + ) as mock_check_plan, patch.object( + terraform_command, "_exec_plan", return_value="changes" + ) as mock_exec_plan, patch.object( + terraform_command, "_check_apply_or_destroy", return_value=True + ) as mock_check_apply_or_destroy, patch.object( + terraform_command, "_exec_apply_or_destroy" + ) as mock_exec_apply_or_destroy: + + terraform_command.exec() + + mock_download_plugins.assert_called_once() + mock_prep_modules.assert_called_once_with( + terraform_command._terraform_modules_dir, + terraform_command._temp_dir, + required=False, + ) + mock_prep_and_init.assert_called_once_with(def_iter) + mock_check_plan.assert_called_once_with(definition) + mock_exec_plan.assert_called_once_with(definition) + mock_check_apply_or_destroy.assert_called_once_with("changes", definition) + mock_exec_apply_or_destroy.assert_called_once_with(definition) + + def test_exec_with_invalid_limit(self, terraform_command): + with patch.object( + terraform_command.definitions, + "limited", + side_effect=ValueError("Invalid limit"), + ), patch("click.secho") as mock_secho: + with pytest.raises(SystemExit): + terraform_command.exec() + mock_secho.assert_called_once_with( + "Error with supplied limit: Invalid limit", fg="red" + ) + + def test_exec_without_plan(self, terraform_command, definition): + def_iter = [definition] + + with patch.object( + terraform_command.definitions, "limited", return_value=def_iter + ), patch.object( + terraform_command._plugins, "download" + ) as mock_download_plugins, patch( + "tfworker.commands.terraform.tf_util.prep_modules" + ) as mock_prep_modules, patch.object( + terraform_command, "_prep_and_init" + ) as mock_prep_and_init, patch.object( + terraform_command, "_check_plan", return_value=False + ) as mock_check_plan, patch.object( + terraform_command, "_exec_plan" + ) as mock_exec_plan, patch.object( + terraform_command, "_check_apply_or_destroy", return_value=True + ) as mock_check_apply_or_destroy, patch.object( + terraform_command, "_exec_apply_or_destroy" + ) as mock_exec_apply_or_destroy: + + terraform_command.exec() + + mock_download_plugins.assert_called_once() + mock_prep_modules.assert_called_once_with( + terraform_command._terraform_modules_dir, + terraform_command._temp_dir, + required=False, + ) + mock_prep_and_init.assert_called_once_with(def_iter) + mock_check_plan.assert_called_once_with(definition) + mock_exec_plan.assert_not_called() + mock_check_apply_or_destroy.assert_called_once_with(None, definition) + mock_exec_apply_or_destroy.assert_called_once_with(definition) + + def test_exec_with_no_apply_or_destroy(self, terraform_command, definition): + def_iter = [definition] + + with patch.object( + terraform_command.definitions, "limited", return_value=def_iter + ), patch.object( + terraform_command._plugins, "download" + ) as mock_download_plugins, patch( + "tfworker.commands.terraform.tf_util.prep_modules" + ) as mock_prep_modules, patch.object( + terraform_command, "_prep_and_init" + ) as mock_prep_and_init, patch.object( + terraform_command, "_check_plan", return_value=True + ) as mock_check_plan, patch.object( + terraform_command, "_exec_plan", return_value="changes" + ) as mock_exec_plan, patch.object( + terraform_command, "_check_apply_or_destroy", return_value=False + ) as mock_check_apply_or_destroy, patch.object( + terraform_command, "_exec_apply_or_destroy" + ) as mock_exec_apply_or_destroy: + + terraform_command.exec() + + mock_download_plugins.assert_called_once() + mock_prep_modules.assert_called_once_with( + terraform_command._terraform_modules_dir, + terraform_command._temp_dir, + required=False, + ) + mock_prep_and_init.assert_called_once_with(def_iter) + mock_check_plan.assert_called_once_with(definition) + mock_exec_plan.assert_called_once_with(definition) + mock_check_apply_or_destroy.assert_called_once_with("changes", definition) + mock_exec_apply_or_destroy.assert_not_called() + + def test_exec_with_required_prep_modules(self, terraform_command, definition): + terraform_command._terraform_modules_dir = "/temp/path" + def_iter = [definition] + + with patch.object( + terraform_command.definitions, "limited", return_value=def_iter + ), patch.object( + terraform_command._plugins, "download" + ) as mock_download_plugins, patch( + "tfworker.commands.terraform.tf_util.prep_modules" + ) as mock_prep_modules, patch.object( + terraform_command, "_prep_and_init" + ) as mock_prep_and_init, patch.object( + terraform_command, "_check_plan", return_value=True + ) as mock_check_plan, patch.object( + terraform_command, "_exec_plan", return_value="changes" + ) as mock_exec_plan, patch.object( + terraform_command, "_check_apply_or_destroy", return_value=True + ) as mock_check_apply_or_destroy, patch.object( + terraform_command, "_exec_apply_or_destroy" + ) as mock_exec_apply_or_destroy: + + terraform_command.exec() + + mock_download_plugins.assert_called_once() + mock_prep_modules.assert_called_once_with( + terraform_command._terraform_modules_dir, + terraform_command._temp_dir, + required=True, + ) + mock_prep_and_init.assert_called_once_with(def_iter) + mock_check_plan.assert_called_once_with(definition) + mock_exec_plan.assert_called_once_with(definition) + mock_check_apply_or_destroy.assert_called_once_with("changes", definition) + mock_exec_apply_or_destroy.assert_called_once_with(definition) + + +class TestTerraformCommandPrepAndInit: + + def test_prep_and_init_valid_flow(self, terraform_command, definition): + def_iter = [definition] + + with patch("click.secho") as mock_secho, patch.object( + definition, "prep" + ) as mock_prep, patch.object(terraform_command, "_run") as mock_run: + + terraform_command._prep_and_init(def_iter) + + mock_secho.assert_any_call( + f"preparing definition: {definition.tag}", fg="green" + ) + mock_prep.assert_called_once_with(terraform_command._backend) + mock_run.assert_called_once_with( + definition, "init", debug=terraform_command._show_output + ) + + def test_prep_and_init_with_terraform_error(self, terraform_command, definition): + def_iter = [definition] + + with patch("click.secho") as mock_secho, patch.object( + definition, "prep" + ) as mock_prep, patch.object( + terraform_command, "_run", side_effect=TerraformError + ) as mock_run: + + with pytest.raises(SystemExit): + terraform_command._prep_and_init(def_iter) + + mock_secho.assert_any_call( + f"preparing definition: {definition.tag}", fg="green" + ) + mock_prep.assert_called_once_with(terraform_command._backend) + mock_run.assert_called_once_with( + definition, "init", debug=terraform_command._show_output + ) + mock_secho.assert_any_call("error running terraform init", fg="red") + + +class TestTerraformCommandPlanFunctions: + def test_handle_no_plan_path_true(self, terraform_command, definition): + terraform_command._tf_plan = False + assert terraform_command._handle_no_plan_path(definition) is False + assert definition._ready_to_apply is True + + def test_handle_no_plan_path_false(self, terraform_command, definition): + terraform_command._tf_plan = True + assert terraform_command._handle_no_plan_path(definition) is True + assert definition._ready_to_apply is False + + def test_prepare_plan_file(self, terraform_command, definition): + plan_file = terraform_command._prepare_plan_file(definition) + assert definition.plan_file == plan_file + assert plan_file == pathlib.Path("/path/to/plan/deployment_test_tag.tfplan") + + def test_validate_plan_path_valid(self, terraform_command): + with patch("pathlib.Path.exists", return_value=True), patch( + "pathlib.Path.is_dir", return_value=True + ): + terraform_command._validate_plan_path(pathlib.Path("/valid/path")) + + def test_validate_plan_path_invalid(self, terraform_command): + with patch("pathlib.Path.exists", return_value=False), patch( + "pathlib.Path.is_dir", return_value=False + ), pytest.raises(SystemExit): + terraform_command._validate_plan_path(pathlib.Path("/invalid/path")) + + def test_run_handlers(self, terraform_command, definition): + with patch.object( + terraform_command, "_execute_handlers", return_value=None + ) as mock_execute_handlers: + terraform_command._run_handlers( + definition, "plan", "check", pathlib.Path("/path/to/planfile") + ) + mock_execute_handlers.assert_called_once_with( + action="plan", + stage="check", + deployment="deployment", + definition=definition.tag, + definition_path=definition.fs_path, + planfile=pathlib.Path("/path/to/planfile"), + ) + + def test_run_handlers_with_error(self, terraform_command, definition): + error = HandlerError("Handler failed") + error.terminate = False + with patch.object( + terraform_command, "_execute_handlers", side_effect=error + ), patch("click.secho"): + terraform_command._run_handlers( + definition, "plan", "check", pathlib.Path("/path/to/planfile") + ) + + def test_run_handlers_with_fatal_error(self, terraform_command, definition): + error = HandlerError("Fatal handler error") + error.terminate = True + with patch.object( + terraform_command, "_execute_handlers", side_effect=error + ), patch("click.secho"), pytest.raises(SystemExit): + terraform_command._run_handlers( + definition, "plan", "check", pathlib.Path("/path/to/planfile") + ) + + def test_should_plan_no_tf_plan(self, terraform_command, definition): + terraform_command._tf_plan = False + plan_file = pathlib.Path("/path/to/empty.tfplan") + assert terraform_command._should_plan(definition, plan_file) is False + assert definition._ready_to_apply is True + + def test_should_plan_empty_plan_file(self, terraform_command, definition): + plan_file = pathlib.Path("/path/to/empty.tfplan") + with patch("pathlib.Path.exists", return_value=True), patch( + "pathlib.Path.stat", return_value=MagicMock(st_size=0) + ): + assert terraform_command._should_plan(definition, plan_file) is True + assert definition._ready_to_apply is False + + def test_should_plan_existing_valid_plan_file(self, terraform_command, definition): + plan_file = pathlib.Path("/path/to/valid.tfplan") + with patch("pathlib.Path.exists", return_value=True), patch( + "pathlib.Path.stat", return_value=MagicMock(st_size=100) + ): + assert terraform_command._should_plan(definition, plan_file) is False + assert definition._ready_to_apply is True + + def test_should_plan_no_existing_plan_file(self, terraform_command, definition): + plan_file = pathlib.Path("/path/to/nonexistent.tfplan") + with patch("pathlib.Path.exists", return_value=False): + assert terraform_command._should_plan(definition, plan_file) is True + assert definition._ready_to_apply is False + + def test_check_plan_no_plan_path(self, terraform_command, definition): + terraform_command._plan_file_path = None + with patch.object( + terraform_command, "_handle_no_plan_path", return_value=False + ) as mock_handle_no_plan_path: + assert terraform_command._check_plan(definition) is False + mock_handle_no_plan_path.assert_called_once_with(definition) + + def test_check_plan_with_plan_path(self, terraform_command, definition): + plan_file = pathlib.Path("/path/to/plan/deployment_test_tag.tfplan") + with patch.object( + terraform_command, "_prepare_plan_file", return_value=plan_file + ) as mock_prepare_plan_file, patch.object( + terraform_command, "_validate_plan_path" + ) as mock_validate_plan_path, patch.object( + terraform_command, "_run_handlers" + ) as mock_run_handlers, patch.object( + terraform_command, "_should_plan", return_value=True + ) as mock_should_plan: + assert terraform_command._check_plan(definition) is True + mock_prepare_plan_file.assert_called_once_with(definition) + mock_validate_plan_path.assert_called_once_with(plan_file.parent) + mock_run_handlers.assert_called_once() + mock_should_plan.assert_called_once_with(definition, plan_file) + + +if __name__ == "__main__": + pytest.main() diff --git a/tests/commands/test_version.py b/tests/commands/test_version.py index 25bae6f..693f7cd 100644 --- a/tests/commands/test_version.py +++ b/tests/commands/test_version.py @@ -14,19 +14,13 @@ from unittest import mock -from pkg_resources import DistributionNotFound - from tfworker.commands.version import VersionCommand -def mock_get_distribution(package: str): - raise DistributionNotFound - - -class TestVersionCommand: - def test_exec(self, capsys): - vc = VersionCommand() - vc._version = "1.2.3" - vc.exec() +def test_version_command(capsys): + with mock.patch("tfworker.commands.version.get_version") as mock_get_version: + mock_get_version.return_value = "1.2.3" + command = VersionCommand() + command.exec() text = capsys.readouterr() assert text.out == "terraform-worker version 1.2.3\n" diff --git a/tests/conftest.py b/tests/conftest.py index 68a236a..78a84b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections import os import random import string diff --git a/tests/handlers/test_exceptions.py b/tests/handlers/test_exceptions.py index 18e3e8a..435e3d3 100644 --- a/tests/handlers/test_exceptions.py +++ b/tests/handlers/test_exceptions.py @@ -1,5 +1,3 @@ -import pytest - from tfworker.handlers.exceptions import HandlerError, UnknownHandler diff --git a/tests/handlers/test_trivy.py b/tests/handlers/test_trivy.py index 71aff5b..ac8a87f 100644 --- a/tests/handlers/test_trivy.py +++ b/tests/handlers/test_trivy.py @@ -148,7 +148,6 @@ def test__scan_plan_success_with_options(self, mock_click, mock_pipe_exec): "skip_dirs": [], "severity": "CRITICAL", "cache_dir": "/path/to/cache", - "stream_output": True, "quiet": False, "debug": True, "stream_output": False, @@ -195,7 +194,6 @@ def test__scan_planfile_success_with_options(self, mock_click, mock_pipe_exec): "skip_dirs": [], "severity": "CRITICAL", "cache_dir": "/path/to/cache", - "stream_output": True, "quiet": False, "debug": True, "stream_output": False, diff --git a/tests/test_cli.py b/tests/test_cli.py index f264bb1..7a20f8c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,14 +14,12 @@ import os import tempfile -from unittest import mock from unittest.mock import patch import pytest from click.testing import CliRunner import tfworker.cli -from tfworker.commands import CleanCommand class TestCLI: @@ -123,7 +121,7 @@ def test_validate_working_dir_does_not_exist(self, capfd): def test_validate_working_dir_not_empty(self, capfd): """ensure non empty dirs fail""" with tempfile.TemporaryDirectory() as tmpd: - with open(os.path.join(tmpd, "test"), "w+") as tmpf: + with open(os.path.join(tmpd, "test"), "w+"): with pytest.raises(SystemExit) as e: tfworker.cli.validate_working_dir(tmpd) out, err = capfd.readouterr() @@ -169,7 +167,7 @@ def test_cli_clean_command(self, mock_request, test_config_file): from tfworker.cli import cli runner = CliRunner() - result = runner.invoke(cli, ["--config-file", test_config_file, "clean", "foo"]) + runner.invoke(cli, ["--config-file", test_config_file, "clean", "foo"]) mock_request.assert_called_once() assert mock_request.method_calls[0][0] == "().exec" @@ -179,7 +177,7 @@ def test_cli_version_command(self, mock_request, test_config_file): from tfworker.cli import cli runner = CliRunner() - result = runner.invoke(cli, ["version"]) + runner.invoke(cli, ["version"]) mock_request.assert_called_once() assert mock_request.method_calls[0][0] == "().exec" @@ -197,9 +195,7 @@ def test_cli_terraform_command(self, mock_request, test_config_file): ) assert result.exit_code == 0 # the three steps the cli should execute - assert mock_request.method_calls[0][0] == "().plugins.download" - assert mock_request.method_calls[1][0] == "().prep_modules" - assert mock_request.method_calls[2][0] == "().exec" + assert mock_request.method_calls[0][0] == "().exec" @patch("tfworker.cli.EnvCommand", autospec=True) def test_cli_env_command(self, mock_request, test_config_file): @@ -207,6 +203,6 @@ def test_cli_env_command(self, mock_request, test_config_file): from tfworker.cli import cli runner = CliRunner() - result = runner.invoke(cli, ["--config-file", test_config_file, "env"]) + runner.invoke(cli, ["--config-file", test_config_file, "env"]) mock_request.assert_called_once() assert mock_request.method_calls[0][0] == "().exec" diff --git a/tests/test_definitions.py b/tests/test_definitions.py index d5301d1..a53ed45 100644 --- a/tests/test_definitions.py +++ b/tests/test_definitions.py @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import collections -import os - import pytest from tfworker.definitions import Definition diff --git a/tests/util/test_system.py b/tests/util/test_system.py index f0cda0f..3758e94 100644 --- a/tests/util/test_system.py +++ b/tests/util/test_system.py @@ -134,17 +134,17 @@ def test_which_not_found(self): def test_get_version(self): with mock.patch( - "tfworker.util.system.get_distribution", + "tfworker.util.system.importlib_metadata.distribution", side_effect=mock_distribution, ): assert get_version() == "1.2.3" def test_get_version_unknown(self): - from pkg_resources import DistributionNotFound + from importlib_metadata import PackageNotFoundError with mock.patch( - "tfworker.util.system.get_distribution", - side_effect=DistributionNotFound, + "tfworker.util.system.importlib_metadata.distribution", + side_effect=PackageNotFoundError, ): assert get_version() == "unknown" diff --git a/tests/util/test_terraform_util.py b/tests/util/test_terraform_util.py new file mode 100644 index 0000000..53db826 --- /dev/null +++ b/tests/util/test_terraform_util.py @@ -0,0 +1,77 @@ +import shutil + +import pytest + +from tfworker.util.terraform import prep_modules + + +def test_prep_modules(tmp_path): + test_file_content = "test" + + module_path = tmp_path / "terraform-modules" + module_path.mkdir() + + target_path = tmp_path / "target" + target_path.mkdir() + + # Create a test module directory with a file + test_module_dir = module_path / "test_module_dir" + test_module_dir.mkdir() + test_module_file = test_module_dir / "test_module_file.tf" + with open(test_module_file, "w") as f: + f.write(test_file_content) + test_module_ignored_file = test_module_dir / "test_module_ignored_file.txt" + test_module_ignored_file.touch() + test_module_default_ignored_file = test_module_dir / "terraform.tfstate" + test_module_default_ignored_file.touch() + + prep_modules(str(module_path), str(target_path)) + + final_target_path = target_path / "terraform-modules" / "test_module_dir" + + # check the target path exists + assert final_target_path.exists() + + # check the file is copied to the target directory + assert (final_target_path / "test_module_file.tf").exists() + + # check the file content is the same + with open(final_target_path / "test_module_file.tf") as f: + assert f.read() == test_file_content + + # check that the ignored file is not copied to the target directory + assert not (final_target_path / "terraform.tfstate").exists() + + # remove the contents of the target directory + shutil.rmtree(target_path) + assert not target_path.exists() + + # Use a custom ignore pattern + prep_modules(str(module_path), str(target_path), ignore_patterns=["*.txt"]) + + # ensure the default ignored file is copied + assert (final_target_path / "terraform.tfstate").exists() + + # ensure the custom ignored file is not copied + assert not (final_target_path / "test_module_ignored_file.txt").exists() + + +def test_prep_modules_not_found(tmp_path): + module_path = tmp_path / "terraform-modules" + target_path = tmp_path / "target" + + prep_modules(str(module_path), str(target_path)) + + # check the target path does not exist + assert not target_path.exists() + + +def test_prep_modules_required(tmp_path): + module_path = tmp_path / "terraform-modules" + target_path = tmp_path / "target" + + with pytest.raises(SystemExit): + prep_modules(str(module_path), str(target_path), required=True) + + # check the target path does not exist + assert not target_path.exists() diff --git a/tfworker/authenticators/__init__.py b/tfworker/authenticators/__init__.py index 0d982a6..f341f4f 100644 --- a/tfworker/authenticators/__init__.py +++ b/tfworker/authenticators/__init__.py @@ -31,7 +31,7 @@ def __len__(self): return len(self._authenticators) def __getitem__(self, value): - if type(value) == int: + if type(value) is int: return self._authenticators[list(self._authenticators.keys())[value]] return self._authenticators[value] diff --git a/tfworker/backends/base.py b/tfworker/backends/base.py index 550de42..2d4186c 100644 --- a/tfworker/backends/base.py +++ b/tfworker/backends/base.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from abc import ABCMeta, abstractmethod, abstractproperty +from abc import ABCMeta, abstractmethod from tfworker import JSONType diff --git a/tfworker/backends/s3.py b/tfworker/backends/s3.py index b1f9564..704a32d 100644 --- a/tfworker/backends/s3.py +++ b/tfworker/backends/s3.py @@ -14,7 +14,6 @@ import json import os -import sys from contextlib import closing from pathlib import Path from uuid import uuid4 @@ -468,7 +467,7 @@ def _check_plan(self, planfile: Path, definition: str, **kwargs): # verify the lineage and serial from the planfile matches the statefile if not self._verify_lineage(planfile, statefile): click.secho( - f"planfile lineage does not match statefile, remote plan is unsuitable and will be removed", + "planfile lineage does not match statefile, remote plan is unsuitable and will be removed", fg="red", ) self._s3_delete_plan(remotefile) @@ -505,7 +504,6 @@ def _post_plan( def _pre_apply(self, planfile: Path, definition: str, **kwargs): """_pre_apply runs before the apply is started, it should remove the planfile from the backend""" - logfile = planfile.with_suffix(".log") remotefile = f"{self._authenticator.prefix}/{definition}/{planfile.name}" remotelog = remotefile.replace(".tfplan", ".log") if self._s3_delete_plan(remotefile): diff --git a/tfworker/cli.py b/tfworker/cli.py index b0bc11d..6a1afd4 100644 --- a/tfworker/cli.py +++ b/tfworker/cli.py @@ -15,7 +15,6 @@ import os -import struct import sys from pathlib import Path @@ -228,7 +227,6 @@ def cli(context, **kwargs): """CLI for the worker utility.""" validate_host() validate_working_dir(kwargs.get("working_dir", None)) - config_file = kwargs["config_file"] context.obj = RootCommand(args=kwargs) @@ -358,12 +356,6 @@ def terraform(rootc, *args, **kwargs): click.secho(f"building deployment {kwargs.get('deployment')}", fg="green") click.secho(f"working in directory: {tfc.temp_dir}", fg="yellow") - # common setup required for all definitions - click.secho("preparing provider plugins", fg="green") - tfc.plugins.download() - click.secho("preparing modules", fg="green") - tfc.prep_modules() - tfc.exec() sys.exit(0) diff --git a/tfworker/commands/root.py b/tfworker/commands/root.py index 5cf32bc..f597968 100644 --- a/tfworker/commands/root.py +++ b/tfworker/commands/root.py @@ -16,8 +16,6 @@ import os import pathlib import platform -import re -import shutil import tempfile from pathlib import Path from typing import Union @@ -259,7 +257,7 @@ def ordered_config_load(config: str) -> dict: return yaml.load(config, Loader=yaml.FullLoader) except yaml.YAMLError as e: click.secho(f"error loading yaml/json: {e}", fg="red") - click.secho(f"the configuration that caused the error was\n:", fg="red") + click.secho("the configuration that caused the error was\n:", fg="red") for i, line in enumerate(config.split("\n")): click.secho(f"{i+1}: {line}", fg="red") raise SystemExit(1) diff --git a/tfworker/commands/terraform.py b/tfworker/commands/terraform.py index 3b6adff..e1dbcf3 100644 --- a/tfworker/commands/terraform.py +++ b/tfworker/commands/terraform.py @@ -17,10 +17,10 @@ import os import pathlib import re -import shutil import click +import tfworker.util.terraform as tf_util from tfworker.commands.base import BaseCommand from tfworker.definitions import Definition from tfworker.handlers.exceptions import HandlerError @@ -69,6 +69,9 @@ def __init__(self, rootc, **kwargs): self._terraform_modules_dir = self._resolve_arg("terraform_modules_dir") self._terraform_output = dict() + ############## + # Properties # + ############## @property def plan_for(self): """plan_for will either be apply or destroy, indicating what action is being planned for""" @@ -78,92 +81,157 @@ def plan_for(self): def tf_version_major(self): return self._tf_version_major - def prep_modules(self): - """Puts the modules sub directories into place.""" + ################## + # Public methods # + ################## + def exec(self) -> None: + """exec handles running the terraform chain, it the primary method called by the CLI - if self._terraform_modules_dir: - mod_source = self._terraform_modules_dir - mod_path = pathlib.Path(mod_source) - if not mod_path.exists(): - click.secho( - f'The specified terraform-modules directory "{mod_source}" does not exists', - fg="red", - ) - raise SystemExit(1) - else: - mod_source = f"{self._repository_path}/terraform-modules".replace("//", "/") - mod_path = pathlib.Path(mod_source) - if not mod_path.exists(): - click.secho( - "The terraform-modules directory does not exist. Skipping.", - fg="green", - ) - return - mod_destination = f"{self._temp_dir}/terraform-modules".replace("//", "/") - click.secho( - f"copying modules from {mod_source} to {mod_destination}", fg="yellow" - ) - shutil.copytree( - mod_source, - mod_destination, - symlinks=True, - ignore=shutil.ignore_patterns("test", ".terraform", "terraform.tfstate*"), - ) - - def _prep_and_init(self, def_iter: iter = None) -> None: - """_prep_and_init prepares the modules and runs terraform init""" + Returns: + None + """ + # generate an iterator for the specified definitions (or all if no limit is specified) try: - def_iter = self.definitions.limited() + # convert the definitions iterator to a list to allow reusing the iterator + def_iter = list(self.definitions.limited()) except ValueError as e: click.secho(f"Error with supplied limit: {e}", fg="red") raise SystemExit(1) + # download the providers + self._plugins.download() + + # prepare the modules, they are required if the modules dir is specified, otherwise they are optional + tf_util.prep_modules( + self._terraform_modules_dir, + self._temp_dir, + required=(self._terraform_modules_dir is not None), + ) + + # prepare the definitions and run terraform init + self._prep_and_init(def_iter) + + for definition in def_iter: + # Execute plan if needed + changes = ( + self._exec_plan(definition) if self._check_plan(definition) else None + ) + + # execute apply or destroy if needed + if self._check_apply_or_destroy(changes, definition): + self._exec_apply_or_destroy(definition) + + ################### + # Private methods # + ################### + def _prep_and_init(self, def_iter: list[Definition]) -> None: + """Prepares the definition and runs terraform init + + Args: + def_iter: an iterator of definitions to prepare + + Returns: + None + """ for definition in def_iter: - # copy definition files / templates etc. click.secho(f"preparing definition: {definition.tag}", fg="green") definition.prep(self._backend) - # run terraform init try: self._run(definition, "init", debug=self._show_output) except TerraformError: click.secho("error running terraform init", fg="red") raise SystemExit(1) - def _check_plan(self, definition: Definition) -> (bool, bool): - """determines if a plan is needed""" - # when no plan path is specified, it's straight forward - if self._plan_file_path is None: - if self._tf_plan is False: - definition._ready_to_apply = True - return False - else: - definition._ready_to_apply = False - return True + def _check_plan(self, definition: Definition) -> bool: + """ + Determines if a plan is needed for the provided definition - # a lot more to consider when a plan path is specified - plan_path = pathlib.Path.absolute(pathlib.Path(self._plan_file_path)) - plan_file = pathlib.Path( - f"{plan_path}/{self._deployment}_{definition.tag}.tfplan" - ) + Args: + definition: the definition to check for a plan + + Returns: + bool: True if a plan is needed, False otherwise + """ + if not self._plan_file_path: + return self._handle_no_plan_path(definition) + + plan_file = self._prepare_plan_file(definition) + self._validate_plan_path(plan_file.parent) + self._run_handlers(definition, "plan", "check", plan_file=plan_file) + + return self._should_plan(definition, plan_file) + + def _handle_no_plan_path(self, definition: Definition) -> bool: + """Handles the case where no plan path is specified, saved plans are not possible + + Args: + definition: the definition to check for a plan + + Returns: + bool: True if a plan is needed, False otherwise + """ + + if not self._tf_plan: + definition._ready_to_apply = True + return False + definition._ready_to_apply = False + return True + + def _prepare_plan_file(self, definition: Definition) -> pathlib.Path: + """Prepares the plan file for the definition + + Args: + definition: the definition to prepare the plan file for + + Returns: + pathlib.Path: the path to the plan file + """ + plan_path = pathlib.Path(self._plan_file_path).resolve() + plan_file = plan_path / f"{self._deployment}_{definition.tag}.tfplan" definition.plan_file = plan_file click.secho(f"using plan file:{plan_file}", fg="yellow") + return plan_file + + def _validate_plan_path(self, plan_path: pathlib.Path) -> None: + """Validates the plan path + + Args: + plan_path: the path to the plan file + Returns: + None + """ if not (plan_path.exists() and plan_path.is_dir()): click.secho( f'plan path "{plan_path}" is not suitable, it is not an existing directory' ) - raise SystemExit() + raise SystemExit(1) + + def _run_handlers( + self, definition, action, stage, plan_file=None, **kwargs + ) -> None: + """Runs the handlers for the given action and stage - # run all the handlers with with action plan and stage check + Args: + definition: the definition to run the handlers for + action: the action to run the handlers for + stage: the stage to run the handlers for + plan_file: the plan file to pass to the handlers + kwargs: additional keyword arguments to pass to the handlers + + Returns: + None + """ try: self._execute_handlers( - action="plan", - stage="check", + action=action, + stage=stage, deployment=self._deployment, definition=definition.tag, definition_path=definition.fs_path, planfile=plan_file, + **kwargs, ) except HandlerError as e: if e.terminate: @@ -171,12 +239,11 @@ def _check_plan(self, definition: Definition) -> (bool, bool): raise SystemExit(1) click.secho(f"handler error: {e}", fg="red") - # if --no-plan is specified, skip planning step regardless of other conditions - if self._tf_plan is False: + def _should_plan(self, definition: Definition, plan_file: pathlib.Path) -> bool: + if not self._tf_plan: definition._ready_to_apply = True return False - # planning was requested, check if existing plan is suitable if plan_file.exists(): if plan_file.stat().st_size == 0: click.secho( @@ -192,7 +259,6 @@ def _check_plan(self, definition: Definition) -> (bool, bool): definition._ready_to_apply = True return False - # All of the false conditions have been returned, so we need to plan definition._ready_to_apply = False return True @@ -254,7 +320,7 @@ def _exec_plan(self, definition) -> bool: except HandlerError as e: click.secho(f"{e}", fg="red") if e.terminate: - click.secho(f"error is fatal, terminating", fg="red") + click.secho("error is fatal, terminating", fg="red") raise SystemExit(1) if not changes: @@ -355,26 +421,6 @@ def _exec_apply_or_destroy(self, definition) -> None: fg="green", ) - def exec(self): - """exec handles running the terraform chain""" - try: - def_iter = self.definitions.limited() - except ValueError as e: - click.secho(f"Error with supplied limit: {e}", fg="red") - raise SystemExit(1) - - # prepare the modules and run terraform init - self._prep_and_init(def_iter) - - for definition in def_iter: - changes = None - # exec planning step if we should - if self._check_plan(definition): - changes = self._exec_plan(definition) - # exec apply step if we should - if self._check_apply_or_destroy(changes, definition): - self._exec_apply_or_destroy(definition) - def _run( self, definition, command, debug=False, plan_action="init", plan_file=None ): @@ -515,6 +561,9 @@ def _run( raise SystemExit(2) return True + ################## + # Static methods # + ################## @staticmethod def hook_exec( phase, @@ -791,7 +840,7 @@ def _make_state_cache( json.loads(stdout) except json.JSONDecodeError: raise HookError( - f"Error parsing terraform state; output is not in json format" + "Error parsing terraform state; output is not in json format" ) # write the cache to disk diff --git a/tfworker/definitions.py b/tfworker/definitions.py index f97f6b2..db86f52 100644 --- a/tfworker/definitions.py +++ b/tfworker/definitions.py @@ -287,7 +287,7 @@ def __len__(self): return len(self._definitions) def __getitem__(self, value): - if type(value) == int: + if type(value) is int: return self._definitions[list(self._definitions.keys())[value]] return self._definitions[value] diff --git a/tfworker/handlers/__init__.py b/tfworker/handlers/__init__.py index 4db3165..d8ef323 100644 --- a/tfworker/handlers/__init__.py +++ b/tfworker/handlers/__init__.py @@ -1,9 +1,10 @@ import collections -from .base import BaseHandler -from .bitbucket import BitbucketHandler -from .exceptions import HandlerError, UnknownHandler -from .trivy import TrivyHandler +# Make all of the handlers available from the handlers module +from .base import BaseHandler # noqa: F401 +from .bitbucket import BitbucketHandler # noqa: F401 +from .exceptions import HandlerError, UnknownHandler # noqa: F401 +from .trivy import TrivyHandler # noqa: F401 class HandlersCollection(collections.abc.Mapping): @@ -31,7 +32,7 @@ def __len__(self): return len(self._handlers) def __getitem__(self, value): - if type(value) == int: + if type(value) is int: return self._handlers[list(self._handlers.keys())[value]] return self._handlers[value] diff --git a/tfworker/handlers/bitbucket.py b/tfworker/handlers/bitbucket.py index 3330040..4a3e7a5 100644 --- a/tfworker/handlers/bitbucket.py +++ b/tfworker/handlers/bitbucket.py @@ -1,6 +1,5 @@ import os -from atlassian import Bitbucket from atlassian.bitbucket import Cloud from .base import BaseHandler diff --git a/tfworker/handlers/trivy.py b/tfworker/handlers/trivy.py index b61ad95..da503d1 100644 --- a/tfworker/handlers/trivy.py +++ b/tfworker/handlers/trivy.py @@ -2,7 +2,6 @@ from pathlib import Path import click -import yaml from ..util.system import pipe_exec, strip_ansi from .base import BaseHandler @@ -201,7 +200,7 @@ def _handle_results(self, exit_code, stdout, stderr, planfile): click.secho(f"Removing planfile: {planfile}", fg="yellow") os.remove(planfile) raise HandlerError( - f"trivy scan required; aborting execution", terminate=True + "trivy scan required; aborting execution", terminate=True ) def _raise_if_not_ready(self): diff --git a/tfworker/plugins.py b/tfworker/plugins.py index 1587092..cc30ad6 100644 --- a/tfworker/plugins.py +++ b/tfworker/plugins.py @@ -18,17 +18,9 @@ import os import shutil import urllib -import zipfile import click -from tenacity import ( - RetryError, - retry, - retry_if_not_exception_message, - stop_after_attempt, - wait_chain, - wait_fixed, -) +from tenacity import retry, stop_after_attempt, wait_chain, wait_fixed from tfworker.commands.root import get_platform @@ -43,12 +35,13 @@ def __init__(self, body, temp_dir, cache_dir, tf_version_major): self._temp_dir = temp_dir self._cache_dir = cache_dir self._tf_version_major = tf_version_major + self._downloaded = False def __len__(self): return len(self._providers) def __getitem__(self, value): - if type(value) == int: + if type(value) is int: return self._providers[list(self._providers.keys())[value]] return self._providers[value] @@ -66,6 +59,9 @@ def download(self): versions have changed. In production try to remove all all external repositories/sources from the critical path. """ + if self._downloaded: + return + opsys, machine = get_platform() _platform = f"{opsys}_{machine}" @@ -109,6 +105,8 @@ def download(self): click.secho(str(e), fg="red") click.Abort() + self._downloaded = True + class PluginSource: """ diff --git a/tfworker/providers/__init__.py b/tfworker/providers/__init__.py index b780fce..ca70c37 100644 --- a/tfworker/providers/__init__.py +++ b/tfworker/providers/__init__.py @@ -42,7 +42,7 @@ def __len__(self): return len(self._providers) def __getitem__(self, value): - if type(value) == int: + if type(value) is int: return self._providers[list(self._providers.keys())[value]] return self._providers[value] diff --git a/tfworker/providers/base.py b/tfworker/providers/base.py index d02797c..1dbefc5 100644 --- a/tfworker/providers/base.py +++ b/tfworker/providers/base.py @@ -83,10 +83,8 @@ def _hclify(self, s, depth=4): space = " " result = [] if isinstance(s, str): - tmps = s.replace('"', "").replace("'", "") result.append(f"{space * depth}{s}") elif isinstance(s, list): - tmps = [i.replace('"', "").replace("'", "") for i in s] result.append(f"{space * depth}{s}") elif isinstance(s, dict): # unfortunately, HCL doesn't allow for keys to be quoted so a further diff --git a/tfworker/util/system.py b/tfworker/util/system.py index bc21766..4cff798 100644 --- a/tfworker/util/system.py +++ b/tfworker/util/system.py @@ -17,7 +17,7 @@ import subprocess import click -from pkg_resources import DistributionNotFound, get_distribution +import importlib_metadata def strip_ansi(line): @@ -155,7 +155,6 @@ def get_version() -> str: Get the version of the current package """ try: - pkg_info = get_distribution("terraform-worker") - return pkg_info.version - except DistributionNotFound: + return importlib_metadata.version("terraform-worker") + except importlib_metadata.PackageNotFoundError: return "unknown" diff --git a/tfworker/util/terraform.py b/tfworker/util/terraform.py new file mode 100644 index 0000000..b4d162d --- /dev/null +++ b/tfworker/util/terraform.py @@ -0,0 +1,48 @@ +# This file contains functions primarily used by the "TerraformCommand" class +# the goal of moving these functions here is to reduce the responsibility of +# the TerraformCommand class, making it easier to test and maintain +import pathlib +import shutil + +import click + + +def prep_modules( + module_path: str, + target_path: str, + ignore_patterns: list[str] = None, + required: bool = False, +) -> None: + """This puts any terraform modules from the module path in place. By default + it will not generate an error if the module path is not found. If required + is set to True, it will raise an error if the module path is not found. + + Args: + module_path (str): The path to the terraform modules directory + target_path (str): The path to the target directory, /terraform-modules will be appended + ignore_patterns (list(str)): A list of patterns to ignore + required (bool): If the terraform modules directory is required + """ + module_path = pathlib.Path(module_path) + target_path = pathlib.Path(f"{target_path}/terraform-modules".replace("//", "/")) + + if not module_path.exists() and required: + click.secho( + f"The specified terraform-modules directory '{module_path}' does not exists", + fg="red", + ) + raise SystemExit(1) + + if not module_path.exists(): + return + + if ignore_patterns is None: + ignore_patterns = ["test", ".terraform", "terraform.tfstate*"] + + click.secho(f"copying modules from {module_path} to {target_path}", fg="yellow") + shutil.copytree( + module_path, + target_path, + symlinks=True, + ignore=shutil.ignore_patterns(*ignore_patterns), + )