significant changes to handling providers/plugins

- removed custom provider/plugin handling
  - now uses terraform providers mirror command to make cache
  - generates a .terraform.lock.hcl to minimize outputs
  - multiple provider versions can exist fine in cache
  - now /properly/ supports providers not in the hashicorp registry
  - benefits from checking signing of providers upon initial mirror
- fixed provider configuration rendering
  - scans for required_providers
  - injects required_providers config if it does not exist in module
  - only injects provider configuration for used providers
- more conversion to types
- removed custom aws / helm providers
  - they previously required custom configuration
  - tfworker/providers "generic" should work in most cases
- moved types from a single file into a package to better organize
- started implementing pydantic for some types
- added constants for filenames / default TF provider configs
- updated tests to reflect complete removal of legacy "plugins"

.isort.cfg

@@ -1,3 +1,3 @@
 [settings]
-known_third_party = atlassian,boto3,botocore,click,deepdiff,google,hcl2,importlib_metadata,jinja2,mergedeep,moto,pytest,tenacity,yaml
+known_third_party = atlassian,boto3,botocore,click,deepdiff,google,hcl2,jinja2,mergedeep,moto,pytest,yaml
 profile = black

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "terraform-worker"
-version = "0.11.0"
+version = "0.12.0"
 description = "An orchestration tool for Terraform"
 authors = ["Richard Maynard <[email protected]>"]
 packages = [
@@ -41,6 +41,7 @@ markupsafe = "^2.1"
 mergedeep = "^1.3"
 setuptools = "^70.0"
 atlassian-python-api = "^3.41"
+pydantic = "^2.7"
 
 [tool.poetry.dev-dependencies]
 ipython = "^8.24"

tests/commands/test_root.py

@@ -250,7 +250,7 @@ def test_ordered_config_load_invalid(self, capfd):
         """
         expected_error_out = ""
         for i, line in enumerate(config.split("\n")):
-            expected_error_out += f"{i+1}: {line}\n"
+            expected_error_out += f"{i + 1}: {line}\n"
         with pytest.raises(SystemExit) as e:
             ordered_config_load(config)
         out, err = capfd.readouterr()

tests/commands/test_terraform.py

@@ -321,12 +321,13 @@ class TestTerraformCommandExec:
 
     def test_exec_valid_flow(self, terraform_command, definition):
         def_iter = [definition]
+        terraform_command._provider_cache = "/path/to/cache"
 
         with patch.object(
             terraform_command.definitions, "limited", return_value=def_iter
-        ), patch.object(
-            terraform_command._plugins, "download"
-        ) as mock_download_plugins, patch(
+        ), patch(
+            "tfworker.commands.terraform.tf_util.mirror_providers"
+        ) as mock_mirror_providers, patch(
             "tfworker.commands.terraform.tf_util.prep_modules"
         ) as mock_prep_modules, patch.object(
             terraform_command, "_prep_and_init"
@@ -342,7 +343,7 @@ def test_exec_valid_flow(self, terraform_command, definition):
 
             terraform_command.exec()
 
-            mock_download_plugins.assert_called_once()
+            mock_mirror_providers.assert_called_once()
             mock_prep_modules.assert_called_once_with(
                 terraform_command._terraform_modules_dir,
                 terraform_command._temp_dir,
@@ -371,9 +372,7 @@ def test_exec_without_plan(self, terraform_command, definition):
 
         with patch.object(
             terraform_command.definitions, "limited", return_value=def_iter
-        ), patch.object(
-            terraform_command._plugins, "download"
-        ) as mock_download_plugins, patch(
+        ), patch(
             "tfworker.commands.terraform.tf_util.prep_modules"
         ) as mock_prep_modules, patch.object(
             terraform_command, "_prep_and_init"
@@ -389,7 +388,6 @@ def test_exec_without_plan(self, terraform_command, definition):
 
             terraform_command.exec()
 
-            mock_download_plugins.assert_called_once()
             mock_prep_modules.assert_called_once_with(
                 terraform_command._terraform_modules_dir,
                 terraform_command._temp_dir,
@@ -406,9 +404,7 @@ def test_exec_with_no_apply_or_destroy(self, terraform_command, definition):
 
         with patch.object(
             terraform_command.definitions, "limited", return_value=def_iter
-        ), patch.object(
-            terraform_command._plugins, "download"
-        ) as mock_download_plugins, patch(
+        ), patch(
             "tfworker.commands.terraform.tf_util.prep_modules"
         ) as mock_prep_modules, patch.object(
             terraform_command, "_prep_and_init"
@@ -424,7 +420,6 @@ def test_exec_with_no_apply_or_destroy(self, terraform_command, definition):
 
             terraform_command.exec()
 
-            mock_download_plugins.assert_called_once()
             mock_prep_modules.assert_called_once_with(
                 terraform_command._terraform_modules_dir,
                 terraform_command._temp_dir,
@@ -442,9 +437,7 @@ def test_exec_with_required_prep_modules(self, terraform_command, definition):
 
         with patch.object(
             terraform_command.definitions, "limited", return_value=def_iter
-        ), patch.object(
-            terraform_command._plugins, "download"
-        ) as mock_download_plugins, patch(
+        ), patch(
             "tfworker.commands.terraform.tf_util.prep_modules"
         ) as mock_prep_modules, patch.object(
             terraform_command, "_prep_and_init"
@@ -460,7 +453,6 @@ def test_exec_with_required_prep_modules(self, terraform_command, definition):
 
             terraform_command.exec()
 
-            mock_download_plugins.assert_called_once()
             mock_prep_modules.assert_called_once_with(
                 terraform_command._terraform_modules_dir,
                 terraform_command._temp_dir,