python version changes / os.walk fix

pyproject.toml

@@ -27,7 +27,7 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = "^3.11"
 boto3 = "^1.34"
 click = "^8.1"
 jinja2 = "^3.1"

tests/util/test_util_terraform.py

@@ -177,7 +177,7 @@ def test_get_tf_version(
 def mock_mirror_setup():
     mock_mirror_settings = {
         "providers": MagicMock(),
-        "terraform_bin": "/path/to/terraform",
+        'terraform_bin': "/path/to/terraform",
         "working_dir": "/working/dir",
         "cache_dir": "/cache/dir",
         "temp_dir": "/temp/dir",
@@ -208,7 +208,7 @@ def test_mirror_providers(mock_mirror_setup):
 
     result = mirror_providers(
         providers=mock_mirror_settings["providers"],
-        terraform_bin=mock_mirror_settings["terraform_bin"],
+        terraform_bin=mock_mirror_settings['terraform_bin'],
         working_dir=mock_mirror_settings["working_dir"],
         cache_dir=mock_mirror_settings["cache_dir"],
     )
@@ -220,7 +220,7 @@ def test_mirror_providers(mock_mirror_setup):
         mock_mirror_settings["cache_dir"],
     )
     mock_pipe_exec.assert_called_once_with(
-        f"{mock_mirror_settings["terraform_bin"]} providers mirror {mock_mirror_settings['cache_dir']}",
+        f"{mock_mirror_settings['terraform_bin']} providers mirror {mock_mirror_settings['cache_dir']}",
         cwd=mock_mirror_settings["temp_dir"],
         stream_output=True,
     )
@@ -243,7 +243,7 @@ def test_mirror_providers_tf_error(mock_mirror_setup):
     with pytest.raises(SystemExit):
         mirror_providers(
             providers=mock_mirror_settings["providers"],
-            terraform_bin=mock_mirror_settings["terraform_bin"],
+            terraform_bin=mock_mirror_settings['terraform_bin'],
             working_dir=mock_mirror_settings["working_dir"],
             cache_dir=mock_mirror_settings["cache_dir"],
         )
@@ -255,7 +255,7 @@ def test_mirror_providers_tf_error(mock_mirror_setup):
         mock_mirror_settings["cache_dir"],
     )
     mock_pipe_exec.assert_called_once_with(
-        f"{mock_mirror_settings["terraform_bin"]} providers mirror {mock_mirror_settings['cache_dir']}",
+        f"{mock_mirror_settings['terraform_bin']} providers mirror {mock_mirror_settings['cache_dir']}",
         cwd=mock_mirror_settings["temp_dir"],
         stream_output=True,
     )
@@ -273,7 +273,7 @@ def test_mirror_providers_all_in_cache(mock_mirror_setup):
 
     mirror_providers(
         providers=mock_mirror_settings["providers"],
-        terraform_bin=mock_mirror_settings["terraform_bin"],
+        terraform_bin=mock_mirror_settings['terraform_bin'],
         working_dir=mock_mirror_settings["working_dir"],
         cache_dir=mock_mirror_settings["cache_dir"],
     )

tfworker/commands/base.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import click
+import pathlib
 
 from tfworker.authenticators import AuthenticatorsCollection
 from tfworker.backends import BackendError, select_backend
@@ -50,6 +51,8 @@ def __init__(self, rootc, deployment="undefined", limit=tuple(), **kwargs):
         rootc.load_config()
 
         self._provider_cache = self._resolve_arg("provider_cache")
+        if self._provider_cache is not None:
+            self._provider_cache = pathlib.Path(self._provider_cache).resolve()
 
         (self._tf_version_major, self._tf_version_minor) = self._resolve_arg(
             "tf_version"

tfworker/commands/terraform.py

@@ -438,7 +438,12 @@ def _run(
 
         color_str = "-no-color" if self._use_colors is False else ""
         params = {
-            "init": f"-input=false {color_str} -plugin-dir={plugin_dir} -lockfile=readonly",
+            "init": f"-input=false {color_str} -plugin-dir={plugin_dir}",
+            # -lockfile=readonly is ideal, but many of our modules are not
+            # only partially defining the required providers; they need to specify all
+            # required providers, or none, and let the worker generate the requirements
+            # based on the deployment_config.yaml.j2
+            # "init": f"-input=false {color_str} -plugin-dir={plugin_dir} -lockfile=readonly",
             "plan": f"-input=false -detailed-exitcode {color_str}",
             "apply": f"-input=false {color_str} -auto-approve",
             "destroy": f"-input=false {color_str} -auto-approve",

tfworker/definitions.py

@@ -107,7 +107,7 @@ def fs_path(self):
     @property
     def provider_names(self):
         try:
-            return list(find_required_providers(self.path).keys())
+            return list(find_required_providers(self.fs_path).keys())
         except AttributeError:
             return None
 
@@ -212,14 +212,15 @@ def _prep_terraform_lockfile(self):
         if self._provider_cache is None:
             return
 
-        with open(f"{self._target}/{TF_PROVIDER_DEFAULT_LOCKFILE}", "w") as lockfile:
-            lockfile.write(
-                generate_terraform_lockfile(
-                    providers=self._providers,
-                    included_providers=self.provider_names,
-                    cache_dir=self._provider_cache,
-                )
-            )
+        result = generate_terraform_lockfile(
+            providers=self._providers,
+            included_providers=self.provider_names,
+            cache_dir=self._provider_cache,
+        )
+
+        if result is not None:
+            with open(f"{self._target}/{TF_PROVIDER_DEFAULT_LOCKFILE}", "w") as lockfile:
+                lockfile.write(result)
 
     @staticmethod
     def quote_str(some_string):

tfworker/providers/base.py

@@ -25,6 +25,9 @@ def __init__(self, config: ProviderConfig) -> None:
         self.source = config.requirements.source or f"hashicorp/{self.tag}"
         self._field_filter = []
 
+    def __str__(self):
+        return self.tag
+
     @property
     def gid(self) -> ProviderGID:
         from tfworker.util.terraform import get_provider_gid_from_source

tfworker/providers/providers_collection.py

@@ -14,6 +14,7 @@
 
 import collections
 import copy
+import click
 from typing import List
 
 from tfworker.providers.generic import GenericProvider
@@ -49,6 +50,9 @@ def __getitem__(self, value):
     def __iter__(self):
         return iter(self._providers.values())
 
+    def __str__(self):
+        return str([f"{x.tag}: {str(x.gid)}" for x in self._providers.values()])
+
     def keys(self):
         return self._providers.keys()
 

tfworker/util/terraform.py

@@ -196,8 +196,6 @@ def get_provider_gid_from_source(source: str) -> ProviderGID:
 
     return ProviderGID(hostname=hostname, namespace=namespace, type=ptype)
 
-
-@lru_cache
 def find_required_providers(
     search_dir: str,
 ) -> Union[None, Dict[str, [Dict[str, str]]]]:

tfworker/util/terraform_helpers.py

@@ -1,12 +1,15 @@
 import json
 import os
 import pathlib
+from functools import lru_cache
 from tempfile import TemporaryDirectory
 from typing import Dict, List, Union
 
 import click
 import hcl2
 
+from lark.exceptions import UnexpectedToken
+
 from tfworker.providers.providers_collection import ProvidersCollection
 from tfworker.types import ProviderGID
 from tfworker.util.system import get_platform
@@ -79,10 +82,9 @@ def _write_mirror_configuration(
     Raises:
         IndexError: If there are no providers to mirror.
     """
-    includes = [x for x in providers if _not_in_cache(x.gid, x.version, cache_dir)]
+    includes = [x.tag for x in providers if _not_in_cache(x.gid, x.version, cache_dir)]
     if len(includes) == 0:
         raise IndexError("No providers to mirror")
-    click.secho(f"Mirroring providers: {includes}", fg="yellow")
 
     mirror_configuration = _create_mirror_configuration(
         providers=providers, includes=includes
@@ -166,13 +168,18 @@ def _parse_required_providers(content: dict) -> Union[None, Dict[str, Dict[str,
     return providers
 
 
+@lru_cache
 def _find_required_providers(search_dir: str) -> Dict[str, [Dict[str, str]]]:
     providers = {}
-    for root, _, files in os.walk(search_dir):
+    for root, _, files in os.walk(search_dir, followlinks=True):
         for file in files:
             if file.endswith(".tf"):
                 with open(f"{root}/{file}", "r") as f:
-                    content = hcl2.load(f)
+                    try:
+                        content = hcl2.load(f)
+                    except UnexpectedToken as e:
+                        click.secho(f"skipping {root}/{file}: {e}", fg="blue")
+                        continue
                     new_providers = _parse_required_providers(content)
                     if new_providers is not None:
                         providers.update(new_providers)