From 8268c0346b38be577bfba21fc76b80cedd462763 Mon Sep 17 00:00:00 2001 From: Richard Maynard Date: Mon, 17 Jun 2024 10:27:40 -0500 Subject: [PATCH 1/2] python version changes / os.walk fix --- poetry.lock | 49 +--------------------- pyproject.toml | 2 +- tests/util/test_util_terraform.py | 12 +++--- tfworker/commands/base.py | 3 ++ tfworker/commands/terraform.py | 7 +++- tfworker/definitions.py | 19 +++++---- tfworker/providers/base.py | 3 ++ tfworker/providers/providers_collection.py | 4 ++ tfworker/util/terraform.py | 2 - tfworker/util/terraform_helpers.py | 15 +++++-- 10 files changed, 46 insertions(+), 70 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1495a44..5281f7f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -146,8 +146,6 @@ mypy-extensions = ">=0.4.3" packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -464,9 +462,6 @@ files = [ {file = "coverage-7.5.3.tar.gz", hash = "sha256:04aefca5190d1dc7a53a4c1a5a7f8568811306d7a8ee231c42fb69215571944f"}, ] -[package.dependencies] -tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} - [package.extras] toml = ["tomli"] @@ -603,20 +598,6 @@ files = [ {file = "docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f"}, ] -[[package]] -name = "exceptiongroup" -version = "1.2.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, - {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, -] - -[package.extras] -test = ["pytest (>=6)"] - [[package]] name = "executing" version = "2.0.1" @@ -909,7 +890,6 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} decorator = "*" -exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} jedi = ">=0.16" matplotlib-inline = "*" pexpect = {version = ">4.3", markers = "sys_platform != \"win32\" and sys_platform != \"emscripten\""} @@ -1589,11 +1569,9 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" pluggy = ">=1.5,<2.0" -tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] @@ -1724,7 +1702,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1732,16 +1709,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1758,7 +1727,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1766,7 +1734,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1951,7 +1918,6 @@ sphinxcontrib-htmlhelp = ">=2.0.0" sphinxcontrib-jsmath = "*" sphinxcontrib-qthelp = "*" sphinxcontrib-serializinghtml = ">=1.1.9" -tomli = {version = ">=2", markers = "python_version < \"3.11\""} [package.extras] docs = ["sphinxcontrib-websupport"] @@ -2071,17 +2037,6 @@ pure-eval = "*" [package.extras] tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - [[package]] name = "traitlets" version = "5.14.3" @@ -2259,5 +2214,5 @@ files = [ [metadata] lock-version = "2.0" -python-versions = "^3.10" -content-hash = "9649053b9cee29538584e3e3c5f8ad24012bc3c24632b6082c4c39f6f7db2656" +python-versions = "^3.11" +content-hash = "1b93c05b0096aa4fc3f8ad0e4b09d47664c652fccb41d58349dc2c401a829a4c" diff --git a/pyproject.toml b/pyproject.toml index 378b88a..b95831f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ ] [tool.poetry.dependencies] -python = "^3.10" +python = "^3.11" boto3 = "^1.34" click = "^8.1" jinja2 = "^3.1" diff --git a/tests/util/test_util_terraform.py b/tests/util/test_util_terraform.py index a413fed..9729ca4 100644 --- a/tests/util/test_util_terraform.py +++ b/tests/util/test_util_terraform.py @@ -177,7 +177,7 @@ def test_get_tf_version( def mock_mirror_setup(): mock_mirror_settings = { "providers": MagicMock(), - "terraform_bin": "/path/to/terraform", + 'terraform_bin': "/path/to/terraform", "working_dir": "/working/dir", "cache_dir": "/cache/dir", "temp_dir": "/temp/dir", @@ -208,7 +208,7 @@ def test_mirror_providers(mock_mirror_setup): result = mirror_providers( providers=mock_mirror_settings["providers"], - terraform_bin=mock_mirror_settings["terraform_bin"], + terraform_bin=mock_mirror_settings['terraform_bin'], working_dir=mock_mirror_settings["working_dir"], cache_dir=mock_mirror_settings["cache_dir"], ) @@ -220,7 +220,7 @@ def test_mirror_providers(mock_mirror_setup): mock_mirror_settings["cache_dir"], ) mock_pipe_exec.assert_called_once_with( - f"{mock_mirror_settings["terraform_bin"]} providers mirror {mock_mirror_settings['cache_dir']}", + f"{mock_mirror_settings['terraform_bin']} providers mirror {mock_mirror_settings['cache_dir']}", cwd=mock_mirror_settings["temp_dir"], stream_output=True, ) @@ -243,7 +243,7 @@ def test_mirror_providers_tf_error(mock_mirror_setup): with pytest.raises(SystemExit): mirror_providers( providers=mock_mirror_settings["providers"], - terraform_bin=mock_mirror_settings["terraform_bin"], + terraform_bin=mock_mirror_settings['terraform_bin'], working_dir=mock_mirror_settings["working_dir"], cache_dir=mock_mirror_settings["cache_dir"], ) @@ -255,7 +255,7 @@ def test_mirror_providers_tf_error(mock_mirror_setup): mock_mirror_settings["cache_dir"], ) mock_pipe_exec.assert_called_once_with( - f"{mock_mirror_settings["terraform_bin"]} providers mirror {mock_mirror_settings['cache_dir']}", + f"{mock_mirror_settings['terraform_bin']} providers mirror {mock_mirror_settings['cache_dir']}", cwd=mock_mirror_settings["temp_dir"], stream_output=True, ) @@ -273,7 +273,7 @@ def test_mirror_providers_all_in_cache(mock_mirror_setup): mirror_providers( providers=mock_mirror_settings["providers"], - terraform_bin=mock_mirror_settings["terraform_bin"], + terraform_bin=mock_mirror_settings['terraform_bin'], working_dir=mock_mirror_settings["working_dir"], cache_dir=mock_mirror_settings["cache_dir"], ) diff --git a/tfworker/commands/base.py b/tfworker/commands/base.py index 22bddbf..a8c25a1 100644 --- a/tfworker/commands/base.py +++ b/tfworker/commands/base.py @@ -13,6 +13,7 @@ # limitations under the License. import click +import pathlib from tfworker.authenticators import AuthenticatorsCollection from tfworker.backends import BackendError, select_backend @@ -50,6 +51,8 @@ def __init__(self, rootc, deployment="undefined", limit=tuple(), **kwargs): rootc.load_config() self._provider_cache = self._resolve_arg("provider_cache") + if self._provider_cache is not None: + self._provider_cache = pathlib.Path(self._provider_cache).resolve() (self._tf_version_major, self._tf_version_minor) = self._resolve_arg( "tf_version" diff --git a/tfworker/commands/terraform.py b/tfworker/commands/terraform.py index ffd9bf2..44362b6 100644 --- a/tfworker/commands/terraform.py +++ b/tfworker/commands/terraform.py @@ -438,7 +438,12 @@ def _run( color_str = "-no-color" if self._use_colors is False else "" params = { - "init": f"-input=false {color_str} -plugin-dir={plugin_dir} -lockfile=readonly", + "init": f"-input=false {color_str} -plugin-dir={plugin_dir}", + # -lockfile=readonly is ideal, but many of our modules are not + # only partially defining the required providers; they need to specify all + # required providers, or none, and let the worker generate the requirements + # based on the deployment_config.yaml.j2 + # "init": f"-input=false {color_str} -plugin-dir={plugin_dir} -lockfile=readonly", "plan": f"-input=false -detailed-exitcode {color_str}", "apply": f"-input=false {color_str} -auto-approve", "destroy": f"-input=false {color_str} -auto-approve", diff --git a/tfworker/definitions.py b/tfworker/definitions.py index 09c4c06..5b5fff8 100644 --- a/tfworker/definitions.py +++ b/tfworker/definitions.py @@ -107,7 +107,7 @@ def fs_path(self): @property def provider_names(self): try: - return list(find_required_providers(self.path).keys()) + return list(find_required_providers(self.fs_path).keys()) except AttributeError: return None @@ -212,14 +212,15 @@ def _prep_terraform_lockfile(self): if self._provider_cache is None: return - with open(f"{self._target}/{TF_PROVIDER_DEFAULT_LOCKFILE}", "w") as lockfile: - lockfile.write( - generate_terraform_lockfile( - providers=self._providers, - included_providers=self.provider_names, - cache_dir=self._provider_cache, - ) - ) + result = generate_terraform_lockfile( + providers=self._providers, + included_providers=self.provider_names, + cache_dir=self._provider_cache, + ) + + if result is not None: + with open(f"{self._target}/{TF_PROVIDER_DEFAULT_LOCKFILE}", "w") as lockfile: + lockfile.write(result) @staticmethod def quote_str(some_string): diff --git a/tfworker/providers/base.py b/tfworker/providers/base.py index 0e14d6a..a7d7143 100644 --- a/tfworker/providers/base.py +++ b/tfworker/providers/base.py @@ -25,6 +25,9 @@ def __init__(self, config: ProviderConfig) -> None: self.source = config.requirements.source or f"hashicorp/{self.tag}" self._field_filter = [] + def __str__(self): + return self.tag + @property def gid(self) -> ProviderGID: from tfworker.util.terraform import get_provider_gid_from_source diff --git a/tfworker/providers/providers_collection.py b/tfworker/providers/providers_collection.py index 2d9d1ed..9cd16ac 100644 --- a/tfworker/providers/providers_collection.py +++ b/tfworker/providers/providers_collection.py @@ -14,6 +14,7 @@ import collections import copy +import click from typing import List from tfworker.providers.generic import GenericProvider @@ -49,6 +50,9 @@ def __getitem__(self, value): def __iter__(self): return iter(self._providers.values()) + def __str__(self): + return str([f"{x.tag}: {str(x.gid)}" for x in self._providers.values()]) + def keys(self): return self._providers.keys() diff --git a/tfworker/util/terraform.py b/tfworker/util/terraform.py index 11bb8f1..2b56a0d 100644 --- a/tfworker/util/terraform.py +++ b/tfworker/util/terraform.py @@ -196,8 +196,6 @@ def get_provider_gid_from_source(source: str) -> ProviderGID: return ProviderGID(hostname=hostname, namespace=namespace, type=ptype) - -@lru_cache def find_required_providers( search_dir: str, ) -> Union[None, Dict[str, [Dict[str, str]]]]: diff --git a/tfworker/util/terraform_helpers.py b/tfworker/util/terraform_helpers.py index f6d535e..807ae43 100644 --- a/tfworker/util/terraform_helpers.py +++ b/tfworker/util/terraform_helpers.py @@ -1,12 +1,15 @@ import json import os import pathlib +from functools import lru_cache from tempfile import TemporaryDirectory from typing import Dict, List, Union import click import hcl2 +from lark.exceptions import UnexpectedToken + from tfworker.providers.providers_collection import ProvidersCollection from tfworker.types import ProviderGID from tfworker.util.system import get_platform @@ -79,10 +82,9 @@ def _write_mirror_configuration( Raises: IndexError: If there are no providers to mirror. """ - includes = [x for x in providers if _not_in_cache(x.gid, x.version, cache_dir)] + includes = [x.tag for x in providers if _not_in_cache(x.gid, x.version, cache_dir)] if len(includes) == 0: raise IndexError("No providers to mirror") - click.secho(f"Mirroring providers: {includes}", fg="yellow") mirror_configuration = _create_mirror_configuration( providers=providers, includes=includes @@ -166,13 +168,18 @@ def _parse_required_providers(content: dict) -> Union[None, Dict[str, Dict[str, return providers +@lru_cache def _find_required_providers(search_dir: str) -> Dict[str, [Dict[str, str]]]: providers = {} - for root, _, files in os.walk(search_dir): + for root, _, files in os.walk(search_dir, followlinks=True): for file in files: if file.endswith(".tf"): with open(f"{root}/{file}", "r") as f: - content = hcl2.load(f) + try: + content = hcl2.load(f) + except UnexpectedToken as e: + click.secho(f"skipping {root}/{file}: {e}", fg="blue") + continue new_providers = _parse_required_providers(content) if new_providers is not None: providers.update(new_providers) From 0d6ce86c446e45143108378ee689f0667e0fe846 Mon Sep 17 00:00:00 2001 From: Richard Maynard Date: Mon, 17 Jun 2024 14:22:29 -0500 Subject: [PATCH 2/2] Change the .tfvars file name In order to be loaded without a -var parameter, the file needs to end with .auto.tfvars. This was changed when moving the value to a constant, in order to try and be more consistent and predictable. --- tfworker/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfworker/constants.py b/tfworker/constants.py index 90ad2be..10c4535 100644 --- a/tfworker/constants.py +++ b/tfworker/constants.py @@ -30,7 +30,7 @@ TF_STATE_CACHE_NAME = "worker_state_cache.json" WORKER_LOCALS_FILENAME = "worker_generated_locals.tf" WORKER_TF_FILENAME = "worker_generated_terraform.tf" -WORKER_TFVARS_FILENAME = "worker_generated.tfvars" +WORKER_TFVARS_FILENAME = "worker_generated.auto.tfvars" RESERVED_FILES = [ WORKER_LOCALS_FILENAME, WORKER_TF_FILENAME,