use s3 bucket to prepopulate all possible remote states (#28)

* use s3 bucket to prepopulate all possible remote states

* remove debugging cruft

* add new root argument to root test fixtures

tests/conftest.py

@@ -111,6 +111,7 @@ def grootc():
             "backend_region": "us-central1",
             "backend_bucket": "test_gcp_bucket",
             "backend_prefix": "terraform/test-0002",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "gcp_test_config.yaml"
             ),
@@ -132,6 +133,7 @@ def grootc_no_create_backend_bucket():
             "backend_region": "us-central1",
             "backend_bucket": "test_gcp_bucket",
             "backend_prefix": "terraform/test-0002",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "gcp_test_config.yaml"
             ),
@@ -157,6 +159,7 @@ def rootc(s3_client, dynamodb_client, sts_client, create_backend_bucket=True):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "test_config.yaml"
             ),
@@ -182,6 +185,7 @@ def rootc_no_create_backend_bucket(s3_client, dynamodb_client, sts_client):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "test_config.yaml"
             ),
@@ -207,6 +211,7 @@ def json_base_rootc(s3_client, dynamodb_client, sts_client):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "base_config_test.json"
             ),
@@ -231,6 +236,7 @@ def yaml_base_rootc(s3_client, dynamodb_client, sts_client):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "base_config_test.yaml"
             ),
@@ -255,6 +261,7 @@ def hcl_base_rootc(s3_client, dynamodb_client, sts_client):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "base_config_test.hcl"
             ),
@@ -284,6 +291,7 @@ def rootc_options(s3_client, dynamodb_client, sts_client):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "test_config_with_options.yaml"
             ),

tfworker/backends/base.py

@@ -36,6 +36,10 @@ def data_hcl(self, exclude: list) -> str:
     def clean(self, deployment: str, limit: tuple) -> str:
         pass
 
+    @abstractmethod
+    def remotes(self) -> list:
+        pass
+
 
 class Backends:
     s3 = "s3"

tfworker/backends/gcs.py

@@ -95,6 +95,10 @@ def _clean_prefix(self, prefix: str) -> None:
             else:
                 raise BackendError(f"state file at: {b.name} is not empty")
 
+    def remotes(self) -> list:
+        """ this is unimplemented here """
+        raise NotImplementedError
+
     def _get_state_list(self) -> list:
         """
         _get_state_list returns a list of states inside of the prefix, since this is looking for state/folders only

tfworker/backends/s3.py

@@ -112,6 +112,23 @@ def __init__(self, authenticators, definitions, deployment=None):
                     "Backend bucket not found and --no-create-backend-bucket specified."
                 )
 
+        # Generate a list of all files in the bucket, at the desired prefix for the deployment
+        s3_paginator = self._s3_client.get_paginator("list_objects_v2").paginate(
+            Bucket=self._authenticator.bucket,
+            Prefix=self._authenticator.prefix,
+        )
+
+        self._bucket_files = set()
+        for page in s3_paginator:
+            if "Contents" in page:
+                for key in page["Contents"]:
+                    # just append the last part of the prefix to the list
+                    self._bucket_files.add(key["Key"].split("/")[-2])
+
+    def remotes(self) -> list:
+        """ return a list of the remote bucket keys """
+        return list(self._bucket_files)
+
     def _check_table_exists(self, name: str) -> bool:
         """check if a supplied dynamodb table exists"""
         if name in self._ddb_client.list_tables()["TableNames"]:

tfworker/cli.py

@@ -180,6 +180,12 @@ def validate_working_dir(fpath):
     default=const.DEFAULT_AWS_REGION,
     help="Region where terraform rootc/lock bucket exists",
 )
+@click.option(
+    "--backend-use-all-remotes/--no-backend-use-all-remotes",
+    default=False,
+    envvar="WORKER_BACKEND_USE_ALL_REMOTES",
+    help="Generate remote data sources based on all definition paths present in the backend",
+)
 @click.option(
     "--create-backend-bucket/--no-create-backend-bucket",
     default=True,
@@ -330,7 +336,13 @@ def version():
 @click.pass_obj
 def terraform(rootc, *args, **kwargs):
     """execute terraform orchestration"""
-    tfc = TerraformCommand(rootc, *args, **kwargs)
+    try:
+        tfc = TerraformCommand(rootc, *args, **kwargs)
+    except FileNotFoundError as e:
+        click.secho(
+            f"terraform binary not found: {e.filename}", fg="red", err=True
+        )
+        raise SystemExit(1)
 
     click.secho(f"building deployment {kwargs.get('deployment')}", fg="green")
     click.secho(f"working in directory: {tfc.temp_dir}", fg="yellow")

tfworker/definitions.py

@@ -52,6 +52,7 @@ def __init__(
         tf_version_major,
         limited=False,
         template_callback=None,
+        use_backend_remotes=False,
     ):
         self.tag = definition
         self._body = body
@@ -76,6 +77,8 @@ def __init__(
         self._target = f"{self._temp_dir}/definitions/{self.tag}".replace("//", "/")
         self._template_callback = template_callback
 
+        self._use_backend_remotes = use_backend_remotes
+
     @property
     def body(self):
         return self._body
@@ -156,7 +159,11 @@ def prep(self, backend):
                 tflocals.write("}\n\n")
 
         # create remote data sources, and required providers
-        remotes = list(map(lambda x: x.split(".")[0], self._remote_vars.values()))
+        if self._use_backend_remotes:
+            remotes = backend.remotes()
+        else:
+            remotes = list(map(lambda x: x.split(".")[0], self._remote_vars.values()))
+
         with open(f"{self._target}/worker_terraform.tf", "w+") as tffile:
             tffile.write(f"{self._providers.hcl(self.provider_names)}\n\n")
             tffile.write(TERRAFORM_TPL.format(f"{backend.hcl(self.tag)}", ""))
@@ -251,6 +258,7 @@ def __init__(
                 tf_version_major,
                 True if limit and definition in limit else False,
                 template_callback=self.render_templates,
+                use_backend_remotes=self._root_args.backend_use_all_remotes
             )
 
     def __len__(self):

tfworker/plugins.py

@@ -21,11 +21,10 @@
 import zipfile
 
 import click
-from tenacity import retry, stop_after_attempt, wait_chain, wait_fixed
+from tenacity import retry, stop_after_attempt, wait_chain, wait_fixed, RetryError, retry_if_not_exception_message
 
 from tfworker.commands.root import get_platform
 
-
 class PluginSourceParseException(Exception):
     pass
 
@@ -88,16 +87,19 @@ def download(self):
 
             if cache_hit is False:
                 # the provider cache is not populated, download and put the plugin in place
-                download_from_remote(
-                    uri,
-                    plugin_dir,
-                    self._cache_dir,
-                    provider_path,
-                    file_name,
-                    name,
-                    details,
-                )
-
+                try:
+                    download_from_remote(
+                        uri,
+                        plugin_dir,
+                        self._cache_dir,
+                        provider_path,
+                        file_name,
+                        name,
+                        details,
+                    )
+                except PluginSourceParseException as e:
+                    click.secho(str(e), fg="red")
+                    click.Abort()
 
 class PluginSource:
     """
@@ -140,6 +142,7 @@ def __repr__(self):
         wait_fixed(10),
     ),
     stop=stop_after_attempt(3),
+    reraise=True,
 )
 def download_from_remote(
     uri, plugin_dir, cache_dir, provider_path, file_name, name, details
@@ -158,10 +161,15 @@ def download_from_remote(
     )
 
     # download the remote file
-    with urllib.request.urlopen(uri) as response, open(
-        f"{plugin_dir}/{file_name}", "wb"
-    ) as plug_file:
-        shutil.copyfileobj(response, plug_file)
+    try:
+        with urllib.request.urlopen(uri) as response, open(
+            f"{plugin_dir}/{file_name}", "wb"
+        ) as plug_file:
+            shutil.copyfileobj(response, plug_file)
+    except urllib.error.HTTPError as e:
+        raise PluginSourceParseException(
+            f"{e} while downloading plugin: {name}:{details['version']} from {uri}"
+        )
 
     # put the file into the working provider directory and cache if necessary
     files = glob.glob(