Merge pull request #83 from affinipay/master

support remote backend plan storage and more

Makefile

@@ -13,11 +13,11 @@ format: init
 
 test: init
 	poetry run pytest -p no:warnings
-	poetry run coverage report --fail-under=70 -m --skip-empty
+	poetry run coverage report --fail-under=60 -m --skip-empty
 
 dep-test: init
 	poetry run pytest
-	poetry run coverage report --fail-under=70 -m --skip-empty
+	poetry run coverage report --fail-under=60 -m --skip-empty
 
 clean:
 	rm -rf build dist .eggs terraform_worker.egg-info

pyproject.toml

@@ -37,7 +37,7 @@ requests = "^2.28"
 google-cloud-storage = "^2.7.0"
 python-hcl2 = "^4.3.0"
 lark = "^1.1"
-pyyaml = "^5.4"
+pyyaml = "^6.0.1"
 markupsafe = "2.1"
 mergedeep = "1.3.4"
 setuptools = "67.6.0"

tests/commands/test_terraform.py

@@ -230,7 +230,7 @@ def test_worker_options(self, tf_13cmd_options):
     #                 )
 
     def test_no_create_backend_bucket_fails_gcs(self, grootc_no_create_backend_bucket):
-        with pytest.raises(BackendError):
+        with pytest.raises(SystemExit):
             with mock.patch(
                 "tfworker.commands.base.BaseCommand.get_terraform_version",
                 side_effect=lambda x: (13, 3),

tests/conftest.py

@@ -111,6 +111,7 @@ def grootc():
             "backend_region": "us-central1",
             "backend_bucket": "test_gcp_bucket",
             "backend_prefix": "terraform/test-0002",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "gcp_test_config.yaml"
             ),
@@ -132,6 +133,7 @@ def grootc_no_create_backend_bucket():
             "backend_region": "us-central1",
             "backend_bucket": "test_gcp_bucket",
             "backend_prefix": "terraform/test-0002",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "gcp_test_config.yaml"
             ),
@@ -157,6 +159,7 @@ def rootc(s3_client, dynamodb_client, sts_client, create_backend_bucket=True):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "test_config.yaml"
             ),
@@ -182,6 +185,7 @@ def rootc_no_create_backend_bucket(s3_client, dynamodb_client, sts_client):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "test_config.yaml"
             ),
@@ -207,6 +211,7 @@ def json_base_rootc(s3_client, dynamodb_client, sts_client):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "base_config_test.json"
             ),
@@ -231,6 +236,7 @@ def yaml_base_rootc(s3_client, dynamodb_client, sts_client):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "base_config_test.yaml"
             ),
@@ -255,6 +261,7 @@ def hcl_base_rootc(s3_client, dynamodb_client, sts_client):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "base_config_test.hcl"
             ),
@@ -284,6 +291,7 @@ def rootc_options(s3_client, dynamodb_client, sts_client):
             "backend_region": "us-west-2",
             "backend_bucket": "test_bucket",
             "backend_prefix": "terraform/test-0001",
+            "backend_use_all_remotes": False,
             "config_file": os.path.join(
                 os.path.dirname(__file__), "fixtures", "test_config_with_options.yaml"
             ),

tests/util/test_copier.py

@@ -33,6 +33,7 @@
 else:
     C_ROOT_PATH = "/tmp/test/"
 
+
 @pytest.fixture(scope="session")
 def register_test_copier():
     @CopyFactory.register("testfixture")
@@ -317,6 +318,15 @@ def test_copy(self, request, tmp_path):
         c.copy()
         assert os.path.isfile(f"{str(tmp_path)}/test.tf")
 
+        c = FileSystemCopier(
+            source="/tests/fixtures/definitions/test_a",
+            root_path=f"{request.config.rootdir}",
+            destination=f"{str(tmp_path)}",
+        )
+
+        with pytest.raises(FileNotFoundError):
+            c.copy(sub_path="invalid_path")
+
     def test_local_path(self):
         """tests the local path property"""
 
@@ -333,7 +343,9 @@ def test_local_path(self):
 
         # Ensure file not found error is raised on invalid relative path
         with pytest.raises(FileNotFoundError):
-            FileSystemCopier(source="some/invalid/path", root_path=os.getcwd()).local_path
+            FileSystemCopier(
+                source="some/invalid/path", root_path=os.getcwd()
+            ).local_path
 
         # Ensure file not found error is raised on invalid absolute path
         with pytest.raises(FileNotFoundError):
@@ -348,10 +360,7 @@ def test_type_match(self, request):
         # this should return true because the source is a valid directory
         assert FileSystemCopier.type_match(source) is True
         # this should return false because the full path to source does not exist inside of root_path
-        assert (
-            FileSystemCopier.type_match("/some/invalid/path")
-            is False
-        )
+        assert FileSystemCopier.type_match("/some/invalid/path") is False
         # this should return true because the full path to source exists inside of root_path
         assert (
             FileSystemCopier.type_match(

tfworker/backends/__init__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .base import Backends
+from .base import BackendError, Backends, BaseBackend  # noqa
 from .gcs import GCSBackend  # noqa
 from .s3 import S3Backend  # noqa
 

tfworker/backends/base.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from abc import ABCMeta, abstractmethod
+from abc import ABCMeta, abstractmethod, abstractproperty
 
 from tfworker import JSONType
 
@@ -22,6 +22,7 @@ class BackendError(Exception):
 
 
 class BaseBackend(metaclass=ABCMeta):
+    plan_storage = False
     tag = "base"
 
     @abstractmethod
@@ -36,6 +37,14 @@ def data_hcl(self, exclude: list) -> str:
     def clean(self, deployment: str, limit: tuple) -> str:
         pass
 
+    @abstractmethod
+    def remotes(self) -> list:
+        pass
+
+    @property
+    def handlers(self) -> dict:
+        return {}
+
 
 class Backends:
     s3 = "s3"

tfworker/backends/gcs.py

@@ -16,6 +16,7 @@
 
 import click
 from google.api_core import page_iterator
+from google.auth.exceptions import DefaultCredentialsError
 from google.cloud import storage
 from google.cloud.exceptions import Conflict, NotFound
 
@@ -39,14 +40,17 @@ def __init__(self, authenticators, definitions, deployment=None):
             if not self._gcs_prefix.endswith("/"):
                 self._gcs_prefix = f"{self._gcs_prefix}/"
 
-            if self._authenticator.creds_path:
-                self._storage_client = storage.Client.from_service_account_json(
-                    self._authenticator.creds_path
-                )
-            else:
-                self._storage_client = storage.Client(
-                    project=self._authenticator.project
-                )
+            try:
+                if self._authenticator.creds_path:
+                    self._storage_client = storage.Client.from_service_account_json(
+                        self._authenticator.creds_path
+                    )
+                else:
+                    self._storage_client = storage.Client(
+                        project=self._authenticator.project
+                    )
+            except DefaultCredentialsError as e:
+                raise BackendError(f"Unable to authenticate to GCS, error: {str(e)}")
 
             try:
                 self._storage_client.get_bucket(self._gcs_bucket)
@@ -95,6 +99,10 @@ def _clean_prefix(self, prefix: str) -> None:
             else:
                 raise BackendError(f"state file at: {b.name} is not empty")
 
+    def remotes(self) -> list:
+        """this is unimplemented here"""
+        raise NotImplementedError
+
     def _get_state_list(self) -> list:
         """
         _get_state_list returns a list of states inside of the prefix, since this is looking for state/folders only