Update terraform testing procedure

appveyor-ubuntu.yml

@@ -260,10 +260,10 @@ for:
   -
     matrix:
       only:
-        - configuration: DeployIntegTesting
+        - configuration: AllTerraformBuildTests
 
     test_script:
-      - sh: "pytest -vv tests/integration/deploy -n 4 --reruns 4 --dist=loadgroup --json-report --json-report-file=TEST_REPORT-integration-deploy.json"
+      - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
 
   # Integ testing package
   -

appveyor-windows.yml

@@ -261,10 +261,10 @@ for:
   #Integ testing deploy
   - matrix:
       only:
-        - configuration: DeployIntegTesting
+        - configuration: AllTerraformTests
 
     test_script:
-      - ps: "pytest -vv tests/integration/deploy -n 4 --reruns 4 --dist=loadgroup --json-report --json-report-file=TEST_REPORT-integration-deploy.json"
+      - ps: "pytest -vv tests/integration/buildcmd/test_build_terraform_applications.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
 
   # Integ testing package
   - matrix:

tests/integration/buildcmd/test_build_terraform_applications.py

@@ -1,3 +1,4 @@
+import datetime
 import os
 import logging
 import shutil
@@ -19,6 +20,7 @@
 
 from tests.integration.buildcmd.build_integ_base import BuildIntegBase
 from tests.testing_utils import CI_OVERRIDE, IS_WINDOWS, RUN_BY_CANARY
+from tests.testing_utils import run_command as static_run_command
 
 LOG = logging.getLogger(__name__)
 S3_SLEEP = 3
@@ -30,11 +32,14 @@ class BuildTerraformApplicationIntegBase(BuildIntegBase):
     build_in_container = False
     function_identifier = None
     override = False
+    terraform_application_execution_path = None
 
     @classmethod
     def setUpClass(cls):
         super(BuildTerraformApplicationIntegBase, cls).setUpClass()
         cls.terraform_application_path = str(Path(cls.test_data_path, cls.terraform_application))
+        cls.terraform_application_execution_path = str(Path(__file__).resolve().parent.joinpath(str(uuid.uuid4()).replace("-", "")[:10]))
+        shutil.copytree(Path(cls.terraform_application_path), Path(cls.terraform_application_execution_path))
         if cls.build_in_container:
             cls.client = docker.from_env()
             cls.image_name = "sam-terraform-python-build"
@@ -57,7 +62,7 @@ def setUpClass(cls):
     def setUp(self):
         super().setUp()
         shutil.rmtree(Path(self.working_dir))
-        shutil.copytree(Path(self.terraform_application_path), Path(self.working_dir))
+        shutil.copytree(Path(self.terraform_application_execution_path), Path(self.working_dir))
 
     def run_command(self, command_list, env=None, input=None, timeout=None, override_dir=None):
         running_dir = override_dir if override_dir else self.working_dir
@@ -90,9 +95,9 @@ def build_with_prepare_hook(self):
 
         build_cmd_list = self.get_command_list(**command_list_parameters)
         _, stderr, return_code = self.run_command(
-            build_cmd_list, override_dir=self.terraform_application_path, env=environment_variables
+            build_cmd_list, override_dir=self.terraform_application_execution_path, env=environment_variables
         )
-        LOG.info(stderr)
+        LOG.info(stderr.decode("utf-8"))
         self.assertEqual(return_code, 0)
 
     def _verify_invoke_built_function(self, function_logical_id, overrides, expected_result):
@@ -123,6 +128,13 @@ def _verify_invoke_built_function(self, function_logical_id, overrides, expected
         LOG.info("sam local invoke stderr: %s", stderr.decode("utf-8"))
         self.assertEqual(json.loads(process_stdout), expected_result)
 
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cls.terraform_application_execution_path and shutil.rmtree(cls.terraform_application_execution_path, ignore_errors=True)
+
+    def tearDown(self):
+        super(BuildTerraformApplicationIntegBase, self).tearDown()
+
 
 class BuildTerraformApplicationS3BackendIntegBase(BuildTerraformApplicationIntegBase):
     @classmethod
@@ -142,8 +154,25 @@ def setUpClass(cls):
         if not cls.pre_created_bucket:
             cls.s3_bucket.create()
             time.sleep(S3_SLEEP)
-
         super().setUpClass()
+        cls.initialize_s3_backend()
+
+    @classmethod
+    def initialize_s3_backend(cls):
+        cls.backend_key = f"terraform-backend/{str(uuid.uuid4())}"
+        cls.backendconfig_path = str(Path(cls.terraform_application_execution_path) / "backend.conf")
+        with open(cls.backendconfig_path, "w") as f:
+            f.write(f'bucket="{cls.bucket_name}"\n')
+            f.write(f'key="{cls.backend_key}"\n')
+            f.write(f'region="{cls.region_name}"')
+
+        # We have to init the terraform project with specifying the S3 backend first
+        _, stderr, _ = static_run_command(
+            ["terraform", "init", f"-backend-config={cls.backendconfig_path}", "-reconfigure", "-input=false"],
+            cwd=cls.terraform_application_execution_path
+        )
+        if stderr:
+            LOG.info(stderr)
 
     @classmethod
     def tearDownClass(cls):
@@ -155,19 +184,6 @@ def tearDownClass(cls):
 
     def setUp(self):
         super().setUp()
-        self.backend_key = f"terraform-backend/{str(uuid.uuid4())}"
-        self.backendconfig_path = str(Path(self.working_dir) / "backend.conf")
-        with open(self.backendconfig_path, "w") as f:
-            f.write(f'bucket="{self.bucket_name}"\n')
-            f.write(f'key="{self.backend_key}"\n')
-            f.write(f'region="{self.region_name}"')
-
-        # We have to init the terraform project with specifying the S3 backend first
-        _, stderr, _ = self.run_command(
-            ["terraform", "init", f"-backend-config={self.backendconfig_path}", "-reconfigure", "-input=false"]
-        )
-        if stderr:
-            LOG.info(stderr)
 
     def tearDown(self):
         """Clean up the terraform state file on S3 and remove the backendconfg locally"""
@@ -192,10 +208,11 @@ def tearDown(self):
         (True,),
     ],
 )
+@pytest.mark.xdist_group(name="zip_lambda_local_backend_override")
 class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackendWithOverride(
     BuildTerraformApplicationIntegBase
 ):
-    function_identifier = "module.function7.aws_lambda_function.this[0]"
+    function_identifier = "function9"
     override = True
     terraform_application = (
         Path("terraform/zip_based_lambda_functions_local_backend")
@@ -287,8 +304,9 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o
         (True,),
     ],
 )
+@pytest.mark.xdist_group(name="zip_lambda_local_backend")
 class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend(BuildTerraformApplicationIntegBase):
-    function_identifier = "module.function7.aws_lambda_function.this[0]"
+    function_identifier = "function9"
     terraform_application = (
         Path("terraform/zip_based_lambda_functions_local_backend")
         if not IS_WINDOWS
@@ -378,10 +396,11 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o
         (True,),
     ],
 )
+@pytest.mark.xdist_group(name="zip_lambda_s3_backend")
 class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3BackendWithOverride(
     BuildTerraformApplicationS3BackendIntegBase
 ):
-    function_identifier = "module.function7.aws_lambda_function.this[0]"
+    function_identifier = "function9"
     override = True
     terraform_application = (
         Path("terraform/zip_based_lambda_functions_s3_backend")
@@ -471,8 +490,9 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o
         (True,),
     ],
 )
+@pytest.mark.xdist_group(name="zip_lambda_s3_backend_override")
 class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend(BuildTerraformApplicationS3BackendIntegBase):
-    function_identifier = "module.function7.aws_lambda_function.this[0]"
+    function_identifier = "function9"
     terraform_application = (
         Path("terraform/zip_based_lambda_functions_s3_backend")
         if not IS_WINDOWS
@@ -539,8 +559,10 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o
             command_list_parameters["build_image"] = self.docker_tag
         build_cmd_list = self.get_command_list(**command_list_parameters)
         LOG.info("command list: %s", build_cmd_list)
+        start = time.time()
         _, stderr, return_code = self.run_command(build_cmd_list)
-        LOG.info(stderr)
+        end = time.time()
+        LOG.info(f"END: {end}, DURATION {end - start}")
         self.assertEqual(return_code, 0)
 
         self._verify_invoke_built_function(

tests/integration/buildcmd/test_build_terraform_applications_other_cases.py

@@ -140,6 +140,7 @@ def test_build_no_s3_config(self):
     "Skip Terraform test cases unless running in CI",
 )
 class TestBuildTerraformApplicationsWithImageBasedLambdaFunctionAndLocalBackend(BuildTerraformApplicationIntegBase):
+    function_identifier = "aws_lambda_function.function_with_non_image_uri"
     terraform_application = Path("terraform/image_based_lambda_functions_local_backend")
     functions = [
         "aws_lambda_function.function_with_non_image_uri",
@@ -155,7 +156,9 @@ class TestBuildTerraformApplicationsWithImageBasedLambdaFunctionAndLocalBackend(
 
     @parameterized.expand(functions)
     def test_build_and_invoke_lambda_functions(self, function_identifier):
-        build_cmd_list = self.get_command_list(hook_name="terraform", function_identifier=function_identifier)
+        build_cmd_list = self.get_command_list(
+            hook_name="terraform", function_identifier=function_identifier, skip_prepare_infra=True
+        )
         LOG.info("command list: %s", build_cmd_list)
         _, stderr, return_code = self.run_command(build_cmd_list)
         LOG.info(stderr)
@@ -180,6 +183,7 @@ def test_build_and_invoke_lambda_functions(self, function_identifier):
 class TestBuildTerraformApplicationsWithImageBasedLambdaFunctionAndS3Backend(
     BuildTerraformApplicationS3BackendIntegBase
 ):
+    function_identifier = "aws_lambda_function.function_with_non_image_uri"
     terraform_application = Path("terraform/image_based_lambda_functions_s3_backend")
     functions = [
         "aws_lambda_function.function_with_non_image_uri",
@@ -195,7 +199,9 @@ class TestBuildTerraformApplicationsWithImageBasedLambdaFunctionAndS3Backend(
 
     @parameterized.expand(functions)
     def test_build_and_invoke_lambda_functions(self, function_identifier):
-        build_cmd_list = self.get_command_list(hook_name="terraform", function_identifier=function_identifier)
+        build_cmd_list = self.get_command_list(
+            hook_name="terraform", function_identifier=function_identifier, skip_prepare_infra=True
+        )
         LOG.info("command list: %s", build_cmd_list)
         _, stderr, return_code = self.run_command(build_cmd_list)
         LOG.info(stderr)