From fff4382998c4446a657437b5a55220d752e54d87 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 27 Sep 2023 09:59:12 -0700 Subject: [PATCH 1/4] test: Refactor Terraform build tests --- appveyor-ubuntu.yml | 4 + appveyor-windows.yml | 4 + .../integration/buildcmd/build_integ_base.py | 4 + .../test_build_terraform_applications.py | 397 +++++++++++++----- 4 files changed, 293 insertions(+), 116 deletions(-) diff --git a/appveyor-ubuntu.yml b/appveyor-ubuntu.yml index 6778031a89..4fd912ed12 100644 --- a/appveyor-ubuntu.yml +++ b/appveyor-ubuntu.yml @@ -183,6 +183,7 @@ for: - sh: "terraform -version" - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackendWithOverride_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" # Local ZIP Terraform Build In Container integ testing - @@ -200,6 +201,7 @@ for: - sh: "terraform -version" - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackendWithOverride_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" # S3 ZIP Terraform Build integ testing - @@ -217,6 +219,7 @@ for: - sh: "terraform -version" - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3BackendWithOverride_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" # S3 ZIP Terraform Build In Container integ testing - @@ -234,6 +237,7 @@ for: - sh: "terraform -version" - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3BackendWithOverride_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" # Other Terraform Build In Container integ testing - diff --git a/appveyor-windows.yml b/appveyor-windows.yml index 2aae5baa39..8b0a422b11 100644 --- a/appveyor-windows.yml +++ b/appveyor-windows.yml @@ -205,6 +205,7 @@ for: - "terraform -version" - ps: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + - ps: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackendWithOverride_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" # Local ZIP Terraform Build In Container integ testing - matrix: @@ -217,6 +218,7 @@ for: - "terraform -version" - ps: "pytest -vv tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + - ps: "pytest -vv tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackendWithOverride_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" # S3 ZIP Terraform Build integ testing - matrix: @@ -229,6 +231,7 @@ for: - "terraform -version" - ps: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + - ps: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3BackendWithOverride_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" # S3 ZIP Terraform Build In Container integ testing - matrix: @@ -241,6 +244,7 @@ for: - "terraform -version" - ps: "pytest -vv tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + - ps: "pytest -vv tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3BackendWithOverride_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" # Other Terraform Build integ testing - matrix: diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py index d6dad53a1a..415c2ce715 100644 --- a/tests/integration/buildcmd/build_integ_base.py +++ b/tests/integration/buildcmd/build_integ_base.py @@ -86,6 +86,7 @@ def get_command_list( config_file=None, save_params=False, project_root_dir=None, + skip_prepare_infra=None, ): command_list = [self.cmd, "build"] @@ -158,6 +159,9 @@ def get_command_list( if project_root_dir is not None: command_list += ["--terraform-project-root-path", project_root_dir] + if skip_prepare_infra is True: + command_list += ["--skip-prepare-infra"] + return command_list def verify_docker_container_cleanedup(self, runtime): diff --git a/tests/integration/buildcmd/test_build_terraform_applications.py b/tests/integration/buildcmd/test_build_terraform_applications.py index a0b1bbdcf0..b651ffb2e6 100644 --- a/tests/integration/buildcmd/test_build_terraform_applications.py +++ b/tests/integration/buildcmd/test_build_terraform_applications.py @@ -2,6 +2,7 @@ import logging import shutil import json +import tempfile import uuid import time @@ -12,6 +13,7 @@ import boto3 import docker +import pytest from parameterized import parameterized, parameterized_class @@ -26,6 +28,8 @@ class BuildTerraformApplicationIntegBase(BuildIntegBase): terraform_application: Optional[Path] = None template = None build_in_container = False + function_identifier = None + override = False @classmethod def setUpClass(cls): @@ -55,8 +59,9 @@ def setUp(self): shutil.rmtree(Path(self.working_dir)) shutil.copytree(Path(self.terraform_application_path), Path(self.working_dir)) - def run_command(self, command_list, env=None, input=None, timeout=None): - process = Popen(command_list, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env, cwd=self.working_dir) + def run_command(self, command_list, env=None, input=None, timeout=None, override_dir=None): + running_dir = override_dir if override_dir else self.working_dir + process = Popen(command_list, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env, cwd=running_dir) try: (stdout, stderr) = process.communicate(input=input, timeout=timeout) return stdout, stderr, process.returncode @@ -64,6 +69,32 @@ def run_command(self, command_list, env=None, input=None, timeout=None): process.kill() raise + @pytest.fixture(scope="class", autouse=True) + def build_with_prepare_hook(self): + if not self.function_identifier: + # This doesn't make sense for all tests that inherit this base class, skip for those + return + + command_list_parameters = {"function_identifier": self.function_identifier, "hook_name": "terraform"} + if self.build_in_container: + command_list_parameters["use_container"] = True + command_list_parameters["build_image"] = self.docker_tag + if self.override: + command_list_parameters[ + "container_env_var" + ] = "TF_VAR_HELLO_FUNCTION_SRC_CODE=./artifacts/HelloWorldFunction2" + + environment_variables = os.environ.copy() + if self.override: + environment_variables["TF_VAR_HELLO_FUNCTION_SRC_CODE"] = "./artifacts/HelloWorldFunction2" + + build_cmd_list = self.get_command_list(**command_list_parameters) + _, stderr, return_code = self.run_command( + build_cmd_list, override_dir=self.terraform_application_path, env=environment_variables + ) + LOG.info(stderr) + self.assertEqual(return_code, 0) + def _verify_invoke_built_function(self, function_logical_id, overrides, expected_result): LOG.info("Invoking built function '{}'".format(function_logical_id)) @@ -161,49 +192,31 @@ def tearDown(self): (True,), ], ) -class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend(BuildTerraformApplicationIntegBase): +class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackendWithOverride( + BuildTerraformApplicationIntegBase +): + function_identifier = "module.function7.aws_lambda_function.this[0]" + override = True terraform_application = ( Path("terraform/zip_based_lambda_functions_local_backend") if not IS_WINDOWS else Path("terraform/zip_based_lambda_functions_local_backend_windows") ) functions = [ - ("module.function9.aws_lambda_function.this[0]", "hello world 9 - override version", True), - ("function9", "hello world 9 - override version", True), - ("module.function9.aws_lambda_function.this[0]", "hello world 9", False), - ("function9", "hello world 9", False), - ("aws_lambda_function.function6", "hello world 6 - override version", True), - ("function6", "hello world 6 - override version", True), - ("aws_lambda_function.function6", "hello world 6", False), - ("function6", "hello world 6", False), - ("aws_lambda_function.function5", "hello world 5 - override version", True), - ("function5", "hello world 5 - override version", True), - ("aws_lambda_function.function5", "hello world 5", False), - ("function5", "hello world 5", False), - ("aws_lambda_function.function4", "hello world 4 - override version", True), - ("function4", "hello world 4 - override version", True), - ("aws_lambda_function.function4", "hello world 4", False), - ("function4", "hello world 4", False), - ("aws_lambda_function.function3", "hello world 3 - override version", True), - ("function3", "hello world 3 - override version", True), - ("aws_lambda_function.function3", "hello world 3", False), - ("function3", "hello world 3", False), - ("module.function2.aws_lambda_function.this", "hello world 2 - override version", True), - ("function2", "hello world 2 - override version", True), - ("aws_lambda_function.function1", "hello world 1 - override version", True), - ("function1", "hello world 1 - override version", True), - ("module.function2.aws_lambda_function.this", "hello world 2", False), - ("function2", "hello world 2", False), - ("aws_lambda_function.function1", "hello world 1", False), - ("function1", "hello world 1", False), - ("aws_lambda_function.from_localfile", "[]", False), - ("aws_lambda_function.from_s3", "[]", False), - ("module.level1_lambda.aws_lambda_function.this", "[]", False), - ("module.level1_lambda.module.level2_lambda.aws_lambda_function.this", "[]", False), - ("my_function_from_localfile", "[]", False), - ("my_function_from_s3", "[]", False), - ("my_level1_lambda", "[]", False), - ("my_level2_lambda", "[]", False), + ("module.function9.aws_lambda_function.this[0]", "hello world 9 - override version"), + ("function9", "hello world 9 - override version",), + ("aws_lambda_function.function6", "hello world 6 - override version"), + ("function6", "hello world 6 - override version"), + ("aws_lambda_function.function5", "hello world 5 - override version"), + ("function5", "hello world 5 - override version"), + ("aws_lambda_function.function4", "hello world 4 - override version"), + ("function4", "hello world 4 - override version"), + ("aws_lambda_function.function3", "hello world 3 - override version"), + ("function3", "hello world 3 - override version"), + ("module.function2.aws_lambda_function.this", "hello world 2 - override version"), + ("function2", "hello world 2 - override version"), + ("aws_lambda_function.function1", "hello world 1 - override version"), + ("function1", "hello world 1 - override version"), ] @classmethod @@ -219,43 +232,38 @@ def setUpClass(cls): # testing on Windows, and only test them on linux. # check the Serverless TF issue https://github.com/terraform-aws-modules/terraform-aws-lambda/issues/142 cls.functions += [ - ("module.function7.aws_lambda_function.this[0]", "hello world 7 - override version", True), - ("function7", "hello world 7 - override version", True), - ("module.function7.aws_lambda_function.this[0]", "hello world 7", False), - ("function7", "hello world 7", False), - ("module.function8.aws_lambda_function.this[0]", "hello world 8 - override version", True), - ("function8", "hello world 8 - override version", True), - ("module.function8.aws_lambda_function.this[0]", "hello world 8", False), - ("function8", "hello world 8", False), - ("module.function10.aws_lambda_function.this[0]", "hello world 10 - override version", True), - ("function10", "hello world 10 - override version", True), - ("module.function10.aws_lambda_function.this[0]", "hello world 10", False), - ("function10", "hello world 10", False), - ("module.function11.aws_lambda_function.this[0]", "hello world 11 - override version", True), - ("function11", "hello world 11 - override version", True), - ("module.function11.aws_lambda_function.this[0]", "hello world 11", False), - ("function11", "hello world 11", False), + ("module.function7.aws_lambda_function.this[0]", "hello world 7 - override version"), + ("function7", "hello world 7 - override version"), + ("module.function8.aws_lambda_function.this[0]", "hello world 8 - override version"), + ("function8", "hello world 8 - override version"), + ("module.function10.aws_lambda_function.this[0]", "hello world 10 - override version"), + ("function10", "hello world 10 - override version"), + ("module.function11.aws_lambda_function.this[0]", "hello world 11 - override version"), + ("function11", "hello world 11 - override version"), ] super().setUpClass() @parameterized.expand(functions) - def test_build_and_invoke_lambda_functions(self, function_identifier, expected_output, should_override_code): + def test_build_and_invoke_lambda_functions(self, function_identifier, expected_output): command_list_parameters = { "hook_name": "terraform", "function_identifier": function_identifier, + "skip_prepare_infra": True, } if self.build_in_container: command_list_parameters["use_container"] = True command_list_parameters["build_image"] = self.docker_tag - if should_override_code: + if self.override: command_list_parameters[ "container_env_var" ] = "TF_VAR_HELLO_FUNCTION_SRC_CODE=./artifacts/HelloWorldFunction2" - build_cmd_list = self.get_command_list(**command_list_parameters) - LOG.info("command list: %s", build_cmd_list) + environment_variables = os.environ.copy() - if should_override_code: + if self.override: environment_variables["TF_VAR_HELLO_FUNCTION_SRC_CODE"] = "./artifacts/HelloWorldFunction2" + + build_cmd_list = self.get_command_list(**command_list_parameters) + LOG.info("command list: %s", build_cmd_list) stdout, stderr, return_code = self.run_command(build_cmd_list, env=environment_variables) LOG.info("sam build stdout: %s", stdout.decode("utf-8")) LOG.info("sam build stderr: %s", stderr.decode("utf-8")) @@ -279,49 +287,122 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o (True,), ], ) -class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend(BuildTerraformApplicationS3BackendIntegBase): +class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend(BuildTerraformApplicationIntegBase): + function_identifier = "module.function7.aws_lambda_function.this[0]" + terraform_application = ( + Path("terraform/zip_based_lambda_functions_local_backend") + if not IS_WINDOWS + else Path("terraform/zip_based_lambda_functions_local_backend_windows") + ) + functions = [ + ("module.function9.aws_lambda_function.this[0]", "hello world 9"), + ("function9", "hello world 9"), + ("aws_lambda_function.function6", "hello world 6"), + ("function6", "hello world 6"), + ("aws_lambda_function.function5", "hello world 5"), + ("function5", "hello world 5"), + ("aws_lambda_function.function4", "hello world 4"), + ("function4", "hello world 4"), + ("aws_lambda_function.function3", "hello world 3"), + ("function3", "hello world 3"), + ("module.function2.aws_lambda_function.this", "hello world 2"), + ("function2", "hello world 2"), + ("aws_lambda_function.function1", "hello world 1"), + ("function1", "hello world 1"), + ("aws_lambda_function.from_localfile", "[]"), + ("aws_lambda_function.from_s3", "[]"), + ("module.level1_lambda.aws_lambda_function.this", "[]"), + ("module.level1_lambda.module.level2_lambda.aws_lambda_function.this", "[]"), + ("my_function_from_localfile", "[]"), + ("my_function_from_s3", "[]"), + ("my_level1_lambda", "[]"), + ("my_level2_lambda", "[]"), + ] + + @classmethod + def setUpClass(cls): + if IS_WINDOWS and cls.build_in_container: + # we use this TF project to test sam build in container on windows as we need to run a linux bash script for + # build, and also we need to remove the Serverless TF functions from this project. + # that is why we need to use a new project and not one of the existing linux or windows projects + cls.terraform_application = "terraform/zip_based_lambda_functions_local_backend_container_windows" + if not IS_WINDOWS: + # The following functions are defined using serverless tf module, and since Serverless TF has some issue + # while executing `terraform plan` in windows, we removed these function from the TF projects we used in + # testing on Windows, and only test them on linux. + # check the Serverless TF issue https://github.com/terraform-aws-modules/terraform-aws-lambda/issues/142 + cls.functions += [ + ("module.function7.aws_lambda_function.this[0]", "hello world 7"), + ("function7", "hello world 7"), + ("module.function8.aws_lambda_function.this[0]", "hello world 8"), + ("function8", "hello world 8"), + ("module.function10.aws_lambda_function.this[0]", "hello world 10"), + ("function10", "hello world 10"), + ("module.function11.aws_lambda_function.this[0]", "hello world 11"), + ("function11", "hello world 11"), + ] + super().setUpClass() + + @parameterized.expand(functions) + def test_build_and_invoke_lambda_functions(self, function_identifier, expected_output): + command_list_parameters = { + "hook_name": "terraform", + "function_identifier": function_identifier, + "skip_prepare_infra": True, + } + if self.build_in_container: + command_list_parameters["use_container"] = True + command_list_parameters["build_image"] = self.docker_tag + build_cmd_list = self.get_command_list(**command_list_parameters) + LOG.info("command list: %s", build_cmd_list) + stdout, stderr, return_code = self.run_command(build_cmd_list) + LOG.info("sam build stdout: %s", stdout.decode("utf-8")) + LOG.info("sam build stderr: %s", stderr.decode("utf-8")) + self.assertEqual(return_code, 0) + + self._verify_invoke_built_function( + function_logical_id=function_identifier, + overrides=None, + expected_result={"statusCode": 200, "body": expected_output}, + ) + + +@skipIf( + (not RUN_BY_CANARY and not CI_OVERRIDE), + "Skip Terraform test cases unless running in CI", +) +@parameterized_class( + ("build_in_container",), + [ + (False,), + (True,), + ], +) +class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3BackendWithOverride( + BuildTerraformApplicationS3BackendIntegBase +): + function_identifier = "module.function7.aws_lambda_function.this[0]" + override = True terraform_application = ( Path("terraform/zip_based_lambda_functions_s3_backend") if not IS_WINDOWS else Path("terraform/zip_based_lambda_functions_s3_backend_windows") ) functions = [ - ("module.function9.aws_lambda_function.this[0]", "hello world 9 - override version", True), - ("function9", "hello world 9 - override version", True), - ("module.function9.aws_lambda_function.this[0]", "hello world 9", False), - ("function9", "hello world 9", False), - ("aws_lambda_function.function5", "hello world 5 - override version", True), - ("function5", "hello world 5 - override version", True), - ("aws_lambda_function.function5", "hello world 5", False), - ("function5", "hello world 5", False), - ("aws_lambda_function.function6", "hello world 6 - override version", True), - ("function6", "hello world 6 - override version", True), - ("aws_lambda_function.function6", "hello world 6", False), - ("function6", "hello world 6", False), - ("aws_lambda_function.function4", "hello world 4 - override version", True), - ("function4", "hello world 4 - override version", True), - ("aws_lambda_function.function4", "hello world 4", False), - ("function4", "hello world 4", False), - ("aws_lambda_function.function3", "hello world 3 - override version", True), - ("function3", "hello world 3 - override version", True), - ("aws_lambda_function.function3", "hello world 3", False), - ("function3", "hello world 3", False), - ("module.function2.aws_lambda_function.this", "hello world 2 - override version", True), - ("function2", "hello world 2 - override version", True), - ("aws_lambda_function.function1", "hello world 1 - override version", True), - ("function1", "hello world 1 - override version", True), - ("module.function2.aws_lambda_function.this", "hello world 2", False), - ("function2", "hello world 2", False), - ("aws_lambda_function.function1", "hello world 1", False), - ("function1", "hello world 1", False), - ("aws_lambda_function.from_localfile", "[]", False), - ("aws_lambda_function.from_s3", "[]", False), - ("module.level1_lambda.aws_lambda_function.this", "[]", False), - ("module.level1_lambda.module.level2_lambda.aws_lambda_function.this", "[]", False), - ("my_function_from_localfile", "[]", False), - ("my_function_from_s3", "[]", False), - ("my_level1_lambda", "[]", False), - ("my_level2_lambda", "[]", False), + ("module.function9.aws_lambda_function.this[0]", "hello world 9 - override version"), + ("function9", "hello world 9 - override version"), + ("aws_lambda_function.function5", "hello world 5 - override version"), + ("function5", "hello world 5 - override version"), + ("aws_lambda_function.function6", "hello world 6 - override version"), + ("function6", "hello world 6 - override version"), + ("aws_lambda_function.function4", "hello world 4 - override version"), + ("function4", "hello world 4 - override version"), + ("aws_lambda_function.function3", "hello world 3 - override version"), + ("function3", "hello world 3 - override version"), + ("module.function2.aws_lambda_function.this", "hello world 2 - override version"), + ("function2", "hello world 2 - override version"), + ("aws_lambda_function.function1", "hello world 1 - override version"), + ("function1", "hello world 1 - override version"), ] @classmethod @@ -337,42 +418,35 @@ def setUpClass(cls): # testing on Windows, and only test them on linux. # check the Serverless TF issue https://github.com/terraform-aws-modules/terraform-aws-lambda/issues/142 cls.functions += [ - ("module.function7.aws_lambda_function.this[0]", "hello world 7 - override version", True), - ("function7", "hello world 7 - override version", True), - ("module.function7.aws_lambda_function.this[0]", "hello world 7", False), - ("function7", "hello world 7", False), - ("module.function8.aws_lambda_function.this[0]", "hello world 8 - override version", True), - ("function8", "hello world 8 - override version", True), - ("module.function8.aws_lambda_function.this[0]", "hello world 8", False), - ("function8", "hello world 8", False), - ("module.function10.aws_lambda_function.this[0]", "hello world 10 - override version", True), - ("function10", "hello world 10 - override version", True), - ("module.function10.aws_lambda_function.this[0]", "hello world 10", False), - ("function10", "hello world 10", False), - ("module.function11.aws_lambda_function.this[0]", "hello world 11 - override version", True), - ("function11", "hello world 11 - override version", True), - ("module.function11.aws_lambda_function.this[0]", "hello world 11", False), - ("function11", "hello world 11", False), + ("module.function7.aws_lambda_function.this[0]", "hello world 7 - override version"), + ("function7", "hello world 7 - override version"), + ("module.function8.aws_lambda_function.this[0]", "hello world 8 - override version"), + ("function8", "hello world 8 - override version"), + ("module.function10.aws_lambda_function.this[0]", "hello world 10 - override version"), + ("function10", "hello world 10 - override version"), + ("module.function11.aws_lambda_function.this[0]", "hello world 11 - override version"), + ("function11", "hello world 11 - override version"), ] super().setUpClass() @parameterized.expand(functions) - def test_build_and_invoke_lambda_functions(self, function_identifier, expected_output, should_override_code): + def test_build_and_invoke_lambda_functions(self, function_identifier, expected_output): command_list_parameters = { "hook_name": "terraform", "function_identifier": function_identifier, + "skip_prepare_infra": True, } if self.build_in_container: command_list_parameters["use_container"] = True command_list_parameters["build_image"] = self.docker_tag - if should_override_code: + if self.override: command_list_parameters[ "container_env_var" ] = "TF_VAR_HELLO_FUNCTION_SRC_CODE=./artifacts/HelloWorldFunction2" build_cmd_list = self.get_command_list(**command_list_parameters) LOG.info("command list: %s", build_cmd_list) environment_variables = os.environ.copy() - if should_override_code: + if self.override: environment_variables["TF_VAR_HELLO_FUNCTION_SRC_CODE"] = "./artifacts/HelloWorldFunction2" _, stderr, return_code = self.run_command(build_cmd_list, env=environment_variables) LOG.info(stderr) @@ -383,3 +457,94 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o overrides=None, expected_result={"statusCode": 200, "body": expected_output}, ) + + + +@skipIf( + (not RUN_BY_CANARY and not CI_OVERRIDE), + "Skip Terraform test cases unless running in CI", +) +@parameterized_class( + ("build_in_container",), + [ + (False,), + (True,), + ], +) +class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend(BuildTerraformApplicationS3BackendIntegBase): + function_identifier = "module.function7.aws_lambda_function.this[0]" + terraform_application = ( + Path("terraform/zip_based_lambda_functions_s3_backend") + if not IS_WINDOWS + else Path("terraform/zip_based_lambda_functions_s3_backend_windows") + ) + functions = [ + ("module.function9.aws_lambda_function.this[0]", "hello world 9"), + ("function9", "hello world 9"), + ("aws_lambda_function.function5", "hello world 5"), + ("function5", "hello world 5"), + ("aws_lambda_function.function6", "hello world 6"), + ("function6", "hello world 6"), + ("aws_lambda_function.function4", "hello world 4"), + ("function4", "hello world 4"), + ("aws_lambda_function.function3", "hello world 3"), + ("function3", "hello world 3"), + ("module.function2.aws_lambda_function.this", "hello world 2"), + ("function2", "hello world 2"), + ("aws_lambda_function.function1", "hello world 1"), + ("function1", "hello world 1"), + ("aws_lambda_function.from_localfile", "[]"), + ("aws_lambda_function.from_s3", "[]"), + ("module.level1_lambda.aws_lambda_function.this", "[]"), + ("module.level1_lambda.module.level2_lambda.aws_lambda_function.this", "[]"), + ("my_function_from_localfile", "[]"), + ("my_function_from_s3", "[]"), + ("my_level1_lambda", "[]"), + ("my_level2_lambda", "[]"), + ] + + @classmethod + def setUpClass(cls): + if IS_WINDOWS and cls.build_in_container: + # we use this TF project to test sam build in container on windows as we need to run a linux bash script for + # build, and also we need to remove the Serverless TF functions from this project. + # that is why we need to use a new project and not one of the existing linux or windows projects + cls.terraform_application = "terraform/zip_based_lambda_functions_s3_backend_container_windows" + if not IS_WINDOWS: + # The following functions are defined using serverless tf module, and since Serverless TF has some issue + # while executing `terraform plan` in windows, we removed these function from the TF projects we used in + # testing on Windows, and only test them on linux. + # check the Serverless TF issue https://github.com/terraform-aws-modules/terraform-aws-lambda/issues/142 + cls.functions += [ + ("module.function7.aws_lambda_function.this[0]", "hello world 7"), + ("function7", "hello world 7"), + ("module.function8.aws_lambda_function.this[0]", "hello world 8"), + ("function8", "hello world 8"), + ("module.function10.aws_lambda_function.this[0]", "hello world 10"), + ("function10", "hello world 10"), + ("module.function11.aws_lambda_function.this[0]", "hello world 11"), + ("function11", "hello world 11"), + ] + super().setUpClass() + + @parameterized.expand(functions) + def test_build_and_invoke_lambda_functions(self, function_identifier, expected_output): + command_list_parameters = { + "hook_name": "terraform", + "function_identifier": function_identifier, + "skip_prepare_infra": True, + } + if self.build_in_container: + command_list_parameters["use_container"] = True + command_list_parameters["build_image"] = self.docker_tag + build_cmd_list = self.get_command_list(**command_list_parameters) + LOG.info("command list: %s", build_cmd_list) + _, stderr, return_code = self.run_command(build_cmd_list) + LOG.info(stderr) + self.assertEqual(return_code, 0) + + self._verify_invoke_built_function( + function_logical_id=function_identifier, + overrides=None, + expected_result={"statusCode": 200, "body": expected_output}, + ) From a27a7b05d1c0bc1356b251ebcaa727016a913a9b Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 27 Sep 2023 10:39:24 -0700 Subject: [PATCH 2/4] Change function identifier --- .../test_build_terraform_applications.py | 22 ++++++++++++++----- ...uild_terraform_applications_other_cases.py | 10 +++++++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/integration/buildcmd/test_build_terraform_applications.py b/tests/integration/buildcmd/test_build_terraform_applications.py index b651ffb2e6..aa1ce58f69 100644 --- a/tests/integration/buildcmd/test_build_terraform_applications.py +++ b/tests/integration/buildcmd/test_build_terraform_applications.py @@ -30,11 +30,14 @@ class BuildTerraformApplicationIntegBase(BuildIntegBase): build_in_container = False function_identifier = None override = False + terraform_application_execution_path = None @classmethod def setUpClass(cls): super(BuildTerraformApplicationIntegBase, cls).setUpClass() cls.terraform_application_path = str(Path(cls.test_data_path, cls.terraform_application)) + cls.terraform_application_execution_path = str(Path(__file__).resolve().parent.joinpath(str(uuid.uuid4()).replace("-", "")[:10])) + shutil.copytree(Path(cls.terraform_application_path), Path(cls.terraform_application_execution_path)) if cls.build_in_container: cls.client = docker.from_env() cls.image_name = "sam-terraform-python-build" @@ -57,7 +60,7 @@ def setUpClass(cls): def setUp(self): super().setUp() shutil.rmtree(Path(self.working_dir)) - shutil.copytree(Path(self.terraform_application_path), Path(self.working_dir)) + shutil.copytree(Path(self.terraform_application_execution_path), Path(self.working_dir)) def run_command(self, command_list, env=None, input=None, timeout=None, override_dir=None): running_dir = override_dir if override_dir else self.working_dir @@ -90,7 +93,7 @@ def build_with_prepare_hook(self): build_cmd_list = self.get_command_list(**command_list_parameters) _, stderr, return_code = self.run_command( - build_cmd_list, override_dir=self.terraform_application_path, env=environment_variables + build_cmd_list, override_dir=self.terraform_application_execution_path, env=environment_variables ) LOG.info(stderr) self.assertEqual(return_code, 0) @@ -123,6 +126,13 @@ def _verify_invoke_built_function(self, function_logical_id, overrides, expected LOG.info("sam local invoke stderr: %s", stderr.decode("utf-8")) self.assertEqual(json.loads(process_stdout), expected_result) + @classmethod + def tearDownClass(cls) -> None: + cls.terraform_application_execution_path and shutil.rmtree(cls.terraform_application_execution_path, ignore_errors=True) + + def tearDown(self): + super(BuildTerraformApplicationIntegBase, self).tearDown() + class BuildTerraformApplicationS3BackendIntegBase(BuildTerraformApplicationIntegBase): @classmethod @@ -195,7 +205,7 @@ def tearDown(self): class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackendWithOverride( BuildTerraformApplicationIntegBase ): - function_identifier = "module.function7.aws_lambda_function.this[0]" + function_identifier = "function9" override = True terraform_application = ( Path("terraform/zip_based_lambda_functions_local_backend") @@ -288,7 +298,7 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o ], ) class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend(BuildTerraformApplicationIntegBase): - function_identifier = "module.function7.aws_lambda_function.this[0]" + function_identifier = "function9" terraform_application = ( Path("terraform/zip_based_lambda_functions_local_backend") if not IS_WINDOWS @@ -381,7 +391,7 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3BackendWithOverride( BuildTerraformApplicationS3BackendIntegBase ): - function_identifier = "module.function7.aws_lambda_function.this[0]" + function_identifier = "function9" override = True terraform_application = ( Path("terraform/zip_based_lambda_functions_s3_backend") @@ -472,7 +482,7 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o ], ) class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend(BuildTerraformApplicationS3BackendIntegBase): - function_identifier = "module.function7.aws_lambda_function.this[0]" + function_identifier = "function9" terraform_application = ( Path("terraform/zip_based_lambda_functions_s3_backend") if not IS_WINDOWS diff --git a/tests/integration/buildcmd/test_build_terraform_applications_other_cases.py b/tests/integration/buildcmd/test_build_terraform_applications_other_cases.py index cc6ca5b9fe..e7fb5b36ab 100644 --- a/tests/integration/buildcmd/test_build_terraform_applications_other_cases.py +++ b/tests/integration/buildcmd/test_build_terraform_applications_other_cases.py @@ -140,6 +140,7 @@ def test_build_no_s3_config(self): "Skip Terraform test cases unless running in CI", ) class TestBuildTerraformApplicationsWithImageBasedLambdaFunctionAndLocalBackend(BuildTerraformApplicationIntegBase): + function_identifier = "aws_lambda_function.function_with_non_image_uri" terraform_application = Path("terraform/image_based_lambda_functions_local_backend") functions = [ "aws_lambda_function.function_with_non_image_uri", @@ -155,7 +156,9 @@ class TestBuildTerraformApplicationsWithImageBasedLambdaFunctionAndLocalBackend( @parameterized.expand(functions) def test_build_and_invoke_lambda_functions(self, function_identifier): - build_cmd_list = self.get_command_list(hook_name="terraform", function_identifier=function_identifier) + build_cmd_list = self.get_command_list( + hook_name="terraform", function_identifier=function_identifier, skip_prepare_infra=True + ) LOG.info("command list: %s", build_cmd_list) _, stderr, return_code = self.run_command(build_cmd_list) LOG.info(stderr) @@ -180,6 +183,7 @@ def test_build_and_invoke_lambda_functions(self, function_identifier): class TestBuildTerraformApplicationsWithImageBasedLambdaFunctionAndS3Backend( BuildTerraformApplicationS3BackendIntegBase ): + function_identifier = "aws_lambda_function.function_with_non_image_uri" terraform_application = Path("terraform/image_based_lambda_functions_s3_backend") functions = [ "aws_lambda_function.function_with_non_image_uri", @@ -195,7 +199,9 @@ class TestBuildTerraformApplicationsWithImageBasedLambdaFunctionAndS3Backend( @parameterized.expand(functions) def test_build_and_invoke_lambda_functions(self, function_identifier): - build_cmd_list = self.get_command_list(hook_name="terraform", function_identifier=function_identifier) + build_cmd_list = self.get_command_list( + hook_name="terraform", function_identifier=function_identifier, skip_prepare_infra=True + ) LOG.info("command list: %s", build_cmd_list) _, stderr, return_code = self.run_command(build_cmd_list) LOG.info(stderr) From e911e1716e53cbf71a328fc7c377fd31d673534c Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 27 Sep 2023 13:27:49 -0700 Subject: [PATCH 3/4] Fix S3 issues --- appveyor-ubuntu.yml | 4 +- .../test_build_terraform_applications.py | 44 ++++++++++++------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/appveyor-ubuntu.yml b/appveyor-ubuntu.yml index 4fd912ed12..e75531cc16 100644 --- a/appveyor-ubuntu.yml +++ b/appveyor-ubuntu.yml @@ -260,10 +260,10 @@ for: - matrix: only: - - configuration: DeployIntegTesting + - configuration: AllTerraformBuildTests test_script: - - sh: "pytest -vv tests/integration/deploy -n 4 --reruns 4 --dist=loadgroup --json-report --json-report-file=TEST_REPORT-integration-deploy.json" + - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" # Integ testing package - diff --git a/tests/integration/buildcmd/test_build_terraform_applications.py b/tests/integration/buildcmd/test_build_terraform_applications.py index aa1ce58f69..a089fa8a81 100644 --- a/tests/integration/buildcmd/test_build_terraform_applications.py +++ b/tests/integration/buildcmd/test_build_terraform_applications.py @@ -1,3 +1,4 @@ +import datetime import os import logging import shutil @@ -19,6 +20,7 @@ from tests.integration.buildcmd.build_integ_base import BuildIntegBase from tests.testing_utils import CI_OVERRIDE, IS_WINDOWS, RUN_BY_CANARY +from tests.testing_utils import run_command as static_run_command LOG = logging.getLogger(__name__) S3_SLEEP = 3 @@ -95,7 +97,7 @@ def build_with_prepare_hook(self): _, stderr, return_code = self.run_command( build_cmd_list, override_dir=self.terraform_application_execution_path, env=environment_variables ) - LOG.info(stderr) + LOG.info(stderr.decode("utf-8")) self.assertEqual(return_code, 0) def _verify_invoke_built_function(self, function_logical_id, overrides, expected_result): @@ -152,8 +154,25 @@ def setUpClass(cls): if not cls.pre_created_bucket: cls.s3_bucket.create() time.sleep(S3_SLEEP) - super().setUpClass() + cls.initialize_s3_backend() + + @classmethod + def initialize_s3_backend(cls): + cls.backend_key = f"terraform-backend/{str(uuid.uuid4())}" + cls.backendconfig_path = str(Path(cls.terraform_application_execution_path) / "backend.conf") + with open(cls.backendconfig_path, "w") as f: + f.write(f'bucket="{cls.bucket_name}"\n') + f.write(f'key="{cls.backend_key}"\n') + f.write(f'region="{cls.region_name}"') + + # We have to init the terraform project with specifying the S3 backend first + _, stderr, _ = static_run_command( + ["terraform", "init", f"-backend-config={cls.backendconfig_path}", "-reconfigure", "-input=false"], + cwd=cls.terraform_application_execution_path + ) + if stderr: + LOG.info(stderr) @classmethod def tearDownClass(cls): @@ -165,19 +184,6 @@ def tearDownClass(cls): def setUp(self): super().setUp() - self.backend_key = f"terraform-backend/{str(uuid.uuid4())}" - self.backendconfig_path = str(Path(self.working_dir) / "backend.conf") - with open(self.backendconfig_path, "w") as f: - f.write(f'bucket="{self.bucket_name}"\n') - f.write(f'key="{self.backend_key}"\n') - f.write(f'region="{self.region_name}"') - - # We have to init the terraform project with specifying the S3 backend first - _, stderr, _ = self.run_command( - ["terraform", "init", f"-backend-config={self.backendconfig_path}", "-reconfigure", "-input=false"] - ) - if stderr: - LOG.info(stderr) def tearDown(self): """Clean up the terraform state file on S3 and remove the backendconfg locally""" @@ -202,6 +208,7 @@ def tearDown(self): (True,), ], ) +@pytest.mark.xdist_group(name="zip_lambda_local_backend_override") class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackendWithOverride( BuildTerraformApplicationIntegBase ): @@ -297,6 +304,7 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o (True,), ], ) +@pytest.mark.xdist_group(name="zip_lambda_local_backend") class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend(BuildTerraformApplicationIntegBase): function_identifier = "function9" terraform_application = ( @@ -388,6 +396,7 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o (True,), ], ) +@pytest.mark.xdist_group(name="zip_lambda_s3_backend") class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3BackendWithOverride( BuildTerraformApplicationS3BackendIntegBase ): @@ -481,6 +490,7 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o (True,), ], ) +@pytest.mark.xdist_group(name="zip_lambda_s3_backend_override") class TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend(BuildTerraformApplicationS3BackendIntegBase): function_identifier = "function9" terraform_application = ( @@ -549,8 +559,10 @@ def test_build_and_invoke_lambda_functions(self, function_identifier, expected_o command_list_parameters["build_image"] = self.docker_tag build_cmd_list = self.get_command_list(**command_list_parameters) LOG.info("command list: %s", build_cmd_list) + start = time.time() _, stderr, return_code = self.run_command(build_cmd_list) - LOG.info(stderr) + end = time.time() + LOG.info(f"END: {end}, DURATION {end - start}") self.assertEqual(return_code, 0) self._verify_invoke_built_function( From 938d42fb2a07395cf204577c53e94f48c0a6f3ba Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 27 Sep 2023 13:36:37 -0700 Subject: [PATCH 4/4] Test Windows all Terraform tests --- appveyor-windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor-windows.yml b/appveyor-windows.yml index 8b0a422b11..dfe21d4807 100644 --- a/appveyor-windows.yml +++ b/appveyor-windows.yml @@ -261,10 +261,10 @@ for: #Integ testing deploy - matrix: only: - - configuration: DeployIntegTesting + - configuration: AllTerraformTests test_script: - - ps: "pytest -vv tests/integration/deploy -n 4 --reruns 4 --dist=loadgroup --json-report --json-report-file=TEST_REPORT-integration-deploy.json" + - ps: "pytest -vv tests/integration/buildcmd/test_build_terraform_applications.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" # Integ testing package - matrix: