From f3e6673d3810228dc393cf69c61438b5217374ed Mon Sep 17 00:00:00 2001 From: Vishwanath Hiremath <100623239+vishwahiremat@users.noreply.github.com> Date: Tue, 23 Apr 2024 01:20:52 +0530 Subject: [PATCH] Adding functional test for private terraform repo support and fixing delete issue with private repo (#7436) # Description - Added functional test for terraform recipe using module stored in private git repository. - Adding support for delete functionality for private repos. - Adding required unit tests ## Type of change - This pull request is a minor refactor, code cleanup, test improvement, or other maintenance task and doesn't change the functionality of Radius (issue link optional). Fixes: #issue_number --------- Signed-off-by: Vishwanath Hiremath --- pkg/recipes/driver/terraform.go | 17 +- pkg/recipes/engine/engine.go | 45 ++-- pkg/recipes/engine/engine_test.go | 231 ++++++++++++++++++ .../cloud/resources/recipe_terraform_test.go | 90 +++++++ ...ces-terraform-private-git-repo-redis.bicep | 88 +++++++ test/testutil/testutil.go | 15 ++ 6 files changed, 470 insertions(+), 16 deletions(-) create mode 100644 test/functional-portable/corerp/cloud/resources/testdata/corerp-resources-terraform-private-git-repo-redis.bicep diff --git a/pkg/recipes/driver/terraform.go b/pkg/recipes/driver/terraform.go index cfc216f661..e35d6fdb2d 100644 --- a/pkg/recipes/driver/terraform.go +++ b/pkg/recipes/driver/terraform.go @@ -135,13 +135,28 @@ func (d *terraformDriver) Delete(ctx context.Context, opts DeleteOptions) error logger.Info(fmt.Sprintf("Failed to cleanup Terraform execution directory %q. Err: %s", requestDirPath, err.Error())) } }() - + // Add credential information to .gitconfig if module source is of type git. + if strings.HasPrefix(opts.Definition.TemplatePath, "git::") && !reflect.DeepEqual(opts.BaseOptions.Secrets, v20231001preview.SecretStoresClientListSecretsResponse{}) { + err := addSecretsToGitConfig(opts.BaseOptions.Secrets, &opts.Recipe, opts.Definition.TemplatePath) + if err != nil { + return err + } + } err = d.terraformExecutor.Delete(ctx, terraform.Options{ RootDir: requestDirPath, EnvConfig: &opts.Configuration, ResourceRecipe: &opts.Recipe, EnvRecipe: &opts.Definition, }) + + // Unset credential information from .gitconfig if module source is of type git. + if strings.HasPrefix(opts.Definition.TemplatePath, "git::") && !reflect.DeepEqual(opts.BaseOptions.Secrets, v20231001preview.SecretStoresClientListSecretsResponse{}) { + unsetError := unsetSecretsFromGitConfig(opts.BaseOptions.Secrets, opts.Definition.TemplatePath) + if unsetError != nil { + return unsetError + } + } + if err != nil { return recipes.NewRecipeError(recipes.RecipeDeletionFailed, err.Error(), "", recipes.GetErrorDetails(err)) } diff --git a/pkg/recipes/engine/engine.go b/pkg/recipes/engine/engine.go index c953e13abf..462feff5f9 100644 --- a/pkg/recipes/engine/engine.go +++ b/pkg/recipes/engine/engine.go @@ -92,21 +92,9 @@ func (e *engine) executeCore(ctx context.Context, recipe recipes.ResourceMetadat return nil, nil, err } - secrets := v20231001preview.SecretStoresClientListSecretsResponse{} - driverWithSecrets, ok := driver.(recipedriver.DriverWithSecrets) - if ok { - secretStore, err := driverWithSecrets.FindSecretIDs(ctx, *configuration, *definition) - if err != nil { - return nil, nil, err - } - - // Retrieves the secret values from the secret store ID provided. - if secretStore != "" { - secrets, err = e.options.SecretsLoader.LoadSecrets(ctx, secretStore) - if err != nil { - return nil, nil, recipes.NewRecipeError(recipes.LoadSecretsFailed, fmt.Sprintf("failed to fetch secrets from the secret store resource id %s for Terraform recipe %s deployment: %s", secretStore, definition.TemplatePath, err.Error()), util.RecipeSetupError, recipes.GetErrorDetails(err)) - } - } + secrets, err := e.getRecipeConfigSecrets(ctx, driver, configuration, definition) + if err != nil { + return nil, nil, err } res, err := driver.Execute(ctx, recipedriver.ExecuteOptions{ @@ -164,11 +152,16 @@ func (e *engine) deleteCore(ctx context.Context, recipe recipes.ResourceMetadata return nil, err } + secrets, err := e.getRecipeConfigSecrets(ctx, driver, configuration, definition) + if err != nil { + return nil, err + } err = driver.Delete(ctx, recipedriver.DeleteOptions{ BaseOptions: recipedriver.BaseOptions{ Configuration: *configuration, Recipe: recipe, Definition: *definition, + Secrets: secrets, }, OutputResources: outputResources, }) @@ -219,3 +212,25 @@ func (e *engine) getDriver(ctx context.Context, recipeMetadata recipes.ResourceM } return definition, driver, nil } + +func (e *engine) getRecipeConfigSecrets(ctx context.Context, driver recipedriver.Driver, configuration *recipes.Configuration, definition *recipes.EnvironmentDefinition) (v20231001preview.SecretStoresClientListSecretsResponse, error) { + secrets := v20231001preview.SecretStoresClientListSecretsResponse{} + driverWithSecrets, ok := driver.(recipedriver.DriverWithSecrets) + if !ok { + return secrets, nil + } + + secretStore, err := driverWithSecrets.FindSecretIDs(ctx, *configuration, *definition) + if err != nil { + return v20231001preview.SecretStoresClientListSecretsResponse{}, err + } + + // Retrieves the secret values from the secret store ID provided. + if secretStore != "" { + secrets, err = e.options.SecretsLoader.LoadSecrets(ctx, secretStore) + if err != nil { + return v20231001preview.SecretStoresClientListSecretsResponse{}, recipes.NewRecipeError(recipes.LoadSecretsFailed, fmt.Sprintf("failed to fetch secrets from the secret store resource id %s for Terraform recipe %s deployment: %s", secretStore, definition.TemplatePath, err.Error()), util.RecipeSetupError, recipes.GetErrorDetails(err)) + } + } + return secrets, nil +} diff --git a/pkg/recipes/engine/engine_test.go b/pkg/recipes/engine/engine_test.go index 0e520a791a..aaff0208ee 100644 --- a/pkg/recipes/engine/engine_test.go +++ b/pkg/recipes/engine/engine_test.go @@ -316,6 +316,155 @@ func Test_Engine_Terraform_Success(t *testing.T) { require.NoError(t, err) require.Equal(t, result, recipeResult) } +func Test_Engine_Terraform_Failure(t *testing.T) { + tests := []struct { + name string + errFindSecretRefs error + errLoadSecrets error + errExecute error + }{ + { + name: "find secret references failed", + errFindSecretRefs: fmt.Errorf("failed to parse git url %s", "git://https://dev.azure.com/mongo-recipe/recipe"), + }, + { + name: "failed loading secrets", + errLoadSecrets: fmt.Errorf("%q is a valid resource id but does not refer to a resource", ""), + }, + { + name: "find secret references failed", + errExecute: errors.New("failed to add git config"), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + recipeMetadata := recipes.ResourceMetadata{ + Name: "mongo-azure", + ApplicationID: "/planes/radius/local/resourcegroups/test-rg/providers/applications.core/applications/app1", + EnvironmentID: "/planes/radius/local/resourcegroups/test-rg/providers/applications.core/environments/env1", + ResourceID: "/planes/radius/local/resourceGroups/test-rg/providers/Microsoft.Resources/deployments/recipe", + Parameters: map[string]any{ + "resourceName": "resource1", + }, + } + envConfig := &recipes.Configuration{ + Runtime: recipes.RuntimeConfiguration{ + Kubernetes: &recipes.KubernetesRuntime{ + Namespace: "default", + }, + }, + Providers: datamodel.Providers{ + Azure: datamodel.ProvidersAzure{ + Scope: "scope", + }, + }, + RecipeConfig: datamodel.RecipeConfigProperties{ + Terraform: datamodel.TerraformConfigProperties{ + Authentication: datamodel.AuthConfig{ + Git: datamodel.GitAuthConfig{ + PAT: map[string]datamodel.SecretConfig{ + "dev.azure.com": { + Secret: "/planes/radius/local/resourcegroups/default/providers/Applications.Core/secretStores/azdevopsgit", + }, + }, + }, + }, + }, + }, + } + recipeResult := &recipes.RecipeOutput{ + Resources: []string{"mongoStorageAccount", "mongoDatabase"}, + Secrets: map[string]any{ + "connectionString": "mongodb://testUser:testPassword@testAccount1.mongo.cosmos.azure.com:10255", + }, + Values: map[string]any{ + "host": "testAccount1.mongo.cosmos.azure.com", + "port": 10255, + }, + } + recipeDefinition := &recipes.EnvironmentDefinition{ + Driver: recipes.TemplateKindTerraform, + TemplatePath: "git://https://dev.azure.com/mongo-recipe/recipe", + ResourceType: "Applications.Datastores/mongoDatabases", + } + prevState := []string{ + "/subscriptions/test-sub/resourceGroups/test-rg/providers/System.Test/testResources/test1", + } + ctx := testcontext.New(t) + engine, configLoader, _, driverWithSecrets, secretsLoader := setup(t) + configLoader.EXPECT(). + LoadConfiguration(ctx, recipeMetadata). + Times(1). + Return(envConfig, nil) + configLoader.EXPECT(). + LoadRecipe(ctx, &recipeMetadata). + Times(1). + Return(recipeDefinition, nil) + + if tc.errFindSecretRefs != nil { + driverWithSecrets.EXPECT(). + FindSecretIDs(ctx, *envConfig, *recipeDefinition). + Times(1). + Return("", tc.errFindSecretRefs) + } else { + driverWithSecrets.EXPECT(). + FindSecretIDs(ctx, *envConfig, *recipeDefinition). + Times(1). + Return("/planes/radius/local/resourcegroups/default/providers/Applications.Core/secretStores/azdevopsgit", nil) + + if tc.errLoadSecrets != nil { + secretsLoader.EXPECT(). + LoadSecrets(ctx, gomock.Any()). + Times(1). + Return(v20231001preview.SecretStoresClientListSecretsResponse{}, tc.errLoadSecrets) + } else { + secretsLoader.EXPECT(). + LoadSecrets(ctx, gomock.Any()). + Times(1). + Return(v20231001preview.SecretStoresClientListSecretsResponse{}, nil) + if tc.errExecute != nil { + driverWithSecrets.EXPECT(). + Execute(ctx, recipedriver.ExecuteOptions{ + BaseOptions: recipedriver.BaseOptions{ + Configuration: *envConfig, + Recipe: recipeMetadata, + Definition: *recipeDefinition, + }, + PrevState: prevState, + }). + Times(1). + Return(nil, tc.errExecute) + } else { + driverWithSecrets.EXPECT(). + Execute(ctx, recipedriver.ExecuteOptions{ + BaseOptions: recipedriver.BaseOptions{ + Configuration: *envConfig, + Recipe: recipeMetadata, + Definition: *recipeDefinition, + }, + PrevState: prevState, + }). + Times(1). + Return(recipeResult, nil) + } + } + } + + result, err := engine.Execute(ctx, ExecuteOptions{ + BaseOptions: BaseOptions{ + Recipe: recipeMetadata, + }, + PreviousState: prevState, + }) + if tc.errFindSecretRefs != nil || tc.errLoadSecrets != nil || tc.errExecute != nil { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, result, recipeResult) + } + }) + } +} func Test_Engine_InvalidDriver(t *testing.T) { ctx := testcontext.New(t) @@ -824,3 +973,85 @@ func Test_Engine_Execute_With_Secrets_Success(t *testing.T) { require.NoError(t, err) require.Equal(t, result, recipeResult) } + +func Test_Engine_Delete_With_Secrets_Success(t *testing.T) { + _, _, outputResources := getRecipeInputs() + recipeMetadata := recipes.ResourceMetadata{ + Name: "mongo-azure", + ApplicationID: "/planes/radius/local/resourcegroups/test-rg/providers/applications.core/applications/app1", + EnvironmentID: "/planes/radius/local/resourcegroups/test-rg/providers/applications.core/environments/env1", + ResourceID: "/planes/radius/local/resourceGroups/test-rg/providers/Microsoft.Resources/deployments/recipe", + Parameters: map[string]any{ + "resourceName": "resource1", + }, + } + envConfig := &recipes.Configuration{ + Runtime: recipes.RuntimeConfiguration{ + Kubernetes: &recipes.KubernetesRuntime{ + Namespace: "default", + }, + }, + Providers: datamodel.Providers{ + Azure: datamodel.ProvidersAzure{ + Scope: "scope", + }, + }, + RecipeConfig: datamodel.RecipeConfigProperties{ + Terraform: datamodel.TerraformConfigProperties{ + Authentication: datamodel.AuthConfig{ + Git: datamodel.GitAuthConfig{ + PAT: map[string]datamodel.SecretConfig{ + "dev.azure.com": { + Secret: "/planes/radius/local/resourcegroups/default/providers/Applications.Core/secretStores/azdevopsgit", + }, + }, + }, + }, + }, + }, + } + recipeDefinition := &recipes.EnvironmentDefinition{ + Driver: recipes.TemplateKindTerraform, + TemplatePath: "git://https://dev.azure.com/mongo-recipe/recipe", + ResourceType: "Applications.Datastores/mongoDatabases", + } + ctx := testcontext.New(t) + engine, configLoader, _, driverWithSecrets, secretsLoader := setup(t) + + configLoader.EXPECT(). + LoadRecipe(ctx, &recipeMetadata). + Times(1). + Return(recipeDefinition, nil) + + configLoader.EXPECT(). + LoadConfiguration(ctx, recipeMetadata). + Times(1). + Return(envConfig, nil) + driverWithSecrets.EXPECT(). + FindSecretIDs(ctx, *envConfig, *recipeDefinition). + Times(1). + Return("/planes/radius/local/resourcegroups/default/providers/Applications.Core/secretStores/azdevopsgit", nil) + secretsLoader.EXPECT(). + LoadSecrets(ctx, "/planes/radius/local/resourcegroups/default/providers/Applications.Core/secretStores/azdevopsgit"). + Times(1). + Return(v20231001preview.SecretStoresClientListSecretsResponse{}, nil) + driverWithSecrets.EXPECT(). + Delete(ctx, recipedriver.DeleteOptions{ + BaseOptions: recipedriver.BaseOptions{ + Configuration: *envConfig, + Recipe: recipeMetadata, + Definition: *recipeDefinition, + }, + OutputResources: outputResources, + }). + Times(1). + Return(nil) + + err := engine.Delete(ctx, DeleteOptions{ + BaseOptions: BaseOptions{ + Recipe: recipeMetadata, + }, + OutputResources: outputResources, + }) + require.NoError(t, err) +} diff --git a/test/functional-portable/corerp/cloud/resources/recipe_terraform_test.go b/test/functional-portable/corerp/cloud/resources/recipe_terraform_test.go index ee327ab830..b04e8f7f10 100644 --- a/test/functional-portable/corerp/cloud/resources/recipe_terraform_test.go +++ b/test/functional-portable/corerp/cloud/resources/recipe_terraform_test.go @@ -26,6 +26,7 @@ package resource_test import ( "context" + "strings" "testing" "github.com/stretchr/testify/require" @@ -92,3 +93,92 @@ func Test_TerraformRecipe_AzureStorage(t *testing.T) { test.Test(t) } + +// Test_TerraformPrivateGitModule_KubernetesRedis covers the following terraform recipe scenario: +// +// - Create an extender resource using a Terraform recipe stored in private terraform git repository that deploys Redis on Kubernetes. +// - The recipe deployment creates a Kubernetes deployment and a Kubernetes service. +// +// This test uses a recipe stored in a private repository in radius-project organization and uses PAT from radius account, so it cannot be tested locally. +// To run this test locally: +// - Upload the files from test/testrecipes/test-terraform-recipes/kubernetes-redis/modules to a private repository and update the module source in testutil.GetTerraformPrivateModuleSource() +// - Create a PAT to access the private repository and update testutil.GetGitPAT() to return the generated PAT. +func Test_TerraformPrivateGitModule_KubernetesRedis(t *testing.T) { + template := "testdata/corerp-resources-terraform-private-git-repo-redis.bicep" + name := "corerp-resources-terraform-private-redis" + appName := "corerp-resources-terraform-private-app" + envName := "corerp-resources-terraform-private-env" + redisCacheName := "tf-redis-cache-private" + + secretSuffix, err := corerp.GetSecretSuffix("/planes/radius/local/resourcegroups/kind-radius/providers/Applications.Core/extenders/"+name, envName, appName) + require.NoError(t, err) + test := rp.NewRPTest(t, name, []rp.TestStep{ + { + Executor: step.NewDeployExecutor(template, testutil.GetTerraformPrivateModuleSource(), "appName="+appName, "redisCacheName="+redisCacheName, testutil.GetGitPAT()), + RPResources: &validation.RPResourceSet{ + Resources: []validation.RPResource{ + { + Name: envName, + Type: validation.EnvironmentsResource, + }, + { + Name: appName, + Type: validation.ApplicationsResource, + }, + { + Name: name, + Type: validation.ExtendersResource, + App: appName, + OutputResources: []validation.OutputResourceResponse{ + {ID: "/planes/kubernetes/local/namespaces/corerp-resources-terraform-private-app/providers/apps/Deployment/tf-redis-cache-private"}, + {ID: "/planes/kubernetes/local/namespaces/corerp-resources-terraform-private-app/providers/core/Service/tf-redis-cache-private"}, + }, + }, + }, + }, + K8sObjects: &validation.K8sObjectSet{ + Namespaces: map[string][]validation.K8sObject{ + appName: { + validation.NewK8sServiceForResource(appName, redisCacheName). + ValidateLabels(false), + }, + secretNamespace: { + validation.NewK8sSecretForResourceWithResourceName(secretPrefix + secretSuffix). + ValidateLabels(false), + }, + }, + }, + PostStepVerify: func(ctx context.Context, t *testing.T, test rp.RPTest) { + secret, err := test.Options.K8sClient.CoreV1().Secrets(secretNamespace). + Get(ctx, secretPrefix+secretSuffix, metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, secretNamespace, secret.Namespace) + require.Equal(t, secretPrefix+secretSuffix, secret.Name) + + redis, err := test.Options.ManagementClient.ShowResource(ctx, "Applications.Core/extenders", name) + require.NoError(t, err) + require.NotNil(t, redis) + status := redis.Properties["status"].(map[string]any) + recipe := status["recipe"].(map[string]interface{}) + require.Equal(t, "terraform", recipe["templateKind"].(string)) + expectedTemplatePath := strings.Replace(testutil.GetTerraformPrivateModuleSource(), "privateGitModule=", "", 1) + require.Equal(t, expectedTemplatePath, recipe["templatePath"].(string)) + // At present, it is not possible to verify the template version in functional tests + // This is verified by UTs though + + // Manually delete Kubernetes the secret that stores the Terraform state file now. The next step in the test will be the deletion + // of the portable resource that uses this secret for Terraform recipe. This is to verify that the test and portable resource + // deletion will not fail even though the secret is already deleted. + err = test.Options.K8sClient.CoreV1().Secrets(secretNamespace).Delete(ctx, secretPrefix+secretSuffix, metav1.DeleteOptions{}) + require.NoError(t, err) + }, + }, + }) + + test.PostDeleteVerify = func(ctx context.Context, t *testing.T, test rp.RPTest) { + resourceID := "/planes/radius/local/resourcegroups/kind-radius/providers/Applications.Core/extenders/" + name + corerp.TestSecretDeletion(t, ctx, test, appName, envName, resourceID, secretNamespace, secretPrefix) + } + + test.Test(t) +} diff --git a/test/functional-portable/corerp/cloud/resources/testdata/corerp-resources-terraform-private-git-repo-redis.bicep b/test/functional-portable/corerp/cloud/resources/testdata/corerp-resources-terraform-private-git-repo-redis.bicep new file mode 100644 index 0000000000..c9c071d188 --- /dev/null +++ b/test/functional-portable/corerp/cloud/resources/testdata/corerp-resources-terraform-private-git-repo-redis.bicep @@ -0,0 +1,88 @@ +import radius as radius + +@description('Name of the Redis Cache resource.') +param redisCacheName string + +@description('Name of the Radius Application.') +param appName string + +@secure() +param pat string + +@description('Private Git module source in generic git format.') +param privateGitModule string + +resource env 'Applications.Core/environments@2023-10-01-preview' = { + name: 'corerp-resources-terraform-private-env' + properties: { + compute: { + kind: 'kubernetes' + resourceId: 'self' + namespace: 'corerp-resources-terraform-private-env' + } + recipeConfig: { + terraform: { + authentication: { + git: { + pat: { + 'github.com':{ + secret: moduleSecrets.id + } + } + } + } + } + } + recipes: { + 'Applications.Core/extenders': { + default: { + templateKind: 'terraform' + templatePath: privateGitModule + } + } + } + } +} + +resource app 'Applications.Core/applications@2023-10-01-preview' = { + name: appName + properties: { + environment: env.id + extensions: [ + { + kind: 'kubernetesNamespace' + namespace: appName + } + ] + } +} + +resource webapp 'Applications.Core/extenders@2023-10-01-preview' = { + name: 'corerp-resources-terraform-private-redis' + properties: { + application: app.id + environment: env.id + recipe: { + name: 'default' + parameters: { + redis_cache_name: redisCacheName + } + } + } +} + +resource moduleSecrets 'Applications.Core/secretStores@2023-10-01-preview' = { + name: 'module-secrets' + properties: { + resource: 'test-namespace/github' + type: 'generic' + data: { + username: { + value: 'x-access-token' + } + pat: { + value: pat + } + } + } +} diff --git a/test/testutil/testutil.go b/test/testutil/testutil.go index f645cd31bf..e09c62a8a4 100644 --- a/test/testutil/testutil.go +++ b/test/testutil/testutil.go @@ -125,6 +125,21 @@ func GetTerraformRecipeModuleServerURL() string { return "moduleServer=" + u } +// GetTerraformPrivateModuleSource gets the terraform private git module source to use in tests from the environment variable TF_RECIPE_PRIVATE_GIT_SOURCE. +func GetTerraformPrivateModuleSource() string { + u := os.Getenv("TF_RECIPE_PRIVATE_GIT_SOURCE") + if u == "" { + return "privateGitModule=git::https://github.com/radius-project/terraform-private-modules//kubernetes-redis" + } + return "privateGitModule=" + u +} + +// GetGitPAT gets the personal access token for the git account private modules are stored. +func GetGitPAT() string { + u := os.Getenv("GH_TOKEN") + return "pat=" + u +} + // GetAWSAccountId retrieves the AWS Account ID from the environment and returns it as a string. func GetAWSAccountId() string { awsAccountId := os.Getenv("AWS_ACCOUNT_ID")