Skip to content

Commit

Permalink
Make application and environment configurable
Browse files Browse the repository at this point in the history
This change makes the application and environment of a Recipe or Deployment configurable through the Kubernetes API.

This includes the logic and tests to handle the case where the application or environment change after creation. For this case we need to delete the existing paired resource for two reasons:

- This will result in a different deployment scope (resource group name is computed).
- It is forbidden in our API to change the application or environment of an existing resource.

This PR does not address the case where a Deployment reacts to a Recipe changing its environment or application. That will be handled in a follow-up.
  • Loading branch information
rynowak committed Oct 15, 2023
1 parent 0a71d2c commit 7827e4f
Show file tree
Hide file tree
Showing 9 changed files with 290 additions and 40 deletions.
10 changes: 10 additions & 0 deletions deploy/Chart/crds/radius/radapp.io_recipes.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,16 @@ spec:
spec:
description: RecipeSpec defines the desired state of Recipe
properties:
application:
description: Application is the name of the Radius application to
use. If unset the namespace of the Recipe will be used as the application
name.
type: string
environment:
description: Environment is the name of the Radius environment to
use. If unset the value 'default' will be used as the environment
name.
type: string
secretName:
description: SecretName is the name of a Kubernetes secret to create
once the resource is created.
Expand Down
8 changes: 8 additions & 0 deletions pkg/controller/api/radapp.io/v1alpha3/recipe_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ type RecipeSpec struct {
// SecretName is the name of a Kubernetes secret to create once the resource is created.
// +kubebuilder:validation:Optional
SecretName string `json:"secretName,omitempty"`

// Environment is the name of the Radius environment to use. If unset the value 'default' will be
// used as the environment name.
Environment string `json:"environment,omitempty"`

// Application is the name of the Radius application to use. If unset the namespace of the
// Recipe will be used as the application name.
Application string `json:"application,omitempty"`
}

// RecipePhrase is a string representation of the current status of a Recipe.
Expand Down
10 changes: 8 additions & 2 deletions pkg/controller/reconciler/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ type deploymentAnnotations struct {

// deploymentConfiguration is the configuration of the Deployment provided by the user via annotations.
type deploymentConfiguration struct {
Connections map[string]string
Application string `json:"application,omitempty"`
Environment string `json:"environment,omitempty"`
Connections map[string]string `json:"connections,omitempty"`
}

func (c *deploymentConfiguration) computeHash() (string, error) {
Expand Down Expand Up @@ -107,7 +109,11 @@ func readAnnotations(deployment *appsv1.Deployment) (*deploymentAnnotations, err
return &result, nil
}

result.Configuration = &deploymentConfiguration{Connections: map[string]string{}}
result.Configuration = &deploymentConfiguration{
Environment: deployment.Annotations[AnnotatationRadiusEnvironment],
Application: deployment.Annotations[AnnotationRadiusApplication],
Connections: map[string]string{},
}

for k, v := range deployment.Annotations {
if strings.HasPrefix(k, AnnotationRadiusConnectionPrefix) {
Expand Down
8 changes: 8 additions & 0 deletions pkg/controller/reconciler/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ const (
// AnnotationRadiusConfigurationHash is the name of the annotation that indicates the hash of the configuration.
AnnotationRadiusConfigurationHash = "radapp.io/configuration-hash"

// AnnotationRadiusEnvionment is the name of the annotation that indicates the name of the environment. If unset,
// the value 'default' will be used as the environment name.
AnnotatationRadiusEnvironment = "radapp.io/environment"

// AnnotationRadiusApplication is the name of the annotation that indicates the name of the application. If unset,
// the namespace of the Deployment will be used as the application name.
AnnotationRadiusApplication = "radapp.io/application"

// DeploymentFinalizer is the name of the finalizer added to Deployments.
DeploymentFinalizer = "radapp.io/deployment-finalizer"

Expand Down
64 changes: 49 additions & 15 deletions pkg/controller/reconciler/deployment_reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"encoding/base64"
"fmt"
"strings"
"time"

appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -248,11 +249,15 @@ func (r *DeploymentReconciler) reconcileUpdate(ctx context.Context, deployment *
}
}

// For now the environment name is hardcoded to default.
environmentName := "default"
if annotations.Configuration.Environment != "" {
environmentName = annotations.Configuration.Environment
}

// For now the application name is hardcoded to the namespace.
applicationName := deployment.Namespace
if annotations.Configuration.Application != "" {
applicationName = annotations.Configuration.Application
}

resourceGroupID, environmentID, applicationID, err := resolveDependencies(ctx, r.Radius, "/planes/radius/local", environmentName, applicationName)
if err != nil {
Expand All @@ -269,8 +274,9 @@ func (r *DeploymentReconciler) reconcileUpdate(ctx context.Context, deployment *
//
// 1) err != nil - an error happened, this will be retried next reconcile.
// 2) waiting == true - we're waiting on dependencies, this will be retried next reconcile.
// 3) poller != nil - we've started an operation, this will be checked next reconcile.
poller, waiting, err := r.startPutOperationIfNeeded(ctx, deployment, annotations)
// 3) updatePoller != nil - we've started a PUT operation, this will be checked next reconcile.
// 4) deletePoller != nil - we've started a DELETE operation, this will be checked next reconcile.
updatePoller, deletePoller, waiting, err := r.startPutOrDeleteOperationIfNeeded(ctx, deployment, annotations)
if err != nil {
logger.Error(err, "Unable to create or update resource.")
r.EventRecorder.Event(deployment, corev1.EventTypeWarning, "ResourceError", err.Error())
Expand All @@ -288,9 +294,9 @@ func (r *DeploymentReconciler) reconcileUpdate(ctx context.Context, deployment *
// We don't need to requeue here because we watch Recipes and will be notified when
// the state changes.
return ctrl.Result{}, nil
} else if poller != nil {
} else if updatePoller != nil {
// We've successfully started an operation. Update the status and requeue.
token, err := poller.ResumeToken()
token, err := updatePoller.ResumeToken()
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to get operation token: %w", err)
}
Expand All @@ -302,6 +308,21 @@ func (r *DeploymentReconciler) reconcileUpdate(ctx context.Context, deployment *
return ctrl.Result{}, err
}

return ctrl.Result{Requeue: true, RequeueAfter: r.requeueDelay()}, nil
} else if deletePoller != nil {
// We've successfully started an operation. Update the status and requeue.
token, err := deletePoller.ResumeToken()
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to get operation token: %w", err)
}

annotations.Status.Operation = &radappiov1alpha3.ResourceOperation{ResumeToken: token, OperationKind: radappiov1alpha3.OperationKindDelete}
annotations.Status.Phrase = deploymentPhraseDeleting
err = r.saveState(ctx, deployment, annotations)
if err != nil {
return ctrl.Result{}, err
}

return ctrl.Result{Requeue: true, RequeueAfter: r.requeueDelay()}, nil
}

Expand Down Expand Up @@ -366,15 +387,29 @@ func (r *DeploymentReconciler) reconcileDelete(ctx context.Context, deployment *
return ctrl.Result{}, nil
}

func (r *DeploymentReconciler) startPutOperationIfNeeded(ctx context.Context, deployment *appsv1.Deployment, annotations *deploymentAnnotations) (Poller[v20231001preview.ContainersClientCreateOrUpdateResponse], bool, error) {
func (r *DeploymentReconciler) startPutOrDeleteOperationIfNeeded(ctx context.Context, deployment *appsv1.Deployment, annotations *deploymentAnnotations) (Poller[v20231001preview.ContainersClientCreateOrUpdateResponse], Poller[v20231001preview.ContainersClientDeleteResponse], bool, error) {
logger := ucplog.FromContextOrDiscard(ctx)

resourceID := annotations.Status.Scope + "/providers/Applications.Core/containers/" + deployment.Name

// Check the annotations first to see how the current configuration compares to the desired configuration.
if !annotations.IsUpToDate() {
if annotations.Status.Container != "" && !strings.EqualFold(annotations.Status.Container, resourceID) {
// If we get here it means that the environment or application changed, so we should delete
// the old resource and create a new one.
logger.Info("Container is already created but is out-of-date")

logger.Info("Starting DELETE operation.")
poller, err := deleteContainer(ctx, r.Radius, annotations.Status.Container)
if err != nil {
return nil, nil, false, err
}

return nil, poller, false, err
} else if !annotations.IsUpToDate() {
logger.Info("Container configuration is out-of-date.")
} else if annotations.Status.Container != "" {
logger.Info("Container is already created and is up-to-date.")
return nil, false, nil
return nil, nil, false, nil
}

logger.Info("Starting PUT operation.")
Expand All @@ -397,26 +432,25 @@ func (r *DeploymentReconciler) startPutOperationIfNeeded(ctx context.Context, de
err := r.Client.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: source}, &recipe)
if apierrors.IsNotFound(err) {
logger.Info("Recipe does not exist.", "recipe", source)
return nil, true, nil
return nil, nil, true, nil
} else if err != nil {
return nil, false, fmt.Errorf("failed to fetch recipe %s: %w", source, err)
return nil, nil, false, fmt.Errorf("failed to fetch recipe %s: %w", source, err)
} else if recipe.Status.Resource == "" {
logger.Info("Recipe is not ready.", "recipe", source)
return nil, true, nil
return nil, nil, true, nil
}

properties.Connections[name] = &v20231001preview.ConnectionProperties{
Source: to.Ptr(recipe.Status.Resource),
}
}

resourceID := annotations.Status.Scope + "/providers/Applications.Core/containers/" + deployment.Name
poller, err := createOrUpdateContainer(ctx, r.Radius, resourceID, &properties)
if err != nil {
return nil, false, err
return nil, nil, false, err
}

return poller, false, nil
return poller, nil, false, nil
}

func (r *DeploymentReconciler) startDeleteOperationIfNeeded(ctx context.Context, deployment *appsv1.Deployment, annotations *deploymentAnnotations) (Poller[v20231001preview.ContainersClientDeleteResponse], error) {
Expand Down
77 changes: 74 additions & 3 deletions pkg/controller/reconciler/deployment_reconciler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func Test_DeploymentReconciler_RadiusEnabled_ThenDeploymentDeleted(t *testing.T)
require.NoError(t, err)

// Deployment will be waiting for environment to be created.
createEnvironment(radius)
createEnvironment(radius, "default")

// Deployment will be waiting for container to complete deployment.
annotations := waitForStateUpdating(t, client, name)
Expand All @@ -127,6 +127,77 @@ func Test_DeploymentReconciler_RadiusEnabled_ThenDeploymentDeleted(t *testing.T)
waitForDeploymentDeleted(t, client, name)
}

func Test_DeploymentReconciler_ChangeEnvironmentAndApplication(t *testing.T) {
ctx := testcontext.New(t)
radius, client := SetupDeploymentTest(t)

name := types.NamespacedName{Namespace: "deployment-change-envapp", Name: "test-deployment-change-envapp"}
err := client.Create(ctx, &corev1.Namespace{ObjectMeta: ctrl.ObjectMeta{Name: name.Namespace}})
require.NoError(t, err)

deployment := makeDeployment(name)
deployment.Annotations[AnnotationRadiusEnabled] = "true"
err = client.Create(ctx, deployment)
require.NoError(t, err)

// Deployment will be waiting for environment to be created.
createEnvironment(radius, "default")

// Deployment will be waiting for container to complete deployment.
annotations := waitForStateUpdating(t, client, name)
require.Equal(t, "/planes/radius/local/resourceGroups/default-deployment-change-envapp", annotations.Status.Scope)
require.Equal(t, "/planes/radius/local/resourceGroups/default/providers/Applications.Core/environments/default", annotations.Status.Environment)
require.Equal(t, "/planes/radius/local/resourceGroups/default-deployment-change-envapp/providers/Applications.Core/applications/deployment-change-envapp", annotations.Status.Application)
radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil)

// Deployment will update after operation completes
annotations = waitForStateReady(t, client, name)
require.Equal(t, "/planes/radius/local/resourceGroups/default-deployment-change-envapp/providers/Applications.Core/containers/test-deployment-change-envapp", annotations.Status.Container)

createEnvironment(radius, "new-environment")

// Now update the deployment to change the environment and application.
err = client.Get(ctx, name, deployment)
require.NoError(t, err)

deployment.Annotations[AnnotatationRadiusEnvironment] = "new-environment"
deployment.Annotations[AnnotationRadiusApplication] = "new-application"

err = client.Update(ctx, deployment)
require.NoError(t, err)

// Now the deployment will delete and re-create the resource.

// Deletion of the container is in progress.
annotations = waitForStateDeleting(t, client, name)
radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil)

// Resource should be gone.
_, err = radius.Containers(annotations.Status.Scope).Get(ctx, name.Name, nil)
require.Error(t, err)

// Recipe will be waiting for extender to complete provisioning.
annotations = waitForStateUpdating(t, client, name)
require.Equal(t, "/planes/radius/local/resourceGroups/new-environment-new-application", annotations.Status.Scope)
require.Equal(t, "/planes/radius/local/resourceGroups/new-environment/providers/Applications.Core/environments/new-environment", annotations.Status.Environment)
require.Equal(t, "/planes/radius/local/resourceGroups/new-environment-new-application/providers/Applications.Core/applications/new-application", annotations.Status.Application)
radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil)

// Recipe will update after operation completes
annotations = waitForStateReady(t, client, name)
require.Equal(t, "/planes/radius/local/resourceGroups/new-environment-new-application/providers/Applications.Core/containers/test-deployment-change-envapp", annotations.Status.Container)

err = client.Delete(ctx, deployment)
require.NoError(t, err)

// Deletion of the container is in progress.
annotations = waitForStateDeleting(t, client, name)
radius.CompleteOperation(annotations.Status.Operation.ResumeToken, nil)

// Now deleting of the deployment object can complete.
waitForDeploymentDeleted(t, client, name)
}

// Creates a deployment with Radius enabled.
//
// Then exercises the cleanup path by disabling Radius.
Expand All @@ -144,7 +215,7 @@ func Test_DeploymentReconciler_RadiusEnabled_ThenRadiusDisabled(t *testing.T) {
require.NoError(t, err)

// Deployment will be waiting for environment to be created.
createEnvironment(radius)
createEnvironment(radius, "default")

// Deployment will be waiting for container to complete deployment.
annotations := waitForStateUpdating(t, client, name)
Expand Down Expand Up @@ -199,7 +270,7 @@ func Test_DeploymentReconciler_Connections(t *testing.T) {
require.NoError(t, err)

// Deployment will be waiting for environment to be created.
createEnvironment(radius)
createEnvironment(radius, "default")

// Deployment will be waiting for recipe resources to be created
_ = waitForStateWaiting(t, client, name)
Expand Down
Loading

0 comments on commit 7827e4f

Please sign in to comment.