diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 5ecbb652..374a5534 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -12,6 +12,48 @@ rules: verbs: - create - patch +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - daemonsets/status + verbs: + - get +- apiGroups: + - apps + resources: + - deployments + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments/status + verbs: + - get +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get - apiGroups: - boundless.mirantis.com resources: diff --git a/controllers/addon_controller.go b/controllers/addon_controller.go index b5be2041..28186629 100644 --- a/controllers/addon_controller.go +++ b/controllers/addon_controller.go @@ -4,29 +4,37 @@ import ( "context" "fmt" "strings" + "time" "github.com/go-logr/logr" - boundlessv1alpha1 "github.com/mirantis/boundless-operator/api/v1alpha1" - "github.com/mirantis/boundless-operator/pkg/event" - "github.com/mirantis/boundless-operator/pkg/helm" - "github.com/mirantis/boundless-operator/pkg/manifest" + batch "k8s.io/api/batch/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + boundlessv1alpha1 "github.com/mirantis/boundless-operator/api/v1alpha1" + "github.com/mirantis/boundless-operator/pkg/event" + "github.com/mirantis/boundless-operator/pkg/helm" + "github.com/mirantis/boundless-operator/pkg/manifest" ) const ( kindManifest = "manifest" kindChart = "chart" - BoundlessNamespace = "boundless-system" addonHelmchartFinalizer = "boundless.mirantis.com/helmchart-finalizer" addonManifestFinalizer = "boundless.mirantis.com/manifest-finalizer" + addonIndexName = "helmchartIndex" + helmJobNameTemplate = "helm-install-%s" ) // AddonReconciler reconciles a Addon object @@ -39,6 +47,10 @@ type AddonReconciler struct { //+kubebuilder:rbac:groups=boundless.mirantis.com,resources=addons,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=boundless.mirantis.com,resources=addons/status,verbs=get;update;patch //+kubebuilder:rbac:groups=boundless.mirantis.com,resources=addons/finalizers,verbs=update +//+kubebuilder:rbac:groups=boundless.mirantis.com,resources=manifests,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=boundless.mirantis.com,resources=manifests/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch +//+kubebuilder:rbac:groups=batch,resources=jobs/status,verbs=get //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -81,8 +93,6 @@ func (r *AddonReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl kind = instance.Spec.Kind } - // @TODO: Update addon status only once per reconcile; React to Statuses of HelmChart / Manifests - switch kind { case kindChart: if instance.Spec.Chart == nil { @@ -137,11 +147,22 @@ func (r *AddonReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl if err := hc.CreateHelmChart(chart, instance.Spec.Namespace); err != nil { logger.Error(err, "failed to install addon", "Name", chart.Name, "Version", chart.Version) r.Recorder.AnnotatedEventf(instance, map[string]string{event.AddonAnnotationKey: instance.Name}, event.TypeWarning, event.ReasonFailedCreate, "Failed to Create Chart Addon %s/%s : %s", instance.Spec.Namespace, instance.Name, err) - r.updateStatus(ctx, logger, req.NamespacedName, boundlessv1alpha1.TypeComponentUnhealthy, "Failed to Create HelmChart") return ctrl.Result{Requeue: true}, err } - r.Recorder.AnnotatedEventf(instance, map[string]string{event.AddonAnnotationKey: instance.Name}, event.TypeNormal, event.ReasonSuccessfulCreate, "Created Chart Addon %s/%s", instance.Spec.Namespace, instance.Name) - r.updateStatus(ctx, logger, req.NamespacedName, boundlessv1alpha1.TypeComponentAvailable, "Chart Addon Created") + + // unfortunately the HelmChart CR doesn't have any useful events or status we can monitor + // each helmchart object creates a job that runs the helm install - update status from that instead + jobName := fmt.Sprintf(helmJobNameTemplate, instance.Spec.Chart.Name) + job := &batch.Job{} + err = r.Get(ctx, types.NamespacedName{Namespace: instance.Spec.Namespace, Name: jobName}, job) + if err != nil { + // might need some time for helmchart CR to create job + return ctrl.Result{RequeueAfter: 10 * time.Second}, err + } + + if err := r.updateHelmchartAddonStatus(ctx, logger, req.NamespacedName, job, instance); err != nil { + return ctrl.Result{RequeueAfter: 10 * time.Second}, err + } case kindManifest: if instance.Spec.Manifest == nil { @@ -165,11 +186,10 @@ func (r *AddonReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl // The object is being deleted if controllerutil.ContainsFinalizer(instance, addonManifestFinalizer) { // our finalizer is present, so lets delete the helm chart - if err := mc.DeleteManifest(BoundlessNamespace, instance.Spec.Name, instance.Spec.Manifest.URL); err != nil { + if err := mc.DeleteManifest(boundlessSystemNamespace, instance.Spec.Name, instance.Spec.Manifest.URL); err != nil { // if fail to delete the manifest here, return with error // so that it can be retried r.Recorder.AnnotatedEventf(instance, map[string]string{event.AddonAnnotationKey: instance.Name}, event.TypeWarning, event.ReasonFailedDelete, "Failed to Delete Manifest Addon %s/%s : %s", instance.Spec.Namespace, instance.Name, err) - r.updateStatus(ctx, logger, req.NamespacedName, boundlessv1alpha1.TypeComponentUnhealthy, "Failed to Cleanup Manifest") return ctrl.Result{}, err } @@ -184,16 +204,32 @@ func (r *AddonReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{}, nil } - err = mc.CreateManifest(BoundlessNamespace, instance.Spec.Name, instance.Spec.Manifest.URL) + err = mc.CreateManifest(boundlessSystemNamespace, instance.Spec.Name, instance.Spec.Manifest.URL) if err != nil { logger.Error(err, "failed to install addon via manifest", "URL", instance.Spec.Manifest.URL) r.Recorder.AnnotatedEventf(instance, map[string]string{event.AddonAnnotationKey: instance.Name}, event.TypeWarning, event.ReasonFailedCreate, "Failed to Create Manifest Addon %s/%s : %s", instance.Spec.Namespace, instance.Name, err) - r.updateStatus(ctx, logger, req.NamespacedName, boundlessv1alpha1.TypeComponentUnhealthy, "Failed to Create Manifest") return ctrl.Result{Requeue: true}, err } - r.Recorder.AnnotatedEventf(instance, map[string]string{event.AddonAnnotationKey: instance.Name}, event.TypeNormal, event.ReasonSuccessfulCreate, "Created Manifest Addon %s/%s", instance.Spec.Namespace, instance.Name) - r.updateStatus(ctx, logger, req.NamespacedName, boundlessv1alpha1.TypeComponentAvailable, "Manifest Addon Created") + m := &boundlessv1alpha1.Manifest{} + err = r.Get(ctx, types.NamespacedName{Namespace: boundlessSystemNamespace, Name: instance.Spec.Name}, m) + if err != nil { + if errors.IsNotFound(err) { + // might need some time for CR to be created + r.updateStatus(ctx, logger, req.NamespacedName, boundlessv1alpha1.TypeComponentProgressing, "Awaiting Manifest Resource Creation") + } + return ctrl.Result{}, err + } + + result, err := r.setOwnerReferenceOnManifest(ctx, logger, instance, m) + if err != nil { + return result, err + } + + err = r.updateManifestAddonStatus(ctx, logger, instance, m) + if err != nil { + return result, err + } default: logger.Info("Unknown AddOn kind", "Kind", instance.Spec.Kind) @@ -204,14 +240,127 @@ func (r *AddonReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{Requeue: false}, nil } +// updateManifestAddonStatus checks if the manifest associated with the addon has a status to bubble up to addon and updates addon if so +func (r *AddonReconciler) updateManifestAddonStatus(ctx context.Context, logger logr.Logger, addon *boundlessv1alpha1.Addon, manifest *boundlessv1alpha1.Manifest) error { + if manifest.Status.Type == "" || manifest.Status.Reason == "" { + err := r.updateStatus(ctx, logger, types.NamespacedName{Namespace: addon.Namespace, Name: addon.Name}, boundlessv1alpha1.TypeComponentProgressing, "Awaiting status from manifest object") + if err != nil { + return err + } + // manifest has no status yet so nothing to do + return nil + } + + if manifest.Status.Type == boundlessv1alpha1.TypeComponentAvailable && addon.Status.Type != boundlessv1alpha1.TypeComponentAvailable { + // we are about to update the addon status from not available to available so let's emit an event + r.Recorder.AnnotatedEventf(addon, map[string]string{event.AddonAnnotationKey: addon.Name}, event.TypeNormal, event.ReasonSuccessfulCreate, "Created Manifest Addon %s/%s", addon.Spec.Namespace, addon.Name) + } + + err := r.updateStatus(ctx, logger, types.NamespacedName{Namespace: addon.Namespace, Name: addon.Name}, manifest.Status.Type, manifest.Status.Reason, manifest.Status.Message) + if err != nil { + return err + } + return nil +} + +// setOwnerReferenceOnManifest sets the owner reference on the manifest object to point to the addon object +// This effectively causes the owner addon to be reconciled when the manifest is updated. +func (r *AddonReconciler) setOwnerReferenceOnManifest(ctx context.Context, logger logr.Logger, addon *boundlessv1alpha1.Addon, manifest *boundlessv1alpha1.Manifest) (ctrl.Result, error) { + logger.Info("Set owner ref field on manifest") + if err := controllerutil.SetControllerReference(addon, manifest, r.Scheme); err != nil { + logger.Error(err, "Failed to set owner reference on manifest", "ManifestName", manifest.Name) + return ctrl.Result{}, err + } + + if err := r.Update(ctx, manifest); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + // SetupWithManager sets up the controller with the Manager. func (r *AddonReconciler) SetupWithManager(mgr ctrl.Manager) error { + // attaches an index onto the Addon + // This is done so we can later easily find the addon associated with a particular job + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &boundlessv1alpha1.Addon{}, addonIndexName, func(rawObj client.Object) []string { + addon := rawObj.(*boundlessv1alpha1.Addon) + if isHelmChartAddon(addon) { + jobName := fmt.Sprintf(helmJobNameTemplate, addon.Spec.Chart.Name) + return []string{fmt.Sprintf("%s-%s", addon.Spec.Namespace, jobName)} + } + // don't add this index for non helm-chart type addons + return nil + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&boundlessv1alpha1.Addon{}). + Owns(&boundlessv1alpha1.Manifest{}). + Watches( + &batch.Job{}, // Watch all Job Objects in the cluster + handler.EnqueueRequestsFromMapFunc(r.findAddonForJob), // All jobs trigger this MapFunc, the MapFunc filters which jobs should trigger reconciles to which addons, if any + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), // By default, any Update to job will trigger a run of the MapFunc, limit it to only Resource version updates + ). Complete(r) } -func (r *AddonReconciler) updateStatus(ctx context.Context, logger logr.Logger, namespacedName types.NamespacedName, conditionTypeToApply boundlessv1alpha1.StatusType, reasonToApply string, messageToApply ...string) error { +// isHelmChartAddon checks the provided addon's spec and determines whether this addon is a chart kind +func isHelmChartAddon(addon *boundlessv1alpha1.Addon) bool { + return addon.Spec.Chart != nil && addon.Spec.Chart.Name != "" +} + +// findAddonForJob finds the addons associated with a particular job +// This is done by looking for the addon that was previously indexed in the form jobNamespace-jobName +func (r *AddonReconciler) findAddonForJob(ctx context.Context, job client.Object) []reconcile.Request { + attachedAddonList := &boundlessv1alpha1.AddonList{} + err := r.List(context.TODO(), attachedAddonList, client.MatchingFields{addonIndexName: fmt.Sprintf("%s-%s", job.GetNamespace(), job.GetName())}) + if err != nil { + return []reconcile.Request{} + } + + requests := make([]reconcile.Request, len(attachedAddonList.Items)) + for i, item := range attachedAddonList.Items { + requests[i] = reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + } + } + return requests +} + +// updateHelmchartAddonStatus checks the status of the associated helm chart job and updates the status of the Addon CR accordingly +func (r *AddonReconciler) updateHelmchartAddonStatus(ctx context.Context, logger logr.Logger, namespacedName types.NamespacedName, job *batch.Job, addon *boundlessv1alpha1.Addon) error { + logger.Info("Updating Helm Chart Addon Status") + if job.Status.CompletionTime != nil && job.Status.Succeeded > 0 { + r.Recorder.AnnotatedEventf(addon, map[string]string{event.AddonAnnotationKey: addon.Name}, event.TypeNormal, event.ReasonSuccessfulCreate, "Created Chart Addon %s/%s", addon.Spec.Namespace, addon.Name) + err := r.updateStatus(ctx, logger, namespacedName, boundlessv1alpha1.TypeComponentAvailable, fmt.Sprintf("Helm Chart %s successfully installed", job.Name)) + if err != nil { + return err + } + } else if job.Status.StartTime != nil && job.Status.Failed > 0 { + r.Recorder.AnnotatedEventf(addon, map[string]string{event.AddonAnnotationKey: addon.Name}, event.TypeWarning, event.ReasonFailedCreate, "Helm Chart Addon %s/%s has failed to install", addon.Spec.Namespace, addon.Name) + err := r.updateStatus(ctx, logger, namespacedName, boundlessv1alpha1.TypeComponentUnhealthy, fmt.Sprintf("Helm Chart %s install has failed", job.Name)) + if err != nil { + return err + } + } else { + err := r.updateStatus(ctx, logger, namespacedName, boundlessv1alpha1.TypeComponentProgressing, fmt.Sprintf("Helm Chart %s install still progressing", job.Name)) + if err != nil { + return err + } + } + return nil +} + +// updateStatus queries for a fresh Addon with the provided namespacedName. +// This avoids some errors where we fail to update status because we have an older (stale) version of the object +// It then updates the Addon's status fields with the provided type, reason, and optionally message. +func (r *AddonReconciler) updateStatus(ctx context.Context, logger logr.Logger, namespacedName types.NamespacedName, typeToApply boundlessv1alpha1.StatusType, reasonToApply string, messageToApply ...string) error { + logger.Info("Update status with type and reason", "TypeToApply", typeToApply, "ReasonToApply", reasonToApply) + addon := &boundlessv1alpha1.Addon{} err := r.Get(ctx, namespacedName, addon) if err != nil { @@ -219,7 +368,8 @@ func (r *AddonReconciler) updateStatus(ctx context.Context, logger logr.Logger, return err } - if addon.Status.Type == conditionTypeToApply && addon.Status.Reason == reasonToApply { + nilStatus := boundlessv1alpha1.AddonStatus{} + if addon.Status != nilStatus && addon.Status.Type == typeToApply && addon.Status.Reason == reasonToApply { // avoid infinite reconciliation loops logger.Info("No updates to status needed") return nil @@ -228,7 +378,7 @@ func (r *AddonReconciler) updateStatus(ctx context.Context, logger logr.Logger, logger.Info("Update status for addon", "Name", addon.Name) patch := client.MergeFrom(addon.DeepCopy()) - addon.Status.Type = conditionTypeToApply + addon.Status.Type = typeToApply addon.Status.Reason = reasonToApply if len(messageToApply) > 0 { addon.Status.Message = messageToApply[0] diff --git a/controllers/manifest_controller.go b/controllers/manifest_controller.go index a47234fa..b073a539 100644 --- a/controllers/manifest_controller.go +++ b/controllers/manifest_controller.go @@ -10,15 +10,6 @@ import ( "time" "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/log" - adm_v1 "k8s.io/api/admissionregistration/v1" apps_v1 "k8s.io/api/apps/v1" core_v1 "k8s.io/api/core/v1" @@ -27,16 +18,29 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" boundlessv1alpha1 "github.com/mirantis/boundless-operator/api/v1alpha1" "github.com/mirantis/boundless-operator/pkg/event" ) const ( - actionUpdate = "update" - actionCreate = "create" - actionDelete = "delete" + actionUpdate = "update" + actionCreate = "create" + actionDelete = "delete" + manifestUpdateIndex = "manifestupdateindex" ) // ManifestReconciler reconciles a Manifest object @@ -50,6 +54,10 @@ type ManifestReconciler struct { //+kubebuilder:rbac:groups=boundless.mirantis.com,resources=manifests/status,verbs=get;update;patch //+kubebuilder:rbac:groups=boundless.mirantis.com,resources=manifests/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch +//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch +//+kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get +//+kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch +//+kubebuilder:rbac:groups=apps,resources=daemonsets/status,verbs=get // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -72,6 +80,7 @@ func (r *ManifestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } existing := &boundlessv1alpha1.Manifest{} + err := r.Client.Get(ctx, key, existing) if err != nil { @@ -126,9 +135,11 @@ func (r *ManifestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, nil } - if existing.Spec.Checksum == existing.Spec.NewChecksum && existing.Status.Type == boundlessv1alpha1.TypeComponentAvailable { + if existing.Spec.Checksum == existing.Spec.NewChecksum { logger.Info("checksum is same, no update needed", "Checksum", existing.Spec.Checksum, "NewChecksum", existing.Spec.NewChecksum) - return ctrl.Result{}, nil + // manifest is already installed as specified - get latest status from objects in the cluster + err = r.checkManifestStatus(ctx, logger, req.NamespacedName, existing.Spec.Objects) + return ctrl.Result{}, err } if (existing.Spec.Checksum != existing.Spec.NewChecksum) && (existing.Spec.NewChecksum != "") { @@ -152,7 +163,7 @@ func (r *ManifestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if err := r.Update(ctx, &updatedCRD); err != nil { logger.Error(err, "failed to update manifest crd while update operation") r.Recorder.AnnotatedEventf(existing, map[string]string{event.AddonAnnotationKey: existing.Name}, event.TypeWarning, event.ReasonFailedCreate, "failed to update manifest crd while update operation %s/%s : %s", existing.Namespace, existing.Name, err.Error()) - r.updateStatus(ctx, logger, key, boundlessv1alpha1.TypeComponentUnhealthy, "failed to update manifest crd while update operation", fmt.Sprintf("failed to update manifest crd while update operation : %s", err)) + r.updateStatus(ctx, logger, key, boundlessv1alpha1.TypeComponentUnhealthy, "failed to update manifest crd while update operation ", fmt.Sprintf("failed to update manifest crd while update operation : %s", err)) return ctrl.Result{}, err } @@ -160,10 +171,9 @@ func (r *ManifestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if err := r.UpdateManifestObjects(req, ctx, existing); err != nil { logger.Error(err, "failed to update manifest") r.Recorder.AnnotatedEventf(existing, map[string]string{event.AddonAnnotationKey: existing.Name}, event.TypeWarning, event.ReasonFailedCreate, "failed to update manifest %s/%s : %s", existing.Namespace, existing.Name, err.Error()) - r.updateStatus(ctx, logger, key, boundlessv1alpha1.TypeComponentUnhealthy, "failed to update manifest", fmt.Sprintf("failed to update manifest : %s", err)) + r.updateStatus(ctx, logger, key, boundlessv1alpha1.TypeComponentUnhealthy, "failed to update manifest ", fmt.Sprintf("failed to update manifest : %s", err)) return ctrl.Result{}, err } - } if existing.Spec.NewChecksum == "" { @@ -186,7 +196,6 @@ func (r *ManifestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if err := r.Update(ctx, &updatedCRD); err != nil { logger.Error(err, "failed to update manifest crd while create operation") r.Recorder.AnnotatedEventf(existing, map[string]string{event.AddonAnnotationKey: existing.Name}, event.TypeWarning, event.ReasonFailedCreate, "failed to update manifest crd while create operation %s/%s : %s", existing.Namespace, existing.Name, err.Error()) - r.updateStatus(ctx, logger, key, boundlessv1alpha1.TypeComponentUnhealthy, "failed to update manifest crd while create operation", fmt.Sprintf("failed to update manifest crd while create operation : %s", err)) return ctrl.Result{}, err } @@ -195,7 +204,6 @@ func (r *ManifestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if err != nil { logger.Error(err, "failed to fetch manifest file content for url: %s", "Manifest Url", existing.Spec.Url) r.Recorder.AnnotatedEventf(existing, map[string]string{event.AddonAnnotationKey: existing.Name}, event.TypeWarning, event.ReasonFailedCreate, "failed to fetch manifest file content for url %s/%s : %s", existing.Namespace, existing.Name, err.Error()) - r.updateStatus(ctx, logger, key, boundlessv1alpha1.TypeComponentUnhealthy, "failed to fetch manifest file content for url", fmt.Sprintf("failed to fetch manifest file content for url : %s", err)) return ctrl.Result{RequeueAfter: time.Minute}, err } @@ -204,29 +212,74 @@ func (r *ManifestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if err != nil { logger.Error(err, "failed to create objects for the manifest", "Name", req.Name) r.Recorder.AnnotatedEventf(existing, map[string]string{event.AddonAnnotationKey: existing.Name}, event.TypeWarning, event.ReasonFailedCreate, "failed to create objects for the manifest %s/%s : %s", existing.Namespace, existing.Name, err.Error()) - r.updateStatus(ctx, logger, key, boundlessv1alpha1.TypeComponentUnhealthy, "failed to create objects for the manifest", fmt.Sprintf("failed to create objects for the manifest : %s", err)) return ctrl.Result{}, err } } r.Recorder.AnnotatedEventf(existing, map[string]string{event.AddonAnnotationKey: existing.Name}, event.TypeNormal, event.ReasonSuccessfulCreate, "Created Manifest %s/%s", existing.Namespace, existing.Name) - err = r.updateStatus(ctx, logger, key, boundlessv1alpha1.TypeComponentAvailable, "Manifest Created") - if err != nil { - logger.Error(err, "Failed to update status after manifest creation") - return ctrl.Result{Requeue: true}, err - } - return ctrl.Result{}, nil } // SetupWithManager sets up the controller with the Manager. func (r *ManifestReconciler) SetupWithManager(mgr ctrl.Manager) error { + // attaches an index onto the Manifest + // This is done so we can later easily find the addon associated with a particular deployment or daemonset + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &boundlessv1alpha1.Manifest{}, manifestUpdateIndex, func(rawObj client.Object) []string { + manifest := rawObj.(*boundlessv1alpha1.Manifest) + if manifest.Spec.Objects == nil || len(manifest.Spec.Objects) == 0 { + return nil + } + + var indexes []string + for _, obj := range manifest.Spec.Objects { + if obj.Kind == "DaemonSet" || obj.Kind == "Deployment" { + indexes = append(indexes, fmt.Sprintf("%s-%s", obj.Namespace, obj.Name)) + } + } + return indexes + + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&boundlessv1alpha1.Manifest{}). + Watches( + &apps_v1.DaemonSet{}, + handler.EnqueueRequestsFromMapFunc(r.findAssociatedManifest), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Watches( + &apps_v1.Deployment{}, + handler.EnqueueRequestsFromMapFunc(r.findAssociatedManifest), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } +// findAssociatedManifest finds the manifest tied to a particular object if one exists +// This is done by looking for the manifest that was previously indexed in the form objectNamespace-objectName +func (r *ManifestReconciler) findAssociatedManifest(ctx context.Context, obj client.Object) []reconcile.Request { + attachedManifestList := &boundlessv1alpha1.ManifestList{} + //TODO: this index will clash if we have multiple deployments / daemonsets with the same name and namespace + err := r.List(context.TODO(), attachedManifestList, client.MatchingFields{manifestUpdateIndex: fmt.Sprintf("%s-%s", obj.GetNamespace(), obj.GetName())}) + if err != nil { + return []reconcile.Request{} + } + + requests := make([]reconcile.Request, len(attachedManifestList.Items)) + for i, item := range attachedManifestList.Items { + requests[i] = reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + } + } + return requests +} + func (r *ManifestReconciler) CreateManifestObjects(req ctrl.Request, data []byte, logger logr.Logger, ctx context.Context, existing *boundlessv1alpha1.Manifest) error { apiextensionsv1.AddToScheme(clientgoscheme.Scheme) apiextensionsv1beta1.AddToScheme(clientgoscheme.Scheme) @@ -2164,7 +2217,104 @@ func (r *ManifestReconciler) ReadManifest(req ctrl.Request, url string, logger l } +// checkManifestStatus checks the status of any deployments and daemonsets associated with the namespacedName manifest +// Check the status of the deployment and daemonset and set the manifest to an error state if any errors are found +// If no errors are found, we check if any deployments/daemonsets are still progressing and set the manifest status to Progressing +// Otherwise set the manifest status to Available +// This is not comprehensive and may need to be updated as we support more complex manifests +func (r *ManifestReconciler) checkManifestStatus(ctx context.Context, logger logr.Logger, namespacedName types.NamespacedName, objects []boundlessv1alpha1.ManifestObject) error { + + if objects == nil || len(objects) == 0 { + logger.Info("No manifest objects for manifest") + return nil + } + + // for now focus on getting status from any Deployments or Daemonsets deployed via the manifest since + // they have reliable status fields we can pull from and are most likely to fail + stillProgressing := false + var reasonToApply, messageToApply string + for _, obj := range objects { + kind := obj.Kind + + if kind == "Deployment" { + deployment := &apps_v1.Deployment{} + err := r.Get(ctx, types.NamespacedName{Namespace: obj.Namespace, Name: obj.Name}, deployment) + if err != nil { + return err + } + if deployment.Status.AvailableReplicas == deployment.Status.Replicas && (deployment.Status.Conditions == nil || len(deployment.Status.Conditions) == 0) { + // this deployment is ready + continue + } + latestCondition := deployment.Status.Conditions[0] + if deployment.Status.AvailableReplicas == deployment.Status.Replicas && latestCondition.Type == apps_v1.DeploymentAvailable { + // this deployment is ready + continue + } + + if latestCondition.Type == apps_v1.DeploymentProgressing || latestCondition.Reason == "MinimumReplicasUnavailable" { + stillProgressing = true + reasonToApply = fmt.Sprintf("Deployment %s still progressing", obj.Name) + messageToApply = latestCondition.Message + } else { + // deployment is in error state, so we can update the manifest status that it has issues + err := r.updateStatus(ctx, logger, namespacedName, boundlessv1alpha1.TypeComponentUnhealthy, latestCondition.Reason, latestCondition.Message) + if err != nil { + return err + } + break + } + } else if kind == "DaemonSet" { + daemonset := &apps_v1.DaemonSet{} + err := r.Get(ctx, types.NamespacedName{Namespace: obj.Namespace, Name: obj.Name}, daemonset) + if err != nil { + return err + } + + if daemonset.Status.DesiredNumberScheduled == daemonset.Status.CurrentNumberScheduled && daemonset.Status.DesiredNumberScheduled == daemonset.Status.NumberAvailable { + //daemonset is ready + continue + } + + if daemonset.Status.NumberMisscheduled > 0 { + err := r.updateStatus(ctx, logger, namespacedName, boundlessv1alpha1.TypeComponentUnhealthy, fmt.Sprintf("Daemonset %s failed to schedule pods", daemonset.Name)) + if err != nil { + return err + } + break + } else { + stillProgressing = true + reasonToApply = fmt.Sprintf("Daemonset %s is still progressing", daemonset.Name) + messageToApply = fmt.Sprintf("Daemonset %s is still progressing", daemonset.Name) + } + + } else { + continue + } + + } + + if stillProgressing { + err := r.updateStatus(ctx, logger, namespacedName, boundlessv1alpha1.TypeComponentProgressing, fmt.Sprintf("One or more components still progressing : %s", reasonToApply), messageToApply) + if err != nil { + return err + } + return nil + } + + err := r.updateStatus(ctx, logger, namespacedName, boundlessv1alpha1.TypeComponentAvailable, "Manifest Components Available", "Manifest Components Available") + if err != nil { + return err + } + + return nil +} + +// updateStatus queries for a fresh Manifest with the provided namespacedName. +// It then updates the Manifest's status fields with the provided type, reason, and optionally message. func (r *ManifestReconciler) updateStatus(ctx context.Context, logger logr.Logger, namespacedName types.NamespacedName, typeToApply boundlessv1alpha1.StatusType, reasonToApply string, messageToApply ...string) error { + logger.Info("Update status with type and reason", "TypeToApply", typeToApply, "ReasonToApply", reasonToApply) + manifest := &boundlessv1alpha1.Manifest{} err := r.Get(ctx, namespacedName, manifest) if err != nil { @@ -2172,7 +2322,8 @@ func (r *ManifestReconciler) updateStatus(ctx context.Context, logger logr.Logge return err } - if manifest.Status.Type == typeToApply && manifest.Status.Reason == reasonToApply { + nilStatus := boundlessv1alpha1.ManifestStatus{} + if manifest.Status != nilStatus && manifest.Status.Type == typeToApply && manifest.Status.Reason == reasonToApply { // avoid infinite reconciliation loops logger.Info("No updates to status needed") return nil