From bd4bb27d8bf13a885271478fc1f8b7703c716c31 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 27 Oct 2023 09:14:23 +0200 Subject: [PATCH] Generic sync/suspend/inventory listing (#4096) * Handle unstructured reconcilation of "ks-like" resources - Expose useInventory hook too * Convert the entire inventory from unstructured * Modernise our multi-error handling, golang can do it now * Expose `useListEvents` via npm module --- core/fluxsync/adapters.go | 148 ++++++++++------- core/fluxsync/adapters_test.go | 125 ++++++++++++++ core/server/inventory.go | 121 ++++++++------ core/server/inventory_internal_test.go | 220 +++++++++++++++++++++++++ core/server/suspend.go | 17 +- core/server/sync.go | 66 ++------ core/server/sync_test.go | 6 +- ui/hooks/inventory.ts | 1 + ui/index.ts | 4 + 9 files changed, 542 insertions(+), 166 deletions(-) create mode 100644 core/fluxsync/adapters_test.go create mode 100644 core/server/inventory_internal_test.go diff --git a/core/fluxsync/adapters.go b/core/fluxsync/adapters.go index f403db2e9f..c1d845f991 100644 --- a/core/fluxsync/adapters.go +++ b/core/fluxsync/adapters.go @@ -1,8 +1,6 @@ package fluxsync import ( - "errors" - helmv2 "github.com/fluxcd/helm-controller/api/v2beta1" imgautomationv1 "github.com/fluxcd/image-automation-controller/api/v1beta1" reflectorv1 "github.com/fluxcd/image-reflector-controller/api/v1beta2" @@ -10,6 +8,9 @@ import ( "github.com/fluxcd/pkg/apis/meta" sourcev1 "github.com/fluxcd/source-controller/api/v1" sourcev1b2 "github.com/fluxcd/source-controller/api/v1beta2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -23,7 +24,7 @@ type Reconcilable interface { GetLastHandledReconcileRequest() string AsClientObject() client.Object GroupVersionKind() schema.GroupVersionKind - SetSuspended(suspend bool) + SetSuspended(suspend bool) error DeepCopyClientObject() client.Object } @@ -42,30 +43,6 @@ type Automation interface { SourceRef() SourceRef } -func NewReconcileable(obj client.Object) Reconcilable { - switch o := obj.(type) { - case *kustomizev1.Kustomization: - return KustomizationAdapter{Kustomization: o} - case *helmv2.HelmRelease: - return HelmReleaseAdapter{HelmRelease: o} - case *sourcev1.GitRepository: - return GitRepositoryAdapter{GitRepository: o} - case *sourcev1b2.HelmRepository: - return HelmRepositoryAdapter{HelmRepository: o} - case *sourcev1b2.Bucket: - return BucketAdapter{Bucket: o} - case *sourcev1b2.HelmChart: - return HelmChartAdapter{HelmChart: o} - case *sourcev1b2.OCIRepository: - return OCIRepositoryAdapter{OCIRepository: o} - case *reflectorv1.ImageRepository: - return ImageRepositoryAdapter{ImageRepository: o} - case *imgautomationv1.ImageUpdateAutomation: - return ImageUpdateAutomationAdapter{ImageUpdateAutomation: o} - } - return nil -} - type GitRepositoryAdapter struct { *sourcev1.GitRepository } @@ -82,8 +59,9 @@ func (obj GitRepositoryAdapter) GroupVersionKind() schema.GroupVersionKind { return sourcev1.GroupVersion.WithKind(sourcev1.GitRepositoryKind) } -func (obj GitRepositoryAdapter) SetSuspended(suspend bool) { +func (obj GitRepositoryAdapter) SetSuspended(suspend bool) error { obj.Spec.Suspend = suspend + return nil } func (obj GitRepositoryAdapter) DeepCopyClientObject() client.Object { @@ -106,8 +84,9 @@ func (obj BucketAdapter) GroupVersionKind() schema.GroupVersionKind { return sourcev1b2.GroupVersion.WithKind(sourcev1b2.BucketKind) } -func (obj BucketAdapter) SetSuspended(suspend bool) { +func (obj BucketAdapter) SetSuspended(suspend bool) error { obj.Spec.Suspend = suspend + return nil } func (obj BucketAdapter) DeepCopyClientObject() client.Object { @@ -130,8 +109,9 @@ func (obj HelmChartAdapter) GroupVersionKind() schema.GroupVersionKind { return sourcev1b2.GroupVersion.WithKind(sourcev1b2.HelmChartKind) } -func (obj HelmChartAdapter) SetSuspended(suspend bool) { +func (obj HelmChartAdapter) SetSuspended(suspend bool) error { obj.Spec.Suspend = suspend + return nil } func (obj HelmChartAdapter) DeepCopyClientObject() client.Object { @@ -154,8 +134,9 @@ func (obj HelmRepositoryAdapter) GroupVersionKind() schema.GroupVersionKind { return sourcev1b2.GroupVersion.WithKind(sourcev1b2.HelmRepositoryKind) } -func (obj HelmRepositoryAdapter) SetSuspended(suspend bool) { +func (obj HelmRepositoryAdapter) SetSuspended(suspend bool) error { obj.Spec.Suspend = suspend + return nil } func (obj HelmRepositoryAdapter) DeepCopyClientObject() client.Object { @@ -178,8 +159,9 @@ func (obj OCIRepositoryAdapter) GroupVersionKind() schema.GroupVersionKind { return sourcev1b2.GroupVersion.WithKind(sourcev1b2.OCIRepositoryKind) } -func (obj OCIRepositoryAdapter) SetSuspended(suspend bool) { +func (obj OCIRepositoryAdapter) SetSuspended(suspend bool) error { obj.Spec.Suspend = suspend + return nil } func (obj OCIRepositoryAdapter) DeepCopyClientObject() client.Object { @@ -213,8 +195,9 @@ func (obj HelmReleaseAdapter) GroupVersionKind() schema.GroupVersionKind { return helmv2.GroupVersion.WithKind(helmv2.HelmReleaseKind) } -func (obj HelmReleaseAdapter) SetSuspended(suspend bool) { +func (obj HelmReleaseAdapter) SetSuspended(suspend bool) error { obj.Spec.Suspend = suspend + return nil } func (obj HelmReleaseAdapter) DeepCopyClientObject() client.Object { @@ -246,8 +229,9 @@ func (obj KustomizationAdapter) GroupVersionKind() schema.GroupVersionKind { return kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind) } -func (obj KustomizationAdapter) SetSuspended(suspend bool) { +func (obj KustomizationAdapter) SetSuspended(suspend bool) error { obj.Spec.Suspend = suspend + return nil } func (obj KustomizationAdapter) DeepCopyClientObject() client.Object { @@ -270,8 +254,9 @@ func (obj ImageRepositoryAdapter) GroupVersionKind() schema.GroupVersionKind { return reflectorv1.GroupVersion.WithKind(reflectorv1.ImageRepositoryKind) } -func (obj ImageRepositoryAdapter) SetSuspended(suspend bool) { +func (obj ImageRepositoryAdapter) SetSuspended(suspend bool) error { obj.Spec.Suspend = suspend + return nil } func (obj ImageRepositoryAdapter) DeepCopyClientObject() client.Object { @@ -294,14 +279,61 @@ func (obj ImageUpdateAutomationAdapter) GroupVersionKind() schema.GroupVersionKi return imgautomationv1.GroupVersion.WithKind(imgautomationv1.ImageUpdateAutomationKind) } -func (obj ImageUpdateAutomationAdapter) SetSuspended(suspend bool) { +func (obj ImageUpdateAutomationAdapter) SetSuspended(suspend bool) error { obj.Spec.Suspend = suspend + return nil } func (obj ImageUpdateAutomationAdapter) DeepCopyClientObject() client.Object { return obj.DeepCopy() } +// UnstructuredAdapter implements the Reconcilable interface for unstructured resources. +// The underlying resource gvk should have the standard flux object sync/suspend fields +type UnstructuredAdapter struct { + *unstructured.Unstructured +} + +func (obj UnstructuredAdapter) GetLastHandledReconcileRequest() string { + if val, found, _ := unstructured.NestedString(obj.Object, "status", "lastHandledReconcileAt"); found { + return val + } + return "" +} + +func (obj UnstructuredAdapter) GetConditions() []metav1.Condition { + conditionsSlice, found, err := unstructured.NestedSlice(obj.Object, "status", "conditions") + if !found || err != nil { + return nil + } + + var conditions []metav1.Condition + for _, c := range conditionsSlice { + var condition metav1.Condition + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(c.(map[string]interface{}), &condition); err != nil { + continue + } + conditions = append(conditions, condition) + } + + return conditions +} + +func (obj UnstructuredAdapter) AsClientObject() client.Object { + // Important for the controller-runtime type reflection to work + // We can't return just `obj` here otherwise we get a + // panic: reflect: call of reflect.Value.Elem on struct Value + return obj.Unstructured +} + +func (obj UnstructuredAdapter) SetSuspended(suspend bool) error { + return unstructured.SetNestedField(obj.Object, suspend, "spec", "suspend") +} + +func (obj UnstructuredAdapter) DeepCopyClientObject() client.Object { + return obj.DeepCopy() +} + type sRef struct { apiVersion string name string @@ -325,35 +357,39 @@ func (s sRef) Kind() string { return s.kind } -func ToReconcileable(kind string) (client.ObjectList, Reconcilable, error) { - switch kind { +// ToReconcileable takes a GVK and returns a "Reconcilable" for it. +// The reconcilable can be passed to a controller-runtime client to fetch it +// from the cluster. Once fetched we can query it for the last sync time, whether +// its suspended etc, using the Reconcilable interface. +// +// The generic unstructured case handles "flux like" objects that we don't explicitly +// know about, but which follow the same patterns for suspend/sync as a stadard flux object. +// E.g. `spec.suspend` and `status.lastHandledReconcileRequest` etc. +func ToReconcileable(gvk schema.GroupVersionKind) Reconcilable { + switch gvk.Kind { case kustomizev1.KustomizationKind: - return &kustomizev1.KustomizationList{}, NewReconcileable(&kustomizev1.Kustomization{}), nil - + return KustomizationAdapter{Kustomization: &kustomizev1.Kustomization{}} case helmv2.HelmReleaseKind: - return &helmv2.HelmReleaseList{}, NewReconcileable(&helmv2.HelmRelease{}), nil - + return HelmReleaseAdapter{HelmRelease: &helmv2.HelmRelease{}} + // TODO: remove all these and let them fall through to the Unstructured case? case sourcev1.GitRepositoryKind: - return &sourcev1.GitRepositoryList{}, NewReconcileable(&sourcev1.GitRepository{}), nil - + return GitRepositoryAdapter{GitRepository: &sourcev1.GitRepository{}} case sourcev1b2.BucketKind: - return &sourcev1b2.BucketList{}, NewReconcileable(&sourcev1b2.Bucket{}), nil - + return BucketAdapter{Bucket: &sourcev1b2.Bucket{}} case sourcev1b2.HelmRepositoryKind: - return &sourcev1b2.HelmRepositoryList{}, NewReconcileable(&sourcev1b2.HelmRepository{}), nil - + return HelmRepositoryAdapter{HelmRepository: &sourcev1b2.HelmRepository{}} case sourcev1b2.HelmChartKind: - return &sourcev1b2.HelmChartList{}, NewReconcileable(&sourcev1b2.HelmChart{}), nil - + return HelmChartAdapter{HelmChart: &sourcev1b2.HelmChart{}} case sourcev1b2.OCIRepositoryKind: - return &sourcev1b2.OCIRepositoryList{}, NewReconcileable(&sourcev1b2.OCIRepository{}), nil - + return OCIRepositoryAdapter{OCIRepository: &sourcev1b2.OCIRepository{}} case reflectorv1.ImageRepositoryKind: - return &reflectorv1.ImageRepositoryList{}, NewReconcileable(&reflectorv1.ImageRepository{}), nil - + return ImageRepositoryAdapter{ImageRepository: &reflectorv1.ImageRepository{}} case imgautomationv1.ImageUpdateAutomationKind: - return &imgautomationv1.ImageUpdateAutomationList{}, NewReconcileable(&imgautomationv1.ImageUpdateAutomation{}), nil + return ImageUpdateAutomationAdapter{ImageUpdateAutomation: &imgautomationv1.ImageUpdateAutomation{}} } - return nil, nil, errors.New("could not find source type") + // Return the UnstructuredAdapter for flux-like resources + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + return UnstructuredAdapter{Unstructured: obj} } diff --git a/core/fluxsync/adapters_test.go b/core/fluxsync/adapters_test.go new file mode 100644 index 0000000000..036a146487 --- /dev/null +++ b/core/fluxsync/adapters_test.go @@ -0,0 +1,125 @@ +package fluxsync + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestGetLastHandledReconcileRequest(t *testing.T) { + g := NewGomegaWithT(t) + + obj := &UnstructuredAdapter{ + Unstructured: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "status": map[string]interface{}{ + "lastHandledReconcileAt": "2023-10-20T10:10:10Z", + }, + }, + }, + } + + expected := "2023-10-20T10:10:10Z" + got := obj.GetLastHandledReconcileRequest() + g.Expect(got).To(Equal(expected)) +} + +func TestGetConditions(t *testing.T) { + g := NewGomegaWithT(t) + + condition := v1.Condition{ + Type: "Ready", + Status: "True", + } + unstructuredCondition, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(&condition) + + obj := &UnstructuredAdapter{ + Unstructured: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "status": map[string]interface{}{ + "conditions": []interface{}{unstructuredCondition}, + }, + }, + }, + } + + conditions := obj.GetConditions() + g.Expect(conditions).To(HaveLen(1)) + g.Expect(conditions[0].Type).To(Equal(condition.Type)) + g.Expect(conditions[0].Status).To(Equal(condition.Status)) +} + +func TestSetSuspended(t *testing.T) { + g := NewGomegaWithT(t) + + obj := &UnstructuredAdapter{ + Unstructured: &unstructured.Unstructured{ + Object: make(map[string]interface{}), + }, + } + + err := obj.SetSuspended(true) + g.Expect(err).NotTo(HaveOccurred()) + suspend, _, _ := unstructured.NestedBool(obj.Object, "spec", "suspend") + g.Expect(suspend).To(BeTrue()) +} + +func TestDeepCopyClientObject(t *testing.T) { + g := NewGomegaWithT(t) + + obj := &UnstructuredAdapter{ + Unstructured: &unstructured.Unstructured{ + Object: map[string]interface{}{"key": "value"}, + }, + } + + objCopy := obj.DeepCopyClientObject().(*unstructured.Unstructured) + g.Expect(objCopy.Object).To(Equal(obj.Object)) + g.Expect(objCopy).ToNot(BeIdenticalTo(obj)) +} + +func TestAsClientObjectCompatibilityWithTestClient(t *testing.T) { + g := NewGomegaWithT(t) + + scheme := runtime.NewScheme() + + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + + obj := &UnstructuredAdapter{ + Unstructured: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test-cm", + "namespace": "default", + }, + "data": map[string]interface{}{"key": "value"}, + }, + }, + } + + err := cl.Create(context.TODO(), obj.AsClientObject()) + g.Expect(err).NotTo(HaveOccurred()) + + retrieved := &UnstructuredAdapter{ + Unstructured: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + }, + }, + } + err = cl.Get(context.TODO(), client.ObjectKey{Namespace: "default", Name: "test-cm"}, retrieved.AsClientObject()) + g.Expect(err).NotTo(HaveOccurred()) + + // check the data key + data, _, _ := unstructured.NestedStringMap(retrieved.Object, "data") + g.Expect(data).To(Equal(map[string]string{"key": "value"})) +} diff --git a/core/server/inventory.go b/core/server/inventory.go index 62215a9e5b..cc89592217 100644 --- a/core/server/inventory.go +++ b/core/server/inventory.go @@ -37,29 +37,38 @@ func (cs *coreServer) GetInventory(ctx context.Context, msg *pb.GetInventoryRequ return nil, fmt.Errorf("error getting scoped client for cluster=%s: %w", msg.ClusterName, err) } - var entries []*pb.InventoryEntry + var entries []*unstructured.Unstructured switch msg.Kind { case kustomizev1.KustomizationKind: - entries, err = cs.getKustomizationInventory(ctx, msg.ClusterName, client, msg.Name, msg.Namespace, msg.WithChildren) + entries, err = cs.getKustomizationInventory(ctx, client, msg.Name, msg.Namespace) if err != nil { return nil, fmt.Errorf("failed getting kustomization inventory: %w", err) } case helmv2.HelmReleaseKind: - entries, err = cs.getHelmReleaseInventory(ctx, msg.ClusterName, client, msg.Name, msg.Namespace, msg.WithChildren) + entries, err = cs.getHelmReleaseInventory(ctx, client, msg.Name, msg.Namespace) if err != nil { return nil, fmt.Errorf("failed getting helm Release inventory: %w", err) } default: - return nil, fmt.Errorf("unknown kind: %s", msg.Kind) + gvk, err := cs.primaryKinds.Lookup(msg.Kind) + if err != nil { + return nil, err + } + entries, err = getFluxLikeInventory(ctx, client, msg.Name, msg.Namespace, *gvk) + if err != nil { + return nil, fmt.Errorf("failed getting flux like inventory: %w", err) + } } + resources := cs.getInventoryResources(ctx, msg.ClusterName, client, entries, msg.Namespace, msg.WithChildren) + return &pb.GetInventoryResponse{ - Entries: entries, + Entries: resources, }, nil } -func (cs *coreServer) getKustomizationInventory(ctx context.Context, clusterName string, k8sClient client.Client, name, namespace string, withChildren bool) ([]*pb.InventoryEntry, error) { +func (cs *coreServer) getKustomizationInventory(ctx context.Context, k8sClient client.Client, name, namespace string) ([]*unstructured.Unstructured, error) { kust := &kustomizev1.Kustomization{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -79,46 +88,19 @@ func (cs *coreServer) getKustomizationInventory(ctx context.Context, clusterName return nil, nil } - result := []*pb.InventoryEntry{} - resultMu := sync.Mutex{} - - wg := sync.WaitGroup{} - + objects := []*unstructured.Unstructured{} for _, e := range kust.Status.Inventory.Entries { - wg.Add(1) - - go func(ref kustomizev1.ResourceRef) { - defer wg.Done() - - obj, err := resourceRefToUnstructured(ref) - if err != nil { - cs.logger.Error(err, "failed converting inventory entry", "entry", ref) - return - } - - if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(&obj), &obj); err != nil { - cs.logger.Error(err, "failed to get object", "entry", ref) - return - } - - entry, err := cs.unstructuredToInventoryEntry(ctx, clusterName, k8sClient, obj, namespace, withChildren) - if err != nil { - cs.logger.Error(err, "failed converting inventory entry", "entry", ref) - return - } - - resultMu.Lock() - result = append(result, entry) - resultMu.Unlock() - }(e) + obj, err := resourceRefToUnstructured(e) + if err != nil { + return nil, fmt.Errorf("failed converting inventory entry: %w", err) + } + objects = append(objects, &obj) } - wg.Wait() - - return result, nil + return objects, nil } -func (cs *coreServer) getHelmReleaseInventory(ctx context.Context, clusterName string, k8sClient client.Client, name, namespace string, withChildren bool) ([]*pb.InventoryEntry, error) { +func (cs *coreServer) getHelmReleaseInventory(ctx context.Context, k8sClient client.Client, name, namespace string) ([]*unstructured.Unstructured, error) { release := &helmv2.HelmRelease{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -135,10 +117,10 @@ func (cs *coreServer) getHelmReleaseInventory(ctx context.Context, clusterName s return nil, fmt.Errorf("failed to get helm release objects: %w", err) } - if len(objects) == 0 { - return []*pb.InventoryEntry{}, nil - } + return objects, nil +} +func (cs *coreServer) getInventoryResources(ctx context.Context, clusterName string, k8sClient client.Client, objects []*unstructured.Unstructured, namespace string, withChildren bool) []*pb.InventoryEntry { result := []*pb.InventoryEntry{} resultMu := sync.Mutex{} @@ -150,10 +132,6 @@ func (cs *coreServer) getHelmReleaseInventory(ctx context.Context, clusterName s go func(obj unstructured.Unstructured) { defer wg.Done() - if obj.GetNamespace() == "" { - obj.SetNamespace(release.GetReleaseNamespace()) - } - if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(&obj), &obj); err != nil { cs.logger.Error(err, "failed to get object", "entry", obj) return @@ -173,7 +151,7 @@ func (cs *coreServer) getHelmReleaseInventory(ctx context.Context, clusterName s wg.Wait() - return result, nil + return result } // Returns the list of resources applied in the helm chart. @@ -383,3 +361,48 @@ func sanitizeUnstructuredSecret(obj unstructured.Unstructured) (unstructured.Uns return redactedUnstructured, nil } + +func getFluxLikeInventory(ctx context.Context, k8sClient client.Client, name, namespace string, gvk schema.GroupVersionKind) ([]*unstructured.Unstructured, error) { + // Create an unstructured object with the desired GVK (GroupVersionKind) + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + obj.SetName(name) + obj.SetNamespace(namespace) + + // Get the object from the Kubernetes cluster + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(obj), obj); err != nil { + return nil, fmt.Errorf("failed to get kustomization: %w", err) + } + + return parseInventoryFromUnstructured(obj) +} + +func parseInventoryFromUnstructured(obj *unstructured.Unstructured) ([]*unstructured.Unstructured, error) { + content := obj.UnstructuredContent() + + // Check if status.inventory is present + unstructuredInventory, found, err := unstructured.NestedMap(content, "status", "inventory") + if err != nil { + return nil, fmt.Errorf("error getting status.inventory from object: %w", err) + } + if !found { + return nil, fmt.Errorf("status.inventory not found in object %s/%s", obj.GetNamespace(), obj.GetName()) + } + + resourceInventory := &kustomizev1.ResourceInventory{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredInventory, resourceInventory) + if err != nil { + return nil, fmt.Errorf("error converting inventory to resource inventory: %w", err) + } + + objects := []*unstructured.Unstructured{} + for _, entry := range resourceInventory.Entries { + u, err := resourceRefToUnstructured(entry) + if err != nil { + return nil, fmt.Errorf("error converting resource ref to unstructured: %w", err) + } + objects = append(objects, &u) + } + + return objects, nil +} diff --git a/core/server/inventory_internal_test.go b/core/server/inventory_internal_test.go new file mode 100644 index 0000000000..49446d18b7 --- /dev/null +++ b/core/server/inventory_internal_test.go @@ -0,0 +1,220 @@ +package server + +import ( + "context" + "testing" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1" + sourcev1 "github.com/fluxcd/source-controller/api/v1" + . "github.com/onsi/gomega" + "github.com/weaveworks/weave-gitops/pkg/kube" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestGetFluxLikeInventory(t *testing.T) { + g := NewGomegaWithT(t) + + ctx := context.Background() + + ks := &kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-kustomization", + Namespace: "my-namespace", + }, + Spec: kustomizev1.KustomizationSpec{ + SourceRef: kustomizev1.CrossNamespaceSourceReference{ + Kind: sourcev1.GitRepositoryKind, + }, + }, + Status: kustomizev1.KustomizationStatus{ + Inventory: &kustomizev1.ResourceInventory{ + Entries: []kustomizev1.ResourceRef{ + { + ID: "my-namespace_my-deployment_apps_Deployment", + Version: "v1", + }, + }, + }, + }, + } + + scheme, err := kube.CreateScheme() + g.Expect(err).To(BeNil()) + + k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(ks).Build() + + gvk := kustomizev1.GroupVersion.WithKind("Kustomization") + entries, err := getFluxLikeInventory(ctx, k8sClient, ks.Name, ks.Namespace, gvk) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(entries).To(HaveLen(1)) + + expected := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "name": "my-deployment", + "namespace": "my-namespace", + }, + }, + } + + g.Expect(entries[0]).To(Equal(expected)) +} + +func TestParseInventoryFromUnstructured(t *testing.T) { + // inv lives at status.inventory.entries + stdErr := "status.inventory not found in object my-namespace/my-resource" + testCases := []struct { + name string + obj *unstructured.Unstructured + expected []*unstructured.Unstructured + expectedErr string + }{ + { + name: "no status field", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + // include name to make sure its included in the error + "metadata": map[string]interface{}{ + "name": "my-resource", + "namespace": "my-namespace", + }, + }, + }, + expected: nil, + expectedErr: stdErr, + }, + { + name: "empty status", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "my-resource", + "namespace": "my-namespace", + }, + "status": map[string]interface{}{}, + }, + }, + expected: nil, + expectedErr: stdErr, + }, + { + name: "empty inventory", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "status": map[string]interface{}{ + "inventory": map[string]interface{}{}, + }, + }, + }, + expected: nil, + }, + { + name: "mallformed inventory", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "status": map[string]interface{}{ + "inventory": "hi there", + }, + }, + }, + expectedErr: ".status.inventory accessor error: hi there is of the type string, expected map[string]interface{}", + }, + { + name: "empty entry item", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "status": map[string]interface{}{ + "inventory": map[string]interface{}{ + "entries": []interface{}{ + map[string]interface{}{}, + }, + }, + }, + }, + }, + expected: nil, + expectedErr: "unable to parse stored object metadata: ", + }, + { + name: "invalid inventory", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "status": map[string]interface{}{ + "inventory": map[string]interface{}{ + "entries": []interface{}{ + map[string]interface{}{ + "v": "v1", + "id": "foo", + }, + }, + }, + }, + }, + }, + expected: nil, + expectedErr: "unable to parse stored object metadata: foo", + }, + { + name: "valid inventory", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "status": map[string]interface{}{ + "inventory": map[string]interface{}{ + "entries": []interface{}{ + map[string]interface{}{ + "v": "v1", + "id": "my-namespace_my-deployment_apps_Deployment", + }, + map[string]interface{}{ + "v": "v1", + "id": "my-other-namespace_my-configmap__ConfigMap", + }, + }, + }, + }, + }, + }, + expected: []*unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "name": "my-deployment", + "namespace": "my-namespace", + }, + }, + }, + { + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "my-configmap", + "namespace": "my-other-namespace", + }, + }, + }, + }, + }, + } + + for _, tt := range testCases { + // subtests... + t.Run(tt.name, func(t *testing.T) { + g := NewGomegaWithT(t) + // Parse inventory from unstructured + entries, err := parseInventoryFromUnstructured(tt.obj) + + if err != nil || tt.expectedErr != "" { + g.Expect(err).To(MatchError(ContainSubstring(tt.expectedErr))) + } + + g.Expect(entries).To(ConsistOf(tt.expected)) + }) + } +} diff --git a/core/server/suspend.go b/core/server/suspend.go index 3f3411adc1..adf8767aed 100644 --- a/core/server/suspend.go +++ b/core/server/suspend.go @@ -33,12 +33,14 @@ func (cs *coreServer) ToggleSuspendResource(ctx context.Context, msg *pb.ToggleS Namespace: obj.Namespace, } - obj, err := getReconcilableObject(obj.Kind) + gvk, err := cs.primaryKinds.Lookup(obj.Kind) if err != nil { - respErrors = *multierror.Append(fmt.Errorf("converting to reconcilable source: %w", err), respErrors.Errors...) + respErrors = *multierror.Append(fmt.Errorf("looking up GVK for %q: %w", obj.Kind, err), respErrors.Errors...) continue } + obj := fluxsync.ToReconcileable(*gvk) + log := cs.logger.WithValues( "user", principal.ID, "kind", obj.GroupVersionKind().Kind, @@ -53,7 +55,10 @@ func (cs *coreServer) ToggleSuspendResource(ctx context.Context, msg *pb.ToggleS patch := client.MergeFrom(obj.DeepCopyClientObject()) - obj.SetSuspended(msg.Suspend) + err = obj.SetSuspended(msg.Suspend) + if err != nil { + return nil, err + } if msg.Suspend { log.Info("Suspending resource") @@ -68,9 +73,3 @@ func (cs *coreServer) ToggleSuspendResource(ctx context.Context, msg *pb.ToggleS return &pb.ToggleSuspendResourceResponse{}, respErrors.ErrorOrNil() } - -func getReconcilableObject(kind string) (fluxsync.Reconcilable, error) { - _, s, err := fluxsync.ToReconcileable(kind) - - return s, err -} diff --git a/core/server/sync.go b/core/server/sync.go index d54007f58c..267729a3dd 100644 --- a/core/server/sync.go +++ b/core/server/sync.go @@ -2,15 +2,9 @@ package server import ( "context" + "errors" "fmt" - helmv2 "github.com/fluxcd/helm-controller/api/v2beta1" - imgautomationv1 "github.com/fluxcd/image-automation-controller/api/v1beta1" - reflectorv1 "github.com/fluxcd/image-reflector-controller/api/v1beta2" - kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1" - sourcev1 "github.com/fluxcd/source-controller/api/v1" - sourcev1b2 "github.com/fluxcd/source-controller/api/v1beta2" - "github.com/hashicorp/go-multierror" "github.com/weaveworks/weave-gitops/core/fluxsync" pb "github.com/weaveworks/weave-gitops/pkg/api/core" "github.com/weaveworks/weave-gitops/pkg/server/auth" @@ -19,18 +13,18 @@ import ( func (cs *coreServer) SyncFluxObject(ctx context.Context, msg *pb.SyncFluxObjectRequest) (*pb.SyncFluxObjectResponse, error) { principal := auth.Principal(ctx) - respErrors := multierror.Error{} + var syncErr error for _, sync := range msg.Objects { clustersClient, err := cs.clustersManager.GetImpersonatedClient(ctx, principal) if err != nil { - respErrors = *multierror.Append(fmt.Errorf("error getting impersonating client: %w", err), respErrors.Errors...) + syncErr = errors.Join(syncErr, fmt.Errorf("error getting impersonating client: %w", err)) continue } c, err := clustersClient.Scoped(sync.ClusterName) if err != nil { - respErrors = *multierror.Append(fmt.Errorf("getting cluster client: %w", err), respErrors.Errors...) + syncErr = errors.Join(syncErr, fmt.Errorf("getting cluster client: %w", err)) continue } @@ -39,14 +33,15 @@ func (cs *coreServer) SyncFluxObject(ctx context.Context, msg *pb.SyncFluxObject Namespace: sync.Namespace, } - obj, err := getFluxObject(sync.Kind) + gvk, err := cs.primaryKinds.Lookup(sync.Kind) if err != nil { - respErrors = *multierror.Append(fmt.Errorf("error converting to object: %w", err), respErrors.Errors...) + syncErr = errors.Join(syncErr, fmt.Errorf("looking up GVK for %q: %w", sync.Kind, err)) continue } + obj := fluxsync.ToReconcileable(*gvk) if err := c.Get(ctx, key, obj.AsClientObject()); err != nil { - respErrors = *multierror.Append(fmt.Errorf("error getting object: %w", err), respErrors.Errors...) + syncErr = errors.Join(syncErr, fmt.Errorf("error getting object: %w", err)) continue } @@ -54,13 +49,12 @@ func (cs *coreServer) SyncFluxObject(ctx context.Context, msg *pb.SyncFluxObject if msg.WithSource && isAutomation { sourceRef := automation.SourceRef() - _, sourceObj, err := fluxsync.ToReconcileable(sourceRef.Kind()) - + sourceGVK, err := cs.primaryKinds.Lookup(sourceRef.Kind()) if err != nil { - respErrors = *multierror.Append(fmt.Errorf("getting source type for %q: %w", sourceRef.Kind(), err), respErrors.Errors...) - continue + return nil, err } + sourceObj := fluxsync.ToReconcileable(*sourceGVK) sourceNs := sourceRef.Namespace() // sourceRef.Namespace is an optional field in flux @@ -87,12 +81,12 @@ func (cs *coreServer) SyncFluxObject(ctx context.Context, msg *pb.SyncFluxObject log.Info("Syncing resource") if err := fluxsync.RequestReconciliation(ctx, c, sourceKey, sourceGvk); err != nil { - respErrors = *multierror.Append(fmt.Errorf("requesting source reconciliation: %w", err), respErrors.Errors...) + syncErr = errors.Join(syncErr, fmt.Errorf("requesting source reconciliation: %w", err)) continue } if err := fluxsync.WaitForSync(ctx, c, sourceKey, sourceObj); err != nil { - respErrors = *multierror.Append(fmt.Errorf("syncing source: %w", err), respErrors.Errors...) + syncErr = errors.Join(syncErr, fmt.Errorf("syncing source: %w", err)) continue } } @@ -105,42 +99,16 @@ func (cs *coreServer) SyncFluxObject(ctx context.Context, msg *pb.SyncFluxObject ) log.Info("Syncing resource") - gvk := obj.GroupVersionKind() - if err := fluxsync.RequestReconciliation(ctx, c, key, gvk); err != nil { - respErrors = *multierror.Append(fmt.Errorf("requesting reconciliation: %w", err), respErrors.Errors...) + if err := fluxsync.RequestReconciliation(ctx, c, key, *gvk); err != nil { + syncErr = errors.Join(syncErr, fmt.Errorf("requesting reconciliation: %w", err)) continue } if err := fluxsync.WaitForSync(ctx, c, key, obj); err != nil { - respErrors = *multierror.Append(fmt.Errorf("syncing automation: %w", err), respErrors.Errors...) + syncErr = errors.Join(syncErr, fmt.Errorf("syncing automation: %w", err)) continue } } - return &pb.SyncFluxObjectResponse{}, respErrors.ErrorOrNil() -} - -func getFluxObject(kind string) (fluxsync.Reconcilable, error) { - switch kind { - case kustomizev1.KustomizationKind: - return &fluxsync.KustomizationAdapter{Kustomization: &kustomizev1.Kustomization{}}, nil - case helmv2.HelmReleaseKind: - return &fluxsync.HelmReleaseAdapter{HelmRelease: &helmv2.HelmRelease{}}, nil - case sourcev1.GitRepositoryKind: - return &fluxsync.GitRepositoryAdapter{GitRepository: &sourcev1.GitRepository{}}, nil - case sourcev1b2.BucketKind: - return &fluxsync.BucketAdapter{Bucket: &sourcev1b2.Bucket{}}, nil - case sourcev1b2.HelmChartKind: - return &fluxsync.HelmChartAdapter{HelmChart: &sourcev1b2.HelmChart{}}, nil - case sourcev1b2.HelmRepositoryKind: - return &fluxsync.HelmRepositoryAdapter{HelmRepository: &sourcev1b2.HelmRepository{}}, nil - case sourcev1b2.OCIRepositoryKind: - return &fluxsync.OCIRepositoryAdapter{OCIRepository: &sourcev1b2.OCIRepository{}}, nil - case reflectorv1.ImageRepositoryKind: - return &fluxsync.ImageRepositoryAdapter{ImageRepository: &reflectorv1.ImageRepository{}}, nil - case imgautomationv1.ImageUpdateAutomationKind: - return &fluxsync.ImageUpdateAutomationAdapter{ImageUpdateAutomation: &imgautomationv1.ImageUpdateAutomation{}}, nil - } - - return nil, fmt.Errorf("not supported kind: %s", kind) + return &pb.SyncFluxObjectResponse{}, syncErr } diff --git a/core/server/sync_test.go b/core/server/sync_test.go index 35eadabd8a..7052185b99 100644 --- a/core/server/sync_test.go +++ b/core/server/sync_test.go @@ -88,7 +88,7 @@ func TestSync(t *testing.T) { WithSource: true, }, reconcilable: fluxsync.HelmReleaseAdapter{HelmRelease: hr}, - source: fluxsync.NewReconcileable(helmRepo), + source: fluxsync.HelmRepositoryAdapter{HelmRepository: helmRepo}, }, { name: "kustomization no source", msg: &pb.SyncFluxObjectRequest{ @@ -105,7 +105,7 @@ func TestSync(t *testing.T) { WithSource: true, }, reconcilable: fluxsync.KustomizationAdapter{Kustomization: kust}, - source: fluxsync.NewReconcileable(gitRepo), + source: fluxsync.GitRepositoryAdapter{GitRepository: gitRepo}, }, { name: "gitrepository", msg: &pb.SyncFluxObjectRequest{ @@ -171,7 +171,7 @@ func TestSync(t *testing.T) { WithSource: true, }, reconcilable: fluxsync.HelmReleaseAdapter{HelmRelease: hr}, - source: fluxsync.NewReconcileable(helmRepo), + source: fluxsync.HelmRepositoryAdapter{HelmRepository: helmRepo}, }} for _, tt := range tests { diff --git a/ui/hooks/inventory.ts b/ui/hooks/inventory.ts index 4bad1edac4..96966c863d 100644 --- a/ui/hooks/inventory.ts +++ b/ui/hooks/inventory.ts @@ -40,6 +40,7 @@ export function useGetInventory( opts ); } + function convertEntries(entries: InventoryEntry[]) { return entries.map((obj) => { const parsedObj = new FluxObject(obj); diff --git a/ui/index.ts b/ui/index.ts index d8a1c7aee1..b3a45b8aff 100644 --- a/ui/index.ts +++ b/ui/index.ts @@ -92,6 +92,7 @@ import { } from "./contexts/LinkResolverContext"; import { useListAutomations, useSyncFluxObject } from "./hooks/automations"; import { useDebounce, useRequestState } from "./hooks/common"; +import { useListEvents } from "./hooks/events"; import { useFeatureFlags } from "./hooks/featureflags"; import { useListFluxCrds, @@ -99,6 +100,7 @@ import { useToggleSuspend, } from "./hooks/flux"; import { useCheckCRDInstalled } from "./hooks/imageautomation"; +import { useGetInventory } from "./hooks/inventory"; import useNavigation from "./hooks/navigation"; import { useListAlerts, useListProviders } from "./hooks/notifications"; import { useGetObject, useListObjects } from "./hooks/objects"; @@ -270,9 +272,11 @@ export { useDebounce, useFeatureFlags, useGetObject, + useGetInventory, useLinkResolver, useListAlerts, useListAutomations, + useListEvents, useListFluxCrds, useListFluxRuntimeObjects, useListObjects,