diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 108b643d..934cf03c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,4 +1,7 @@ on: + push: + branches: + - main pull_request: name: Test diff --git a/context/context.go b/context/context.go index 92b6665c..d2735f59 100644 --- a/context/context.go +++ b/context/context.go @@ -340,15 +340,23 @@ var ctxHistograms sync.Map var LatencyBuckets = []float64{ float64(10 * time.Millisecond), - float64(50 * time.Millisecond), float64(100 * time.Millisecond), float64(500 * time.Millisecond), float64(1 * time.Second), - float64(5 * time.Second), - float64(30 * time.Second), - float64(1 * time.Minute), - float64(5 * time.Minute), - float64(30 * time.Minute), + float64(10 * time.Second), +} + +var ShortLatencyBuckets = []float64{ + float64(10 * time.Millisecond), + float64(100 * time.Millisecond), + float64(500 * time.Millisecond), +} + +var LongLatencyBuckets = []float64{ + float64(1 * time.Second), + float64(10 * time.Second), + float64(100 * time.Second), + float64(1000 * time.Second), } func (k Context) Histogram(name string, buckets []float64, labels ...string) Histogram { diff --git a/job/job.go b/job/job.go index 00b57902..0f47bef3 100644 --- a/job/job.go +++ b/job/job.go @@ -218,7 +218,7 @@ func (j *JobRuntime) end() { j.Job.statusRing.Add(j.History) j.Context.Counter("job", "name", j.Job.Name, "id", j.Job.ResourceID, "resource", j.Job.ResourceType, "status", j.History.Status).Add(1) - j.Context.Histogram("job_duration", context.LatencyBuckets, "name", j.Job.Name, "id", j.Job.ResourceID, "resource", j.Job.ResourceType, "status", j.History.Status).Since(j.History.TimeStart) + j.Context.Histogram("job_duration", context.LongLatencyBuckets, "name", j.Job.Name, "id", j.Job.ResourceID, "resource", j.Job.ResourceType, "status", j.History.Status).Since(j.History.TimeStart) } func (j *JobRuntime) Failf(message string, args ...interface{}) { diff --git a/tests/config_traversal_test.go b/tests/config_traversal_test.go index 1a85eec4..177ddfaf 100644 --- a/tests/config_traversal_test.go +++ b/tests/config_traversal_test.go @@ -14,22 +14,39 @@ import ( "gorm.io/gorm/clause" ) +func assertTraverseConfig(from models.ConfigItem, relationType string, direction string, to ...models.ConfigItem) { + got := query.TraverseConfig(DefaultContext, from.ID.String(), relationType, direction) + Expect(got).To(EqualConfigs(to...)) +} + +func traverseTemplate(from models.ConfigItem, relationType string, direction string) string { + templateEnv := map[string]any{ + "configID": from.ID.String(), + } + + template := gomplate.Template{ + Expression: fmt.Sprintf("dyn(catalog.traverse(configID, '%s', '%s')).map(i, i.id).join(' ')", relationType, direction), + } + gotExpr, err := DefaultContext.RunTemplate(template, templateEnv) + Expect(err).ToNot(HaveOccurred()) + return gotExpr +} + var _ = ginkgo.Describe("Config traversal", ginkgo.Ordered, func() { ginkgo.It("should be able to traverse config relationships via types", func() { - configItems := map[string]models.ConfigItem{ - "deployment": {ID: uuid.MustParse("dc451afd-4329-4611-a488-61902ec0189f"), Name: utils.Ptr("canary-checker"), Type: utils.Ptr("Kubernetes::Deployment"), ConfigClass: "Deployment"}, - "helm-release-of-deployment": {ID: uuid.MustParse("f24df74f-b290-4896-814c-ecf611b01127"), Name: utils.Ptr("mission-control"), Type: utils.Ptr("Kubernetes::HelmRelease"), ConfigClass: "HelmRelease"}, - "kustomize-of-helm-release": {ID: uuid.MustParse("9258815c-eca3-4256-9beb-975a54c888ab"), Name: utils.Ptr("aws-demo-infra"), Type: utils.Ptr("Kubernetes::Kustomization"), ConfigClass: "Kustomization"}, - "kustomize-of-kustomize": {ID: uuid.MustParse("1a0daf44-e1e7-42bd-80a7-4c7db187c1c9"), Name: utils.Ptr("aws-demo-bootstrap"), Type: utils.Ptr("Kubernetes::Kustomization"), ConfigClass: "Kustomization"}, - } + deployment := models.ConfigItem{ID: uuid.New(), Name: utils.Ptr("canary-checker"), Type: utils.Ptr("Kubernetes::Deployment"), ConfigClass: "Deployment"} + helmRelease := models.ConfigItem{ID: uuid.New(), Name: utils.Ptr("mission-control"), Type: utils.Ptr("Kubernetes::HelmRelease"), ConfigClass: "HelmRelease"} + kustomize := models.ConfigItem{ID: uuid.New(), Name: utils.Ptr("aws-demo-infra"), Type: utils.Ptr("Kubernetes::Kustomization"), ConfigClass: "Kustomization"} + bootstrap := models.ConfigItem{ID: uuid.New(), Name: utils.Ptr("aws-demo-bootstrap"), Type: utils.Ptr("Kubernetes::Kustomization"), ConfigClass: "Kustomization"} + all := []models.ConfigItem{deployment, helmRelease, kustomize, bootstrap} ctx := DefaultContext - err := ctx.DB().Save(lo.Values(configItems)).Error + err := ctx.DB().Save(all).Error Expect(err).ToNot(HaveOccurred()) configRelations := []models.ConfigRelationship{ - {ConfigID: configItems["helm-release-of-deployment"].ID.String(), RelatedID: configItems["deployment"].ID.String(), Relation: "HelmReleaseDeployment"}, - {ConfigID: configItems["kustomize-of-helm-release"].ID.String(), RelatedID: configItems["helm-release-of-deployment"].ID.String(), Relation: "KustomizationHelmRelease"}, - {ConfigID: configItems["kustomize-of-kustomize"].ID.String(), RelatedID: configItems["kustomize-of-helm-release"].ID.String(), Relation: "KustomizationKustomization"}, + {ConfigID: helmRelease.ID.String(), RelatedID: deployment.ID.String(), Relation: "HelmReleaseDeployment"}, + {ConfigID: kustomize.ID.String(), RelatedID: helmRelease.ID.String(), Relation: "KustomizationHelmRelease"}, + {ConfigID: bootstrap.ID.String(), RelatedID: kustomize.ID.String(), Relation: "KustomizationKustomization"}, } err = ctx.DB().Clauses(clause.OnConflict{DoNothing: true}).Save(configRelations).Error Expect(err).ToNot(HaveOccurred()) @@ -37,92 +54,46 @@ var _ = ginkgo.Describe("Config traversal", ginkgo.Ordered, func() { err = query.SyncConfigCache(DefaultContext) Expect(err).ToNot(HaveOccurred()) - got := query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::HelmRelease", "incoming") - Expect(got).ToNot(BeNil()) - Expect(got[0].ID.String()).To(Equal(configItems["helm-release-of-deployment"].ID.String())) + assertTraverseConfig(deployment, "Kubernetes::HelmRelease", "incoming", helmRelease) - got = query.TraverseConfig(DefaultContext, configItems["helm-release-of-deployment"].ID.String(), "Kubernetes::Kustomization", "incoming") - Expect(got).ToNot(BeNil()) - Expect(len(got)).To(Equal(2)) - Expect(got[0].ID.String()).To(Equal(configItems["kustomize-of-helm-release"].ID.String())) - Expect(got[1].ID.String()).To(Equal(configItems["kustomize-of-kustomize"].ID.String())) + assertTraverseConfig(helmRelease, "Kubernetes::Kustomization", "incoming", kustomize, bootstrap) - got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::HelmRelease/Kubernetes::Kustomization", "incoming") - Expect(got).ToNot(BeNil()) - Expect(len(got)).To(Equal(2)) - Expect(got[0].ID.String()).To(Equal(configItems["kustomize-of-helm-release"].ID.String())) - Expect(got[1].ID.String()).To(Equal(configItems["kustomize-of-kustomize"].ID.String())) + assertTraverseConfig(deployment, "Kubernetes::Kustomization", "incoming", bootstrap, kustomize) - got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::Kustomization", "incoming") - Expect(got).ToNot(BeNil()) - Expect(len(got)).To(Equal(2)) - Expect(got[0].ID.String()).To(Equal(configItems["kustomize-of-helm-release"].ID.String())) - Expect(got[1].ID.String()).To(Equal(configItems["kustomize-of-kustomize"].ID.String())) + assertTraverseConfig(deployment, "Kubernetes::HelmRelease/Kubernetes::Kustomization", "incoming", kustomize, bootstrap) - got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::Kustomization/Kubernetes::Kustomization", "incoming") + got := query.TraverseConfig(DefaultContext, deployment.ID.String(), "Kubernetes::Kustomization/Kubernetes::Kustomization", "incoming") Expect(got).ToNot(BeNil()) // This should only return 1 object since we are // passing explicit path for the boostrap kustomization Expect(len(got)).To(Equal(1)) - Expect(got[0].ID.String()).To(Equal(configItems["kustomize-of-kustomize"].ID.String())) + Expect(got[0].ID.String()).To(Equal(bootstrap.ID.String())) - got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::Pod", "incoming") - Expect(got).To(BeNil()) + assertTraverseConfig(deployment, "Kubernetes::Pod", "incoming") - got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::Node", "incoming") - Expect(got).To(BeNil()) + assertTraverseConfig(deployment, "Kubernetes::Node", "incoming") + assertTraverseConfig(deployment, "Kubernetes::HelmRelease/Kubernetes::Node", "incoming") - got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::HelmRelease/Kubernetes::Node", "incoming") - Expect(got).To(BeNil()) + assertTraverseConfig(helmRelease, "Kubernetes::Deployment", "outgoing", deployment) - got = query.TraverseConfig(DefaultContext, configItems["helm-release-of-deployment"].ID.String(), "Kubernetes::Deployment", "outgoing") - Expect(got).ToNot(BeNil()) - Expect(got[0].ID.String()).To(Equal(configItems["deployment"].ID.String())) + assertTraverseConfig(kustomize, "Kubernetes::HelmRelease", "outgoing", helmRelease) - got = query.TraverseConfig(DefaultContext, configItems["kustomize-of-helm-release"].ID.String(), "Kubernetes::HelmRelease", "outgoing") - Expect(got).ToNot(BeNil()) - Expect(got[0].ID.String()).To(Equal(configItems["helm-release-of-deployment"].ID.String())) + assertTraverseConfig(kustomize, "Kubernetes::Deployment", "outgoing", deployment) - got = query.TraverseConfig(DefaultContext, configItems["kustomize-of-helm-release"].ID.String(), "Kubernetes::Deployment", "outgoing") - Expect(got).ToNot(BeNil()) - Expect(got[0].ID.String()).To(Equal(configItems["deployment"].ID.String())) + Expect(traverseTemplate(deployment, "Kubernetes::HelmRelease", "incoming")). + To(Equal(helmRelease.ID.String())) - // Test with CEL Exprs - templateEnv := map[string]any{ - "configID": configItems["deployment"].ID.String(), - "configIDKustomize": configItems["kustomize-of-helm-release"].ID.String(), - } + Expect(traverseTemplate(deployment, "Kubernetes::Kustomization", "incoming")). + To(Equal(fmt.Sprintf("%s %s", kustomize.ID, bootstrap.ID))) - template := gomplate.Template{ - Expression: "catalog.traverse(configID, 'Kubernetes::HelmRelease', 'incoming')[0].id", - } - gotExpr, err := DefaultContext.RunTemplate(template, templateEnv) - Expect(err).ToNot(HaveOccurred()) - Expect(gotExpr).To(Equal(configItems["helm-release-of-deployment"].ID.String())) - - template = gomplate.Template{ - Expression: "catalog.traverse(configID, 'Kubernetes::Kustomization', 'incoming')[0].name", - } - gotExpr, err = DefaultContext.RunTemplate(template, templateEnv) - Expect(err).ToNot(HaveOccurred()) - Expect(gotExpr).To(Equal(*configItems["kustomize-of-helm-release"].Name)) + Expect(traverseTemplate(deployment, "Kubernetes::Pod", "incoming")). + To(BeEmpty()) - template = gomplate.Template{ - Expression: "catalog.traverse(configID, 'Kubernetes::Pod', 'incoming')[0].name", - } - gotExpr, err = DefaultContext.RunTemplate(template, templateEnv) - Expect(err).To(HaveOccurred()) - Expect(gotExpr).To(Equal("")) - - template = gomplate.Template{ - Expression: "catalog.traverse(configIDKustomize, 'Kubernetes::Deployment', 'outgoing')[0].name", - } - gotExpr, err = DefaultContext.RunTemplate(template, templateEnv) - Expect(err).ToNot(HaveOccurred()) - Expect(gotExpr).To(Equal(*configItems["deployment"].Name)) + Expect(traverseTemplate(kustomize, "Kubernetes::Deployment", "outgoing")). + To(Equal(deployment.ID.String())) // Testing struct templater - t := DefaultContext.NewStructTemplater(map[string]any{"id": configItems["deployment"].ID.String()}, "", nil) + t := DefaultContext.NewStructTemplater(map[string]any{"id": deployment.ID.String()}, "", nil) inlineStruct := struct { Name string Type string @@ -133,14 +104,14 @@ var _ = ginkgo.Describe("Config traversal", ginkgo.Ordered, func() { err = t.Walk(&inlineStruct) Expect(err).ToNot(HaveOccurred()) - Expect(inlineStruct.Name).To(Equal(fmt.Sprintf("Name is %s", *configItems["kustomize-of-helm-release"].Name))) - Expect(inlineStruct.Type).To(Equal(fmt.Sprintf("Type is %s", *configItems["kustomize-of-helm-release"].Type))) + Expect(inlineStruct.Name).To(Equal(fmt.Sprintf("Name is %s", *kustomize.Name))) + Expect(inlineStruct.Type).To(Equal(fmt.Sprintf("Type is %s", *kustomize.Type))) // Cleanup for normal tests to pass - err = ctx.DB().Where("config_id in ?", lo.Map(lo.Values(configItems), func(c models.ConfigItem, _ int) string { return c.ID.String() })).Delete(&models.ConfigRelationship{}).Error + err = ctx.DB().Where("config_id in ?", lo.Map(all, func(c models.ConfigItem, _ int) string { return c.ID.String() })).Delete(&models.ConfigRelationship{}).Error Expect(err).ToNot(HaveOccurred()) - err = ctx.DB().Delete(lo.Values(configItems)).Error + err = ctx.DB().Delete(all).Error Expect(err).ToNot(HaveOccurred()) }) diff --git a/tests/matchers.go b/tests/matchers.go new file mode 100644 index 00000000..daae9b6a --- /dev/null +++ b/tests/matchers.go @@ -0,0 +1,45 @@ +package tests + +import ( + "fmt" + + "github.com/flanksource/duty/models" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" + "github.com/samber/lo" +) + +type EqualsConfigItems struct { + Expected []models.ConfigItem +} + +func (matcher *EqualsConfigItems) Match(actual interface{}) (bool, error) { + to, ok := actual.([]models.ConfigItem) + if !ok { + return false, fmt.Errorf("EqualsConfigItems must be passed []models.ConfigItem. Got\n%+s", actual) + } + + got := lo.Map(to, + func(i models.ConfigItem, _ int) string { return i.ID.String() }) + + expected := lo.Map(matcher.Expected, + func(i models.ConfigItem, _ int) string { return i.ID.String() }) + Expect(len(got)).To(Equal(len(expected))) + if lo.Every(got, expected) { + return true, nil + } + return false, nil +} + +func (matcher *EqualsConfigItems) FailureMessage(actual interface{}) string { + return fmt.Sprintf("Expected %s to equal %s", actual, matcher.Expected) +} + +func (matcher *EqualsConfigItems) NegatedFailureMessage(actual interface{}) string { + return fmt.Sprintf("Expected %s to not equal %s", actual, matcher.Expected) +} +func EqualConfigs(expected ...models.ConfigItem) types.GomegaMatcher { + return &EqualsConfigItems{ + Expected: expected, + } +} diff --git a/upstream/controllers.go b/upstream/controllers.go index 0cbb6d31..667eccc5 100644 --- a/upstream/controllers.go +++ b/upstream/controllers.go @@ -33,7 +33,7 @@ func AgentAuthMiddleware(agentCache *cache.Cache) func(echo.HandlerFunc) echo.Ha return next(c) } - histogram := ctx.Histogram("agent_auth_middleware", context.LatencyBuckets, StatusLabel, "") + histogram := ctx.Histogram("agent_auth_middleware", context.ShortLatencyBuckets, StatusLabel, "") agentName := c.QueryParam(AgentNameQueryParam) if agentName == "" { @@ -141,7 +141,7 @@ func PingHandler(c echo.Context) error { start := time.Now() ctx := c.Request().Context().(context.Context) - histogram := ctx.Histogram("push_queue_ping_handler", context.LatencyBuckets, StatusLabel, "", AgentLabel, ctx.Agent().ID.String()) + histogram := ctx.Histogram("push_queue_ping_handler", context.ShortLatencyBuckets, StatusLabel, "", AgentLabel, ctx.Agent().ID.String()) if err := UpdateAgentLastSeen(ctx, ctx.Agent().ID); err != nil { histogram.Label(StatusLabel, StatusError).Since(start) diff --git a/views/006_config_views.sql b/views/006_config_views.sql index abd84b90..d45e3ec4 100644 --- a/views/006_config_views.sql +++ b/views/006_config_views.sql @@ -438,7 +438,7 @@ IF type_filter NOT IN ('incoming', 'outgoing', 'all') THEN END IF; IF type_filter = 'outgoing' THEN - RETURN query + RETURN query WITH RECURSIVE cte (config_id, related_id, relation, direction, depth) AS ( SELECT parent.config_id, parent.related_id, parent.relation, 'outgoing', 1::int FROM config_relationships parent @@ -455,10 +455,10 @@ IF type_filter = 'outgoing' THEN AND deleted_at IS NULL ) CYCLE config_id SET is_cycle USING path SELECT DISTINCT cte.config_id, cte.related_id, cte.relation as "relation_type", type_filter as "direction", cte.depth - FROM cte + FROM cte ORDER BY cte.depth asc; ELSIF type_filter = 'incoming' THEN - RETURN query + RETURN query WITH RECURSIVE cte (config_id, related_id, relation, direction, depth) AS ( SELECT parent.config_id, parent.related_id as related_id, parent.relation, 'incoming', 1::int FROM config_relationships parent @@ -475,13 +475,13 @@ ELSIF type_filter = 'incoming' THEN AND deleted_at IS NULL ) CYCLE config_id SET is_cycle USING path SELECT DISTINCT cte.config_id, cte.related_id, cte.relation AS "relation_type", type_filter as "direction", cte.depth - FROM cte + FROM cte ORDER BY cte.depth asc; ELSE RETURN query - SELECT * FROM config_relationships_recursive(config_id, 'incoming', max_depth, incoming_relation, outgoing_relation) - UNION - SELECT * FROM config_relationships_recursive(config_id, 'outgoing', max_depth, incoming_relation, outgoing_relation); + SELECT * FROM config_relationships_recursive(config_id, 'incoming', max_depth, incoming_relation, outgoing_relation) + UNION + SELECT * FROM config_relationships_recursive(config_id, 'outgoing', max_depth, incoming_relation, outgoing_relation); END IF; END; @@ -532,7 +532,7 @@ BEGIN LEFT JOIN edges ON edges.id = all_ids.id GROUP BY all_ids.id ) - SELECT + SELECT configs.id, configs.name, configs.type, @@ -550,7 +550,7 @@ BEGIN configs.health, configs.ready, configs.status - FROM configs + FROM configs LEFT JOIN grouped_related_ids ON configs.id = grouped_related_ids.id WHERE configs.id IN (SELECT DISTINCT all_ids.id FROM all_ids) AND (include_deleted_configs OR configs.deleted_at IS NULL)