chore: tune metric latency buckets

.github/workflows/test.yaml

@@ -1,4 +1,7 @@
 on:
+  push:
+    branches:
+      - main
   pull_request:
 
 name: Test

context/context.go

@@ -340,15 +340,23 @@ var ctxHistograms sync.Map
 
 var LatencyBuckets = []float64{
 	float64(10 * time.Millisecond),
-	float64(50 * time.Millisecond),
 	float64(100 * time.Millisecond),
 	float64(500 * time.Millisecond),
 	float64(1 * time.Second),
-	float64(5 * time.Second),
-	float64(30 * time.Second),
-	float64(1 * time.Minute),
-	float64(5 * time.Minute),
-	float64(30 * time.Minute),
+	float64(10 * time.Second),
+}
+
+var ShortLatencyBuckets = []float64{
+	float64(10 * time.Millisecond),
+	float64(100 * time.Millisecond),
+	float64(500 * time.Millisecond),
+}
+
+var LongLatencyBuckets = []float64{
+	float64(1 * time.Second),
+	float64(10 * time.Second),
+	float64(100 * time.Second),
+	float64(1000 * time.Second),
 }
 
 func (k Context) Histogram(name string, buckets []float64, labels ...string) Histogram {

job/job.go

@@ -218,7 +218,7 @@ func (j *JobRuntime) end() {
 	j.Job.statusRing.Add(j.History)
 
 	j.Context.Counter("job", "name", j.Job.Name, "id", j.Job.ResourceID, "resource", j.Job.ResourceType, "status", j.History.Status).Add(1)
-	j.Context.Histogram("job_duration", context.LatencyBuckets, "name", j.Job.Name, "id", j.Job.ResourceID, "resource", j.Job.ResourceType, "status", j.History.Status).Since(j.History.TimeStart)
+	j.Context.Histogram("job_duration", context.LongLatencyBuckets, "name", j.Job.Name, "id", j.Job.ResourceID, "resource", j.Job.ResourceType, "status", j.History.Status).Since(j.History.TimeStart)
 }
 
 func (j *JobRuntime) Failf(message string, args ...interface{}) {

tests/config_traversal_test.go

@@ -14,115 +14,86 @@ import (
 	"gorm.io/gorm/clause"
 )
 
+func assertTraverseConfig(from models.ConfigItem, relationType string, direction string, to ...models.ConfigItem) {
+	got := query.TraverseConfig(DefaultContext, from.ID.String(), relationType, direction)
+	Expect(got).To(EqualConfigs(to...))
+}
+
+func traverseTemplate(from models.ConfigItem, relationType string, direction string) string {
+	templateEnv := map[string]any{
+		"configID": from.ID.String(),
+	}
+
+	template := gomplate.Template{
+		Expression: fmt.Sprintf("dyn(catalog.traverse(configID, '%s', '%s')).map(i, i.id).join(' ')", relationType, direction),
+	}
+	gotExpr, err := DefaultContext.RunTemplate(template, templateEnv)
+	Expect(err).ToNot(HaveOccurred())
+	return gotExpr
+}
+
 var _ = ginkgo.Describe("Config traversal", ginkgo.Ordered, func() {
 	ginkgo.It("should be able to traverse config relationships via types", func() {
-		configItems := map[string]models.ConfigItem{
-			"deployment":                 {ID: uuid.MustParse("dc451afd-4329-4611-a488-61902ec0189f"), Name: utils.Ptr("canary-checker"), Type: utils.Ptr("Kubernetes::Deployment"), ConfigClass: "Deployment"},
-			"helm-release-of-deployment": {ID: uuid.MustParse("f24df74f-b290-4896-814c-ecf611b01127"), Name: utils.Ptr("mission-control"), Type: utils.Ptr("Kubernetes::HelmRelease"), ConfigClass: "HelmRelease"},
-			"kustomize-of-helm-release":  {ID: uuid.MustParse("9258815c-eca3-4256-9beb-975a54c888ab"), Name: utils.Ptr("aws-demo-infra"), Type: utils.Ptr("Kubernetes::Kustomization"), ConfigClass: "Kustomization"},
-			"kustomize-of-kustomize":     {ID: uuid.MustParse("1a0daf44-e1e7-42bd-80a7-4c7db187c1c9"), Name: utils.Ptr("aws-demo-bootstrap"), Type: utils.Ptr("Kubernetes::Kustomization"), ConfigClass: "Kustomization"},
-		}
+		deployment := models.ConfigItem{ID: uuid.New(), Name: utils.Ptr("canary-checker"), Type: utils.Ptr("Kubernetes::Deployment"), ConfigClass: "Deployment"}
+		helmRelease := models.ConfigItem{ID: uuid.New(), Name: utils.Ptr("mission-control"), Type: utils.Ptr("Kubernetes::HelmRelease"), ConfigClass: "HelmRelease"}
+		kustomize := models.ConfigItem{ID: uuid.New(), Name: utils.Ptr("aws-demo-infra"), Type: utils.Ptr("Kubernetes::Kustomization"), ConfigClass: "Kustomization"}
+		bootstrap := models.ConfigItem{ID: uuid.New(), Name: utils.Ptr("aws-demo-bootstrap"), Type: utils.Ptr("Kubernetes::Kustomization"), ConfigClass: "Kustomization"}
+		all := []models.ConfigItem{deployment, helmRelease, kustomize, bootstrap}
 		ctx := DefaultContext
-		err := ctx.DB().Save(lo.Values(configItems)).Error
+		err := ctx.DB().Save(all).Error
 		Expect(err).ToNot(HaveOccurred())
 
 		configRelations := []models.ConfigRelationship{
-			{ConfigID: configItems["helm-release-of-deployment"].ID.String(), RelatedID: configItems["deployment"].ID.String(), Relation: "HelmReleaseDeployment"},
-			{ConfigID: configItems["kustomize-of-helm-release"].ID.String(), RelatedID: configItems["helm-release-of-deployment"].ID.String(), Relation: "KustomizationHelmRelease"},
-			{ConfigID: configItems["kustomize-of-kustomize"].ID.String(), RelatedID: configItems["kustomize-of-helm-release"].ID.String(), Relation: "KustomizationKustomization"},
+			{ConfigID: helmRelease.ID.String(), RelatedID: deployment.ID.String(), Relation: "HelmReleaseDeployment"},
+			{ConfigID: kustomize.ID.String(), RelatedID: helmRelease.ID.String(), Relation: "KustomizationHelmRelease"},
+			{ConfigID: bootstrap.ID.String(), RelatedID: kustomize.ID.String(), Relation: "KustomizationKustomization"},
 		}
 		err = ctx.DB().Clauses(clause.OnConflict{DoNothing: true}).Save(configRelations).Error
 		Expect(err).ToNot(HaveOccurred())
 
 		err = query.SyncConfigCache(DefaultContext)
 		Expect(err).ToNot(HaveOccurred())
 
-		got := query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::HelmRelease", "incoming")
-		Expect(got).ToNot(BeNil())
-		Expect(got[0].ID.String()).To(Equal(configItems["helm-release-of-deployment"].ID.String()))
+		assertTraverseConfig(deployment, "Kubernetes::HelmRelease", "incoming", helmRelease)
 
-		got = query.TraverseConfig(DefaultContext, configItems["helm-release-of-deployment"].ID.String(), "Kubernetes::Kustomization", "incoming")
-		Expect(got).ToNot(BeNil())
-		Expect(len(got)).To(Equal(2))
-		Expect(got[0].ID.String()).To(Equal(configItems["kustomize-of-helm-release"].ID.String()))
-		Expect(got[1].ID.String()).To(Equal(configItems["kustomize-of-kustomize"].ID.String()))
+		assertTraverseConfig(helmRelease, "Kubernetes::Kustomization", "incoming", kustomize, bootstrap)
 
-		got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::HelmRelease/Kubernetes::Kustomization", "incoming")
-		Expect(got).ToNot(BeNil())
-		Expect(len(got)).To(Equal(2))
-		Expect(got[0].ID.String()).To(Equal(configItems["kustomize-of-helm-release"].ID.String()))
-		Expect(got[1].ID.String()).To(Equal(configItems["kustomize-of-kustomize"].ID.String()))
+		assertTraverseConfig(deployment, "Kubernetes::Kustomization", "incoming", bootstrap, kustomize)
 
-		got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::Kustomization", "incoming")
-		Expect(got).ToNot(BeNil())
-		Expect(len(got)).To(Equal(2))
-		Expect(got[0].ID.String()).To(Equal(configItems["kustomize-of-helm-release"].ID.String()))
-		Expect(got[1].ID.String()).To(Equal(configItems["kustomize-of-kustomize"].ID.String()))
+		assertTraverseConfig(deployment, "Kubernetes::HelmRelease/Kubernetes::Kustomization", "incoming", kustomize, bootstrap)
 
-		got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::Kustomization/Kubernetes::Kustomization", "incoming")
+		got := query.TraverseConfig(DefaultContext, deployment.ID.String(), "Kubernetes::Kustomization/Kubernetes::Kustomization", "incoming")
 		Expect(got).ToNot(BeNil())
 		// This should only return 1 object since we are
 		// passing explicit path for the boostrap kustomization
 		Expect(len(got)).To(Equal(1))
-		Expect(got[0].ID.String()).To(Equal(configItems["kustomize-of-kustomize"].ID.String()))
+		Expect(got[0].ID.String()).To(Equal(bootstrap.ID.String()))
 
-		got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::Pod", "incoming")
-		Expect(got).To(BeNil())
+		assertTraverseConfig(deployment, "Kubernetes::Pod", "incoming")
 
-		got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::Node", "incoming")
-		Expect(got).To(BeNil())
+		assertTraverseConfig(deployment, "Kubernetes::Node", "incoming")
+		assertTraverseConfig(deployment, "Kubernetes::HelmRelease/Kubernetes::Node", "incoming")
 
-		got = query.TraverseConfig(DefaultContext, configItems["deployment"].ID.String(), "Kubernetes::HelmRelease/Kubernetes::Node", "incoming")
-		Expect(got).To(BeNil())
+		assertTraverseConfig(helmRelease, "Kubernetes::Deployment", "outgoing", deployment)
 
-		got = query.TraverseConfig(DefaultContext, configItems["helm-release-of-deployment"].ID.String(), "Kubernetes::Deployment", "outgoing")
-		Expect(got).ToNot(BeNil())
-		Expect(got[0].ID.String()).To(Equal(configItems["deployment"].ID.String()))
+		assertTraverseConfig(kustomize, "Kubernetes::HelmRelease", "outgoing", helmRelease)
 
-		got = query.TraverseConfig(DefaultContext, configItems["kustomize-of-helm-release"].ID.String(), "Kubernetes::HelmRelease", "outgoing")
-		Expect(got).ToNot(BeNil())
-		Expect(got[0].ID.String()).To(Equal(configItems["helm-release-of-deployment"].ID.String()))
+		assertTraverseConfig(kustomize, "Kubernetes::Deployment", "outgoing", deployment)
 
-		got = query.TraverseConfig(DefaultContext, configItems["kustomize-of-helm-release"].ID.String(), "Kubernetes::Deployment", "outgoing")
-		Expect(got).ToNot(BeNil())
-		Expect(got[0].ID.String()).To(Equal(configItems["deployment"].ID.String()))
+		Expect(traverseTemplate(deployment, "Kubernetes::HelmRelease", "incoming")).
+			To(Equal(helmRelease.ID.String()))
 
-		// Test with CEL Exprs
-		templateEnv := map[string]any{
-			"configID":          configItems["deployment"].ID.String(),
-			"configIDKustomize": configItems["kustomize-of-helm-release"].ID.String(),
-		}
+		Expect(traverseTemplate(deployment, "Kubernetes::Kustomization", "incoming")).
+			To(Equal(fmt.Sprintf("%s %s", kustomize.ID, bootstrap.ID)))
 
-		template := gomplate.Template{
-			Expression: "catalog.traverse(configID, 'Kubernetes::HelmRelease', 'incoming')[0].id",
-		}
-		gotExpr, err := DefaultContext.RunTemplate(template, templateEnv)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(gotExpr).To(Equal(configItems["helm-release-of-deployment"].ID.String()))
-
-		template = gomplate.Template{
-			Expression: "catalog.traverse(configID, 'Kubernetes::Kustomization', 'incoming')[0].name",
-		}
-		gotExpr, err = DefaultContext.RunTemplate(template, templateEnv)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(gotExpr).To(Equal(*configItems["kustomize-of-helm-release"].Name))
+		Expect(traverseTemplate(deployment, "Kubernetes::Pod", "incoming")).
+			To(BeEmpty())
 
-		template = gomplate.Template{
-			Expression: "catalog.traverse(configID, 'Kubernetes::Pod', 'incoming')[0].name",
-		}
-		gotExpr, err = DefaultContext.RunTemplate(template, templateEnv)
-		Expect(err).To(HaveOccurred())
-		Expect(gotExpr).To(Equal(""))
-
-		template = gomplate.Template{
-			Expression: "catalog.traverse(configIDKustomize, 'Kubernetes::Deployment', 'outgoing')[0].name",
-		}
-		gotExpr, err = DefaultContext.RunTemplate(template, templateEnv)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(gotExpr).To(Equal(*configItems["deployment"].Name))
+		Expect(traverseTemplate(kustomize, "Kubernetes::Deployment", "outgoing")).
+			To(Equal(deployment.ID.String()))
 
 		// Testing struct templater
-		t := DefaultContext.NewStructTemplater(map[string]any{"id": configItems["deployment"].ID.String()}, "", nil)
+		t := DefaultContext.NewStructTemplater(map[string]any{"id": deployment.ID.String()}, "", nil)
 		inlineStruct := struct {
 			Name string
 			Type string
@@ -133,14 +104,14 @@ var _ = ginkgo.Describe("Config traversal", ginkgo.Ordered, func() {
 
 		err = t.Walk(&inlineStruct)
 		Expect(err).ToNot(HaveOccurred())
-		Expect(inlineStruct.Name).To(Equal(fmt.Sprintf("Name is %s", *configItems["kustomize-of-helm-release"].Name)))
-		Expect(inlineStruct.Type).To(Equal(fmt.Sprintf("Type is %s", *configItems["kustomize-of-helm-release"].Type)))
+		Expect(inlineStruct.Name).To(Equal(fmt.Sprintf("Name is %s", *kustomize.Name)))
+		Expect(inlineStruct.Type).To(Equal(fmt.Sprintf("Type is %s", *kustomize.Type)))
 
 		// Cleanup for normal tests to pass
-		err = ctx.DB().Where("config_id in ?", lo.Map(lo.Values(configItems), func(c models.ConfigItem, _ int) string { return c.ID.String() })).Delete(&models.ConfigRelationship{}).Error
+		err = ctx.DB().Where("config_id in ?", lo.Map(all, func(c models.ConfigItem, _ int) string { return c.ID.String() })).Delete(&models.ConfigRelationship{}).Error
 		Expect(err).ToNot(HaveOccurred())
 
-		err = ctx.DB().Delete(lo.Values(configItems)).Error
+		err = ctx.DB().Delete(all).Error
 		Expect(err).ToNot(HaveOccurred())
 	})
 

tests/matchers.go

@@ -0,0 +1,45 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/flanksource/duty/models"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/types"
+	"github.com/samber/lo"
+)
+
+type EqualsConfigItems struct {
+	Expected []models.ConfigItem
+}
+
+func (matcher *EqualsConfigItems) Match(actual interface{}) (bool, error) {
+	to, ok := actual.([]models.ConfigItem)
+	if !ok {
+		return false, fmt.Errorf("EqualsConfigItems must be passed []models.ConfigItem. Got\n%+s", actual)
+	}
+
+	got := lo.Map(to,
+		func(i models.ConfigItem, _ int) string { return i.ID.String() })
+
+	expected := lo.Map(matcher.Expected,
+		func(i models.ConfigItem, _ int) string { return i.ID.String() })
+	Expect(len(got)).To(Equal(len(expected)))
+	if lo.Every(got, expected) {
+		return true, nil
+	}
+	return false, nil
+}
+
+func (matcher *EqualsConfigItems) FailureMessage(actual interface{}) string {
+	return fmt.Sprintf("Expected %s to equal %s", actual, matcher.Expected)
+}
+
+func (matcher *EqualsConfigItems) NegatedFailureMessage(actual interface{}) string {
+	return fmt.Sprintf("Expected %s to not equal %s", actual, matcher.Expected)
+}
+func EqualConfigs(expected ...models.ConfigItem) types.GomegaMatcher {
+	return &EqualsConfigItems{
+		Expected: expected,
+	}
+}

upstream/controllers.go

@@ -33,7 +33,7 @@ func AgentAuthMiddleware(agentCache *cache.Cache) func(echo.HandlerFunc) echo.Ha
 				return next(c)
 			}
 
-			histogram := ctx.Histogram("agent_auth_middleware", context.LatencyBuckets, StatusLabel, "")
+			histogram := ctx.Histogram("agent_auth_middleware", context.ShortLatencyBuckets, StatusLabel, "")
 
 			agentName := c.QueryParam(AgentNameQueryParam)
 			if agentName == "" {
@@ -141,7 +141,7 @@ func PingHandler(c echo.Context) error {
 	start := time.Now()
 	ctx := c.Request().Context().(context.Context)
 
-	histogram := ctx.Histogram("push_queue_ping_handler", context.LatencyBuckets, StatusLabel, "", AgentLabel, ctx.Agent().ID.String())
+	histogram := ctx.Histogram("push_queue_ping_handler", context.ShortLatencyBuckets, StatusLabel, "", AgentLabel, ctx.Agent().ID.String())
 
 	if err := UpdateAgentLastSeen(ctx, ctx.Agent().ID); err != nil {
 		histogram.Label(StatusLabel, StatusError).Since(start)

views/006_config_views.sql

@@ -438,7 +438,7 @@ IF type_filter NOT IN ('incoming', 'outgoing', 'all') THEN
 END IF;
 
 IF type_filter = 'outgoing' THEN
-	RETURN query
+  RETURN query
       WITH RECURSIVE cte (config_id, related_id, relation, direction, depth) AS (
         SELECT parent.config_id, parent.related_id, parent.relation, 'outgoing', 1::int
         FROM config_relationships parent
@@ -455,10 +455,10 @@ IF type_filter = 'outgoing' THEN
             AND deleted_at IS NULL
       ) CYCLE config_id SET is_cycle USING path
       SELECT DISTINCT cte.config_id, cte.related_id, cte.relation as "relation_type", type_filter as "direction", cte.depth
-      FROM cte 
+      FROM cte
       ORDER BY cte.depth asc;
 ELSIF type_filter = 'incoming' THEN
-	RETURN query
+  RETURN query
       WITH RECURSIVE cte (config_id, related_id, relation, direction, depth) AS (
         SELECT parent.config_id, parent.related_id as related_id, parent.relation, 'incoming', 1::int
         FROM config_relationships parent
@@ -475,13 +475,13 @@ ELSIF type_filter = 'incoming' THEN
             AND deleted_at IS NULL
       ) CYCLE config_id SET is_cycle USING path
       SELECT DISTINCT cte.config_id, cte.related_id, cte.relation AS "relation_type", type_filter as "direction", cte.depth
-      FROM cte 
+      FROM cte
       ORDER BY cte.depth asc;
 ELSE
   RETURN query
-   		SELECT * FROM config_relationships_recursive(config_id, 'incoming', max_depth, incoming_relation, outgoing_relation)
-   		UNION
-   		SELECT * FROM config_relationships_recursive(config_id, 'outgoing', max_depth, incoming_relation, outgoing_relation);
+      SELECT * FROM config_relationships_recursive(config_id, 'incoming', max_depth, incoming_relation, outgoing_relation)
+      UNION
+      SELECT * FROM config_relationships_recursive(config_id, 'outgoing', max_depth, incoming_relation, outgoing_relation);
 END IF;
 
 END;
@@ -532,7 +532,7 @@ BEGIN
       LEFT JOIN edges ON edges.id = all_ids.id
       GROUP BY all_ids.id
     )
-      SELECT 
+      SELECT
         configs.id,
         configs.name,
         configs.type,
@@ -550,7 +550,7 @@ BEGIN
         configs.health,
         configs.ready,
         configs.status
-      FROM configs 
+      FROM configs
       LEFT JOIN grouped_related_ids ON configs.id = grouped_related_ids.id
       WHERE configs.id IN (SELECT DISTINCT all_ids.id FROM all_ids)
       AND (include_deleted_configs OR configs.deleted_at IS NULL)