diff --git a/query/grammar/grammar_parser.go b/query/grammar/grammar_parser.go index d96aa10a..1d2502d8 100644 --- a/query/grammar/grammar_parser.go +++ b/query/grammar/grammar_parser.go @@ -144,12 +144,19 @@ func stringFromChars(chars interface{}) string { return str } -func ParsePEG(peg string) (any, error) { - +func ParsePEG(peg string) (*types.QueryField, error) { stats := Stats{} v, err := Parse("", []byte(peg), Statistics(&stats, "no match")) + if err != nil { + return nil, fmt.Errorf("error parsing peg: %w", err) + } logger.Infof(logger.Pretty(stats)) - return v, err + + rv, ok := v.(*types.QueryField) + if !ok { + return nil, fmt.Errorf("return type not types.QueryField") + } + return rv, nil } diff --git a/query/grammar/grammar_test.go b/query/grammar/grammar_test.go index 01bd1f0a..f2f07b8c 100644 --- a/query/grammar/grammar_test.go +++ b/query/grammar/grammar_test.go @@ -9,11 +9,9 @@ import ( var _ = Describe("grammer", func() { It("parses", func() { - result, err := ParsePEG("john:doe metadata.name=bob metadata.name!=harry spec.status.reason!=\"failed reson\" -jane johnny type!=pod type!=replicaset namespace!=\"a,b,c\"") logger.Infof(logger.Pretty(result)) Expect(err).To(BeNil()) }) - }) diff --git a/query/models.go b/query/models.go index 12221afc..00f3a2a8 100644 --- a/query/models.go +++ b/query/models.go @@ -5,7 +5,9 @@ import ( "strconv" "strings" + "github.com/flanksource/commons/logger" "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" "github.com/flanksource/duty/types" "github.com/google/uuid" "github.com/pkg/errors" @@ -51,7 +53,7 @@ var ConfigQueryModel = QueryModel{ JSONColumn: "config", Custom: map[string]func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error){ "limit": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) { - if i, err := strconv.Atoi(fmt.Sprintf("%s", val)); err == nil { + if i, err := strconv.Atoi(val); err == nil { return tx.Limit(i), nil } else { return nil, err @@ -61,8 +63,8 @@ var ConfigQueryModel = QueryModel{ return tx.Order(clause.OrderByColumn{Column: clause.Column{Name: sort}}), nil }, "offset": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) { - if i, err := strconv.Atoi(fmt.Sprintf("%s", val)); err == nil { - return tx.Limit(i), nil + if i, err := strconv.Atoi(val); err == nil { + return tx.Offset(i), nil } else { return nil, err } @@ -91,6 +93,77 @@ var ConfigQueryModel = QueryModel{ }, } +var ComponentQueryModel = QueryModel{ + Table: "components", + JSONColumn: "component", + Custom: map[string]func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error){ + "limit": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) { + if i, err := strconv.Atoi(val); err == nil { + return tx.Limit(i), nil + } else { + return nil, err + } + }, + "sort": func(ctx context.Context, tx *gorm.DB, sort string) (*gorm.DB, error) { + return tx.Order(clause.OrderByColumn{Column: clause.Column{Name: sort}}), nil + }, + "offset": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) { + if i, err := strconv.Atoi(val); err == nil { + return tx.Offset(i), nil + } else { + return nil, err + } + }, + "component_config_traverse": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) { + // search: component_config_traverse=72143d48-da4a-477f-bac1-1e9decf188a6,outgoing + // Args should be componentID, direction and types (compID,direction) + args := strings.Split(val, ",") + componentID := args[0] + direction := "outgoing" + if len(args) > 1 { + direction = args[1] + } + // NOTE: Direction is not supported as of now + _ = direction + tx = tx.Where("id IN (SELECT id from lookup_component_config_id_related_components(?))", componentID) + return tx, nil + }, + }, + Columns: []string{ + "name", "topology_id", "type", "status", "health", + }, + LabelsColumn: "labels", + Aliases: map[string]string{ + "created": "created_at", + "updated": "updated_at", + "deleted": "deleted_at", + "scraped": "last_scraped_time", + "agent": "agent_id", + "component_type": "type", + "namespace": "@namespace", + }, + + FieldMapper: map[string]func(ctx context.Context, id string) (any, error){ + "agent_id": AgentMapper, + "created_at": DateMapper, + "updated_at": DateMapper, + "deleted_at": DateMapper, + "last_scraped_time": DateMapper, + }, +} + +func GetModelFromTable(table string) (QueryModel, error) { + switch table { + case models.ConfigItem{}.TableName(): + return ConfigQueryModel, nil + case models.Component{}.TableName(): + logger.Infof("Inside comp get") + return ComponentQueryModel, nil + default: + return QueryModel{}, fmt.Errorf("invalid table") + } +} + func (qm QueryModel) Apply(ctx context.Context, q types.QueryField, tx *gorm.DB) (*gorm.DB, []clause.Expression, error) { if tx == nil { tx = ctx.DB().Table(qm.Table) @@ -103,7 +176,7 @@ func (qm QueryModel) Apply(ctx context.Context, q types.QueryField, tx *gorm.DB) q.Field = alias } - val := fmt.Sprintf("%s", q.Value) + val := fmt.Sprint(q.Value) if mapper, ok := qm.FieldMapper[q.Field]; ok { if q.Value, err = mapper(ctx, val); err != nil { return nil, nil, err diff --git a/query/resource_selector.go b/query/resource_selector.go index a0b7b956..7fc5f1d4 100644 --- a/query/resource_selector.go +++ b/query/resource_selector.go @@ -15,11 +15,13 @@ import ( "github.com/samber/lo" "golang.org/x/sync/errgroup" "gorm.io/gorm" + "gorm.io/gorm/clause" "k8s.io/apimachinery/pkg/selection" "github.com/flanksource/duty/api" "github.com/flanksource/duty/context" "github.com/flanksource/duty/pkg/kube/labels" + "github.com/flanksource/duty/query/grammar" "github.com/flanksource/duty/types" ) @@ -134,22 +136,42 @@ func SetResourceSelectorClause(ctx context.Context, resourceSelector types.Resou // We call setSearchQueryParams as it sets those params that // might later be used by the query + + // TODO: support funcs if resourceSelector.Search != "" { - if strings.Contains(resourceSelector.Search, "=") { - setSearchQueryParams(&resourceSelector) - } else { - var prefixQueries []*gorm.DB - if resourceSelector.Name == "" { - prefixQueries = append(prefixQueries, ctx.DB().Where("name ILIKE ?", resourceSelector.Search+"%")) - } - if resourceSelector.TagSelector == "" && table == "config_items" { - prefixQueries = append(prefixQueries, ctx.DB().Where("EXISTS (SELECT 1 FROM jsonb_each_text(tags) WHERE value ILIKE ?)", resourceSelector.Search+"%")) - } - if resourceSelector.LabelSelector == "" { - prefixQueries = append(prefixQueries, ctx.DB().Where("EXISTS (SELECT 1 FROM jsonb_each_text(labels) WHERE value ILIKE ?)", resourceSelector.Search+"%")) - } + qf, err := grammar.ParsePEG(resourceSelector.Search) + if err != nil { + return nil, fmt.Errorf("") + } + qm, err := GetModelFromTable(table) + if err != nil { + return nil, fmt.Errorf("") + } - query = OrQueries(query, prefixQueries...) + var clauses []clause.Expression + query, clauses, err = qm.Apply(ctx, *qf, query) + if err != nil { + return nil, fmt.Errorf("") + } + query = query.Clauses(clauses...) + + if false { + if strings.Contains(resourceSelector.Search, "=") { + setSearchQueryParams(&resourceSelector) + } else { + var prefixQueries []*gorm.DB + if resourceSelector.Name == "" { + prefixQueries = append(prefixQueries, ctx.DB().Where("name ILIKE ?", resourceSelector.Search+"%")) + } + if resourceSelector.TagSelector == "" && table == "config_items" { + prefixQueries = append(prefixQueries, ctx.DB().Where("EXISTS (SELECT 1 FROM jsonb_each_text(tags) WHERE value ILIKE ?)", resourceSelector.Search+"%")) + } + if resourceSelector.LabelSelector == "" { + prefixQueries = append(prefixQueries, ctx.DB().Where("EXISTS (SELECT 1 FROM jsonb_each_text(labels) WHERE value ILIKE ?)", resourceSelector.Search+"%")) + } + + query = OrQueries(query, prefixQueries...) + } } } @@ -277,6 +299,7 @@ func setSearchQueryParams(rs *types.ResourceSelector) { } switch items[0] { + // TODO(yash): Move this to components query model case "component_config_traverse": // search: component_config_traverse=72143d48-da4a-477f-bac1-1e9decf188a6,outgoing // Args should be componentID, direction and types (compID,direction) diff --git a/tests/query_resource_selector_test.go b/tests/query_resource_selector_test.go index 674a036f..110a9377 100644 --- a/tests/query_resource_selector_test.go +++ b/tests/query_resource_selector_test.go @@ -5,7 +5,9 @@ import ( ginkgo "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" + //"github.com/flanksource/commons/logger" "github.com/flanksource/duty/models" "github.com/flanksource/duty/query" "github.com/flanksource/duty/tests/fixtures/dummy" @@ -282,3 +284,42 @@ var _ = ginkgo.Describe("Resoure Selector limits", ginkgo.Ordered, func() { } }) }) + +var _ = ginkgo.FDescribe("Resoure Selector from PEG", ginkgo.Ordered, func() { + ginkgo.BeforeAll(func() { + _ = query.SyncConfigCache(DefaultContext) + }) + + ginkgo.FIt("should query configs", func() { + //description: "labels | IN Query", + //query: query.SearchResourcesRequest{ + //Configs: []types.ResourceSelector{{LabelSelector: "app in (frontend,backend)"}}, + //}, + //[]models.ConfigItem{dummy.EC2InstanceA, dummy.EC2InstanceB}, + + rs := types.ResourceSelector{ + Search: `name="node-b" type="Kubernetes::Node"`, + } + + ci, err := query.FindConfigsByResourceSelector(DefaultContext, 1, rs) + Expect(err).To(BeNil()) + + Expect(len(ci)).To(Equal(1)) + Expect(lo.FromPtr(ci[0].Name)).To(Equal(lo.FromPtr(dummy.KubernetesNodeB.Name))) + + }) + + ginkgo.FIt("should query components", func() { + rs := types.ResourceSelector{ + Search: `type="Application"`, + } + + comps, err := query.FindComponents(DefaultContext, -1, rs) + Expect(err).To(BeNil()) + + Expect(len(comps)).To(Equal(4)) + //Expect(lo.FromPtr(ci[0].Name)).To(Equal(lo.FromPtr(dummy.KubernetesNodeB.Name))) + + }) + +}) diff --git a/types/filters.go b/types/filters.go index c84090c6..9ff935b8 100644 --- a/types/filters.go +++ b/types/filters.go @@ -74,7 +74,6 @@ func ParseFilteringQueryV2(query string, decodeURL bool) (FilteringQuery, error) } var q expressions - q = result.expressions if strings.HasPrefix(item, "!") { q = result.Not item = strings.TrimPrefix(item, "!") @@ -86,6 +85,8 @@ func ParseFilteringQueryV2(query string, decodeURL bool) (FilteringQuery, error) } else { q.In = append(q.In, item) } + + result.expressions = q } return result, nil diff --git a/types/resource_selector.go b/types/resource_selector.go index 8ca34cd1..9379a565 100644 --- a/types/resource_selector.go +++ b/types/resource_selector.go @@ -169,7 +169,7 @@ func ParseFilteringQuery(query string, decodeURL bool) (in []interface{}, notIN } func (q QueryField) ToClauses() ([]clause.Expression, error) { - val := fmt.Sprintf("%s", q.Value) + val := fmt.Sprint(q.Value) filters, err := ParseFilteringQueryV2(val, false) if err != nil { @@ -177,14 +177,16 @@ func (q QueryField) ToClauses() ([]clause.Expression, error) { } var clauses []clause.Expression - if q.Op == Eq { + switch q.Op { + case Eq: clauses = append(clauses, filters.ToExpression(q.Field)...) - } else if q.Op == Neq { + case Neq: clauses = append(clauses, clause.Not(filters.ToExpression(q.Field)...)) - } else if q.Op == Lt { + case Lt: clauses = append(clauses, clause.Lt{Column: q.Field, Value: q.Value}) - } else if q.Op == Gt { + case Gt: clauses = append(clauses, clause.Gt{Column: q.Field, Value: q.Value}) + default: return nil, fmt.Errorf("invalid operator: %s", q.Op) }