Skip to content

Commit

Permalink
feat: add pagination and more filtering in catalog changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@adityathebe
adityathebe committed Mar 12, 2024
1 parent d6e51ad commit 9f1fc5c
Show file tree
Hide file tree
Showing 3 changed files with 151 additions and 18 deletions.
8 changes: 4 additions & 4 deletions query/commons.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@ func parseAndBuildFilteringQuery(query string, field string) ([]string, map[stri

in, notIN, prefixes, suffixes := ParseFilteringQuery(query)
if len(in) > 0 {
clauses = append(clauses, fmt.Sprintf("%s IN @field_in", field))
args["field_in"] = in
clauses = append(clauses, fmt.Sprintf("%s IN @%s_field_in", field, field))
args[fmt.Sprintf("%s_field_in", field)] = in
}

if len(notIN) > 0 {
clauses = append(clauses, fmt.Sprintf("%s NOT IN @field_not_in", field))
args["field_not_in"] = notIN
clauses = append(clauses, fmt.Sprintf("%s NOT IN @%s_field_not_in", field, field))
args[fmt.Sprintf("%s_field_not_in", field)] = notIN
}

for i, p := range prefixes {
Expand Down
81 changes: 73 additions & 8 deletions query/config_changes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,40 @@ const (
)

type CatalogChangesSearchRequest struct {
CatalogID uuid.UUID `query:"id"`
ConfigType string `query:"config_type"`
ChangeType string `query:"type"`
From string `query:"from"`
CatalogID uuid.UUID `query:"id"`
ConfigType string `query:"config_type"`
ChangeType string `query:"type"`
Severity string `query:"severity"`
IncludeDeletedConfigs bool `query:"include_deleted_configs"`

// From date in datemath format
From string `query:"from"`
// To date in datemath format
To string `query:"to"`

PageSize int `query:"page_size"`
Page int `query:"page"`
SortBy string `query:"sort_by"`

// upstream | downstream | both
Recursive string `query:"recursive"`

fromParsed time.Time
toParsed time.Time
}

func (t *CatalogChangesSearchRequest) SetDefaults() {
if t.PageSize <= 0 {
t.PageSize = 50
}

if t.Page <= 0 {
t.Page = 1
}

if t.From == "" && t.To == "" {
t.From = "now-2d"
}
}

func (t *CatalogChangesSearchRequest) Validate() error {
Expand All @@ -49,6 +74,18 @@ func (t *CatalogChangesSearchRequest) Validate() error {
}
}

if t.To != "" {
if expr, err := datemath.Parse(t.To); err != nil {
return fmt.Errorf("invalid 'to' param: %w", err)
} else {
t.toParsed = expr.Time()
}
}

if !t.fromParsed.IsZero() && !t.toParsed.IsZero() && !t.fromParsed.Before(t.toParsed) {
return fmt.Errorf("'from' must be before 'to'")
}

return nil
}

Expand All @@ -65,17 +102,19 @@ func (t *CatalogChangesSearchResponse) Summarize() {
}

func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (*CatalogChangesSearchResponse, error) {
req.SetDefaults()
if err := req.Validate(); err != nil {
return nil, api.Errorf(api.EINVALID, "bad request: %v", err)
}

args := map[string]any{
"catalog_id": req.CatalogID,
"recursive": req.Recursive,
"catalog_id": req.CatalogID,
"recursive": req.Recursive,
"include_deleted_configs": req.IncludeDeletedConfigs,
}

var clauses []string
query := "SELECT cc.* FROM related_changes_recursive(@catalog_id, @recursive) cc"
query := "SELECT cc.* FROM related_changes_recursive(@catalog_id, @recursive, @include_deleted_configs) cc"
if req.Recursive == "" {
query = "SELECT cc.* FROM config_changes cc"
clauses = append(clauses, "cc.config_id = @catalog_id")
Expand All @@ -95,15 +134,41 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (*
args = collections.MergeMap(args, _args)
}

if req.Severity != "" {
_clauses, _args := parseAndBuildFilteringQuery(req.Severity, "cc.severity")
clauses = append(clauses, _clauses...)
args = collections.MergeMap(args, _args)
}

if !req.fromParsed.IsZero() {
clauses = append(clauses, "cc.created_at >= @from")
clauses = append(clauses, "cc.created_at > @from")
args["from"] = req.fromParsed
}

if !req.toParsed.IsZero() {
clauses = append(clauses, "cc.created_at < @to")
args["to"] = req.toParsed
}

if len(clauses) > 0 {
query += fmt.Sprintf(" WHERE %s", strings.Join(clauses, " AND "))
}

if req.SortBy != "" {
var sortOrder = "ASC"
if strings.HasPrefix(string(req.SortBy), "-") {
sortOrder = "DESC"
req.SortBy = strings.TrimPrefix(string(req.SortBy), "-")
}

query += fmt.Sprintf(" ORDER BY @sortby %s", sortOrder)
args["sortby"] = req.SortBy
}

query += " LIMIT @page_size OFFSET @page_number"
args["page_size"] = req.PageSize
args["page_number"] = (req.Page - 1) * req.PageSize

var output CatalogChangesSearchResponse
if err := ctx.DB().Raw(query, args).Find(&output.Changes).Error; err != nil {
return nil, err
Expand Down
80 changes: 74 additions & 6 deletions tests/config_changes_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,12 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {

// Create changes for each config
var (
UChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now()), ConfigID: U.ID.String(), Summary: ".name.U", ChangeType: "RegisterNode", Source: "test-changes"}
VChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour)), ConfigID: V.ID.String(), Summary: ".name.V", ChangeType: "diff", Source: "test-changes"}
WChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 2)), ConfigID: W.ID.String(), Summary: ".name.W", ChangeType: "Pulled", Source: "test-changes"}
XChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 3)), ConfigID: X.ID.String(), Summary: ".name.X", ChangeType: "diff", Source: "test-changes"}
YChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 4)), ConfigID: Y.ID.String(), Summary: ".name.Y", ChangeType: "diff", Source: "test-changes"}
ZChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 5)), ConfigID: Z.ID.String(), Summary: ".name.Z", ChangeType: "Pulled", Source: "test-changes"}
UChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now()), Severity: "info", ConfigID: U.ID.String(), Summary: ".name.U", ChangeType: "RegisterNode", Source: "test-changes"}
VChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour)), Severity: "warn", ConfigID: V.ID.String(), Summary: ".name.V", ChangeType: "diff", Source: "test-changes"}
WChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 2)), Severity: "low", ConfigID: W.ID.String(), Summary: ".name.W", ChangeType: "Pulled", Source: "test-changes"}
XChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 3)), Severity: "info", ConfigID: X.ID.String(), Summary: ".name.X", ChangeType: "diff", Source: "test-changes"}
YChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 4)), Severity: "warn", ConfigID: Y.ID.String(), Summary: ".name.Y", ChangeType: "diff", Source: "test-changes"}
ZChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 5)), Severity: "info", ConfigID: Z.ID.String(), Summary: ".name.Z", ChangeType: "Pulled", Source: "test-changes"}

changes = []models.ConfigChange{UChange, VChange, WChange, XChange, YChange, ZChange}
)
Expand Down Expand Up @@ -216,6 +216,73 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {
})
})

ginkgo.It("Severity filter", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
Severity: "!info",
})
Expect(err).To(BeNil())
Expect(len(response.Changes)).To(Equal(3))
Expect(response.Summary["Pulled"]).To(Equal(1))
Expect(response.Summary["diff"]).To(Equal(2))
})

ginkgo.Context("Pagination", func() {
ginkgo.It("Page size", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
SortBy: "summary",
PageSize: 2,
})
Expect(err).To(BeNil())
Expect(len(response.Changes)).To(Equal(2))
changes := lo.Map(response.Changes, func(c models.ConfigChange, _ int) string { return c.Summary })
Expect(changes).To(Equal([]string{".name.U", ".name.V"}))
})

ginkgo.It("Page number", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
SortBy: "summary",
PageSize: 2,
Page: 2,
})
Expect(err).To(BeNil())
Expect(len(response.Changes)).To(Equal(2))
changes := lo.Map(response.Changes, func(c models.ConfigChange, _ int) string { return c.Summary })
Expect(changes).To(Equal([]string{".name.W", ".name.X"}))
})
})

ginkgo.Context("Sorting", func() {
ginkgo.It("Ascending", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
SortBy: "change_type",
})
Expect(err).To(BeNil())
Expect(len(response.Changes)).To(Equal(6))
changes := lo.Map(response.Changes, func(c models.ConfigChange, _ int) string { return c.ChangeType })
Expect(changes).To(Equal([]string{"diff", "diff", "diff", "Pulled", "Pulled", "RegisterNode"}))
})

ginkgo.It("Descending", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
SortBy: "-change_type",
})
Expect(err).To(BeNil())
Expect(len(response.Changes)).To(Equal(6))
changes := lo.Map(response.Changes, func(c models.ConfigChange, _ int) string { return c.ChangeType })
Expect(changes).To(Equal([]string{"RegisterNode", "Pulled", "Pulled", "diff", "diff", "diff"}))
})
})

ginkgo.Context("recursive mode", func() {
ginkgo.It("upstream", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
Expand Down Expand Up @@ -258,6 +325,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
From: "now-65m",
To: "now-1s",
})
Expect(err).To(BeNil())
Expect(len(response.Changes)).To(Equal(2))
Expand Down

0 comments on commit 9f1fc5c

Please sign in to comment.