From 0f15f5ccbee7fda1557d2a10ffe482a9b752179c Mon Sep 17 00:00:00 2001 From: sayedppqq Date: Thu, 21 Dec 2023 12:40:44 +0600 Subject: [PATCH] works fine with mongodb dashboards Signed-off-by: sayedppqq --- pkg/cmds/grafana_dashboard.go | 8 +- pkg/cmds/root.go | 6 ++ pkg/dashboard/db.go | 181 ++++++++++++++++++++++++++++++++-- pkg/dashboard/helper.go | 76 ++++++++++++++ 4 files changed, 260 insertions(+), 11 deletions(-) create mode 100644 pkg/dashboard/helper.go diff --git a/pkg/cmds/grafana_dashboard.go b/pkg/cmds/grafana_dashboard.go index 0e825a707..bfcd7321b 100644 --- a/pkg/cmds/grafana_dashboard.go +++ b/pkg/cmds/grafana_dashboard.go @@ -25,20 +25,24 @@ var dashboardExample = templates.Examples(` * redis `) -func NewCmdDashboardCMD(f cmdutil.Factory) *cobra.Command { +func NewCmdDashboard(f cmdutil.Factory) *cobra.Command { var branch string + var prom dashboard.PromSvc cmd := &cobra.Command{ Use: "dashboard", Short: i18n.T("Check availability of a grafana dashboard"), Long: dashboardLong, Run: func(cmd *cobra.Command, args []string) { - dashboard.Run(f, args, branch) + dashboard.Run(f, args, branch, prom) }, Example: dashboardExample, DisableFlagsInUseLine: true, DisableAutoGenTag: true, } cmd.Flags().StringVarP(&branch, "branch", "b", "master", "branch name of the github repo") + cmd.Flags().StringVarP(&prom.Name, "prom-svc-name", "", "", "name of the prometheus service") + cmd.Flags().StringVarP(&prom.Namespace, "prom-svc-namespace", "", "", "namespace of the prometheus service") + cmd.Flags().IntVarP(&prom.Port, "prom-svc-port", "", 9090, "port of the prometheus service") return cmd } diff --git a/pkg/cmds/root.go b/pkg/cmds/root.go index 7e7ddfe22..8d8c3fa58 100644 --- a/pkg/cmds/root.go +++ b/pkg/cmds/root.go @@ -113,6 +113,12 @@ func NewKubeDBCommand(in io.Reader, out, err io.Writer) *cobra.Command { NewCmdGenApb(f), }, }, + { + Message: "Check availability of a grafana dashboard", + Commands: []*cobra.Command{ + NewCmdDashboard(f), + }, + }, } filters := []string{"options"} diff --git a/pkg/dashboard/db.go b/pkg/dashboard/db.go index d22c485a1..3326c0926 100644 --- a/pkg/dashboard/db.go +++ b/pkg/dashboard/db.go @@ -1,23 +1,186 @@ package dashboard import ( + "context" + "encoding/json" + "fmt" + "github.com/prometheus/client_golang/api" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + "io/ioutil" cmdutil "k8s.io/kubectl/pkg/cmd/util" + "kubedb.dev/cli/pkg/lib" "log" + "net/http" + "strconv" + "strings" + "time" ) -type DashboardOpts struct { - Branch string - Database string - Dashboard string +var metricLabels = make(map[string][]string) + +type queryInformation struct { + metric string + labelNames []string +} +type PromSvc struct { + Name string + Namespace string + Port int } -func Run(f cmdutil.Factory, args []string, branch string) { +func Run(f cmdutil.Factory, args []string, branch string, prom PromSvc) { if len(args) < 2 { log.Fatal("Enter database and grafana dashboard name as argument") } - dashboardOpts := DashboardOpts{ - Branch: branch, - Database: args[0], - Dashboard: args[1], + + database := args[0] + dashboard := args[1] + + url := getURL(branch, database, dashboard) + + dashboardData := getDashboard(url) + + var queries []queryInformation + if panels, ok := dashboardData["panels"].([]interface{}); ok { + for _, panel := range panels { + if targets, ok := panel.(map[string]interface{})["targets"].([]interface{}); ok { + for _, target := range targets { + if expr, ok := target.(map[string]interface{})["expr"]; ok { + if expr != "" { + query := expr.(string) + queries = append(queries, getMetricAndLabels(query)...) + } + } + } + } + } + } + + config, err := f.ToRESTConfig() + if err != nil { + log.Fatal(err) + } + // Port forwarding cluster prometheus service for that grafana dashboard's prom datasource. + tunnel, err := lib.TunnelToDBService(config, prom.Name, prom.Namespace, prom.Port) + if err != nil { + log.Fatal(err) + } + defer tunnel.Close() + + promClient := getPromClient(strconv.Itoa(tunnel.Local)) + + var unknownMetrics, unknownLabels []string + + for _, query := range queries { + metricName := query.metric + for _, labelKey := range query.labelNames { + + endTime := time.Now() + + result, _, err := promClient.Query(context.TODO(), metricName, endTime) + if err != nil { + log.Fatal("Error querying Prometheus:", err, " metric: ", metricName) + } + + matrix := result.(model.Vector) + if len(matrix) > 0 { + // Check if the label exists for any result in the matrix + labelExists := false + + for _, sample := range matrix { + if sample.Metric != nil { + if _, ok := sample.Metric[model.LabelName(labelKey)]; ok { + labelExists = true + break + } + } + } + + if !labelExists { + unknownLabels = uniqueAppend(unknownLabels, fmt.Sprintf(`label: "%s" metric: "%s"`, labelKey, metricName)) + } + } else { + unknownMetrics = uniqueAppend(unknownMetrics, metricName) + } + } + } + if len(unknownMetrics) > 0 { + fmt.Print("List of unknown metrics:\n", strings.Join(unknownMetrics, "\n")) } + if len(unknownLabels) > 0 { + fmt.Print("List of unknown labels:\n", strings.Join(unknownLabels, "\n")) + } + if len(unknownMetrics) == 0 && len(unknownLabels) == 0 { + fmt.Println("All metrics found") + } +} + +func getURL(branch, database, dashboard string) string { + return fmt.Sprintf("https://raw.githubusercontent.com/appscode/grafana-dashboards/%s/%s/%s.json", branch, database, dashboard) +} +func getDashboard(url string) map[string]interface{} { + var dashboardData map[string]interface{} + response, err := http.Get(url) + if err != nil { + log.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + log.Fatalf("Error fetching url. status : %s", response.Status) + } + body, err := ioutil.ReadAll(response.Body) + if err != nil { + log.Fatal("Error reading JSON file: ", err) + } + + err = json.Unmarshal(body, &dashboardData) + if err != nil { + log.Fatal("Error unmarshalling JSON data:", err) + } + return dashboardData +} + +// Steps: +// - if current character is '{' +// - extract metric name by matching metric regex +// - get label selector substring inside { } +// - get label name from this substring by matching label regex +// - move i to its closing bracket position. +func getMetricAndLabels(query string) []queryInformation { + var queries []queryInformation + for i := 0; i < len(query); i++ { + if query[i] == '{' { + j := i + for { + if j-1 < 0 || (!matchMetricRegex(rune(query[j-1]))) { + break + } + j-- + } + metric := query[j:i] + labelSelector, closingPosition := substringInsideLabelSelector(query, i) + labelNames := getLabelNames(labelSelector) + queries = append(queries, queryInformation{ + metric: metric, + labelNames: labelNames, + }) + i = closingPosition + } + } + return queries +} + +func getPromClient(localPort string) v1.API { + prometheusURL := fmt.Sprintf("http://localhost:%s/", localPort) + + client, err := api.NewClient(api.Config{ + Address: prometheusURL, + }) + if err != nil { + log.Fatal("Error creating Prometheus client:", err) + } + + // Create a new Prometheus API client + return v1.NewAPI(client) } diff --git a/pkg/dashboard/helper.go b/pkg/dashboard/helper.go new file mode 100644 index 000000000..a7ec8f252 --- /dev/null +++ b/pkg/dashboard/helper.go @@ -0,0 +1,76 @@ +package dashboard + +import ( + "regexp" + "strings" + "unicode" +) + +func excludeQuotedSubstrings(input string) string { + // Define the regular expression pattern to match string inside double quotation + re := regexp.MustCompile(`"[^"]*"`) + + // Replace all quoted substring with an empty string + result := re.ReplaceAllString(input, "") + + return result +} +func excludeNonAlphanumericUnderscore(input string) string { + // Define the regular expression pattern to match non-alphanumeric characters except underscore + pattern := `[^a-zA-Z0-9_]` + re := regexp.MustCompile(pattern) + + // Replace non-alphanumeric or underscore characters with an empty string + result := re.ReplaceAllString(input, "") + + return result +} + +// Labels may contain ASCII letters, numbers, as well as underscores. They must match the regex [a-zA-Z_][a-zA-Z0-9_]* +// So we need to split the selector string by comma. then extract label name with the help of the regex format +// Ref: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels +func getLabelNames(labelSelector string) []string { + var labelNames []string + unQuoted := excludeQuotedSubstrings(labelSelector) + commaSeparated := strings.Split(unQuoted, ",") + for _, s := range commaSeparated { + labelName := excludeNonAlphanumericUnderscore(s) + labelNames = append(labelNames, labelName) + } + return labelNames +} + +// Finding valid bracket sequence from startPosition +func substringInsideLabelSelector(query string, startPosition int) (string, int) { + balance := 0 + closingPosition := startPosition + for i := startPosition; i < len(query); i++ { + if query[i] == '{' { + balance++ + } + if query[i] == '}' { + balance-- + } + if balance == 0 { + closingPosition = i + break + } + } + return query[startPosition+1 : closingPosition], closingPosition +} + +// Metric names may contain ASCII letters, digits, underscores, and colons. It must match the regex [a-zA-Z_:][a-zA-Z0-9_:]* +// So we can use this if the character is in a metric name +// Ref: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels +func matchMetricRegex(char rune) bool { + return unicode.IsLetter(char) || unicode.IsDigit(char) || char == '_' || char == ':' +} + +func uniqueAppend(slice []string, valueToAdd string) []string { + for _, existingValue := range slice { + if existingValue == valueToAdd { + return slice + } + } + return append(slice, valueToAdd) +}