diff --git a/pkg/cmds/monitor.go b/pkg/cmds/monitor.go index fe181bb0e..3d00ecc93 100644 --- a/pkg/cmds/monitor.go +++ b/pkg/cmds/monitor.go @@ -82,7 +82,7 @@ var alertLong = templates.LongDesc(` var alertExample = templates.Examples(` kubectl dba monitor get-alerts [DATABASE] [DATABASE_NAME] -n [NAMESPACE] \ - --prom-svc=[PROM_SVC_NAME] --prom-svc-namespace=[PROM_SVC_NS] --prom-svc-port=[PROM_SVC_PORT] + --prom-svc-name=[PROM_SVC_NAME] --prom-svc-namespace=[PROM_SVC_NS] --prom-svc-port=[PROM_SVC_PORT] # Get triggered alert for a specific mongodb kubectl dba monitor get-alerts mongodb sample-mongodb -n demo \ @@ -121,11 +121,13 @@ var dashboardLong = templates.LongDesc(` `) var dashboardExample = templates.Examples(` - kubectl dba monitor dashboard [DATABASE] [DASHBOARD_NAME] \ - --prom-svc=[PROM_SVC_NAME] --prom-svc-namespace=[PROM_SVC_NS] --prom-svc-port=[PROM_SVC_PORT] + kubectl dba monitor dashboard [DATABASE] [DATABASE_NAME] -n [NAMESPACE] \ + [DASHBOARD_NAME] --file=[FILE_CONTAINING_DASHBOARD_JSON] \ + --prom-svc-name=[PROM_SVC_NAME] --prom-svc-namespace=[PROM_SVC_NS] --prom-svc-port=[PROM_SVC_PORT] # Check availability of a postgres grafana dashboard - kubectl-dba monitor dashboard postgres postgres_databases_dashboard \ + kubectl-dba monitor dashboard postgres pg15 -n demo \ + --file=/home/arnob/yamls/summary.json \ --prom-svc-name=prometheus-kube-prometheus-prometheus --prom-svc-namespace=monitoring --prom-svc-port=9090 Valid dashboards include: @@ -141,20 +143,24 @@ var dashboardExample = templates.Examples(` `) func DashboardCMD(f cmdutil.Factory) *cobra.Command { - var branch string + var ( + branch string + file string + ) cmd := &cobra.Command{ Use: "dashboard", Short: i18n.T("Check availability of a grafana dashboard"), Long: dashboardLong, Run: func(cmd *cobra.Command, args []string) { - dashboard.Run(f, args, branch, prom) + dashboard.Run(f, args, branch, file, prom) }, Example: dashboardExample, DisableFlagsInUseLine: true, DisableAutoGenTag: true, } cmd.Flags().StringVarP(&branch, "branch", "b", "master", "branch name of the github repo") + cmd.Flags().StringVarP(&file, "file", "f", "", "absolute or relative path of the file containing dashboard") return cmd } @@ -165,7 +171,7 @@ var connectionLong = templates.LongDesc(` var connectionExample = templates.Examples(` kubectl dba monitor check-connection [DATABASE] [DATABASE_NAME] -n [NAMESPACE] \ - --prom-svc=[PROM_SVC_NAME] --prom-svc-namespace=[PROM_SVC_NS] --prom-svc-port=[PROM_SVC_PORT] + --prom-svc-name=[PROM_SVC_NAME] --prom-svc-namespace=[PROM_SVC_NS] --prom-svc-port=[PROM_SVC_PORT] # Check connection status for different targets with prometheus server for a specific postgres database kubectl dba monitor check-connection mongodb sample_mg -n demo \ diff --git a/pkg/monitor/connection/helper.go b/pkg/monitor/connection/helper.go index d801dd2a2..0f93a858b 100644 --- a/pkg/monitor/connection/helper.go +++ b/pkg/monitor/connection/helper.go @@ -25,7 +25,7 @@ import ( const ( cAdvisorMetric = "container_cpu_usage_seconds_total" - kubeletMetric = "kubelet_active_pods" + kubeletMetric = "kubelet_running_pods" ksmMetric = "kube_pod_status_phase" nodeExporterMetric = "node_cpu_seconds_total" ) diff --git a/pkg/monitor/dashboard/db.go b/pkg/monitor/dashboard/db.go index 247146187..2d2b4d9b4 100644 --- a/pkg/monitor/dashboard/db.go +++ b/pkg/monitor/dashboard/db.go @@ -40,17 +40,36 @@ type missingOpts struct { panelTitle []string } -func Run(f cmdutil.Factory, args []string, branch string, prom monitor.PromSvc) { +func Run(f cmdutil.Factory, args []string, branch, file string, prom monitor.PromSvc) { if len(args) < 2 { - log.Fatal("Enter database and grafana dashboard name as argument") + log.Fatal("Enter db object's name as an argument") + } + database := monitor.ConvertedResourceToPlural(args[0]) + dbName := args[1] + namespace, _, err := f.ToRawKubeConfigLoader().Namespace() + if err != nil { + _ = fmt.Errorf("failed to get current namespace") + return } - database := monitor.ConvertedResourceToSingular(args[0]) - dashboard := args[1] - - url := getURL(branch, database, dashboard) + db, err := getDB(f, database, namespace, dbName) + if err != nil { + fmt.Printf("failed to get %s database %s/%s. error %s \n", database, namespace, dbName, err.Error()) + return + } - dashboardData := getDashboard(url) + database = monitor.ConvertedResourceToSingular(database) + var dashboardData map[string]interface{} + if file == "" { + if len(args) < 3 { + log.Fatal("Enter dashboard name as third argument") + } + dashboard := args[2] + url := getURL(branch, database, dashboard) + dashboardData = getDashboardFromURL(url) + } else { + dashboardData = getDashboardFromFile(file) + } queries := parseAllExpressions(dashboardData) @@ -105,6 +124,7 @@ func Run(f cmdutil.Factory, args []string, branch string, prom monitor.PromSvc) unknown[metricName].panelTitle = uniqueAppend(unknown[metricName].panelTitle, query.panelTitle) } } + unknown = ignoreModeSpecificExpressions(unknown, database, db) if len(unknown) > 0 { fmt.Println("Missing Information:") for metric, opts := range unknown { diff --git a/pkg/monitor/dashboard/elasticsearch.go b/pkg/monitor/dashboard/elasticsearch.go new file mode 100644 index 000000000..5c192aa67 --- /dev/null +++ b/pkg/monitor/dashboard/elasticsearch.go @@ -0,0 +1,145 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dashboard + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func ignoreElasticsearchModeSpecificExpressions(unknown map[string]*missingOpts, db *unstructured.Unstructured) map[string]*missingOpts { + master := []string{ + "kubedb_com_elasticsearch_master_node_replicas", + "kubedb_com_elasticsearch_master_node_storage_class_info", + "kubedb_com_elasticsearch_master_node_max_unavailable", + } + data := []string{ + "kubedb_com_elasticsearch_data_node_replicas", + "kubedb_com_elasticsearch_data_node_storage_class_info", + "kubedb_com_elasticsearch_data_node_max_unavailable", + } + datacontent := []string{ + "kubedb_com_elasticsearch_datacontent_node_replicas", + "kubedb_com_elasticsearch_datacontent_node_storage_class_info", + "kubedb_com_elasticsearch_datacontent_node_max_unavailable", + } + datahot := []string{ + "kubedb_com_elasticsearch_datahot_node_replicas", + "kubedb_com_elasticsearch_datahot_node_storage_class_info", + "kubedb_com_elasticsearch_datahot_node_max_unavailable", + } + datawarm := []string{ + "kubedb_com_elasticsearch_datawarm_node_replicas", + "kubedb_com_elasticsearch_datawarm_node_storage_class_info", + "kubedb_com_elasticsearch_datawarm_node_max_unavailable", + } + datacold := []string{ + "kubedb_com_elasticsearch_datacold_node_replicas", + "kubedb_com_elasticsearch_datacold_node_storage_class_info", + "kubedb_com_elasticsearch_datacold_node_max_unavailable", + } + datafrozen := []string{ + "kubedb_com_elasticsearch_datafrozen_node_replicas", + "kubedb_com_elasticsearch_datafrozen_node_storage_class_info", + "kubedb_com_elasticsearch_datafrozen_node_max_unavailable", + } + ingest := []string{ + "kubedb_com_elasticsearch_ingest_node_replicas", + "kubedb_com_elasticsearch_ingest_node_storage_class_info", + "kubedb_com_elasticsearch_ingest_node_max_unavailable", + } + ml := []string{ + "kubedb_com_elasticsearch_ml_node_replicas", + "kubedb_com_elasticsearch_ml_node_storage_class_info", + "kubedb_com_elasticsearch_ml_node_max_unavailable", + } + transform := []string{ + "kubedb_com_elasticsearch_transform_node_replicas", + "kubedb_com_elasticsearch_transform_node_storage_class_info", + "kubedb_com_elasticsearch_transform_node_max_unavailable", + } + coordinating := []string{ + "kubedb_com_elasticsearch_coordinating_node_replicas", + "kubedb_com_elasticsearch_coordinating_node_storage_class_info", + "kubedb_com_elasticsearch_coordinating_node_max_unavailable", + } + + has := func(shared []string, expr string) bool { + for _, s := range shared { + if expr == s { + return true + } + } + return false + } + + isSet := func(db *unstructured.Unstructured, typ string) bool { + spec, found, err := unstructured.NestedMap(db.Object, "spec") + if err != nil || !found { + return false + } + + topo, found, err := unstructured.NestedMap(spec, "topology") + if err != nil || !found { + return false + } + + typed, found, err := unstructured.NestedMap(topo, typ) + if err != nil || !found { + return false + } + return len(typed) > 0 + } + + ret := make(map[string]*missingOpts) + for s, o := range unknown { + if has(master, s) && !isSet(db, "master") { + continue + } + if has(data, s) && !isSet(db, "data") { + continue + } + if has(datacontent, s) && !isSet(db, "dataContent") { + continue + } + if has(datahot, s) && !isSet(db, "dataHot") { + continue + } + if has(datawarm, s) && !isSet(db, "dataWarm") { + continue + } + if has(datacold, s) && !isSet(db, "dataCold") { + continue + } + if has(datafrozen, s) && !isSet(db, "dataFrozen") { + continue + } + if has(ingest, s) && !isSet(db, "ingest") { + continue + } + if has(ml, s) && !isSet(db, "ml") { + continue + } + if has(transform, s) && !isSet(db, "transform") { + continue + } + if has(coordinating, s) && !isSet(db, "coordinating") { + continue + } + ret[s] = o + } + return ret +} diff --git a/pkg/monitor/dashboard/helper.go b/pkg/monitor/dashboard/helper.go index 7da7d35ac..1f6ec04e1 100644 --- a/pkg/monitor/dashboard/helper.go +++ b/pkg/monitor/dashboard/helper.go @@ -17,18 +17,44 @@ limitations under the License. package dashboard import ( + "context" "encoding/json" "fmt" "io" "log" "net/http" + "os" + + api "kubedb.dev/apimachinery/apis/kubedb/v1alpha2" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + cmdutil "k8s.io/kubectl/pkg/cmd/util" ) +func getDB(f cmdutil.Factory, resource, ns, name string) (*unstructured.Unstructured, error) { + config, err := f.ToRESTConfig() + if err != nil { + return nil, err + } + + dc, err := dynamic.NewForConfig(config) + if err != nil { + return nil, err + } + + gvk := api.SchemeGroupVersion + dbRes := schema.GroupVersionResource{Group: gvk.Group, Version: gvk.Version, Resource: resource} + return dc.Resource(dbRes).Namespace(ns).Get(context.TODO(), name, metav1.GetOptions{}) +} + func getURL(branch, database, dashboard string) string { return fmt.Sprintf("https://raw.githubusercontent.com/appscode/grafana-dashboards/%s/%s/%s.json", branch, database, dashboard) } -func getDashboard(url string) map[string]interface{} { +func getDashboardFromURL(url string) map[string]interface{} { var dashboardData map[string]interface{} response, err := http.Get(url) if err != nil { @@ -50,6 +76,19 @@ func getDashboard(url string) map[string]interface{} { return dashboardData } +func getDashboardFromFile(file string) map[string]interface{} { + body, err := os.ReadFile(file) + if err != nil { + log.Fatal("Error on ReadFile:", err) + } + var dashboardData map[string]interface{} + err = json.Unmarshal(body, &dashboardData) + if err != nil { + log.Fatal("Error unmarshalling JSON data:", err) + } + return dashboardData +} + func uniqueAppend(slice []string, valueToAdd string) []string { for _, existingValue := range slice { if existingValue == valueToAdd { @@ -58,3 +97,13 @@ func uniqueAppend(slice []string, valueToAdd string) []string { } return append(slice, valueToAdd) } + +func ignoreModeSpecificExpressions(unknown map[string]*missingOpts, database string, db *unstructured.Unstructured) map[string]*missingOpts { + if database == api.ResourceSingularMongoDB { + return ignoreMongoDBModeSpecificExpressions(unknown, db) + } + if database == api.ResourceSingularElasticsearch { + return ignoreElasticsearchModeSpecificExpressions(unknown, db) + } + return unknown +} diff --git a/pkg/monitor/dashboard/mongodb.go b/pkg/monitor/dashboard/mongodb.go new file mode 100644 index 000000000..aadfe09f3 --- /dev/null +++ b/pkg/monitor/dashboard/mongodb.go @@ -0,0 +1,68 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dashboard + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func ignoreMongoDBModeSpecificExpressions(unknown map[string]*missingOpts, db *unstructured.Unstructured) map[string]*missingOpts { + shared := []string{ + "kubedb_com_mongodb_shard_shards", + "kubedb_com_mongodb_shard_replicas", + "kubedb_com_mongodb_mongos_replicas", + "kubedb_com_mongodb_configsvr_replicas", + } + + general := []string{ + "kubedb_com_mongodb_replicas", + } + + has := func(shared []string, expr string) bool { + for _, s := range shared { + if expr == s { + return true + } + } + return false + } + + isShardTopologySet := func(db *unstructured.Unstructured) bool { + spec, found, err := unstructured.NestedMap(db.Object, "spec") + if err != nil || !found { + return false + } + + shardTopology, found, err := unstructured.NestedMap(spec, "shardTopology") + if err != nil || !found { + return false + } + return len(shardTopology) > 0 + } + + sharded := isShardTopologySet(db) + ret := make(map[string]*missingOpts) + for s, o := range unknown { + if has(shared, s) && !sharded { + continue + } else if has(general, s) && sharded { + continue + } + ret[s] = o + } + return ret +}