diff --git a/pkg/cmds/grafana_dashboard.go b/pkg/cmds/grafana_dashboard.go new file mode 100644 index 000000000..6ea70dac7 --- /dev/null +++ b/pkg/cmds/grafana_dashboard.go @@ -0,0 +1,65 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmds + +import ( + "kubedb.dev/cli/pkg/dashboard" + + "github.com/spf13/cobra" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +var dashboardLong = templates.LongDesc(` + Check availability of metrics in prometheus server used in a grafana dashboard. + `) + +var dashboardExample = templates.Examples(` + # Check availability of mongodb-summary-dashboard grafana dashboard of mongodb + kubectl dba dashboard mongodb mongodb-summary-dashboard + + Valid dashboards include: + * elasticsearch + * mongodb + * mariadb + * mysql + * postgres + * redis +`) + +func NewCmdDashboard(f cmdutil.Factory) *cobra.Command { + var branch string + var prom dashboard.PromSvc + cmd := &cobra.Command{ + Use: "dashboard", + Short: i18n.T("Check availability of a grafana dashboard"), + Long: dashboardLong, + + Run: func(cmd *cobra.Command, args []string) { + dashboard.Run(f, args, branch, prom) + }, + Example: dashboardExample, + DisableFlagsInUseLine: true, + DisableAutoGenTag: true, + } + cmd.Flags().StringVarP(&branch, "branch", "b", "master", "branch name of the github repo") + cmd.Flags().StringVarP(&prom.Name, "prom-svc-name", "", "", "name of the prometheus service") + cmd.Flags().StringVarP(&prom.Namespace, "prom-svc-namespace", "", "", "namespace of the prometheus service") + cmd.Flags().IntVarP(&prom.Port, "prom-svc-port", "", 9090, "port of the prometheus service") + return cmd +} diff --git a/pkg/cmds/root.go b/pkg/cmds/root.go index 7e7ddfe22..7e92b68a7 100644 --- a/pkg/cmds/root.go +++ b/pkg/cmds/root.go @@ -113,6 +113,12 @@ func NewKubeDBCommand(in io.Reader, out, err io.Writer) *cobra.Command { NewCmdGenApb(f), }, }, + { + Message: "Check availability of metrics", + Commands: []*cobra.Command{ + NewCmdDashboard(f), + }, + }, } filters := []string{"options"} diff --git a/pkg/dashboard/db.go b/pkg/dashboard/db.go new file mode 100644 index 000000000..e75efa4c6 --- /dev/null +++ b/pkg/dashboard/db.go @@ -0,0 +1,131 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dashboard + +import ( + "context" + "fmt" + "log" + "strconv" + "strings" + "time" + + "kubedb.dev/cli/pkg/lib" + + "github.com/prometheus/common/model" + cmdutil "k8s.io/kubectl/pkg/cmd/util" +) + +type queryOpts struct { + metric string + panelTitle string + labelNames []string +} +type missingOpts struct { + labelName []string + panelTitle []string +} +type PromSvc struct { + Name string + Namespace string + Port int +} + +func Run(f cmdutil.Factory, args []string, branch string, prom PromSvc) { + if len(args) < 2 { + log.Fatal("Enter database and grafana dashboard name as argument") + } + + database := args[0] + dashboard := args[1] + + url := getURL(branch, database, dashboard) + + dashboardData := getDashboard(url) + + queries := parseAllExpressions(dashboardData) + + config, err := f.ToRESTConfig() + if err != nil { + log.Fatal(err) + } + // Port forwarding cluster prometheus service for that grafana dashboard's prom datasource. + tunnel, err := lib.TunnelToDBService(config, prom.Name, prom.Namespace, prom.Port) + if err != nil { + log.Fatal(err) + } + defer tunnel.Close() + + promClient := getPromClient(strconv.Itoa(tunnel.Local)) + + // var unknown []missingOpts + unknown := make(map[string]*missingOpts) + + for _, query := range queries { + metricName := query.metric + endTime := time.Now() + + result, _, err := promClient.Query(context.TODO(), metricName, endTime) + if err != nil { + log.Fatal("Error querying Prometheus:", err, " metric: ", metricName) + } + + matrix := result.(model.Vector) + + if len(matrix) > 0 { + for _, labelKey := range query.labelNames { + // Check if the label exists for any result in the matrix + labelExists := false + + for _, sample := range matrix { + if sample.Metric != nil { + if _, ok := sample.Metric[model.LabelName(labelKey)]; ok { + labelExists = true + break + } + } + } + + if !labelExists { + if unknown[metricName] == nil { + unknown[metricName] = &missingOpts{labelName: []string{}, panelTitle: []string{}} + } + unknown[metricName].labelName = uniqueAppend(unknown[metricName].labelName, labelKey) + unknown[metricName].panelTitle = uniqueAppend(unknown[metricName].panelTitle, query.panelTitle) + } + } + } else { + if unknown[metricName] == nil { + unknown[metricName] = &missingOpts{labelName: []string{}, panelTitle: []string{}} + } + unknown[metricName].panelTitle = uniqueAppend(unknown[metricName].panelTitle, query.panelTitle) + } + } + if len(unknown) > 0 { + fmt.Println("Missing Information:") + for metric, opts := range unknown { + fmt.Println("---------------------------------------------------") + fmt.Printf("Metric: %s \n", metric) + if len(opts.labelName) > 0 { + fmt.Printf("Missing Lables: %s \n", strings.Join(opts.labelName, ", ")) + } + fmt.Printf("Effected Panel: %s \n", strings.Join(opts.panelTitle, ", ")) + } + } else { + fmt.Println("All metrics found") + } +} diff --git a/pkg/dashboard/helper.go b/pkg/dashboard/helper.go new file mode 100644 index 000000000..96b577af8 --- /dev/null +++ b/pkg/dashboard/helper.go @@ -0,0 +1,77 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dashboard + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + + "github.com/prometheus/client_golang/api" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" +) + +func getURL(branch, database, dashboard string) string { + return fmt.Sprintf("https://raw.githubusercontent.com/appscode/grafana-dashboards/%s/%s/%s.json", branch, database, dashboard) +} + +func getDashboard(url string) map[string]interface{} { + var dashboardData map[string]interface{} + response, err := http.Get(url) + if err != nil { + log.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + log.Fatalf("Error fetching url. status : %s", response.Status) + } + body, err := io.ReadAll(response.Body) + if err != nil { + log.Fatal("Error reading JSON file: ", err) + } + + err = json.Unmarshal(body, &dashboardData) + if err != nil { + log.Fatal("Error unmarshalling JSON data:", err) + } + return dashboardData +} + +func getPromClient(localPort string) v1.API { + prometheusURL := fmt.Sprintf("http://localhost:%s/", localPort) + + client, err := api.NewClient(api.Config{ + Address: prometheusURL, + }) + if err != nil { + log.Fatal("Error creating Prometheus client:", err) + } + + // Create a new Prometheus API client + return v1.NewAPI(client) +} + +func uniqueAppend(slice []string, valueToAdd string) []string { + for _, existingValue := range slice { + if existingValue == valueToAdd { + return slice + } + } + return append(slice, valueToAdd) +} diff --git a/pkg/dashboard/parser.go b/pkg/dashboard/parser.go new file mode 100644 index 000000000..7292099d0 --- /dev/null +++ b/pkg/dashboard/parser.go @@ -0,0 +1,134 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dashboard + +import ( + "log" + "regexp" + "strings" + "unicode" +) + +func parseAllExpressions(dashboardData map[string]interface{}) []queryOpts { + var queries []queryOpts + if panels, ok := dashboardData["panels"].([]interface{}); ok { + for _, panel := range panels { + if targets, ok := panel.(map[string]interface{})["targets"].([]interface{}); ok { + title, ok := panel.(map[string]interface{})["title"].(string) + if !ok { + log.Fatal("panel's title found empty") + } + for _, target := range targets { + if expr, ok := target.(map[string]interface{})["expr"]; ok { + if expr != "" { + query := expr.(string) + queries = append(queries, parseSingleExpression(query, title)...) + } + } + } + } + } + } + return queries +} + +// Steps: +// - if current character is '{' +// - extract metric name by matching metric regex +// - get label selector substring inside { } +// - get label name from this substring by matching label regex +// - move i to its closing bracket position. +func parseSingleExpression(query, title string) []queryOpts { + var queries []queryOpts + for i := 0; i < len(query); i++ { + if query[i] == '{' { + j := i + for { + if j-1 < 0 || (!matchMetricRegex(rune(query[j-1]))) { + break + } + j-- + } + metric := query[j:i] + fullLabelString, closingPosition := getFullLabelString(query, i) + labelNames := parseLabelNames(fullLabelString) + queries = append(queries, queryOpts{ + metric: metric, + labelNames: labelNames, + panelTitle: title, + }) + i = closingPosition + } + } + return queries +} + +func matchMetricRegex(char rune) bool { // Must match the regex [a-zA-Z_:][a-zA-Z0-9_:]* + return unicode.IsLetter(char) || unicode.IsDigit(char) || char == '_' || char == ':' +} + +// Finding valid bracket sequence from startPosition +func getFullLabelString(query string, startPosition int) (string, int) { + balance := 0 + closingPosition := startPosition + for i := startPosition; i < len(query); i++ { + if query[i] == '{' { + balance++ + } + if query[i] == '}' { + balance-- + } + if balance == 0 { + closingPosition = i + break + } + } + return query[startPosition+1 : closingPosition], closingPosition +} + +// Labels may contain ASCII letters, numbers, as well as underscores. They must match the regex [a-zA-Z_][a-zA-Z0-9_]* +// So we need to split the selector string by comma. then extract label name with the help of the regex format +// Ref: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels +func parseLabelNames(fullLabelString string) []string { + // Define the regular expression pattern to match string inside double quotation + // Replace all quoted substring with an empty string + excludeQuotedSubstrings := func(input string) string { + re := regexp.MustCompile(`"[^"]*"`) + result := re.ReplaceAllString(input, "") + return result + } + + // Define the regular expression pattern to match non-alphanumeric characters except underscore + // Replace non-alphanumeric or underscore characters with an empty string + excludeNonAlphanumericUnderscore := func(input string) string { + pattern := `[^a-zA-Z0-9_]` + re := regexp.MustCompile(pattern) + result := re.ReplaceAllString(input, "") + return result + } + + var labelNames []string + unQuoted := excludeQuotedSubstrings(fullLabelString) + commaSeparated := strings.Split(unQuoted, ",") + for _, s := range commaSeparated { + labelName := excludeNonAlphanumericUnderscore(s) + if labelName != "" { + labelNames = append(labelNames, labelName) + } + } + return labelNames +}