From f74f863beab032392cc9bf1d51a6b0bc7ab8cf3f Mon Sep 17 00:00:00 2001 From: Ling Jin <7138436+3AceShowHand@users.noreply.github.com> Date: Fri, 6 Sep 2024 19:26:53 +0800 Subject: [PATCH] metrics(ticdc): fix some missing db sorter related metrics (#11547) close pingcap/tiflow#11574 --- cdc/scheduler/internal/v3/agent/agent.go | 2 +- .../v3/replication/replication_manager.go | 2 + metrics/grafana/ticdc.json | 728 +++++------------- pkg/config/db.go | 21 +- pkg/config/debug.go | 11 + pkg/config/kvclient.go | 22 +- pkg/config/server_config.go | 32 +- 7 files changed, 271 insertions(+), 547 deletions(-) diff --git a/cdc/scheduler/internal/v3/agent/agent.go b/cdc/scheduler/internal/v3/agent/agent.go index 664c479a039..d2dd1bbfa7f 100644 --- a/cdc/scheduler/internal/v3/agent/agent.go +++ b/cdc/scheduler/internal/v3/agent/agent.go @@ -214,7 +214,7 @@ func (a *agent) Tick(ctx context.Context) (*schedulepb.Barrier, error) { outboundMessages = append(outboundMessages, responses...) - if err := a.sendMsgs(ctx, outboundMessages); err != nil { + if err = a.sendMsgs(ctx, outboundMessages); err != nil { return nil, errors.Trace(err) } diff --git a/cdc/scheduler/internal/v3/replication/replication_manager.go b/cdc/scheduler/internal/v3/replication/replication_manager.go index c3583606c85..cb042689690 100644 --- a/cdc/scheduler/internal/v3/replication/replication_manager.go +++ b/cdc/scheduler/internal/v3/replication/replication_manager.go @@ -902,6 +902,8 @@ func (r *Manager) CleanMetrics() { slowestTableStageCheckpointTsLagHistogramVec.Reset() slowestTableStageResolvedTsLagHistogramVec.Reset() slowestTableRegionGaugeVec.Reset() + slowestTablePullerResolvedTs.Reset() + slowestTablePullerResolvedTsLag.Reset() } // SetReplicationSetForTests is only used in tests. diff --git a/metrics/grafana/ticdc.json b/metrics/grafana/ticdc.json index b9233e44ef2..aa159db19e3 100644 --- a/metrics/grafana/ticdc.json +++ b/metrics/grafana/ticdc.json @@ -131,7 +131,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1724839764339, + "iteration": 1725343906113, "links": [], "panels": [ { @@ -3973,7 +3973,7 @@ "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-barrier", + "legendFormat": "{{namespace}}-{{changefeed}}-barrier-lag", "refId": "C" }, { @@ -3981,7 +3981,7 @@ "expr": "max(ticdc_scheduler_slow_table_puller_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)", "hide": false, "interval": "", - "legendFormat": "{{namespace}}-{{changefeed}}-puller", + "legendFormat": "{{namespace}}-{{changefeed}}-puller-lag", "refId": "A" } ], @@ -4764,7 +4764,7 @@ "h": 4, "w": 12, "x": 0, - "y": 46 + "y": 4 }, "hiddenSeries": false, "id": 218, @@ -4811,7 +4811,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Puller output events/s", + "title": "Puller output events / s", "tooltip": { "shared": true, "sort": 0, @@ -4867,7 +4867,7 @@ "h": 4, "w": 12, "x": 12, - "y": 46 + "y": 4 }, "hiddenSeries": false, "id": 229, @@ -4972,7 +4972,7 @@ "h": 4, "w": 12, "x": 0, - "y": 50 + "y": 8 }, "hiddenSeries": false, "id": 228, @@ -5019,7 +5019,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Sorter output events/s", + "title": "Sorter output events / s", "tooltip": { "shared": true, "sort": 0, @@ -5075,7 +5075,7 @@ "h": 4, "w": 12, "x": 12, - "y": 50 + "y": 8 }, "hiddenSeries": false, "id": 220, @@ -5178,7 +5178,7 @@ "h": 4, "w": 12, "x": 0, - "y": 54 + "y": 12 }, "hiddenSeries": false, "id": 219, @@ -5225,7 +5225,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Mounter output events/s", + "title": "Mounter output events / s", "tooltip": { "shared": true, "sort": 0, @@ -5281,7 +5281,7 @@ "h": 4, "w": 12, "x": 12, - "y": 54 + "y": 12 }, "hiddenSeries": false, "id": 224, @@ -5384,7 +5384,7 @@ "h": 4, "w": 12, "x": 0, - "y": 58 + "y": 16 }, "hiddenSeries": false, "id": 223, @@ -5431,7 +5431,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Table sink output events/s", + "title": "Table sink output events / s", "tooltip": { "shared": true, "sort": 0, @@ -5487,7 +5487,7 @@ "h": 4, "w": 12, "x": 12, - "y": 58 + "y": 16 }, "hiddenSeries": false, "id": 221, @@ -5590,7 +5590,7 @@ "h": 4, "w": 12, "x": 0, - "y": 62 + "y": 20 }, "hiddenSeries": false, "id": 664, @@ -5638,7 +5638,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Sink flush rows/s", + "title": "Sink flush rows / s", "tooltip": { "shared": true, "sort": 2, @@ -5694,7 +5694,7 @@ "h": 4, "w": 12, "x": 12, - "y": 62 + "y": 20 }, "hiddenSeries": false, "id": 665, @@ -7190,7 +7190,7 @@ "h": 7, "w": 12, "x": 0, - "y": 95 + "y": 7 }, "hiddenSeries": false, "id": 653, @@ -7298,7 +7298,7 @@ "h": 7, "w": 12, "x": 12, - "y": 95 + "y": 7 }, "hiddenSeries": false, "id": 628, @@ -7401,7 +7401,7 @@ "h": 7, "w": 12, "x": 0, - "y": 102 + "y": 14 }, "hiddenSeries": false, "id": 627, @@ -7504,7 +7504,7 @@ "h": 7, "w": 12, "x": 12, - "y": 102 + "y": 14 }, "hiddenSeries": false, "id": 630, @@ -7607,7 +7607,7 @@ "h": 7, "w": 12, "x": 0, - "y": 109 + "y": 21 }, "hiddenSeries": false, "id": 629, @@ -7710,7 +7710,7 @@ "h": 7, "w": 12, "x": 12, - "y": 109 + "y": 21 }, "hiddenSeries": false, "id": 631, @@ -7811,7 +7811,7 @@ "h": 7, "w": 12, "x": 0, - "y": 116 + "y": 28 }, "hiddenSeries": false, "id": 626, @@ -7912,7 +7912,7 @@ "h": 7, "w": 12, "x": 12, - "y": 116 + "y": 28 }, "hiddenSeries": false, "id": 709, @@ -8010,7 +8010,7 @@ "h": 7, "w": 12, "x": 0, - "y": 123 + "y": 35 }, "hiddenSeries": false, "id": 710, @@ -8108,7 +8108,7 @@ "h": 7, "w": 12, "x": 12, - "y": 123 + "y": 35 }, "hiddenSeries": false, "id": 711, @@ -8214,7 +8214,7 @@ "h": 7, "w": 12, "x": 0, - "y": 130 + "y": 42 }, "hiddenSeries": false, "id": 712, @@ -8321,7 +8321,7 @@ "h": 7, "w": 12, "x": 12, - "y": 130 + "y": 42 }, "hiddenSeries": false, "id": 741, @@ -8365,7 +8365,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Claim Check Send Message Count / Second", + "title": "Claim Check Send Message Count / s", "tooltip": { "shared": true, "sort": 0, @@ -8419,7 +8419,7 @@ "h": 7, "w": 12, "x": 0, - "y": 137 + "y": 49 }, "hiddenSeries": false, "id": 740, @@ -8525,7 +8525,7 @@ "h": 7, "w": 12, "x": 12, - "y": 137 + "y": 49 }, "hiddenSeries": false, "id": 731, @@ -8623,7 +8623,7 @@ "h": 7, "w": 12, "x": 0, - "y": 144 + "y": 56 }, "hiddenSeries": false, "id": 733, @@ -8721,7 +8721,7 @@ "h": 7, "w": 12, "x": 12, - "y": 144 + "y": 56 }, "hiddenSeries": false, "id": 735, @@ -8819,7 +8819,7 @@ "h": 8, "w": 12, "x": 0, - "y": 151 + "y": 63 }, "hiddenSeries": false, "id": 739, @@ -8917,7 +8917,7 @@ "h": 8, "w": 12, "x": 12, - "y": 151 + "y": 63 }, "hiddenSeries": false, "id": 737, @@ -9511,7 +9511,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -9614,7 +9614,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -9724,7 +9724,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 1, "points": true, "renderer": "flot", @@ -9837,7 +9837,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 1, "points": true, "renderer": "flot", @@ -9950,7 +9950,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 1, "points": true, "renderer": "flot", @@ -10063,7 +10063,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 1, "points": true, "renderer": "flot", @@ -10169,7 +10169,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": true, "renderer": "flot", @@ -10279,7 +10279,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 1, "points": true, "renderer": "flot", @@ -10391,7 +10391,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 1, "points": true, "renderer": "flot", @@ -10504,7 +10504,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 1, "points": true, "renderer": "flot", @@ -10611,7 +10611,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -10730,7 +10730,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -10854,7 +10854,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -11652,7 +11652,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -11747,7 +11747,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -11856,7 +11856,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -12037,7 +12037,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -12129,7 +12129,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "CPU usage of LevelDB sorter", + "description": "The count and duration of write delay", "fieldConfig": { "defaults": { "links": [] @@ -12145,12 +12145,11 @@ "y": 17 }, "hiddenSeries": false, - "id": 286, + "id": 275, "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, - "hideEmpty": true, "max": true, "min": false, "rightSide": false, @@ -12167,13 +12166,13 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { - "alias": "/.*sorter-[0-9]+/", + "alias": "/.*count.*/", "yaxis": 2 } ], @@ -12182,18 +12181,30 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_actor_worker_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", name=~\"sorter.*\"}[1m])) by (name, instance)", + "exemplar": true, + "expr": "sum(rate(ticdc_db_write_delay_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{name}}", + "legendFormat": "{{instance}}-duration", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_db_write_delay_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-count", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CPU usage", + "title": "Write delay", "tooltip": { "shared": true, "sort": 0, @@ -12209,7 +12220,7 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "dtdurations", "label": null, "logBase": 1, "max": null, @@ -12217,10 +12228,10 @@ "show": true }, { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, - "max": "1.2", + "max": null, "min": null, "show": true } @@ -12353,7 +12364,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -12414,124 +12425,6 @@ "alignLevel": null } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "The count and duration of write delay", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 23 - }, - "hiddenSeries": false, - "id": 275, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*count.*/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_db_write_delay_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-duration", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_db_write_delay_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-count", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Write delay", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "dtdurations", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, { "cards": { "cardPadding": 0, @@ -12547,7 +12440,7 @@ }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "The time of sorter iterator read", + "description": "The time of sorter compact", "fieldConfig": { "defaults": {}, "overrides": [] @@ -12555,13 +12448,13 @@ "gridPos": { "h": 6, "w": 8, - "x": 0, - "y": 29 + "x": 16, + "y": 23 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 278, + "id": 285, "legend": { "alignAsTable": true, "avg": false, @@ -12582,7 +12475,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[1m])) by (le)", + "expr": "sum(rate(ticdc_sorter_db_compact_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -12591,7 +12484,7 @@ "refId": "A" } ], - "title": "Read duration - First", + "title": "Compact duration", "tooltip": { "show": true, "showHistogram": true @@ -12639,13 +12532,13 @@ "gridPos": { "h": 6, "w": 8, - "x": 8, + "x": 0, "y": 29 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 282, + "id": 278, "legend": { "alignAsTable": true, "avg": false, @@ -12666,7 +12559,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[1m])) by (le)", + "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[1m])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -12675,7 +12568,7 @@ "refId": "A" } ], - "title": "Read duration - Next", + "title": "Read duration - First", "tooltip": { "show": true, "showHistogram": true @@ -12723,13 +12616,13 @@ "gridPos": { "h": 6, "w": 8, - "x": 16, + "x": 8, "y": 29 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 281, + "id": 282, "legend": { "alignAsTable": true, "avg": false, @@ -12750,7 +12643,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[1m])) by (le)", + "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[1m])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -12759,7 +12652,7 @@ "refId": "A" } ], - "title": "Read duration - Release", + "title": "Read duration - Next", "tooltip": { "show": true, "showHistogram": true @@ -12823,7 +12716,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -12923,7 +12816,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -12983,190 +12876,6 @@ "align": false, "alignLevel": null } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 35 - }, - "hiddenSeries": false, - "id": 287, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[1m])) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-sorter", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Read OPS - Release", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 10, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The time of sorter compact", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 41 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 285, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "maxPerRow": 3, - "repeatDirection": "h", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_compact_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Compact duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null } ], "title": "DB", @@ -13202,7 +12911,7 @@ "h": 7, "w": 12, "x": 0, - "y": 12 + "y": 120 }, "hiddenSeries": false, "id": 15, @@ -13227,7 +12936,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -13332,7 +13041,7 @@ "h": 7, "w": 12, "x": 12, - "y": 12 + "y": 120 }, "hiddenSeries": false, "id": 29, @@ -13358,7 +13067,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -13437,7 +13146,7 @@ "h": 7, "w": 12, "x": 0, - "y": 19 + "y": 127 }, "hiddenSeries": false, "id": 28, @@ -13463,7 +13172,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -13559,7 +13268,7 @@ "h": 7, "w": 12, "x": 12, - "y": 19 + "y": 127 }, "hiddenSeries": false, "id": 31, @@ -13585,7 +13294,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -13674,7 +13383,7 @@ "h": 7, "w": 12, "x": 0, - "y": 26 + "y": 134 }, "hiddenSeries": false, "id": 188, @@ -13700,7 +13409,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -13778,7 +13487,7 @@ "h": 7, "w": 12, "x": 12, - "y": 26 + "y": 134 }, "hiddenSeries": false, "id": 17, @@ -13802,7 +13511,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -13881,7 +13590,7 @@ "h": 7, "w": 12, "x": 0, - "y": 33 + "y": 141 }, "hiddenSeries": false, "id": 177, @@ -13907,7 +13616,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -13985,7 +13694,7 @@ "h": 7, "w": 12, "x": 12, - "y": 33 + "y": 141 }, "hiddenSeries": false, "id": 459, @@ -14013,7 +13722,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -14097,7 +13806,7 @@ "h": 7, "w": 12, "x": 0, - "y": 40 + "y": 148 }, "heatmap": {}, "hideZeroBuckets": true, @@ -14174,7 +13883,7 @@ "h": 7, "w": 12, "x": 12, - "y": 40 + "y": 148 }, "hiddenSeries": false, "id": 251, @@ -14200,7 +13909,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -14278,7 +13987,7 @@ "h": 7, "w": 12, "x": 0, - "y": 47 + "y": 155 }, "hiddenSeries": false, "id": 453, @@ -14306,7 +14015,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -14386,7 +14095,7 @@ "h": 7, "w": 12, "x": 12, - "y": 47 + "y": 155 }, "hiddenSeries": false, "id": 460, @@ -14414,7 +14123,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -14495,7 +14204,7 @@ "h": 7, "w": 12, "x": 0, - "y": 54 + "y": 162 }, "hiddenSeries": false, "id": 5, @@ -14519,7 +14228,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -14598,7 +14307,7 @@ "h": 7, "w": 12, "x": 12, - "y": 54 + "y": 162 }, "hiddenSeries": false, "id": 252, @@ -14624,7 +14333,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -14708,7 +14417,7 @@ "h": 7, "w": 12, "x": 0, - "y": 61 + "y": 169 }, "heatmap": {}, "hideZeroBuckets": true, @@ -14790,7 +14499,7 @@ "h": 7, "w": 12, "x": 12, - "y": 61 + "y": 169 }, "heatmap": {}, "hideZeroBuckets": true, @@ -14865,7 +14574,7 @@ "h": 7, "w": 12, "x": 0, - "y": 68 + "y": 176 }, "hiddenSeries": false, "id": 53, @@ -14890,7 +14599,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -14979,7 +14688,7 @@ "h": 7, "w": 12, "x": 12, - "y": 68 + "y": 176 }, "hiddenSeries": false, "id": 106, @@ -15004,7 +14713,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -15091,7 +14800,7 @@ "h": 7, "w": 12, "x": 0, - "y": 75 + "y": 183 }, "hiddenSeries": false, "id": 707, @@ -15113,7 +14822,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -15189,7 +14898,7 @@ "h": 7, "w": 12, "x": 12, - "y": 75 + "y": 183 }, "hiddenSeries": false, "id": 10039, @@ -15211,7 +14920,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -15301,7 +15010,7 @@ "h": 7, "w": 12, "x": 0, - "y": 13 + "y": 121 }, "hiddenSeries": false, "id": 289, @@ -15326,7 +15035,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -15403,7 +15112,7 @@ "h": 7, "w": 12, "x": 12, - "y": 13 + "y": 121 }, "hiddenSeries": false, "id": 114, @@ -15428,7 +15137,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -15525,7 +15234,7 @@ "h": 7, "w": 12, "x": 0, - "y": 20 + "y": 128 }, "heatmap": {}, "hideZeroBuckets": true, @@ -15593,7 +15302,7 @@ "h": 7, "w": 12, "x": 12, - "y": 20 + "y": 128 }, "hiddenSeries": false, "id": 264, @@ -15613,7 +15322,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -15705,7 +15414,7 @@ "h": 7, "w": 12, "x": 0, - "y": 27 + "y": 135 }, "heatmap": {}, "hideZeroBuckets": true, @@ -15775,7 +15484,7 @@ "h": 7, "w": 12, "x": 12, - "y": 27 + "y": 135 }, "hiddenSeries": false, "id": 258, @@ -15795,7 +15504,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -15890,7 +15599,7 @@ "h": 7, "w": 12, "x": 0, - "y": 34 + "y": 142 }, "heatmap": {}, "hideZeroBuckets": true, @@ -15956,7 +15665,7 @@ "h": 7, "w": 12, "x": 12, - "y": 34 + "y": 142 }, "hiddenSeries": false, "id": 260, @@ -15976,7 +15685,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -16063,7 +15772,7 @@ "h": 7, "w": 12, "x": 0, - "y": 41 + "y": 149 }, "hiddenSeries": false, "id": 291, @@ -16090,7 +15799,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -16171,7 +15880,7 @@ "h": 7, "w": 12, "x": 12, - "y": 41 + "y": 149 }, "hiddenSeries": false, "id": 290, @@ -16196,7 +15905,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -16316,7 +16025,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -16454,7 +16163,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -16474,9 +16183,11 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "sum(rate(tikv_cdc_grpc_message_sent_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[30s])) by (instance, type)", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", "refId": "A", @@ -16571,7 +16282,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -16679,7 +16390,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -16700,67 +16411,28 @@ "steppedLine": false, "targets": [ { - "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "tikv-{{instance}}", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"cdc.*\"}) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "cdc-{{instance}}", - "refId": "B", - "step": 10 - }, - { + "exemplar": true, "expr": "(avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)) - (avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", db=\"kv\"}) by(instance))", - "format": "time_series", "hide": false, - "intervalFactor": 2, + "interval": "", "legendFormat": "tikv-{{instance}}", - "refId": "C", - "step": 10 + "refId": "A" }, { + "exemplar": true, "expr": "sum(tikv_cdc_sink_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", "hide": false, - "intervalFactor": 2, + "interval": "", "legendFormat": "sink-{{instance}}", - "refId": "D", - "step": 10 + "refId": "B" }, { + "exemplar": true, "expr": "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", "hide": false, - "intervalFactor": 2, + "interval": "", "legendFormat": "old-value-{{instance}}", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(tikv_cdc_sink_memory_capacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "sink-cap-{{instance}}", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "old-value-cap-{{instance}}", - "refId": "G", - "step": 10 + "refId": "C" } ], "thresholds": [], @@ -16853,7 +16525,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 1, "points": false, "renderer": "flot", @@ -16876,10 +16548,12 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", "format": "time_series", "hide": false, "instant": false, + "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-min-resolved-lag", "refId": "A", @@ -16895,9 +16569,11 @@ "step": 10 }, { - "expr": "avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", + "exemplar": true, + "expr": "sum(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-min-resolved-ts", "refId": "C", @@ -16994,7 +16670,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 1, "points": true, "renderer": "flot", @@ -17005,7 +16681,7 @@ "targets": [ { "exemplar": true, - "expr": "avg(tikv_cdc_min_resolved_ts_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", + "expr": "sum(tikv_cdc_min_resolved_ts_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "format": "time_series", "hide": false, "interval": "", @@ -17019,7 +16695,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Min resolved Region", + "title": "Min resolved Region ID", "tooltip": { "msResolution": false, "shared": true, @@ -17104,7 +16780,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -17114,8 +16790,10 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "histogram_quantile(0.99999, sum(rate(tikv_cdc_resolved_ts_gap_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))", "format": "time_series", + "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-p9999", "refId": "A" @@ -17220,7 +16898,7 @@ "refId": "A" } ], - "title": "Initial scan duration", + "title": "Incremental scan duration", "tooltip": { "show": true, "showHistogram": true @@ -17292,7 +16970,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -17302,18 +16980,28 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "histogram_quantile(0.9999, sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-p9999", + "legendFormat": "p9999-{{instance}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance) / sum(rate(tikv_cdc_scan_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{instance}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Initial scan duration percentile", + "title": "Incremental scan duration percentile", "tooltip": { "shared": true, "sort": 0, @@ -17397,7 +17085,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -17412,17 +17100,21 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"ongoing\"}) by (type, instance)", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "refId": "A" }, { - "expr": "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=~\"abort|finish\"}) by (instance)", + "exemplar": true, + "expr": "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=~\"abort|finish|ongoing\"}) by (instance)", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-pending", "refId": "B" @@ -17432,7 +17124,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Initial scan tasks status", + "title": "Incremental scan tasks count", "tooltip": { "shared": true, "sort": 0, @@ -17517,7 +17209,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -17527,9 +17219,11 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "avg(tikv_cdc_captured_region_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 2, "legendFormat": "tikv-{{instance}}-total", "refId": "A", @@ -17637,7 +17331,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -17647,9 +17341,11 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "sum(rate(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}[30s])) by (instance)", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 2, "legendFormat": "tikv-{{instance}}", "refId": "A", @@ -17660,7 +17356,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CDC scan speed", + "title": "Incremental scan speed / s", "tooltip": { "msResolution": false, "shared": true, @@ -17729,7 +17425,7 @@ "current": true, "hideEmpty": true, "hideZero": true, - "max": true, + "max": false, "min": false, "rightSide": false, "show": true, @@ -17748,7 +17444,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -17758,9 +17454,11 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "sum(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}) by (instance)", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 2, "legendFormat": "tikv-{{instance}}", "refId": "A", @@ -17771,7 +17469,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CDC total scan bytes", + "title": "Incremental scan total bytes", "tooltip": { "msResolution": false, "shared": true, @@ -17856,7 +17554,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 1, "points": true, "renderer": "flot", @@ -17993,7 +17691,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 1, "points": false, "renderer": "flot", @@ -18124,7 +17822,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -18314,7 +18012,7 @@ }, "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -21953,5 +21651,5 @@ "timezone": "browser", "title": "${DS_TEST-CLUSTER}-TiCDC", "uid": "YiGL8hBZ1", - "version": 60 + "version": 61 } diff --git a/pkg/config/db.go b/pkg/config/db.go index f9b70136448..ecf46587d84 100644 --- a/pkg/config/db.go +++ b/pkg/config/db.go @@ -13,7 +13,11 @@ package config -import "github.com/pingcap/tiflow/pkg/errors" +import ( + "math" + + "github.com/pingcap/tiflow/pkg/errors" +) // DBConfig represents db sorter config. type DBConfig struct { @@ -51,6 +55,21 @@ type DBConfig struct { CompactionL0Trigger int `toml:"compaction-l0-trigger" json:"compaction-l0-trigger"` } +// NewDefaultDBConfig return the default db configuration +func NewDefaultDBConfig() *DBConfig { + return &DBConfig{ + Count: 8, + // Following configs are optimized for write/read throughput. + // Users should not change them. + MaxOpenFiles: 10000, + BlockSize: 65536, + WriterBufferSize: 8388608, + Compression: "snappy", + WriteL0PauseTrigger: math.MaxInt32, + CompactionL0Trigger: 16, // Based on a performance test on 4K tables. + } +} + // ValidateAndAdjust validates and adjusts the db configuration func (c *DBConfig) ValidateAndAdjust() error { if c.Compression != "none" && c.Compression != "snappy" { diff --git a/pkg/config/debug.go b/pkg/config/debug.go index d44cb1e71ba..00e38f841c1 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -14,6 +14,8 @@ package config import ( + "time" + "github.com/pingcap/errors" ) @@ -60,3 +62,12 @@ type PullerConfig struct { // LogRegionDetails determines whether logs Region details or not in puller and kv-client. LogRegionDetails bool `toml:"log-region-details" json:"log-region-details"` } + +// NewDefaultPullerConfig return the default puller configuration +func NewDefaultPullerConfig() *PullerConfig { + return &PullerConfig{ + EnableResolvedTsStuckDetection: false, + ResolvedTsStuckInterval: TomlDuration(5 * time.Minute), + LogRegionDetails: false, + } +} diff --git a/pkg/config/kvclient.go b/pkg/config/kvclient.go index f6a9c49af09..17d17143127 100644 --- a/pkg/config/kvclient.go +++ b/pkg/config/kvclient.go @@ -13,7 +13,11 @@ package config -import "github.com/pingcap/tiflow/pkg/errors" +import ( + "time" + + "github.com/pingcap/tiflow/pkg/errors" +) // KVClientConfig represents config for kv client type KVClientConfig struct { @@ -34,6 +38,22 @@ type KVClientConfig struct { RegionRetryDuration TomlDuration `toml:"region-retry-duration" json:"region-retry-duration"` } +// NewDefaultKVClientConfig return the default kv client configuration +func NewDefaultKVClientConfig() *KVClientConfig { + return &KVClientConfig{ + EnableMultiplexing: true, + WorkerConcurrent: 8, + GrpcStreamConcurrent: 1, + AdvanceIntervalInMs: 300, + FrontierConcurrent: 8, + WorkerPoolSize: 0, // 0 will use NumCPU() * 2 + RegionScanLimit: 40, + // The default TiKV region election timeout is [10s, 20s], + // Use 1 minute to cover region leader missing. + RegionRetryDuration: TomlDuration(time.Minute), + } +} + // ValidateAndAdjust validates and adjusts the kv client configuration func (c *KVClientConfig) ValidateAndAdjust() error { if c.RegionScanLimit <= 0 { diff --git a/pkg/config/server_config.go b/pkg/config/server_config.go index 14cfd9cec11..108d2cb3efe 100644 --- a/pkg/config/server_config.go +++ b/pkg/config/server_config.go @@ -16,7 +16,6 @@ package config import ( "encoding/json" "fmt" - "math" "net" "regexp" "strings" @@ -112,39 +111,14 @@ var defaultServerConfig = &ServerConfig{ CacheSizeInMB: 128, // By default, use 128M memory as sorter cache. }, Security: &security.Credential{}, - KVClient: &KVClientConfig{ - EnableMultiplexing: true, - WorkerConcurrent: 8, - GrpcStreamConcurrent: 1, - AdvanceIntervalInMs: 300, - FrontierConcurrent: 8, - WorkerPoolSize: 0, // 0 will use NumCPU() * 2 - RegionScanLimit: 40, - // The default TiKV region election timeout is [10s, 20s], - // Use 1 minute to cover region leader missing. - RegionRetryDuration: TomlDuration(time.Minute), - }, + KVClient: NewDefaultKVClientConfig(), Debug: &DebugConfig{ - DB: &DBConfig{ - Count: 8, - // Following configs are optimized for write/read throughput. - // Users should not change them. - MaxOpenFiles: 10000, - BlockSize: 65536, - WriterBufferSize: 8388608, - Compression: "snappy", - WriteL0PauseTrigger: math.MaxInt32, - CompactionL0Trigger: 16, // Based on a performance test on 4K tables. - }, + DB: NewDefaultDBConfig(), Messages: defaultMessageConfig.Clone(), Scheduler: NewDefaultSchedulerConfig(), CDCV2: &CDCV2{Enable: false}, - Puller: &PullerConfig{ - EnableResolvedTsStuckDetection: false, - ResolvedTsStuckInterval: TomlDuration(5 * time.Minute), - LogRegionDetails: false, - }, + Puller: NewDefaultPullerConfig(), }, ClusterID: "default", GcTunerMemoryThreshold: DisableMemoryLimit,