{ "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 34, "iteration": 1667987821492, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 123248, "panels": [], "title": "Proxy", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of searching vectors.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 1 }, "hiddenSeries": false, "id": 123250, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_search_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Search Vector Count Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of inserting vectors.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 1 }, "hiddenSeries": false, "id": 123315, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_insert_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Insert Vector Count Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of search request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 1 }, "hiddenSeries": false, "id": 123264, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "hide": false, "interval": "", "legendFormat": "p99-{{query_type}}-{{pod}}-{{node_id}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{query_type}}-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Search Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of search request for a specified collection over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 7 }, "hiddenSeries": false, "id": 123377, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_collection_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))", "hide": false, "interval": "", "legendFormat": "p99-{{query_type}}-{{pod}}-{{node_id}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_collection_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_collection_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{query_type}}-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Collection Search Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of mutation request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 7 }, "hiddenSeries": false, "id": 123320, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{msg_type}}-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)", "hide": false, "interval": "", "legendFormat": "avg-{{msg_type}}-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Mutation Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of mutation request for a specified collection over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 7 }, "hiddenSeries": false, "id": 123378, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_collection_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_collection_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_collection_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Collection Mutation Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of wait search result over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 13 }, "hiddenSeries": false, "id": 123348, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_wait_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "hide": false, "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_sq_wait_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_wait_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Wait Search Result Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of reduce search result over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 13 }, "hiddenSeries": false, "id": 123316, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_reduce_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_sq_reduce_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_reduce_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Reduce Search Result Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of decode search result over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 13 }, "hiddenSeries": false, "id": 123317, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_decode_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "hide": false, "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_sq_decode_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_decode_resultlatency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Decode Search Result Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Average, maximum and minimum values of the number of msgstream objects created on all physical topics.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 123319, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "avg(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "max(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "min(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-min", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "Msg Stream Object Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of send mutation over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 19 }, "hiddenSeries": false, "id": 123321, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_send_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{msg_type}}-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)", "hide": false, "interval": "", "legendFormat": "avg-{{msg_type}}-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Mutation Send Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Average cache hits per minute of cache operations.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 19 }, "hiddenSeries": false, "id": 123322, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", cache_state=\"hit\"}[2m])/120) by(cache_name, pod, node_id) / sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(cache_name, pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}-{{cache_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Cache Hit rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "max": "1", "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of update cache over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 25 }, "hiddenSeries": false, "id": 123323, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_cache_update_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_cache_update_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_cache_update_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Cache Update Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 25 }, "hiddenSeries": false, "id": 123325, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "avg(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "max(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "min(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-min", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "", "hide": false, "interval": "", "legendFormat": "", "refId": "D" } ], "thresholds": [], "timeRegions": [], "title": "Produced Timetick Lag Behind Now", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile of latency for applying PK over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 25 }, "hiddenSeries": false, "id": 123326, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_pk_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_apply_pk_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_pk_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Apply PK Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of apply timestamp over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 31 }, "hiddenSeries": false, "id": 123327, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_timestamp_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_apply_timestamp_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_timestamp_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Apply Timestamp Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of success requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 31 }, "hiddenSeries": false, "id": 123329, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"success\"}[2m])/120) by(function_name, pod, node_id)", "interval": "", "legendFormat": "{{function_name}}-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Success Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of faild requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 31 }, "hiddenSeries": false, "id": 123379, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by(function_name, pod, node_id)", "interval": "", "legendFormat": "{{function_name}}-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Faild Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 37 }, "hiddenSeries": false, "id": 123381, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, function_name) (rate(milvus_proxy_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{function_name}}-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name) / sum(increase(milvus_proxy_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name)", "hide": false, "interval": "", "legendFormat": "avg-{{function_name}}-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of bytes received in proxy", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 37 }, "hiddenSeries": false, "id": 123368, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_receive_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Received Byte Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of bytes sent back to the client by the Proxy in response to a Search or Query request.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 37 }, "hiddenSeries": false, "id": 123369, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_send_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Send Byte Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "id": 123155, "panels": [], "title": "Root Coordinator", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of proxy nodes which has register with etcd", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 44 }, "hiddenSeries": false, "id": 123141, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_rootcoord_proxy_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Proxy Node Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 44 }, "hiddenSeries": false, "id": 123384, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "avg(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "max(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "min(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-min", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "Produced Timetick Lag Behind Now", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency for rootcoord to finish synchronizing timestamp messages to all pchanels.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 44 }, "hiddenSeries": false, "id": 123338, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-latency", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_sync_timetick_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_rootcoord_sync_timetick_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", "hide": false, "interval": "", "legendFormat": "avg-latency", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Produced Timetick Time Taken", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of DDL request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 50 }, "hiddenSeries": false, "id": 123337, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{function_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_ddl_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name) / sum(increase(milvus_rootcoord_ddl_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name)", "hide": false, "interval": "", "legendFormat": "avg-{{function_name}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "DDL Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "RootCoord stores pre-assigned timestamps in the metastore", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 50 }, "hiddenSeries": false, "id": 123340, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "milvus_rootcoord_timestamp_saved{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "tiestamp", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Timestamp Saved", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of DDL request", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 50 }, "hiddenSeries": false, "id": 123347, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, function_name)", "interval": "", "legendFormat": "{{function_name}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "DDL Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1456", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1457", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "RoootCoord current latest timestamp", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 56 }, "hiddenSeries": false, "id": 123339, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "milvus_rootcoord_timestamp{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "tiestamp", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Timestamp", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The number of DML channels", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 56 }, "hiddenSeries": false, "id": 123345, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_rootcoord_dml_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "DML Channel Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of IDs assigned by RootCoord", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 56 }, "hiddenSeries": false, "id": 123221, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_id_alloc_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120)", "interval": "", "legendFormat": "total", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "ID Alloc Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1456", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1457", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The number of partitions.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 62 }, "hiddenSeries": false, "id": 123344, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_rootcoord_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Partition Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The max time tick delay of flow graphs. unit ms", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 62 }, "hiddenSeries": false, "id": 123383, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_rootcoord_time_tick_delay{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (role_name, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{role_name}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Time Tick Delay", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The number of collections.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 62 }, "hiddenSeries": false, "id": 123342, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_rootcoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Collection Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The number of credential", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 68 }, "hiddenSeries": false, "id": 123343, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_rootcoord_credential_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Credential Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The number of Msgstream objects.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 68 }, "hiddenSeries": false, "id": 123346, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_rootcoord_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Msgstream Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 74 }, "id": 123246, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of loaded collections.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 3 }, "hiddenSeries": false, "id": 123288, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querycoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Collection Loaded Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of loaded entities.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 3 }, "hiddenSeries": false, "id": 123289, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querycoord_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Entity Loaded Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of load requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 3 }, "hiddenSeries": false, "id": 123291, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querycoord_load_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)", "interval": "", "legendFormat": "{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Load Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of relaase requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 123292, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querycoord_release_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)", "interval": "", "legendFormat": "{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Release Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of load request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 9 }, "hiddenSeries": false, "id": 123294, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_load_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-latency", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querycoord_load_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_load_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", "hide": false, "interval": "", "legendFormat": "avg-latency", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Load Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of release request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 9 }, "hiddenSeries": false, "id": 123313, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_release_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-latency", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querycoord_release_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_release_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", "hide": false, "interval": "", "legendFormat": "avg-latency", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Release Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of sub-load task.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 15 }, "hiddenSeries": false, "id": 123295, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querycoord_child_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Sub-Load Task", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number pf parent loading task.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 15 }, "hiddenSeries": false, "id": 123296, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querycoord_parent_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Parent Load Task", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of sub-load task request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 15 }, "hiddenSeries": false, "id": 123298, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_child_task_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-latency", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querycoord_child_task_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_child_task_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", "hide": false, "interval": "", "legendFormat": "avg-latency", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Sub-Load Task Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of Query nodes which has register with etcd.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 21 }, "hiddenSeries": false, "id": 123297, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querycoord_querynode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Query Node Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Query Coordinator", "type": "row" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 75 }, "id": 123244, "panels": [], "title": "Query Node", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of the loaded collections in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 76 }, "hiddenSeries": false, "id": 123299, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Collection Loaded Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of the loaded partitions in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 76 }, "hiddenSeries": false, "id": 123303, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Partition Loaded Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of the loaded segment in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 76 }, "hiddenSeries": false, "id": 123305, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, segment_state)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}-{{segment_state}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Segment Loaded Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 82 }, "hiddenSeries": false, "id": 123385, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-min", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "TimeTick Lag Behind Now (Consumed Insert)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 82 }, "hiddenSeries": false, "id": 123386, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-min", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "TimeTick Lag Behind Now (Consumed Delete)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of consuming message", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 82 }, "hiddenSeries": false, "id": 123387, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-all", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Consumed Message Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "cps", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of the queryable entities in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 88 }, "hiddenSeries": false, "id": 123365, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Queryable Entity Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of dml virtual channels for QueryNode watch", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 88 }, "hiddenSeries": false, "id": 123304, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_dml_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "DML Virtual Channel", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of delta virtual channels for QueryNode watch", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 88 }, "hiddenSeries": false, "id": 123306, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_delta_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Delta Virtual Channel", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of consumers in the QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 94 }, "hiddenSeries": false, "id": 123307, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Consumer Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of searching requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 94 }, "hiddenSeries": false, "id": 123350, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (query_type, status, pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}-{{query_type}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Search Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of search and query request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 94 }, "hiddenSeries": false, "id": 123366, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Search Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of search or query in queue over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 100 }, "hiddenSeries": false, "id": 123372, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Search in Queue Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of search segment over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 100 }, "hiddenSeries": false, "id": 123374, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": false, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}-{{segment_state}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}_{{segment_state}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Search Segment Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 100 }, "hiddenSeries": false, "id": 123310, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_core_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_core_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Segcore Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of search or query reduce over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 106 }, "hiddenSeries": false, "id": 123367, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_reduce_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Search Reduce Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of load segment over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 106 }, "hiddenSeries": false, "id": 123311, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_load_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_load_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_load_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Load Segment Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of flowgraph", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 106 }, "hiddenSeries": false, "id": 123312, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Flowgraph Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The length of the task queue for unsolved read requests", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 112 }, "hiddenSeries": false, "id": 123351, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_read_task_unsolved_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Unsolved Read Task Length", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The length of the task queue of read requests to be executed", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 112 }, "hiddenSeries": false, "id": 123356, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_read_task_ready_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Ready Read Task Length", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of read requests currently being executed in parallel", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 112 }, "hiddenSeries": false, "id": 123357, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_read_task_concurrency{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Parallel Read Task Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "cpu utilization under scheduler evaluation", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 118 }, "hiddenSeries": false, "id": 123358, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_querynode_estimate_cpu_usage{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Estimate CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The number of original tasks contained in the merged search task", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 118 }, "hiddenSeries": false, "id": 123352, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_group_size_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Search Group Size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of queries for search requests", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 118 }, "hiddenSeries": false, "id": 123361, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Search NQ", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of queries for the merged search requests", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 124 }, "hiddenSeries": false, "id": 123360, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_group_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-milvus_querynode_search_group_nq", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Search Group NQ", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Top_K for search requests", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 124 }, "hiddenSeries": false, "id": 123359, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Search Top_K", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Top_K for the merged search requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 124 }, "hiddenSeries": false, "id": 123362, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_group_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Search Group Top_K", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of evicted read requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 130 }, "hiddenSeries": false, "id": 123364, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_read_evicted_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Evicted Read Requests Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "id": 123397, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 130 }, "type": "graph", "title": "Knowhere Search Top_K", "datasource": { "uid": "$datasource", "type": "prometheus" }, "thresholds": [], "pluginVersion": "8.5.20", "description": "Top_K for knowhere search requests", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "aliasColors": {}, "dashLength": 10, "fill": 1, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "pointradius": 2, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(knowhere_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(knowhere_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(knowhere_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "timeRegions": [], "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "xaxis": { "mode": "time", "show": true, "values": [], "name": null, "buckets": null }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false }, "bars": false, "dashes": false, "fillGradient": 0, "hiddenSeries": false, "percentage": false, "points": false, "stack": false, "steppedLine": false, "timeFrom": null, "timeShift": null }, { "id": 123395, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 130 }, "type": "graph", "title": "Knowhere Search Count", "datasource": { "uid": "$datasource", "type": "prometheus" }, "thresholds": [], "pluginVersion": "8.5.20", "description": "Total number of knowhere search cnt.", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "aliasColors": {}, "dashLength": 10, "fill": 1, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "pointradius": 2, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(knowhere_search_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "timeRegions": [], "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "xaxis": { "mode": "time", "show": true, "values": [], "name": null, "buckets": null }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false }, "bars": false, "dashes": false, "fillGradient": 0, "hiddenSeries": false, "percentage": false, "points": false, "stack": false, "steppedLine": false, "timeFrom": null, "timeShift": null }, { "id": 123396, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 136 }, "type": "graph", "title": "Knowhere Range Search Count", "datasource": { "uid": "$datasource", "type": "prometheus" }, "thresholds": [], "pluginVersion": "8.5.20", "description": "Total number of knowhere range search cnt.", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "aliasColors": {}, "dashLength": 10, "fill": 1, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "pointradius": 2, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(knowhere_range_search_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "timeRegions": [], "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "xaxis": { "mode": "time", "show": true, "values": [], "name": null, "buckets": null }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false }, "bars": false, "dashes": false, "fillGradient": 0, "hiddenSeries": false, "percentage": false, "points": false, "stack": false, "steppedLine": false, "timeFrom": null, "timeShift": null }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 136 }, "id": 123172, "panels": [], "title": "Data Coordinator", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of data nodes which has register with etcd.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 137 }, "hiddenSeries": false, "id": 123207, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_datacoord_datanode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "total", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Data Node Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 137 }, "hiddenSeries": false, "id": 123390, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "avg(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "max(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "min(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-min", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "TimeTick Lag Behind Now From DataNode", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The number of rows of valid data accumulated in DataCoord that flushed.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 137 }, "hiddenSeries": false, "id": 123269, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_datacoord_stored_rows_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "row", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Stored Rows", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of data flushed.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 143 }, "hiddenSeries": false, "id": 123371, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_datacoord_stored_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Stored Rows Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of segments with different states in the Meta of DataCoord", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 143 }, "hiddenSeries": false, "id": 123267, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_datacoord_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (segment_state)", "interval": "", "intervalFactor": 2, "legendFormat": "{{segment_state}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Segment Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "binlog size of all collections/segments, unit byte", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 143 }, "hiddenSeries": false, "id": 123382, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_datacoord_stored_binlog_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, "legendFormat": "size", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Stored Binlog Size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 149 }, "id": 123242, "panels": [], "title": "Data Node", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of flowgraph", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 150 }, "hiddenSeries": false, "id": 123272, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_datanode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Flowgraph Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of prodecer created on the DataNode", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 150 }, "hiddenSeries": false, "id": 123277, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_datanode_producer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Producer Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 150 }, "hiddenSeries": false, "id": 123393, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "avg(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "max(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "min(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-min", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "TimeTick Lag Behind Now (Consumed All)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now, which send to DataCoord", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 156 }, "hiddenSeries": false, "id": 123388, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "avg(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "max(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "min(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "{{pod}}-{{node_id}}-min", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "Produced TimeTick Lag Behind Now For DC", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of consuming message", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 156 }, "hiddenSeries": false, "id": 123389, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Consumed Message Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "cps", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "forward delete and timetick message to delta channel latency", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 156 }, "hiddenSeries": false, "id": 123394, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_forward_delete_msg_time_taken_ms_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "hide": false, "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_forward_delete_msg_time_taken_ms_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_forward_delete_msg_time_taken_ms_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "Forward Delete&Timetick Message latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of consumers created on the DataNode", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 162 }, "hiddenSeries": false, "id": 123276, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_datanode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Consumer Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of encode the data in the buffer over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 162 }, "hiddenSeries": false, "id": 123282, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_encode_buffer_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_encode_buffer_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Encode Buffer Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of messages consumed for insert and delete operation.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 162 }, "hiddenSeries": false, "id": 123274, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_msg_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Msg Rows Consumed Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "cps", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total number of segment that has been not flushed.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 168 }, "hiddenSeries": false, "id": 123280, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_datanode_unflushed_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Unflushed Segment Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of auto flush operate.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 168 }, "hiddenSeries": false, "id": 123285, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_autoflush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Autoflush Operate Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of each message that has been flushed.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 168 }, "hiddenSeries": false, "id": 123275, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Flush Data Size Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of flush operete.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 174 }, "hiddenSeries": false, "id": 123284, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_flush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Flush Operate Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of flush requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 174 }, "hiddenSeries": false, "id": 123286, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_flush_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Flush Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of writte the data in buffer to storage over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 174 }, "hiddenSeries": false, "id": 123283, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_save_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_save_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Save Data Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of compaction over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 180 }, "hiddenSeries": false, "id": 123314, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "hide": false, "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_compaction_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_compaction_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Compaction Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 186 }, "id": 123223, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of building index requests recived by IndexCoord.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 7 }, "hiddenSeries": false, "id": 123225, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_indexcoord_index_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)", "interval": "", "legendFormat": "{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Index Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:372", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:373", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of indexing tasks with different states in IndexCoord's meta", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 7 }, "hiddenSeries": false, "id": 123227, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_indexcoord_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (index_task_status)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index_task_status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Index Task Count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:401", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:402", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of index nodes which has register with etcd", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 7 }, "hiddenSeries": false, "id": 123229, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(milvus_indexcoord_index_node_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "total", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Index Node Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:479", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:480", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Index Coordinator", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 187 }, "id": 123231, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "per-second increasing rate of index tasks recevied by IndexNode", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 8 }, "hiddenSeries": false, "id": 123233, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", "interval": "", "legendFormat": "{{pod}}-{{node_id}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Index Task Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency ofr load the FieldData over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 8 }, "hiddenSeries": false, "id": 123235, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_load_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_load_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_load_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Load Field Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of decod the FieldData over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 8 }, "hiddenSeries": false, "id": 123238, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_decode_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_decode_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_decode_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Decode Field Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:190", "format": "short", "label": "", "logBase": 1, "show": true }, { "$$hashKey": "object:191", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of build index over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 14 }, "hiddenSeries": false, "id": 123237, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_build_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_build_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_build_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Build Index Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:246", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:247", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile and average latency of encode index over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 14 }, "hiddenSeries": false, "id": 123239, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_encode_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_encode_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_encode_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Encode Index Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:275", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:276", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "The 99th percentile of latency for save index over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 14 }, "hiddenSeries": false, "id": 123240, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_save_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_save_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_save_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Save Index Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:331", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:332", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Index Node", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 188 }, "id": 123157, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Total cpu usage of all milvus components.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 183 }, "hiddenSeries": false, "id": 123202, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "sum(rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m]))", "interval": "", "intervalFactor": 2, "legendFormat": "cpu usage", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "CPU Usage Total", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:8411", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:8412", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 183 }, "hiddenSeries": false, "id": 123204, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})", "interval": "", "intervalFactor": 2, "legendFormat": "memory", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Memory Total", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1340", "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1341", "format": "bytes", "logBase": 1, "show": false } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "Number of goroutines that currently exist.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 183 }, "hiddenSeries": false, "id": 123206, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})", "interval": "", "intervalFactor": 2, "legendFormat": "goroutines", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Goroutines Total", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1281", "format": "short", "label": "", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1282", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "process cpu usage", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 189 }, "hiddenSeries": false, "id": 123135, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Process memory of milvus pod.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 189 }, "hiddenSeries": false, "id": 123133, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Memory", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "format": "bytes", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "Number of goroutines that currently exist.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 189 }, "hiddenSeries": false, "id": 123125, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Goroutines", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "Number of bytes allocated and still in use.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 195 }, "hiddenSeries": false, "id": 123137, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "go_memstats_alloc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Go Allocated Memory", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "format": "bytes", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "GC Max duration seconds.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 195 }, "hiddenSeries": false, "id": 123131, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "go_gc_duration_seconds{quantile=\"1\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", "interval": "", "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "GC Max duration seconds", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "Number of OS threads created.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 195 }, "hiddenSeries": false, "id": 123127, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "go_threads{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "OS Threads", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "Next GC Bytes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 201 }, "hiddenSeries": false, "id": 123129, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "go_memstats_next_gc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Next GC Bytes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "Number of process opened fds.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 201 }, "hiddenSeries": false, "id": 123211, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "process_open_fds{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Process Opened Fds", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "The fraction of this program's available CPU time used by the GC since the program started.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 201 }, "hiddenSeries": false, "id": 123212, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "go_memstats_gc_cpu_fraction{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "GC CPU time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "Rate of memory frees.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 207 }, "hiddenSeries": false, "id": 123213, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "rate(go_memstats_frees_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Memory Free Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "Rate of go memory mallocs.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 207 }, "hiddenSeries": false, "id": 123215, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "rate(go_memstats_mallocs_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[2m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Go Memory mallocs rate.", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "Number of allocated heap objects.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 207 }, "hiddenSeries": false, "id": 123216, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "go_memstats_heap_objects{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Allocated Heap Objects", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Rate of bytes allocated, even if freed.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 213 }, "hiddenSeries": false, "id": 123217, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "rate(go_memstats_alloc_bytes_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Memory Allocated Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:8539", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:8540", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "Number of heap bytes released to OS.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 213 }, "hiddenSeries": false, "id": 123218, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "go_memstats_heap_released_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Go Heap released", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "uid": "$datasource" }, "description": "Number of heap bytes waiting to be used.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 213 }, "hiddenSeries": false, "id": 123219, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "go_memstats_heap_idle_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Go Heap idle", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of heap bytes that be used.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 219 }, "hiddenSeries": false, "id": 123375, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "go_memstats_heap_inuse_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Go Heap in Use", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "description": "Number of container threads created.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 219 }, "hiddenSeries": false, "id": 123376, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "exemplar": true, "expr": "container_threads{image=\"\", namespace=\"$namespace\"}", "interval": "", "intervalFactor": 2, "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Container Threads", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Runtime", "type": "row" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "id": 123402, "panels": [], "title": "storage", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 14, "x": 5, "y": 5 }, "id": 123406, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.90, milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", "legendFormat": "90% {{persistent_data_op_type}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.75, milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", "hide": false, "legendFormat": "75% {{persistent_data_op_type}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.5, milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", "hide": false, "legendFormat": "50% {{persistent_data_op_type}}", "range": true, "refId": "C" } ], "title": "[go]kv size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 14, "x": 5, "y": 15 }, "id": 123446, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Name", "sortDesc": false }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"get\"}[2m])", "legendFormat": "{{le}}", "range": true, "refId": "A" } ], "title": "[go]get", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 14, "x": 5, "y": 25 }, "id": 123447, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"put\"}[2m])", "legendFormat": "{{le}}", "range": true, "refId": "A" } ], "title": "[go]put", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "list::success", "remove::total" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, "id": 123404, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(milvus_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status!=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", "legendFormat": "{{persistent_data_op_type}}::{{status}}", "range": true, "refId": "A" } ], "title": "[go]total/success request", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, "id": 123444, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, pod, persistent_data_op_type) (rate(milvus_storage_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "legendFormat": "{{persistent_data_op_type}}", "range": true, "refId": "A" } ], "title": "[go]pct99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, "id": 123405, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(milvus_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", "legendFormat": "{{persistent_data_op_type}}::{{status}}", "range": true, "refId": "A" } ], "title": "[go]failed request", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, "id": 123407, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(milvus_storage_request_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type) / sum(increase(milvus_storage_request_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type)", "legendFormat": "{{persistent_data_op_type}}", "range": true, "refId": "A" } ], "title": "[go]avg", "type": "timeseries" },{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 14, "x": 5, "y": 5 }, "id": 123506, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.90, internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", "legendFormat": "90% {{persistent_data_op_type}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.75, internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", "hide": false, "legendFormat": "75% {{persistent_data_op_type}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.5, internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", "hide": false, "legendFormat": "50% {{persistent_data_op_type}}", "range": true, "refId": "C" } ], "title": "[cpp]kv size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 14, "x": 5, "y": 15 }, "id": 123546, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Name", "sortDesc": false }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"get\"}[2m])", "legendFormat": "{{le}}", "range": true, "refId": "A" } ], "title": "[cpp]get", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 14, "x": 5, "y": 25 }, "id": 123547, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"put\"}[2m])", "legendFormat": "{{le}}", "range": true, "refId": "A" } ], "title": "[cpp]put", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "list::success", "remove::total" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, "id": 123504, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(internal_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status!=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", "legendFormat": "{{persistent_data_op_type}}::{{status}}", "range": true, "refId": "A" } ], "title": "[cpp]total/success request", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, "id": 123544, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, pod, persistent_data_op_type) (rate(internal_storage_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "legendFormat": "{{persistent_data_op_type}}", "range": true, "refId": "A" } ], "title": "[cpp]pct99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, "id": 123505, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(internal_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", "legendFormat": "{{persistent_data_op_type}}::{{status}}", "range": true, "refId": "A" } ], "title": "[cpp]failed request", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, "id": 123507, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(internal_storage_request_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type) / sum(increase(internal_storage_request_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type)", "legendFormat": "{{persistent_data_op_type}}", "range": true, "refId": "A" } ], "title": "[cpp]avg", "type": "timeseries" } ], "refresh": false, "schemaVersion": 34, "style": "dark", "tags": [ "milvus2.0" ], "templating": { "list": [ { "current": { "selected": false, "text": "prometheus", "value": "prometheus" }, "hide": 0, "includeAll": false, "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "current": { "selected": true, "text": "milvus", "value": "milvus" }, "hide": 2, "name": "app_name", "options": [ { "selected": true, "text": "milvus", "value": "milvus" } ], "query": "milvus", "skipUrlSync": false, "type": "textbox" }, { "current": { "selected": true, "text": "chaos-testing", "value": "chaos-testing" }, "datasource": { "uid": "$datasource" }, "definition": "label_values(kube_pod_info, namespace)", "hide": 0, "includeAll": false, "multi": false, "name": "namespace", "options": [], "query": { "query": "label_values(kube_pod_info, namespace)", "refId": "prometheus-namespace-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "text": "bulk-insert-test", "value": "bulk-insert-test" }, "datasource": { "uid": "$datasource" }, "definition": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)", "hide": 0, "includeAll": false, "multi": false, "name": "instance", "options": [], "query": { "query": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": false, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "$datasource" }, "definition": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"}, collection_name)", "hide": 0, "includeAll": true, "multi": false, "name": "collection", "options": [], "query": { "query": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"}, collection_name)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": { "selected": false, "text": "bulk-insert-test-milvus-standalone-55968cfc55-cxnps", "value": "bulk-insert-test-milvus-standalone-55968cfc55-cxnps" }, "datasource": { "uid": "$datasource" }, "definition": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)", "hide": 2, "includeAll": false, "multi": true, "name": "pod", "options": [], "query": { "query": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)", "refId": "prometheus-pod-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "isNone": true, "selected": false, "text": "None", "value": "" }, "datasource": { "uid": "$datasource" }, "definition": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)", "hide": 2, "includeAll": false, "multi": true, "name": "component", "options": [], "query": { "query": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": { "hidden": false, "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ], "type": "timepicker" }, "timezone": "browser", "title": "Milvus2.0", "uid": "uLf5cJ3Gz", "version": 5, "weekStart": "" }