https://pastein.ru/t/bjE
Загрузка данных
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"id": 1,
"type": "stat",
"title": "Members reporting a leader",
"gridPos": {
"x": 0,
"y": 0,
"w": 6,
"h": 5
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"unit": "short",
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 2
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto"
},
"pluginVersion": "11.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(etcd_server_has_leader:15m_min{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"})",
"format": "time_series",
"legendFormat": "",
"range": true,
"refId": "A"
}
],
"description": "Number of selected etcd instances whose retained 15m minimum says a leader was present. This is a leader-presence signal, not a proof of quorum; a completely missing member series cannot be counted."
},
{
"id": 2,
"type": "stat",
"title": "Members reporting healthy",
"gridPos": {
"x": 6,
"y": 0,
"w": 6,
"h": 5
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"unit": "short",
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 2
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto"
},
"pluginVersion": "11.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(etcd_server_health_success:15m_min{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"})",
"format": "time_series",
"legendFormat": "",
"range": true,
"refId": "A"
}
],
"description": "Number of selected etcd instances whose retained 15m minimum reports successful health. Missing series are not counted, so this is not a complete quorum calculation."
},
{
"id": 3,
"type": "stat",
"title": "DB / quota now",
"gridPos": {
"x": 12,
"y": 0,
"w": 6,
"h": 5
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 2
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto"
},
"pluginVersion": "11.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "100 * max(etcd_mvcc_db_total_size_in_bytes:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}) / min(etcd_server_quota_backend_bytes:15m_min{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"})",
"format": "time_series",
"legendFormat": "",
"range": true,
"refId": "A"
}
],
"description": "Worst selected DB-size maximum divided by the smallest selected backend quota. Useful as a conservative cluster-level saturation signal."
},
{
"id": 4,
"type": "stat",
"title": "DB / quota headroom",
"gridPos": {
"x": 18,
"y": 0,
"w": 6,
"h": 5
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 2
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto"
},
"pluginVersion": "11.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "100 - (100 * max(etcd_mvcc_db_total_size_in_bytes:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}) / min(etcd_server_quota_backend_bytes:15m_min{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}))",
"format": "time_series",
"legendFormat": "",
"range": true,
"refId": "A"
}
],
"description": "Approximate remaining quota headroom using the same conservative max/min calculation."
},
{
"id": 10,
"type": "timeseries",
"title": "Backend quota utilization by instance",
"gridPos": {
"x": 0,
"y": 6,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"lineWidth": 1,
"fillOpacity": 8,
"gradientMode": "none",
"showPoints": "auto",
"pointSize": 5,
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
},
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "100 * etcd_mvcc_db_total_size_in_bytes:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"} / etcd_server_quota_backend_bytes:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}}",
"range": true,
"refId": "A"
}
],
"description": "Long-term backend saturation trend. Uses retained 15m max DB size and 15m average quota; it does not reconstruct raw samples."
},
{
"id": 11,
"type": "timeseries",
"title": "DB size growth trend (estimated)",
"gridPos": {
"x": 12,
"y": 6,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"unit": "Bps",
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"lineWidth": 1,
"fillOpacity": 8,
"gradientMode": "none",
"showPoints": "auto",
"pointSize": 5,
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
},
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "deriv(etcd_mvcc_db_total_size_in_bytes:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[7d])",
"format": "time_series",
"legendFormat": "{{instance}}",
"range": true,
"refId": "A"
}
],
"description": "Estimated slope of the retained 15m average DB-size series over 7 days. Useful for trend/character-of-growth, but not equivalent to a raw-sample rate."
},
{
"id": 12,
"type": "timeseries",
"title": "Consensus pressure — proposals pending",
"gridPos": {
"x": 0,
"y": 13,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"lineWidth": 1,
"fillOpacity": 8,
"gradientMode": "none",
"showPoints": "auto",
"pointSize": 5,
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
},
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "etcd_server_proposals_pending:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} avg",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "etcd_server_proposals_pending:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} max",
"range": true,
"refId": "B"
}
],
"description": "Retained average and maximum proposal backlog. Persistent growth or high maxima indicate increasing consensus pressure."
},
{
"id": 13,
"type": "timeseries",
"title": "Failed proposals — cumulative",
"gridPos": {
"x": 12,
"y": 13,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"lineWidth": 1,
"fillOpacity": 8,
"gradientMode": "none",
"showPoints": "auto",
"pointSize": 5,
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
},
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "etcd_server_proposals_failed_total:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} avg",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "etcd_server_proposals_failed_total:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} max",
"range": true,
"refId": "B"
}
],
"description": "Cumulative failed-proposal counter as retained by the aggregation rule. A rising line indicates failures have accumulated; an exact failure rate cannot be reconstructed from min/avg/max alone."
},
{
"id": 14,
"type": "timeseries",
"title": "Resident memory",
"gridPos": {
"x": 0,
"y": 20,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"unit": "bytes(IEC)",
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"lineWidth": 1,
"fillOpacity": 8,
"gradientMode": "none",
"showPoints": "auto",
"pointSize": 5,
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
},
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "process_resident_memory_bytes:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} avg",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "process_resident_memory_bytes:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} max",
"range": true,
"refId": "B"
}
],
"description": "Physical RAM resident to the etcd process. Useful for long-term memory-growth and outlier detection."
},
{
"id": 15,
"type": "timeseries",
"title": "Request load — estimated trend",
"gridPos": {
"x": 12,
"y": 20,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"lineWidth": 1,
"fillOpacity": 8,
"gradientMode": "none",
"showPoints": "auto",
"pointSize": 5,
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
},
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "deriv(grpc_server_handled_total:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[6h])",
"format": "time_series",
"legendFormat": "{{instance}}",
"range": true,
"refId": "A"
}
],
"description": "Estimated slope of the retained gRPC handled counter. This is a workload-change signal, not a precise request rate."
},
{
"id": 16,
"type": "timeseries",
"title": "CPU time growth — estimated",
"gridPos": {
"x": 0,
"y": 27,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"unit": "s/s",
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"lineWidth": 1,
"fillOpacity": 8,
"gradientMode": "none",
"showPoints": "auto",
"pointSize": 5,
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
},
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "deriv(process_cpu_seconds_total:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[6h])",
"format": "time_series",
"legendFormat": "{{instance}}",
"range": true,
"refId": "A"
}
],
"description": "Estimated change rate of the retained 15m average CPU-time counter. It is intentionally labeled estimated; it is not a raw process_cpu_seconds_total rate."
},
{
"id": 99,
"type": "text",
"title": "What this dashboard can and cannot prove",
"gridPos": {
"x": 12,
"y": 27,
"w": 12,
"h": 7
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {
"mode": "markdown",
"content": "### Long-term retention model\n\nThis dashboard uses only retained `:15m_min`, `:15m_avg` and `:15m_max` series.\n\n**Covered well**\n- leader/health presence signals\n- consensus pressure from pending proposals\n- backend growth and quota saturation\n- resident memory\n- long-term workload and CPU trends as **estimates**\n\n**Not reconstructed**\n- exact quorum membership\n- exact counter `rate()` / `increase()`\n- WAL fsync / backend commit p95/p99 latency\n\nThe current aggregation rule stores only min/avg/max and has `keepInput: false`, so histogram quantiles and exact counter rates cannot be recovered from the retained data."
},
"pluginVersion": "11.3.2"
}
],
"preload": false,
"refresh": "30m",
"schemaVersion": 40,
"tags": [
"etcd",
"DevOps",
"Kubernetes",
"long-term",
"VictoriaMetrics"
],
"templating": {
"list": [
{
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(etcd_server_id:15m_avg{job=\"kube-etcd\"}, cluster)",
"includeAll": true,
"label": "CLUSTER",
"multi": false,
"name": "cluster",
"options": [],
"query": "label_values(etcd_server_id:15m_avg{job=\"kube-etcd\"}, cluster)",
"refresh": 1,
"type": "query"
},
{
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(etcd_server_id:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\"}, instance)",
"includeAll": true,
"label": "INSTANCE",
"multi": true,
"name": "instance",
"options": [],
"query": "label_values(etcd_server_id:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\"}, instance)",
"refresh": 2,
"type": "query"
},
{
"current": {
"text": "VM-BDO-Infra-Prod-Aggr",
"value": "cf53h5ntafwg0f"
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"hide": 2,
"label": "DATASOURCE",
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "VM-BDO-Infra-Prod-Aggr",
"type": "datasource"
}
]
},
"time": {
"from": "now-30d",
"to": "now"
},
"timepicker": {},
"timezone": "Europe/Moscow",
"title": "K8s Etcd Long-Term Health",
"uid": "k8s-etcd-longterm-health",
"version": 1,
"weekStart": ""
}