https://pastein.ru/t/CS0
Загрузка данных
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"id": 1,
"type": "stat",
"title": "Members",
"gridPos": {
"x": 0,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"refId": "A",
"expr": "count(etcd_server_id{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"})",
"legendFormat": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "green",
"value": 3
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area"
}
},
{
"id": 2,
"type": "stat",
"title": "Healthy members",
"gridPos": {
"x": 4,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"refId": "A",
"expr": "sum(etcd_server_health_success{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"})",
"legendFormat": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "green",
"value": 3
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area"
}
},
{
"id": 3,
"type": "stat",
"title": "Leader members",
"gridPos": {
"x": 8,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"refId": "A",
"expr": "sum(etcd_server_has_leader{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"})",
"legendFormat": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 2
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area"
}
},
{
"id": 4,
"type": "stat",
"title": "DB quota",
"gridPos": {
"x": 12,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"refId": "A",
"expr": "100 * max(etcd_mvcc_db_total_size_in_bytes{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}) / max(etcd_server_quota_backend_bytes{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"})",
"legendFormat": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area"
}
},
{
"id": 5,
"type": "stat",
"title": "Failed proposals / 1h",
"gridPos": {
"x": 16,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"refId": "A",
"expr": "sum(increase(etcd_server_proposals_failed_total{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[1h]))",
"legendFormat": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 5
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area"
}
},
{
"id": 6,
"type": "stat",
"title": "Leader changes / 1h",
"gridPos": {
"x": 20,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"refId": "A",
"expr": "sum(increase(etcd_server_leader_changes_seen_total{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[1h]))",
"legendFormat": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 5
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area"
}
},
{
"id": 7,
"type": "timeseries",
"title": "WAL fsync latency",
"gridPos": {
"x": 0,
"y": 4,
"w": 24,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.50, sum by (le,instance) (rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m])))",
"legendFormat": "p50 {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le,instance) (rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m])))",
"legendFormat": "p95 {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le,instance) (rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m])))",
"legendFormat": "p99 {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 8,
"spanNulls": true,
"showPoints": "never",
"stacking": {
"mode": "none",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 8,
"type": "timeseries",
"title": "Backend commit latency",
"gridPos": {
"x": 0,
"y": 11,
"w": 24,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.50, sum by (le,instance) (rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m])))",
"legendFormat": "p50 {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le,instance) (rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m])))",
"legendFormat": "p95 {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le,instance) (rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m])))",
"legendFormat": "p99 {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 8,
"spanNulls": true,
"showPoints": "never",
"stacking": {
"mode": "none",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 9,
"type": "timeseries",
"title": "Raft proposals",
"gridPos": {
"x": 0,
"y": 18,
"w": 24,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sum by (instance) (rate(etcd_server_proposals_committed_total{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m]))",
"legendFormat": "committed {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "B",
"expr": "sum by (instance) (rate(etcd_server_proposals_applied_total{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m]))",
"legendFormat": "applied {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "C",
"expr": "etcd_server_proposals_pending{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"legendFormat": "pending {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "D",
"expr": "sum by (instance) (rate(etcd_server_proposals_failed_total{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m]))",
"legendFormat": "failed {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 8,
"spanNulls": true,
"showPoints": "never",
"stacking": {
"mode": "none",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 10,
"type": "timeseries",
"title": "Leader changes",
"gridPos": {
"x": 0,
"y": 25,
"w": 24,
"h": 6
},
"targets": [
{
"refId": "A",
"expr": "sum by (instance) (increase(etcd_server_leader_changes_seen_total{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "changes {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 8,
"spanNulls": true,
"showPoints": "never",
"stacking": {
"mode": "none",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 11,
"type": "timeseries",
"title": "DB size / in-use / quota",
"gridPos": {
"x": 0,
"y": 31,
"w": 24,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "etcd_mvcc_db_total_size_in_bytes{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"legendFormat": "DB total {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "B",
"expr": "etcd_mvcc_db_total_size_in_use_in_bytes{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"legendFormat": "DB in-use {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "C",
"expr": "etcd_server_quota_backend_bytes{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"legendFormat": "Quota {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 8,
"spanNulls": true,
"showPoints": "never",
"stacking": {
"mode": "none",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 12,
"type": "timeseries",
"title": "Peer round-trip time",
"gridPos": {
"x": 0,
"y": 38,
"w": 24,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.50, sum by (le,instance) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m])))",
"legendFormat": "p50 {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le,instance) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m])))",
"legendFormat": "p95 {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le,instance) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m])))",
"legendFormat": "p99 {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 8,
"spanNulls": true,
"showPoints": "never",
"stacking": {
"mode": "none",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 13,
"type": "timeseries",
"title": "gRPC request rate",
"gridPos": {
"x": 0,
"y": 45,
"w": 24,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sum by (grpc_method,instance) (rate(grpc_server_handled_total{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m]))",
"legendFormat": "{{grpc_method}} {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 8,
"spanNulls": true,
"showPoints": "never",
"stacking": {
"mode": "none",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 14,
"type": "timeseries",
"title": "Process CPU / RSS",
"gridPos": {
"x": 0,
"y": 52,
"w": 24,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sum by (instance) (rate(process_cpu_seconds_total{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[5m]))",
"legendFormat": "CPU {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
},
{
"refId": "B",
"expr": "process_resident_memory_bytes{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"legendFormat": "RSS {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 1,
"fillOpacity": 8,
"spanNulls": true,
"showPoints": "never",
"stacking": {
"mode": "none",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
}
],
"refresh": "15s",
"schemaVersion": 39,
"tags": [
"kubernetes",
"etcd",
"victoriametrics"
],
"templating": {
"list": [
{
"name": "DS_VICTORIAMETRICS",
"label": "VictoriaMetrics",
"type": "datasource",
"query": "prometheus",
"current": {},
"hide": 0
},
{
"name": "cluster",
"label": "Cluster",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
},
"definition": "label_values(etcd_server_id{job=\"kube-etcd\"}, cluster)",
"query": {
"query": "label_values(etcd_server_id{job=\"kube-etcd\"}, cluster)",
"refId": "StandardVariableQuery"
},
"includeAll": true,
"multi": false,
"allValue": ".*"
},
{
"name": "instance",
"label": "Instance",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "${DS_VICTORIAMETRICS}"
},
"definition": "label_values(etcd_server_id{job=\"kube-etcd\",cluster=~\"$cluster\"}, instance)",
"query": {
"query": "label_values(etcd_server_id{job=\"kube-etcd\",cluster=~\"$cluster\"}, instance)",
"refId": "StandardVariableQuery"
},
"includeAll": true,
"multi": true,
"allValue": ".*"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Kubernetes etcd — VictoriaMetrics v2",
"uid": "k8s-etcd-vm-v2",
"version": 1,
"weekStart": ""
}