https://pastein.ru/t/_jE
Загрузка данных
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [
{
"asDropdown": false,
"icon": "external link",
"includeVars": true,
"keepTime": true,
"tags": [
"etcd"
],
"targetBlank": false,
"title": "etcd link",
"tooltip": "",
"type": "dashboards",
"url": ""
}
],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 0,
"y": 0
},
"id": 4,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto"
},
"pluginVersion": "11.3.2",
"targets": [
{
"refId": "A",
"expr": "100 * max(etcd_mvcc_db_total_size_in_bytes:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}) / min(etcd_server_quota_backend_bytes:15m_min{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"})",
"format": "time_series",
"legendFormat": "max DB / min quota"
}
],
"title": "DB quota usage",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 8,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.2",
"targets": [
{
"refId": "A",
"expr": "etcd_mvcc_db_total_size_in_bytes:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} avg"
},
{
"refId": "B",
"expr": "etcd_mvcc_db_total_size_in_bytes:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} max"
}
],
"title": "DB size",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 16,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.2",
"targets": [
{
"refId": "A",
"expr": "etcd_server_quota_backend_bytes:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} avg"
},
{
"refId": "B",
"expr": "etcd_server_quota_backend_bytes:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} max"
}
],
"title": "Backend quota",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "dashed"
}
},
"mappings": [],
"thresholds": {
"mode": "percentage",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "etcd_server_quota_backend_bytes"
},
"properties": [
{
"id": "custom.lineStyle",
"value": {
"dash": [
10,
10
],
"fill": "dash"
}
},
{
"id": "color",
"value": {
"fixedColor": "dark-red",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 8
},
"id": 5,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"refId": "A",
"expr": "etcd_mvcc_db_total_size_in_bytes:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "DB {{instance}}"
},
{
"refId": "B",
"expr": "etcd_server_quota_backend_bytes:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "Quota {{instance}}"
}
],
"title": "DB size vs Backend quota by instance",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 16
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"refId": "A",
"expr": "etcd_server_proposals_pending:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} avg"
},
{
"refId": "B",
"expr": "etcd_server_proposals_pending:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} max"
}
],
"title": "Proposals pending",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 16
},
"id": 19,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"refId": "A",
"expr": "etcd_server_proposals_failed_total:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} avg"
},
{
"refId": "B",
"expr": "etcd_server_proposals_failed_total:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} max"
}
],
"title": "Failed proposals — cumulative counter",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 24
},
"id": 16,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"refId": "A",
"expr": "process_cpu_seconds_total:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} avg"
},
{
"refId": "B",
"expr": "process_cpu_seconds_total:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} max"
}
],
"title": "CPU",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 24
},
"id": 17,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.2",
"targets": [
{
"refId": "A",
"expr": "process_resident_memory_bytes:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} avg"
},
{
"refId": "B",
"expr": "process_resident_memory_bytes:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}} max"
}
],
"title": "Resident memory",
"type": "timeseries"
},
{
"id": 119,
"title": "DB size growth trend",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"targets": [
{
"refId": "A",
"expr": "deriv(etcd_mvcc_db_total_size_in_bytes:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}[7d])",
"format": "time_series",
"legendFormat": "{{instance}} bytes/s"
}
],
"fieldConfig": {
"defaults": {
"unit": "Bps"
},
"overrides": []
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 32
}
},
{
"id": 120,
"title": "DB quota utilization trend",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"targets": [
{
"refId": "A",
"expr": "100 * etcd_mvcc_db_total_size_in_bytes:15m_max{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"} / etcd_server_quota_backend_bytes:15m_avg{job=\"kube-etcd\",cluster=~\"$cluster\",instance=~\"$instance\"}",
"format": "time_series",
"legendFormat": "{{instance}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"max": 100
},
"overrides": []
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 32
}
},
{
"id": 121,
"title": "Storage model",
"type": "text",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"options": {
"mode": "markdown",
"content": "### Long-term retention model\nThis dashboard queries the stored `:15m_min`, `:15m_avg` and `:15m_max` series.\n\nRaw matched input is not retained (`keepInput: false`).\n\nCounter rates/increases and histogram p95/p99 are intentionally not shown here, because the current aggregation rule stores only min/max/avg. To restore those long-term views, add counter-specific `rate_*`/`increase` and histogram aggregation rules in VictoriaMetrics."
},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 40
}
}
],
"preload": false,
"refresh": "30m",
"schemaVersion": 40,
"tags": [
"etcd",
"DevOps",
"Short-Term",
"k8s"
],
"templating": {
"list": [
{
"current": {
"text": "core-bdo-prod",
"value": "core-bdo-prod"
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(etcd_server_id{job=\"kube-etcd\"}, cluster)",
"includeAll": true,
"label": "CLUSTER",
"name": "cluster",
"options": [],
"query": "label_values(etcd_server_id{job=\"kube-etcd\"}, cluster)",
"refresh": 1,
"regex": "",
"type": "query"
},
{
"current": {
"text": [
"t2ru-k8s-core-401.corp.t2.ru"
],
"value": [
"t2ru-k8s-core-401.corp.t2.ru"
]
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(etcd_server_id{job=\"kube-etcd\",cluster=~\"$cluster\"}, instance)",
"includeAll": true,
"label": "INSTANCE",
"multi": true,
"name": "instance",
"options": [],
"query": "label_values(etcd_server_id{job=\"kube-etcd\",cluster=~\"$cluster\"}, instance)",
"refresh": 2,
"regex": "",
"type": "query"
},
{
"current": {
"text": "VM-BDO-Infra-Prod-Raw",
"value": "cf535ym4p3gn4a"
},
"description": "",
"hide": 2,
"label": "DATASOURCE",
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "VM-BDO-Infra-Prod-Raw",
"type": "datasource"
}
]
},
"time": {
"from": "now-30d",
"to": "now"
},
"timepicker": {},
"timezone": "Europe/Moscow",
"title": "K8s Etcd Aggregated",
"uid": "k8s-etcd-aggregated",
"version": 1,
"weekStart": ""
}