commit 332ca5be344f6bc1647e4499ec6ea342e45dc7d0
parent 5432adbc561d0cdc1f7541760c534923162e92b5
Author: MTRNord <MTRNord@users.noreply.github.com>
Date: Sat, 28 Jun 2025 20:15:10 +0200
Add dashboard
Diffstat:
10 files changed, 26411 insertions(+), 33807 deletions(-)
diff --git a/apps/base/victoria-metrics-k8s-stack/dashboards.yaml b/apps/base/victoria-metrics-k8s-stack/dashboards.yaml
@@ -3,25565 +3,12 @@ kind: ConfigMap
metadata:
labels:
grafana_dashboard: "1"
- annotations:
- grafana_folder: Ceph
- name: ceph-cluster-dashboard
-data:
- ceph-cluster-dashboard.json: |-
- {
- "__inputs": [],
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "5.3.2"
- },
- {
- "id": "graph",
- "name": "Graph",
- "type": "panel",
- "version": "5.0.0"
- },
- {
- "id": "heatmap",
- "name": "Heatmap",
- "type": "panel",
- "version": "5.0.0"
- },
- {
- "id": "singlestat",
- "name": "Singlestat",
- "type": "panel",
- "version": "5.0.0"
- }
- ],
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": "-- Grafana --",
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "showIn": 0,
- "tags": [],
- "type": "dashboard"
- }
- ]
- },
- "description": "Overview of your Ceph cluster.",
- "editable": false,
- "gnetId": 2842,
- "graphTooltip": 0,
- "hideControls": false,
- "id": null,
- "links": [],
- "panels": [
- {
- "collapse": false,
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 2,
- "panels": [],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "CLUSTER STATE",
- "titleSize": "h6",
- "type": "row"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "0": {
- "text": "HEALTHY"
- },
- "1": {
- "text": "WARNING"
- },
- "2": {
- "text": "ERROR"
- }
- },
- "type": "value"
- },
- {
- "id": 1,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#9ac48a"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 1
- },
- {
- "color": "rgba(245, 54, 54, 0.9)",
- "value": 2
- }
- ]
- },
- "unit": "none"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 0,
- "y": 1
- },
- "id": 3,
- "interval": "1m",
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "ceph_health_status{}",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "Ceph health status",
- "transparent": true,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 1,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "Bps"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 3,
- "y": 1
- },
- "id": 4,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(irate(ceph_osd_op_w_in_bytes{}[5m]))",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Write Throughput",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 1,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#d44a3a"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 0
- },
- {
- "color": "#9ac48a",
- "value": 0
- }
- ]
- },
- "unit": "Bps"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 6,
- "y": 1
- },
- "id": 5,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(irate(ceph_osd_op_r_out_bytes{}[5m]))",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Read Throughput",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 2,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(50, 172, 45, 0.97)"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 0.025
- },
- {
- "color": "rgba(245, 54, 54, 0.9)",
- "value": 1
- }
- ]
- },
- "unit": "decbytes"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 9,
- "y": 1
- },
- "id": 6,
- "interval": "1m",
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "ceph_cluster_total_bytes{}",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "Cluster Capacity",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "max": 1,
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(245, 54, 54, 0.9)"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 0.1
- },
- {
- "color": "rgba(50, 172, 45, 0.97)",
- "value": 0.3
- }
- ]
- },
- "unit": "percentunit"
- }
- },
- "gridPos": {
- "h": 6,
- "w": 3,
- "x": 12,
- "y": 1
- },
- "id": 7,
- "interval": "1m",
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showThresholdLabels": false,
- "showThresholdMarkers": true
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "(ceph_cluster_total_bytes{}-ceph_cluster_total_used_bytes{})/ceph_cluster_total_bytes{}",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "Available Capacity",
- "transparent": false,
- "type": "gauge"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 2,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 15,
- "y": 1
- },
- "id": 8,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pool_objects{})",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Number of Objects",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 1,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 18,
- "y": 1
- },
- "id": 9,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "delta"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_osd_op_w_in_bytes{})",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Bytes Written",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 1,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 21,
- "y": 1
- },
- "id": 10,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "delta"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_osd_op_r_out_bytes{})",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Bytes Read",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#9ac48a"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 1
- },
- {
- "color": "#e24d42",
- "value": 1
- }
- ]
- },
- "unit": "none"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 0,
- "y": 4
- },
- "id": 11,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "background",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "count(ALERTS{alertstate=\"firing\",alertname=~\"^Ceph.+\"}) OR vector(0)",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Alerts starting with Ceph",
- "transparent": true,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "ops"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 3,
- "y": 4
- },
- "id": 12,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(irate(ceph_osd_op_w{}[5m]))",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Write IOPS",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#d44a3a"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 0
- },
- {
- "color": "#9ac48a",
- "value": 0
- }
- ]
- },
- "unit": "ops"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 6,
- "y": 4
- },
- "id": 13,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(irate(ceph_osd_op_r{}[5m]))",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Read IOPS",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 2,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(50, 172, 45, 0.97)"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 0.025
- },
- {
- "color": "rgba(245, 54, 54, 0.9)",
- "value": 0.1
- }
- ]
- },
- "unit": "decbytes"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 9,
- "y": 4
- },
- "id": 14,
- "interval": "1m",
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "ceph_cluster_total_used_bytes{}",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Used Capacity",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 2,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#ea6460"
- },
- {
- "color": "#052b51",
- "value": 0
- },
- {
- "color": "#508642",
- "value": 0
- }
- ]
- },
- "unit": "short"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 15,
- "y": 4
- },
- "id": 15,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "diff"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pool_objects)",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Difference",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 128
- }
- ]
- },
- "unit": "short"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 18,
- "y": 4
- },
- "id": 16,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "background",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_mon_num_sessions{})",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Mon Session Num",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(245, 54, 54, 0.9)",
- "value": null
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 2
- },
- {
- "color": "green",
- "value": 3
- }
- ]
- },
- "unit": "none"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 21,
- "y": 4
- },
- "id": 17,
- "interval": "1m",
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "background",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "count(ceph_mon_quorum_status{}) or vector(0)",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "Monitors In Quorum",
- "transparent": false,
- "type": "stat"
- },
- {
- "collapse": false,
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 7
- },
- "id": 18,
- "panels": [],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "OSD STATE",
- "titleSize": "h6",
- "type": "row"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#9ac48a"
- },
- {
- "color": "rgba(237, 40, 40, 0.89)",
- "value": 1
- },
- {
- "color": "rgba(245, 54, 54, 0.9)",
- "value": 1
- }
- ]
- },
- "unit": "none"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 2,
- "x": 0,
- "y": 8
- },
- "id": 19,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "background",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "count(ceph_osd_up{}) - count(ceph_osd_in{})",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "OSDs OUT",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(50, 172, 45, 0.97)"
- },
- {
- "color": "#eab839",
- "value": 1
- },
- {
- "color": "#ea6460",
- "value": 1
- }
- ]
- },
- "unit": "none"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 2,
- "x": 2,
- "y": 8
- },
- "id": 20,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "background",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "count(ceph_osd_up{} == 0.0) OR vector(0)",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "OSDs DOWN",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 2,
- "x": 4,
- "y": 8
- },
- "id": 21,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_osd_up{})",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "OSDs UP",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 2,
- "x": 6,
- "y": 8
- },
- "id": 22,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_osd_in{})",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "OSDs IN",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 1,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(50, 172, 45, 0.97)"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 250
- },
- {
- "color": "rgba(245, 54, 54, 0.9)",
- "value": 300
- }
- ]
- },
- "unit": "none"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 2,
- "x": 8,
- "y": 8
- },
- "id": 23,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "avg(ceph_osd_numpg{})",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "Avg PGs",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 2,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(50, 172, 45, 0.97)"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 10
- },
- {
- "color": "rgba(245, 54, 54, 0.9)",
- "value": 50
- }
- ]
- },
- "unit": "ms"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 10,
- "y": 8
- },
- "id": 24,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "avg(ceph_osd_apply_latency_ms{})",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "Avg Apply Latency",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 2,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(50, 172, 45, 0.97)"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 10
- },
- {
- "color": "rgba(245, 54, 54, 0.9)",
- "value": 50
- }
- ]
- },
- "unit": "ms"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 13,
- "y": 8
- },
- "id": 25,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "avg(ceph_osd_commit_latency_ms{})",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "Avg Commit Latency",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 4,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "color": "#299c46",
- "text": "0"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#299c46"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 1
- },
- {
- "color": "#d44a3a",
- "value": 2
- }
- ]
- },
- "unit": "ms"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 16,
- "y": 8
- },
- "id": 26,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "avg(rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0)",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Avg Op Write Latency",
- "transparent": false,
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 6,
- "links": [],
- "mappings": [
- {
- "id": 0,
- "options": {
- "match": null,
- "result": {
- "color": "#299c46",
- "text": "0"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#299c46"
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 1
- },
- {
- "color": "#d44a3a",
- "value": 2
- }
- ]
- },
- "unit": "ms"
- }
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 19,
- "y": 8
- },
- "id": 27,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "avg(rate(ceph_osd_op_r_latency_sum{}[5m])/rate(ceph_osd_op_r_latency_count{}[5m]) >= 0)",
- "format": "time_series",
- "instant": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Avg Op Read Latency",
- "transparent": false,
- "type": "stat"
- },
- {
- "collapse": false,
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 9
- },
- "id": 28,
- "panels": [],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "CLUSTER STATS",
- "titleSize": "h6",
- "type": "row"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 40,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 0,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "decimals": 2,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Available"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#EAB839",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Total Capacity"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#447EBC",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Used"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#BF1B00",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Used"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#BF1B00",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "total_avail"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#6ED0E0",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "total_space"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#7EB26D",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "total_used"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#890F02",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Total Capacity"
- },
- "properties": [
- {
- "id": "custom.fillOpacity",
- "value": 0
- },
- {
- "id": "custom.lineWidth",
- "value": 3
- },
- {
- "id": "custom.stacking",
- "value": {
- "group": false,
- "mode": "normal"
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 0,
- "y": 10
- },
- "id": 29,
- "interval": "$interval",
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "ceph_cluster_total_bytes{}-ceph_cluster_total_used_bytes{}",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Available",
- "refId": "A",
- "step": 300
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "ceph_cluster_total_used_bytes{}",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Used",
- "refId": "B",
- "step": 300
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "ceph_cluster_total_bytes{}",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Total Capacity",
- "refId": "C",
- "step": 300
- }
- ],
- "title": "Capacity",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Total Capacity"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#7EB26D",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Used"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#BF1B00",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "total_avail"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#6ED0E0",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "total_space"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#7EB26D",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "total_used"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#890F02",
- "mode": "fixed"
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 8,
- "y": 10
- },
- "id": 30,
- "interval": "$interval",
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(irate(ceph_osd_op_w{}[5m]))",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Write",
- "refId": "A",
- "step": 300
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(irate(ceph_osd_op_r{}[5m]))",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Read",
- "refId": "B",
- "step": 300
- }
- ],
- "title": "IOPS",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 16,
- "y": 10
- },
- "id": 31,
- "interval": "$interval",
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(irate(ceph_osd_op_w_in_bytes{}[5m]))",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Write",
- "refId": "A",
- "step": 300
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(irate(ceph_osd_op_r_out_bytes{}[5m]))",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Read",
- "refId": "B",
- "step": 300
- }
- ],
- "title": "Cluster Throughput",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 0,
- "y": 18
- },
- "id": 32,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "(ceph_pool_bytes_used{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{name}}",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "Pool Used Bytes",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 8,
- "y": 18
- },
- "id": 33,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "table",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "(ceph_pool_avail_raw{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{name}} Avail",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "(ceph_pool_stored_raw{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{name}} Stored",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Pool RAW Bytes",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 16,
- "y": 18
- },
- "id": 34,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "(ceph_pool_objects{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{name}}",
- "refId": "A"
- }
- ],
- "title": "Objects Per Pool",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 26
- },
- "id": 35,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "(ceph_pool_quota_bytes{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{name}}",
- "refId": "A"
- }
- ],
- "title": "Pool Quota Bytes",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 26
- },
- "id": 36,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "(ceph_pool_quota_objects{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{name}}",
- "refId": "A"
- }
- ],
- "title": "Pool Objects Quota",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 26
- },
- "id": 37,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "count(ceph_bluestore_kv_commit_lat_count{})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "BlueStore",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "OSD Type Count",
- "type": "timeseries"
- },
- {
- "collapse": true,
- "collapsed": true,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 27
- },
- "id": 38,
- "panels": [
- {
- "columns": [],
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "align": "auto",
- "displayMode": "auto",
- "inspect": false
- },
- "decimals": 2,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Time"
- },
- "properties": [
- {
- "id": "displayName",
- "value": "Time"
- },
- {
- "id": "unit",
- "value": "time: YYYY-MM-DD HH:mm:ss"
- },
- {
- "id": "custom.align"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 28
- },
- "id": 39,
- "links": [],
- "options": {
- "footer": {
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
- },
- "showHeader": true
- },
- "pluginVersion": "9.1.3",
- "styles": "",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "ALERTS{alertstate=\"firing\",alertname=~\"^Ceph.+\"}",
- "format": "table",
- "instant": true,
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Alerts starting with Ceph",
- "transformations": [
- {
- "id": "merge",
- "options": {
- "reducer": []
- }
- }
- ],
- "type": "table"
- },
- {
- "columns": [],
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "align": "auto",
- "displayMode": "auto",
- "filterable": false,
- "inspect": false
- },
- "decimals": 2,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Time"
- },
- "properties": [
- {
- "id": "displayName",
- "value": "Time"
- },
- {
- "id": "unit",
- "value": "time: YYYY-MM-DD HH:mm:ss"
- },
- {
- "id": "custom.align"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 28
- },
- "id": 40,
- "links": [],
- "options": {
- "footer": {
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
- },
- "showHeader": true
- },
- "pluginVersion": "9.1.3",
- "styles": "",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "topk(5,sort_desc(ceph_osd_apply_latency_ms{} + ceph_osd_commit_latency_ms{}))",
- "format": "table",
- "instant": true,
- "intervalFactor": 1,
- "legendFormat": "_auto",
- "refId": "A"
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top Sluggish OSDs",
- "transformations": [
- {
- "id": "merge",
- "options": {
- "reducer": []
- }
- }
- ],
- "type": "table"
- },
- {
- "columns": [],
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "align": "auto",
- "displayMode": "auto",
- "inspect": false
- },
- "decimals": 2,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Time"
- },
- "properties": [
- {
- "id": "displayName",
- "value": "Time"
- },
- {
- "id": "unit",
- "value": "time: YYYY-MM-DD HH:mm:ss"
- },
- {
- "id": "custom.align"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 28
- },
- "id": 41,
- "links": [],
- "options": {
- "footer": {
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
- },
- "showHeader": true
- },
- "pluginVersion": "9.1.3",
- "styles": "",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "ceph_osd_up{} == 0",
- "format": "table",
- "instant": true,
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Down OSDs",
- "transformations": [
- {
- "id": "merge",
- "options": {
- "reducer": []
- }
- }
- ],
- "type": "table"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Alerts",
- "titleSize": "h6",
- "type": "row"
- },
- {
- "collapse": true,
- "collapsed": true,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 28
- },
- "id": 42,
- "panels": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "scaleDistributionLog": 2,
- "type": "log"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 29
- },
- "id": 43,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "max",
- "min"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "node_memory_Active_anon_bytes{}",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{instance}}",
- "refId": "A"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(node_memory_Active_anon_bytes{})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Cluster Memory Usage",
- "refId": "B"
- }
- ],
- "title": "Node Memory Usage",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "scaleDistributionLog": 2,
- "type": "log"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "percent"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 29
- },
- "id": 44,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "avg by(instance)(irate(node_cpu_seconds_total{job='node',mode!=\"idle\"}[$interval])) * 100",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "title": "Node CPU Usage",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 35
- },
- "id": 45,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum by (instance)(irate(node_disk_read_bytes_total{}[$interval]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "title": "Node Out",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 35
- },
- "id": 46,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum by (instance)(irate(node_disk_written_bytes_total{}[$interval]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "title": "Node In",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "percent"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 44
- },
- "id": 47,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "(node_filesystem_free_bytes{ mountpoint=\"/\", device != \"rootfs\"})*100 / (node_filesystem_size_bytes{ mountpoint=\"/\", device != \"rootfs\"})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{instance}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Free Space in root filesystem",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Node Statistics (NodeExporter)",
- "titleSize": "h6",
- "type": "row"
- },
- {
- "collapse": false,
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 45
- },
- "id": 48,
- "panels": [],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "OBJECTS",
- "titleSize": "h6",
- "type": "row"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byRegexp",
- "options": "/^Total.*$/"
- },
- "properties": [
- {
- "id": "custom.stacking",
- "value": {
- "group": false,
- "mode": "normal"
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 12,
- "w": 6,
- "x": 0,
- "y": 46
- },
- "id": 49,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "asc"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pool_objects)",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Total",
- "range": true,
- "refId": "A",
- "step": 300
- }
- ],
- "title": "OSD Type Count",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byRegexp",
- "options": "/^Total.*$/"
- },
- "properties": [
- {
- "id": "custom.stacking",
- "value": {
- "group": false,
- "mode": "normal"
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 12,
- "w": 8,
- "x": 6,
- "y": 46
- },
- "id": 50,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "asc"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_active{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Active",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_clean{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Clean",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_peering{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Peering",
- "range": true,
- "refId": "C"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_degraded{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Degraded",
- "range": true,
- "refId": "D",
- "step": 300
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_stale{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Stale",
- "range": true,
- "refId": "E",
- "step": 300
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_unclean_pgs{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Unclean",
- "range": true,
- "refId": "F",
- "step": 300
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_undersized{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Undersized",
- "range": true,
- "refId": "G",
- "step": 300
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_incomplete{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Incomplete",
- "range": true,
- "refId": "H"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_forced_backfill{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Forced Backfill",
- "range": true,
- "refId": "I"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_forced_recovery{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Forced Recovery",
- "range": true,
- "refId": "J"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_creating{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Creating",
- "range": true,
- "refId": "K"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_wait_backfill{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Wait Backfill",
- "range": true,
- "refId": "L"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_deep{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Deep",
- "range": true,
- "refId": "M"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_scrubbing{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Scrubbing",
- "range": true,
- "refId": "N"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_recovering{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Recovering",
- "range": true,
- "refId": "O"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_repair{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Repair",
- "range": true,
- "refId": "P"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_down{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Down",
- "range": true,
- "refId": "Q"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_peered{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Peered",
- "range": true,
- "refId": "R"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_backfill{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Backfill",
- "range": true,
- "refId": "S"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_remapped{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Remapped",
- "range": true,
- "refId": "T"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_backfill_toofull{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Backfill Toofull",
- "range": true,
- "refId": "U"
- }
- ],
- "title": "PGs State",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byRegexp",
- "options": "/^Total.*$/"
- },
- "properties": [
- {
- "id": "custom.stacking",
- "value": {
- "group": false,
- "mode": "normal"
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 6,
- "w": 10,
- "x": 14,
- "y": 46
- },
- "id": 51,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "asc"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_degraded{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Degraded",
- "range": true,
- "refId": "A",
- "step": 300
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_stale{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Stale",
- "range": true,
- "refId": "B",
- "step": 300
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(ceph_pg_undersized{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Undersized",
- "range": true,
- "refId": "C",
- "step": 300
- }
- ],
- "title": "Stuck PGs",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 10,
- "x": 14,
- "y": 52
- },
- "id": 52,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "sum(irate(ceph_osd_recovery_ops{}[$interval]))",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "OPS",
- "refId": "A",
- "step": 300
- }
- ],
- "title": "Recovery Operations",
- "type": "timeseries"
- },
- {
- "collapse": false,
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 53
- },
- "id": 53,
- "panels": [],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "LATENCY",
- "titleSize": "h6",
- "type": "row"
- },
- {
- "cards": {
- "cardPadding": null,
- "cardRound": null
- },
- "color": {
- "cardColor": "#b4ff00",
- "colorScale": "sqrt",
- "colorScheme": "interpolateOranges",
- "exponent": 0.5,
- "mode": "opacity"
- },
- "dataFormat": "timeseries",
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- }
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 54
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 54,
- "legend": {
- "show": true
- },
- "options": {
- "calculate": true,
- "calculation": {
- "yBuckets": {
- "mode": "count",
- "scale": {
- "log": 2,
- "type": "log"
- },
- "value": "1"
- }
- },
- "cellGap": 2,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#b4ff00",
- "mode": "opacity",
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": false
- },
- "yAxis": {
- "axisPlacement": "left",
- "min": "0",
- "reverse": false,
- "unit": "ms"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "ceph_osd_apply_latency_ms{}",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "OSD Apply Latency Distribution",
- "tooltip": {
- "show": true,
- "showHistogram": false
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "xBucketNumber": null,
- "xBucketSize": "",
- "yAxis": {
- "decimals": null,
- "format": "ms",
- "logBase": 2,
- "max": null,
- "min": "0",
- "show": true,
- "splitFactor": 1
- },
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": 10
- },
- {
- "cards": {
- "cardPadding": null,
- "cardRound": null
- },
- "color": {
- "cardColor": "#65c5db",
- "colorScale": "sqrt",
- "colorScheme": "interpolateOranges",
- "exponent": 0.5,
- "mode": "opacity"
- },
- "dataFormat": "timeseries",
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- }
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 54
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 55,
- "legend": {
- "show": true
- },
- "options": {
- "calculate": true,
- "calculation": {
- "yBuckets": {
- "mode": "count",
- "scale": {
- "log": 2,
- "type": "log"
- }
- }
- },
- "cellGap": 2,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#65c5db",
- "mode": "opacity",
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": false
- },
- "yAxis": {
- "axisPlacement": "left",
- "min": "0",
- "reverse": false,
- "unit": "ms"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "ceph_osd_commit_latency_ms{}",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "OSD Commit Latency Distribution",
- "tooltip": {
- "show": true,
- "showHistogram": false
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "xBucketNumber": null,
- "xBucketSize": "",
- "yAxis": {
- "decimals": null,
- "format": "ms",
- "logBase": 2,
- "max": null,
- "min": "0",
- "show": true,
- "splitFactor": 1
- },
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
- },
- {
- "cards": {
- "cardPadding": null,
- "cardRound": null
- },
- "color": {
- "cardColor": "#806eb7",
- "colorScale": "sqrt",
- "colorScheme": "interpolateOranges",
- "exponent": 0.5,
- "mode": "opacity"
- },
- "dataFormat": "timeseries",
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- }
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 62
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 56,
- "legend": {
- "show": true
- },
- "options": {
- "calculate": true,
- "calculation": {
- "yBuckets": {
- "mode": "count",
- "scale": {
- "log": 2,
- "type": "log"
- }
- }
- },
- "cellGap": 2,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#806eb7",
- "mode": "opacity",
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": false
- },
- "yAxis": {
- "axisPlacement": "left",
- "decimals": 2,
- "min": "0",
- "reverse": false,
- "unit": "ms"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "rate(ceph_osd_op_r_latency_sum{}[5m]) / rate(ceph_osd_op_r_latency_count{}[5m]) >= 0",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "OSD Read Op Latency Distribution",
- "tooltip": {
- "show": true,
- "showHistogram": false
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "xBucketNumber": null,
- "xBucketSize": "",
- "yAxis": {
- "decimals": null,
- "format": "ms",
- "logBase": 2,
- "max": null,
- "min": "0",
- "show": true,
- "splitFactor": 1
- },
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
- },
- {
- "cards": {
- "cardPadding": null,
- "cardRound": null
- },
- "color": {
- "cardColor": "#f9934e",
- "colorScale": "sqrt",
- "colorScheme": "interpolateOranges",
- "exponent": 0.5,
- "mode": "opacity"
- },
- "dataFormat": "timeseries",
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- }
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 62
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 57,
- "legend": {
- "show": true
- },
- "options": {
- "calculate": true,
- "calculation": {
- "yBuckets": {
- "mode": "count",
- "scale": {
- "log": 2,
- "type": "log"
- }
- }
- },
- "cellGap": 2,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#f9934e",
- "mode": "opacity",
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": false
- },
- "yAxis": {
- "axisPlacement": "left",
- "decimals": 2,
- "min": "0",
- "reverse": false,
- "unit": "ms"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "OSD Write Op Latency Distribution",
- "tooltip": {
- "show": true,
- "showHistogram": false
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "xBucketNumber": null,
- "xBucketSize": "",
- "yAxis": {
- "decimals": 2,
- "format": "ms",
- "logBase": 2,
- "max": null,
- "min": "0",
- "show": true,
- "splitFactor": 1
- },
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 70
- },
- "id": 58,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "avg(rate(ceph_osd_op_r_latency_sum{}[5m]) / rate(ceph_osd_op_r_latency_count{}[5m]) >= 0)",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Read",
- "refId": "A"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "avg(rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0)",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Write",
- "refId": "B"
- }
- ],
- "title": "Recovery Operations",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 70
- },
- "id": 59,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "avg(ceph_osd_apply_latency_ms{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "apply",
- "metric": "ceph_osd_perf_apply_latency_seconds",
- "refId": "A",
- "step": 4
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "avg(ceph_osd_commit_latency_ms{})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "commit",
- "metric": "ceph_osd_perf_commit_latency_seconds",
- "refId": "B",
- "step": 4
- }
- ],
- "title": "AVG OSD Apply + Commit Latency",
- "type": "timeseries"
- },
- {
- "collapse": true,
- "collapsed": true,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 71
- },
- "id": 60,
- "panels": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 6,
- "x": 0,
- "y": 72
- },
- "id": 61,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "count by (ceph_version)(ceph_osd_metadata{})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Ceph OSD Versions",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 6,
- "x": 6,
- "y": 72
- },
- "id": 62,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "count by (ceph_version)(ceph_mon_metadata{})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Ceph Mon Versions",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 6,
- "x": 12,
- "y": 72
- },
- "id": 63,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "count by (ceph_version)(ceph_mds_metadata{})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Ceph MDS Versions",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 6,
- "x": 18,
- "y": 72
- },
- "id": 64,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": "${DS_PROMETHEUS}",
- "expr": "count by (ceph_version)(ceph_rgw_metadata{})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Ceph RGW Versions",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ceph Versions",
- "titleSize": "h6",
- "type": "row"
- }
- ],
- "refresh": "1m",
- "rows": [],
- "schemaVersion": 37,
- "style": "dark",
- "tags": [
- "ceph-mixin"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "Prometheus",
- "value": "Prometheus"
- },
- "hide": 0,
- "label": "Data Source",
- "name": "DS_PROMETHEUS",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "auto": true,
- "auto_count": 10,
- "auto_min": "1m",
- "current": {
- "text": "$__auto_interval_interval",
- "value": "$__auto_interval_interval"
- },
- "hide": 0,
- "label": "Interval",
- "name": "interval",
- "options": [
- {
- "selected": true,
- "text": "auto",
- "value": "$__auto_interval_interval"
- },
- {
- "selected": false,
- "text": "5s",
- "value": "5s"
- },
- {
- "selected": false,
- "text": "10s",
- "value": "10s"
- },
- {
- "selected": false,
- "text": "30s",
- "value": "30s"
- },
- {
- "selected": false,
- "text": "1m",
- "value": "1m"
- },
- {
- "selected": false,
- "text": "10m",
- "value": "10m"
- },
- {
- "selected": false,
- "text": "30m",
- "value": "30m"
- },
- {
- "selected": false,
- "text": "1h",
- "value": "1h"
- },
- {
- "selected": false,
- "text": "6h",
- "value": "6h"
- },
- {
- "selected": false,
- "text": "12h",
- "value": "12h"
- },
- {
- "selected": false,
- "text": "1d",
- "value": "1d"
- },
- {
- "selected": false,
- "text": "7d",
- "value": "7d"
- },
- {
- "selected": false,
- "text": "14d",
- "value": "14d"
- },
- {
- "selected": false,
- "text": "30d",
- "value": "30d"
- }
- ],
- "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
- "refresh": 2,
- "type": "interval",
- "valuelabels": {}
- }
- ]
- },
- "time": {
- "from": "now-6h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "",
- "title": "Ceph Cluster",
- "uid": "tbO9LAiZK",
- "version": 0
- }
- ceph-osd-single-dashboard.json: |-
- {
- "__inputs": [],
- "__elements": {},
- "__requires": [
- {
- "type": "panel",
- "id": "gauge",
- "name": "Gauge",
- "version": ""
- },
- {
- "type": "grafana",
- "id": "grafana",
- "name": "Grafana",
- "version": "9.0.5"
- },
- {
- "type": "panel",
- "id": "heatmap",
- "name": "Heatmap",
- "version": ""
- },
- {
- "type": "datasource",
- "id": "prometheus",
- "name": "Prometheus",
- "version": "1.0.0"
- },
- {
- "type": "panel",
- "id": "stat",
- "name": "Stat",
- "version": ""
- },
- {
- "type": "panel",
- "id": "timeseries",
- "name": "Time series",
- "version": ""
- }
- ],
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
- "type": "dashboard"
- }
- ]
- },
- "description": "CEPH OSD Status.",
- "editable": false,
- "fiscalYearStartMonth": 0,
- "gnetId": 5336,
- "graphTooltip": 0,
- "id": null,
- "links": [],
- "liveNow": false,
- "panels": [
- {
- "collapsed": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 11,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "OSD Status / Total OSDs",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "mappings": [
- {
- "options": {
- "from": 0,
- "result": {
- "index": 0,
- "text": "N/A"
- },
- "to": 0
- },
- "type": "range"
- },
- {
- "options": {
- "from": 0,
- "result": {
- "index": 1,
- "text": "DOWN"
- },
- "to": 0.99
- },
- "type": "range"
- },
- {
- "options": {
- "from": 0.99,
- "result": {
- "index": 2,
- "text": "UP"
- },
- "to": 1
- },
- "type": "range"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(245, 54, 54, 0.9)",
- "value": null
- },
- {
- "color": "rgba(237, 40, 40, 0.89)",
- "value": 0
- },
- {
- "color": "rgba(50, 172, 45, 0.97)",
- "value": 1
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 2,
- "x": 0,
- "y": 1
- },
- "id": 6,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "background",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.0.5",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "sum without (instance) (ceph_osd_up{ceph_daemon=\"$osd\"})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "range": true,
- "refId": "A",
- "step": 60
- }
- ],
- "title": "Status",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "mappings": [
- {
- "options": {
- "from": 0,
- "result": {
- "index": 0,
- "text": "N/A"
- },
- "to": 0
- },
- "type": "range"
- },
- {
- "options": {
- "from": 0,
- "result": {
- "index": 1,
- "text": "OUT"
- },
- "to": 0.99
- },
- "type": "range"
- },
- {
- "options": {
- "from": 0.99,
- "result": {
- "index": 2,
- "text": "IN"
- },
- "to": 1
- },
- "type": "range"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(245, 54, 54, 0.9)",
- "value": null
- },
- {
- "color": "rgba(237, 40, 40, 0.89)",
- "value": 0
- },
- {
- "color": "rgba(50, 172, 45, 0.97)",
- "value": 1
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 2,
- "x": 2,
- "y": 1
- },
- "id": 8,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "background",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.0.5",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum without (instance) (ceph_osd_in{ceph_daemon=\"$osd\"})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 60
- }
- ],
- "title": "Available",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "mappings": [
- {
- "options": {
- "from": 0,
- "result": {
- "index": 0,
- "text": "N/A"
- },
- "to": 0
- },
- "type": "range"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(255, 255, 255, 0.9)",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 2,
- "x": 4,
- "y": 1
- },
- "id": 10,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.0.5",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "count without (instance, ceph_daemon) (ceph_osd_up)",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 60
- }
- ],
- "title": "Total OSDs",
- "type": "stat"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 4
- },
- "id": 12,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "refId": "A"
- }
- ],
- "title": "OSD: $osd",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "line"
- }
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "transparent",
- "value": null
- },
- {
- "color": "rgba(216, 200, 27, 0.27)",
- "value": 250
- },
- {
- "color": "rgba(234, 112, 112, 0.22)",
- "value": 300
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byRegexp",
- "options": "/^Average.*/"
- },
- "properties": [
- {
- "id": "custom.fillOpacity",
- "value": 0
- },
- {
- "id": "custom.stacking",
- "value": {
- "group": false,
- "mode": "normal"
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 7,
- "w": 11,
- "x": 0,
- "y": 5
- },
- "id": 5,
- "interval": "$interval",
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum without (instance, ceph_daemon) (ceph_osd_numpg{ceph_daemon=~\"$osd\"})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Number of PGs",
- "refId": "A",
- "step": 60
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "avg without (instance, ceph_daemon) (ceph_osd_numpg)",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Average Number of PGs in the Cluster",
- "refId": "B",
- "step": 60
- }
- ],
- "title": "PGs",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 9,
- "x": 11,
- "y": 5
- },
- "id": 2,
- "interval": "$interval",
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum without (instance, ceph_daemon) (ceph_osd_stat_bytes{ceph_daemon=\"osd.0\"}-ceph_osd_stat_bytes_used{ceph_daemon=\"osd.0\"})",
- "format": "time_series",
- "hide": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Available",
- "metric": "ceph_osd_avail_bytes",
- "refId": "A",
- "step": 60
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum without (instance, ceph_daemon) (ceph_osd_stat_bytes_used{ceph_daemon=~\"$osd\"})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Used",
- "metric": "ceph_osd_avail_bytes",
- "refId": "B",
- "step": 60
- }
- ],
- "title": "OSD Storage",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "max": 100,
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "rgba(50, 172, 45, 0.97)",
- "value": null
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 60
- },
- {
- "color": "rgba(245, 54, 54, 0.9)",
- "value": 80
- }
- ]
- },
- "unit": "percent"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 4,
- "x": 20,
- "y": 5
- },
- "id": 7,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showThresholdLabels": false,
- "showThresholdMarkers": true
- },
- "pluginVersion": "9.0.5",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum without (instance, ceph_daemon) (ceph_osd_stat_bytes_used{ceph_daemon=\"osd.0\"}/ceph_osd_stat_bytes{ceph_daemon=\"osd.0\"})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 60
- }
- ],
- "title": "Utilization",
- "type": "gauge"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 12
- },
- "id": 13,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "refId": "A"
- }
- ],
- "title": "Latency, Storage, Utilization Varience",
- "type": "row"
- },
- {
- "cards": {},
- "color": {
- "cardColor": "#b4ff00",
- "colorScale": "sqrt",
- "colorScheme": "interpolateOranges",
- "exponent": 0.5,
- "mode": "opacity"
- },
- "dataFormat": "timeseries",
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 13
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 83,
- "legend": {
- "show": true
- },
- "links": [],
- "options": {
- "calculate": true,
- "calculation": {
- "yBuckets": {
- "mode": "count",
- "scale": {
- "log": 2,
- "type": "log"
- },
- "value": "1"
- }
- },
- "cellGap": 2,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#b4ff00",
- "mode": "opacity",
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": false
- },
- "yAxis": {
- "axisPlacement": "left",
- "min": "0",
- "reverse": false,
- "unit": "ms"
- }
- },
- "pluginVersion": "9.1.3",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum without (instance) (ceph_osd_apply_latency_ms{ceph_daemon='$osd'})",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "OSD Apply Latency Distribution",
- "tooltip": {
- "show": true,
- "showHistogram": false
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "xBucketSize": "",
- "yAxis": {
- "format": "ms",
- "logBase": 2,
- "min": "0",
- "show": true,
- "splitFactor": 1
- },
- "yBucketBound": "auto",
- "yBucketSize": 10
- },
- {
- "cards": {},
- "color": {
- "cardColor": "#65c5db",
- "colorScale": "sqrt",
- "colorScheme": "interpolateOranges",
- "exponent": 0.5,
- "mode": "opacity"
- },
- "dataFormat": "timeseries",
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 13
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 84,
- "legend": {
- "show": true
- },
- "links": [],
- "options": {
- "calculate": true,
- "calculation": {
- "yBuckets": {
- "mode": "count",
- "scale": {
- "log": 2,
- "type": "log"
- }
- }
- },
- "cellGap": 2,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#65c5db",
- "mode": "opacity",
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": false
- },
- "yAxis": {
- "axisPlacement": "left",
- "min": "0",
- "reverse": false,
- "unit": "ms"
- }
- },
- "pluginVersion": "9.1.3",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum without (instance) (ceph_osd_commit_latency_ms{ceph_daemon='$osd'})",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "OSD Commit Latency Distribution",
- "tooltip": {
- "show": true,
- "showHistogram": false
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "xBucketSize": "",
- "yAxis": {
- "format": "ms",
- "logBase": 2,
- "min": "0",
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "cards": {},
- "color": {
- "cardColor": "#806eb7",
- "colorScale": "sqrt",
- "colorScheme": "interpolateOranges",
- "exponent": 0.5,
- "mode": "opacity"
- },
- "dataFormat": "timeseries",
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 21
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 85,
- "legend": {
- "show": true
- },
- "links": [],
- "options": {
- "calculate": true,
- "calculation": {
- "yBuckets": {
- "mode": "count",
- "scale": {
- "log": 2,
- "type": "log"
- }
- }
- },
- "cellGap": 2,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#806eb7",
- "mode": "opacity",
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": false
- },
- "yAxis": {
- "axisPlacement": "left",
- "decimals": 2,
- "min": "0",
- "reverse": false,
- "unit": "ms"
- }
- },
- "pluginVersion": "9.1.3",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum without (instance) (rate(ceph_osd_op_r_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_r_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "OSD Read Op Latency Distribution",
- "tooltip": {
- "show": true,
- "showHistogram": false
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "xBucketSize": "",
- "yAxis": {
- "decimals": 2,
- "format": "ms",
- "logBase": 2,
- "min": "0",
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "cards": {},
- "color": {
- "cardColor": "#f9934e",
- "colorScale": "sqrt",
- "colorScheme": "interpolateOranges",
- "exponent": 0.5,
- "mode": "opacity"
- },
- "dataFormat": "timeseries",
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 21
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 86,
- "legend": {
- "show": true
- },
- "links": [],
- "options": {
- "calculate": true,
- "calculation": {
- "yBuckets": {
- "mode": "count",
- "scale": {
- "log": 2,
- "type": "log"
- }
- }
- },
- "cellGap": 2,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#f9934e",
- "mode": "opacity",
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": false
- },
- "yAxis": {
- "axisPlacement": "left",
- "decimals": 2,
- "min": "0",
- "reverse": false,
- "unit": "ms"
- }
- },
- "pluginVersion": "9.1.3",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum without (instance) (rate(ceph_osd_op_w_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_w_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
- "format": "time_series",
- "instant": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "OSD Write Op Latency Distribution",
- "tooltip": {
- "show": true,
- "showHistogram": false
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "xBucketSize": "",
- "yAxis": {
- "decimals": 2,
- "format": "ms",
- "logBase": 2,
- "min": "0",
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 29
- },
- "id": 44,
- "links": [],
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "avg without (instance, ceph_daemon) (rate(ceph_osd_op_r_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_r_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "read",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "avg without (instance, ceph_daemon) (rate(ceph_osd_op_w_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_w_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "write",
- "refId": "B"
- }
- ],
- "title": "Avg OSD Op Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 29
- },
- "id": 35,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "avg without (instance, ceph_daemon) (ceph_osd_apply_latency_ms{ceph_daemon='$osd'})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "apply",
- "metric": "ceph_osd_perf_apply_latency_seconds",
- "refId": "A",
- "step": 4
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "avg without (instance, ceph_daemon)(ceph_osd_commit_latency_ms{ceph_daemon='$osd'})",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "commit",
- "metric": "ceph_osd_perf_commit_latency_seconds",
- "refId": "B",
- "step": 4
- }
- ],
- "title": "AVG OSD Apply + Commit Latency",
- "type": "timeseries"
- }
- ],
- "refresh": "5m",
- "schemaVersion": 36,
- "style": "dark",
- "tags": [
- "ceph",
- "osd"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": false,
- "text": "Prometheus",
- "value": "Prometheus"
- },
- "hide": 0,
- "includeAll": false,
- "label": "Data source",
- "multi": false,
- "name": "DS_PROMETHEUS",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "auto": true,
- "auto_count": 10,
- "auto_min": "1m",
- "current": {
- "selected": false,
- "text": "30s",
- "value": "30s"
- },
- "datasource": "Prometheus",
- "hide": 0,
- "includeAll": false,
- "label": "Interval",
- "multi": false,
- "name": "interval",
- "options": [
- {
- "selected": false,
- "text": "auto",
- "value": "$__auto_interval_interval"
- },
- {
- "selected": false,
- "text": "10s",
- "value": "10s"
- },
- {
- "selected": true,
- "text": "30s",
- "value": "30s"
- },
- {
- "selected": false,
- "text": "1m",
- "value": "1m"
- },
- {
- "selected": false,
- "text": "10m",
- "value": "10m"
- },
- {
- "selected": false,
- "text": "30m",
- "value": "30m"
- },
- {
- "selected": false,
- "text": "1h",
- "value": "1h"
- },
- {
- "selected": false,
- "text": "6h",
- "value": "6h"
- },
- {
- "selected": false,
- "text": "12h",
- "value": "12h"
- },
- {
- "selected": false,
- "text": "1d",
- "value": "1d"
- },
- {
- "selected": false,
- "text": "7d",
- "value": "7d"
- },
- {
- "selected": false,
- "text": "14d",
- "value": "14d"
- },
- {
- "selected": false,
- "text": "30d",
- "value": "30d"
- }
- ],
- "query": "10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
- "queryValue": "",
- "refresh": 2,
- "skipUrlSync": false,
- "type": "interval"
- },
- {
- "current": {},
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "definition": "label_values(ceph_osd_up, ceph_daemon)",
- "hide": 0,
- "includeAll": false,
- "label": "OSD",
- "multi": false,
- "name": "osd",
- "options": [],
- "query": {
- "query": "label_values(ceph_osd_up, ceph_daemon)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-3h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "browser",
- "title": "Ceph - OSD (Single)",
- "uid": "Fj5fAfzik",
- "version": 2,
- "weekStart": ""
- }
- ceph-pools-dashboard.json: |-
- {
- "__inputs": [],
- "__elements": {},
- "__requires": [
- {
- "type": "panel",
- "id": "gauge",
- "name": "Gauge",
- "version": ""
- },
- {
- "type": "grafana",
- "id": "grafana",
- "name": "Grafana",
- "version": "9.0.5"
- },
- {
- "type": "datasource",
- "id": "prometheus",
- "name": "Prometheus",
- "version": "1.0.0"
- },
- {
- "type": "panel",
- "id": "timeseries",
- "name": "Time series",
- "version": ""
- }
- ],
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
- "type": "dashboard"
- }
- ]
- },
- "description": "Ceph Pools dashboard.",
- "editable": false,
- "fiscalYearStartMonth": 0,
- "gnetId": 5342,
- "graphTooltip": 0,
- "id": null,
- "links": [],
- "liveNow": false,
- "panels": [
- {
- "collapsed": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 11,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Pool: $pool",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 30,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byRegexp",
- "options": "/^Total.*$/"
- },
- "properties": [
- {
- "id": "custom.fillOpacity",
- "value": 0
- },
- {
- "id": "custom.lineWidth",
- "value": 4
- },
- {
- "id": "custom.stacking",
- "value": {
- "group": false,
- "mode": "normal"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byRegexp",
- "options": "/^Raw.*$/"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#BF1B00",
- "mode": "fixed"
- }
- },
- {
- "id": "custom.fillOpacity",
- "value": 0
- },
- {
- "id": "custom.lineWidth",
- "value": 4
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 7,
- "w": 20,
- "x": 0,
- "y": 1
- },
- "id": 2,
- "interval": "$interval",
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
- ],
- "displayMode": "table",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum((ceph_pool_max_avail) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
- "format": "time_series",
- "hide": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Available - {{ name }}",
- "metric": "ceph_pool_available_bytes",
- "refId": "A",
- "step": 60
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum((ceph_pool_stored) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
- "format": "time_series",
- "hide": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Stored - {{ name }}",
- "metric": "ceph_pool",
- "refId": "B",
- "step": 60
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum((ceph_pool_stored + ceph_pool_max_avail) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
- "format": "time_series",
- "hide": true,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Total - {{ name }}",
- "metric": "ceph_pool",
- "refId": "C",
- "step": 60
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum((ceph_pool_stored_raw) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
- "format": "time_series",
- "hide": false,
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Raw - {{ name }}",
- "metric": "ceph_pool",
- "refId": "D",
- "step": 60
- }
- ],
- "title": "Pool Storage",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "decimals": 2,
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "max": 1,
- "min": 0,
- "thresholds": {
- "mode": "percentage",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "yellow",
- "value": 75
- },
- {
- "color": "red",
- "value": 90
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 4,
- "x": 20,
- "y": 1
- },
- "id": 10,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showThresholdLabels": false,
- "showThresholdMarkers": true
- },
- "pluginVersion": "9.0.5",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum without (instance, pool_id, name) ((ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail)) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"}))",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 60
- }
- ],
- "title": "Usage",
- "type": "gauge"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 8
- },
- "id": 12,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Pool Info",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 9
- },
- "id": 7,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum((ceph_pool_objects) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Objects - {{ name }}",
- "refId": "A",
- "step": 60
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum((ceph_pool_dirty) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Dirty Objects - {{ name }}",
- "refId": "B",
- "step": 60
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum((ceph_pool_quota_objects) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Quota Objects - {{ name }}",
- "refId": "C"
- }
- ],
- "title": "Objects in Pool",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "IOPS",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 9
- },
- "id": 4,
- "interval": "$interval",
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum((irate(ceph_pool_rd[3m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Read - {{ name }}",
- "refId": "B",
- "step": 60
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum((irate(ceph_pool_wr[3m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Write - {{ name }}",
- "refId": "A",
- "step": 60
- }
- ],
- "title": "IOPS",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "Bps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 17
- },
- "id": 5,
- "interval": "$interval",
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum((irate(ceph_pool_rd_bytes[5m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Read Bytes - {{ name }}",
- "refId": "A",
- "step": 60
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum((irate(ceph_pool_wr_bytes[5m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
- "format": "time_series",
- "interval": "$interval",
- "intervalFactor": 1,
- "legendFormat": "Written Bytes - {{ name }}",
- "refId": "B",
- "step": 60
- }
- ],
- "title": "Throughput",
- "type": "timeseries"
- }
- ],
- "refresh": "1m",
- "schemaVersion": 36,
- "style": "dark",
- "tags": [
- "ceph",
- "pools"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": false,
- "text": "Prometheus",
- "value": "Prometheus"
- },
- "hide": 0,
- "includeAll": false,
- "label": "Data source",
- "multi": false,
- "name": "DS_PROMETHEUS",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "auto": true,
- "auto_count": 10,
- "auto_min": "1m",
- "current": {
- "selected": false,
- "text": "30s",
- "value": "30s"
- },
- "datasource": "Prometheus",
- "hide": 0,
- "includeAll": false,
- "label": "Interval",
- "multi": false,
- "name": "interval",
- "options": [
- {
- "selected": false,
- "text": "auto",
- "value": "$__auto_interval_interval"
- },
- {
- "selected": false,
- "text": "10s",
- "value": "10s"
- },
- {
- "selected": true,
- "text": "30s",
- "value": "30s"
- },
- {
- "selected": false,
- "text": "1m",
- "value": "1m"
- },
- {
- "selected": false,
- "text": "10m",
- "value": "10m"
- },
- {
- "selected": false,
- "text": "30m",
- "value": "30m"
- },
- {
- "selected": false,
- "text": "1h",
- "value": "1h"
- },
- {
- "selected": false,
- "text": "6h",
- "value": "6h"
- },
- {
- "selected": false,
- "text": "12h",
- "value": "12h"
- },
- {
- "selected": false,
- "text": "1d",
- "value": "1d"
- },
- {
- "selected": false,
- "text": "7d",
- "value": "7d"
- },
- {
- "selected": false,
- "text": "14d",
- "value": "14d"
- },
- {
- "selected": false,
- "text": "30d",
- "value": "30d"
- }
- ],
- "query": "10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
- "queryValue": "",
- "refresh": 2,
- "skipUrlSync": false,
- "type": "interval"
- },
- {
- "allValue": ".*",
- "current": {},
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "definition": "label_values(ceph_pool_metadata, name)",
- "hide": 0,
- "includeAll": true,
- "label": "Pool",
- "multi": true,
- "name": "pool",
- "options": [],
- "query": {
- "query": "label_values(ceph_pool_metadata, name)",
- "refId": "Prometheus-pool-Variable-Query"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 3,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-3h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "browser",
- "title": "Ceph - Pools",
- "uid": "-gtf0Bzik",
- "version": 2,
- "weekStart": ""
- }
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- labels:
- grafana_dashboard: "1"
- annotations:
- grafana_folder: Matrix
- name: synapse-dashboard
-data:
- synapse-dashboard.json: |-
- {
- "__elements": {},
- "__requires": [
- {
- "type": "grafana",
- "id": "grafana",
- "name": "Grafana",
- "version": "9.2.2"
- },
- {
- "type": "panel",
- "id": "graph",
- "name": "Graph (old)",
- "version": ""
- },
- {
- "type": "panel",
- "id": "heatmap",
- "name": "Heatmap",
- "version": ""
- },
- {
- "type": "datasource",
- "id": "prometheus",
- "name": "Prometheus",
- "version": "1.0.0"
- },
- {
- "type": "panel",
- "id": "timeseries",
- "name": "Time series",
- "version": ""
- }
- ],
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "enable": false,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "limit": 100,
- "name": "Annotations & Alerts",
- "showIn": 0,
- "type": "dashboard"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "enable": true,
- "expr": "changes(process_start_time_seconds{instance=\"$instance\",job=~\"synapse\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"$instance\",job=\"synapse\"}",
- "iconColor": "purple",
- "name": "deploys",
- "titleFormat": "Deployed {{version}}"
- }
- ]
- },
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "id": null,
- "links": [
- {
- "asDropdown": false,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "matrix"
- ],
- "title": "Dashboards",
- "type": "dashboards"
- }
- ],
- "liveNow": false,
- "panels": [
- {
- "collapsed": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 73,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Overview",
- "type": "row"
- },
- {
- "cards": {
- "cardPadding": -1,
- "cardRound": 0
- },
- "color": {
- "cardColor": "#b4ff00",
- "colorScale": "sqrt",
- "colorScheme": "interpolateInferno",
- "exponent": 0.5,
- "mode": "spectrum"
- },
- "dataFormat": "tsbuckets",
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 1
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 189,
- "legend": {
- "show": false
- },
- "links": [],
- "options": {
- "calculate": false,
- "calculation": {},
- "cellGap": -1,
- "cellRadius": 0,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#b4ff00",
- "mode": "scheme",
- "reverse": false,
- "scale": "exponential",
- "scheme": "Inferno",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": false
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": true
- },
- "yAxis": {
- "axisPlacement": "left",
- "reverse": false,
- "unit": "s"
- }
- },
- "pluginVersion": "9.2.2",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le)",
- "format": "heatmap",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{le}}",
- "refId": "A"
- }
- ],
- "title": "Event Send Time (excluding errors, all workers)",
- "tooltip": {
- "show": true,
- "showHistogram": true
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "yAxis": {
- "format": "s",
- "logBase": 2,
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 1
- },
- "hiddenSeries": false,
- "id": 152,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "rightSide": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "connected",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "Avg",
- "fill": 0,
- "linewidth": 3
- },
- {
- "alias": "99%",
- "color": "#C4162A",
- "fillBelowTo": "90%"
- },
- {
- "alias": "90%",
- "color": "#FF7383",
- "fillBelowTo": "75%"
- },
- {
- "alias": "75%",
- "color": "#FFEE52",
- "fillBelowTo": "50%"
- },
- {
- "alias": "50%",
- "color": "#73BF69",
- "fillBelowTo": "25%"
- },
- {
- "alias": "25%",
- "color": "#1F60C4",
- "fillBelowTo": "5%"
- },
- {
- "alias": "5%",
- "lines": false
- },
- {
- "alias": "Average",
- "color": "rgb(255, 255, 255)",
- "lines": true,
- "linewidth": 3
- },
- {
- "alias": "Events",
- "color": "#B877D9",
- "hideTooltip": true,
- "points": true,
- "yaxis": 2,
- "zindex": -3
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "99%",
- "refId": "D"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "90%",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "75%",
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "50%",
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
- "legendFormat": "25%",
- "refId": "F"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
- "legendFormat": "5%",
- "refId": "G"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))",
- "legendFormat": "Average",
- "refId": "H"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size]))",
- "hide": false,
- "instant": false,
- "legendFormat": "Events",
- "refId": "E"
- }
- ],
- "thresholds": [
- {
- "$$hashKey": "object:283",
- "colorMode": "warning",
- "fill": false,
- "line": true,
- "op": "gt",
- "value": 1,
- "yaxis": "left"
- },
- {
- "$$hashKey": "object:284",
- "colorMode": "critical",
- "fill": false,
- "line": true,
- "op": "gt",
- "value": 2,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "Event Send Time Quantiles (excluding errors, all workers)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:255",
- "format": "s",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:256",
- "format": "hertz",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 10
- },
- "hiddenSeries": false,
- "id": 75,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 3,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} ",
- "refId": "A"
- }
- ],
- "thresholds": [
- {
- "$$hashKey": "object:566",
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 1,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "CPU usage",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:538",
- "format": "percentunit",
- "logBase": 1,
- "max": "1.5",
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:539",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 10
- },
- "hiddenSeries": false,
- "id": 198,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 3,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}} {{index}}",
- "refId": "A",
- "step": 20,
- "target": ""
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
- "hide": true,
- "interval": "",
- "legendFormat": "total",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Memory",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "transformations": [],
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:1560",
- "format": "bytes",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:1561",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "axisSoftMax": 1,
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 10,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 19
- },
- "id": 245,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1",
- "legendFormat": "version {{version}}",
- "range": true,
- "refId": "deployed_synapse_versions"
- }
- ],
- "title": "Deployed Synapse versions over time",
- "type": "timeseries"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 19
- },
- "hiddenSeries": false,
- "id": 37,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "$$hashKey": "object:639",
- "alias": "/max$/",
- "color": "#890F02",
- "fill": 0,
- "legend": false
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}}",
- "refId": "A",
- "step": 20
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "hide": true,
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} max",
- "refId": "B",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Open FDs",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:650",
- "format": "none",
- "label": "",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:651",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 26
- },
- "id": 54,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 27
- },
- "hiddenSeries": false,
- "id": 5,
- "legend": {
- "alignAsTable": false,
- "avg": false,
- "current": false,
- "hideEmpty": false,
- "hideZero": false,
- "max": false,
- "min": false,
- "rightSide": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 3,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "$$hashKey": "object:1240",
- "alias": "/user/"
- },
- {
- "$$hashKey": "object:1241",
- "alias": "/system/"
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} system ",
- "metric": "",
- "range": true,
- "refId": "B",
- "step": 20
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} user",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [
- {
- "$$hashKey": "object:1278",
- "colorMode": "custom",
- "fillColor": "rgba(255, 255, 255, 1)",
- "line": true,
- "lineColor": "rgba(216, 200, 27, 0.27)",
- "op": "gt",
- "value": 0.5,
- "yaxis": "left"
- },
- {
- "$$hashKey": "object:1279",
- "colorMode": "custom",
- "fillColor": "rgba(255, 255, 255, 1)",
- "line": true,
- "lineColor": "rgb(87, 6, 16)",
- "op": "gt",
- "value": 0.8,
- "yaxis": "left"
- },
- {
- "$$hashKey": "object:1498",
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 1,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "CPU",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:1250",
- "format": "percentunit",
- "label": "",
- "logBase": 1,
- "max": "1.2",
- "min": 0,
- "show": true
- },
- {
- "$$hashKey": "object:1251",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 27
- },
- "id": 105,
- "interval": "",
- "links": [],
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "8.3.2",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.999, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
- "hide": false,
- "interval": "",
- "legendFormat": "{{job}}-{{index}} 99.9%",
- "refId": "E"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} 99%",
- "refId": "A",
- "step": 20
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 95%",
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 90%",
- "refId": "C"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} mean",
- "refId": "D"
- }
- ],
- "title": "Reactor tick quantiles",
- "type": "timeseries"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 34
- },
- "hiddenSeries": false,
- "id": 34,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}} {{index}}",
- "refId": "A",
- "step": 20,
- "target": ""
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
- "interval": "",
- "legendFormat": "total",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Memory",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "transformations": [],
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "bytes",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 34
- },
- "hiddenSeries": false,
- "id": 49,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/^up/",
- "legend": false,
- "yaxis": 2
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Prometheus scrape time",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "decimals": 0,
- "format": "none",
- "label": "",
- "logBase": 1,
- "max": "0",
- "min": "-1",
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 41
- },
- "hiddenSeries": false,
- "id": 53,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "$$hashKey": "object:116",
- "alias": "/^version .*/",
- "lines": true,
- "linewidth": 6,
- "points": false
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}}",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1",
- "hide": false,
- "legendFormat": "version {{version}}",
- "range": true,
- "refId": "deployed_synapse_versions"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Up",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 41
- },
- "hiddenSeries": false,
- "id": 120,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "hide": false,
- "instant": false,
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "hide": false,
- "instant": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{name}}",
- "refId": "B"
- }
- ],
- "thresholds": [
- {
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 1,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "Stacked CPU usage",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:572",
- "format": "percentunit",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:573",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 48
- },
- "hiddenSeries": false,
- "id": 136,
- "interval": "",
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_http_client_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "legendFormat": "{{job}}-{{index}} {{method}}",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_http_matrixfederationclient_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "legendFormat": "{{job}}-{{index}} {{method}} (federation)",
- "range": true,
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Outgoing HTTP request rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:123",
- "format": "reqps",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:124",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "active threads",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 48
- },
- "id": 207,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "synapse_threadpool_working_threads{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "interval": "",
- "legendFormat": "{{job}}-{{index}} {{name}}",
- "refId": "A"
- }
- ],
- "title": "Threadpool activity",
- "type": "timeseries"
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Process info",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 27
- },
- "id": 56,
- "panels": [
- {
- "cards": {
- "cardPadding": -1,
- "cardRound": 0
- },
- "color": {
- "cardColor": "#b4ff00",
- "colorScale": "sqrt",
- "colorScheme": "interpolateInferno",
- "exponent": 0.5,
- "mode": "spectrum"
- },
- "dataFormat": "tsbuckets",
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 28
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 85,
- "legend": {
- "show": false
- },
- "links": [],
- "options": {
- "calculate": false,
- "calculation": {},
- "cellGap": -1,
- "cellRadius": 0,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#b4ff00",
- "mode": "scheme",
- "reverse": false,
- "scale": "exponential",
- "scheme": "Inferno",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": false
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": true
- },
- "yAxis": {
- "axisPlacement": "left",
- "reverse": false,
- "unit": "s"
- }
- },
- "pluginVersion": "9.2.2",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",
- "format": "heatmap",
- "intervalFactor": 1,
- "legendFormat": "{{le}}",
- "refId": "A"
- }
- ],
- "title": "Event Send Time (Including errors, across all workers)",
- "tooltip": {
- "show": true,
- "showHistogram": true
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "yAxis": {
- "format": "s",
- "logBase": 2,
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 28
- },
- "hiddenSeries": false,
- "id": 33,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size])) without (job,index)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "",
- "refId": "A",
- "step": 20,
- "target": ""
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Events Persisted (all workers)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:102",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:103",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "decimals": 1,
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 37
- },
- "hiddenSeries": false,
- "id": 40,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_storage_events_persisted_by_source_type{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{type}}",
- "refId": "D"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Events/s Local vs Remote",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "decimals": 1,
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 37
- },
- "hiddenSeries": false,
- "id": 46,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_storage_events_persisted_by_event_type{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "format": "time_series",
- "instant": false,
- "intervalFactor": 2,
- "legendFormat": "{{type}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Events/s by Type",
- "tooltip": {
- "shared": false,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {
- "irc-freenode (local)": "#EAB839"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "decimals": 1,
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 44
- },
- "hiddenSeries": false,
- "id": 44,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_storage_events_persisted_by_origin{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{origin_entity}} ({{origin_type}})",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Events/s by Origin",
- "tooltip": {
- "shared": false,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "decimals": 1,
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 44
- },
- "hiddenSeries": false,
- "id": 45,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "sum(rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{origin_entity}} ({{origin_type}})",
- "range": true,
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Memberships/s by Origin",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:232",
- "format": "hertz",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:233",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 51
- },
- "hiddenSeries": false,
- "id": 118,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "repeatDirection": "h",
- "seriesOverrides": [
- {
- "$$hashKey": "object:316",
- "alias": "mean",
- "linewidth": 2
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 99%",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 95%",
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 90%",
- "refId": "C"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 50%",
- "refId": "D"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} mean",
- "range": true,
- "refId": "E"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Event send time quantiles by worker",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:263",
- "format": "s",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:264",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "CPU and DB time spent on most expensive state resolution in a room, summed over all workers. This is a very rough proxy for \"how fast is state res\", but it doesn't accurately represent the system load (e.g. it completely ignores cheap state resolutions).\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 30,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s/s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 51
- },
- "id": 222,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": false,
- "expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))",
- "format": "time_series",
- "hide": false,
- "instant": false,
- "interval": "",
- "legendFormat": "DB time",
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": false,
- "expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))",
- "format": "time_series",
- "hide": false,
- "instant": false,
- "interval": "",
- "legendFormat": "CPU time",
- "refId": "C"
- }
- ],
- "title": "Stateres worst-case",
- "type": "timeseries"
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Event persistence",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 28
- },
- "id": 57,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 2,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 29
- },
- "hiddenSeries": false,
- "id": 4,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideEmpty": false,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [
- {
- "$$hashKey": "object:234",
- "colorMode": "custom",
- "fill": true,
- "fillColor": "rgba(216, 200, 27, 0.27)",
- "op": "gt",
- "value": 100,
- "yaxis": "left"
- },
- {
- "$$hashKey": "object:235",
- "colorMode": "custom",
- "fill": true,
- "fillColor": "rgba(234, 112, 112, 0.22)",
- "op": "gt",
- "value": 250,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "Request Count by arrival time",
- "tooltip": {
- "shared": false,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:206",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:207",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 29
- },
- "hiddenSeries": false,
- "id": 32,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{method}} {{servlet}} {{job}}-{{index}}",
- "refId": "A",
- "step": 20,
- "target": ""
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Top 10 Request Counts",
- "tooltip": {
- "shared": false,
- "sort": 2,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:305",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:306",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 2,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 37
- },
- "hiddenSeries": false,
- "id": 139,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideEmpty": false,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [
- {
- "$$hashKey": "object:135",
- "colorMode": "custom",
- "fill": true,
- "fillColor": "rgba(216, 200, 27, 0.27)",
- "op": "gt",
- "value": 100,
- "yaxis": "left"
- },
- {
- "$$hashKey": "object:136",
- "colorMode": "custom",
- "fill": true,
- "fillColor": "rgba(234, 112, 112, 0.22)",
- "op": "gt",
- "value": 250,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "Total CPU Usage by Endpoint",
- "tooltip": {
- "shared": false,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:107",
- "format": "percentunit",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:108",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 37
- },
- "hiddenSeries": false,
- "id": 52,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideEmpty": false,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [
- {
- "$$hashKey": "object:417",
- "colorMode": "custom",
- "fill": true,
- "fillColor": "rgba(216, 200, 27, 0.27)",
- "op": "gt",
- "value": 100,
- "yaxis": "left"
- },
- {
- "$$hashKey": "object:418",
- "colorMode": "custom",
- "fill": true,
- "fillColor": "rgba(234, 112, 112, 0.22)",
- "op": "gt",
- "value": 250,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "Average CPU Usage by Endpoint",
- "tooltip": {
- "shared": false,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:389",
- "format": "s",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:390",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 45
- },
- "hiddenSeries": false,
- "id": 7,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "DB Usage by endpoint",
- "tooltip": {
- "shared": false,
- "sort": 2,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:488",
- "format": "percentunit",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:489",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 2,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 45
- },
- "hiddenSeries": false,
- "id": 47,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "hideEmpty": false,
- "hideZero": true,
- "max": true,
- "min": false,
- "rightSide": false,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "(sum(rate(synapse_http_server_response_time_seconds_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))/(sum(rate(synapse_http_server_response_time_seconds_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Non-sync avg response time",
- "tooltip": {
- "shared": false,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 53
- },
- "hiddenSeries": false,
- "id": 103,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "Total",
- "color": "rgb(255, 255, 255)",
- "fill": 0,
- "linewidth": 3
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "topk(10,synapse_http_server_in_flight_requests_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
- "interval": "",
- "legendFormat": "Total",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Requests in flight",
- "tooltip": {
- "shared": false,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Requests",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 29
- },
- "id": 97,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 30
- },
- "hiddenSeries": false,
- "id": 99,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{name}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "CPU usage by background jobs",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 30
- },
- "hiddenSeries": false,
- "id": 101,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_background_process_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "hide": false,
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{name}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "DB usage by background jobs (including scheduling time)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 39
- },
- "hiddenSeries": false,
- "id": 138,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
- "legendFormat": "{{job}}-{{index}} {{name}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Background jobs in flight",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Background jobs",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 30
- },
- "id": 81,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 31
- },
- "hiddenSeries": false,
- "id": 79,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "successful txn rate",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_util_metrics_block_count_total{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))",
- "legendFormat": "failed txn rate",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Outgoing federation transaction rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 31
- },
- "hiddenSeries": false,
- "id": 83,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_federation_server_received_pdus_total{instance=~\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "pdus",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_federation_server_received_edus_total{instance=~\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "edus",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Incoming PDU/EDU rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 40
- },
- "hiddenSeries": false,
- "id": 109,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations_count_total{instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "pdus",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_federation_client_sent_edus_total{instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "edus",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Outgoing PDU/EDU rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 40
- },
- "hiddenSeries": false,
- "id": 111,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_federation_client_sent_edus_by_type_total{instance=\"$instance\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{type}}",
- "range": true,
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Outgoing EDUs by type",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:462",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:463",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Triangular growth may indicate a problem with federation sending from the remote host --- but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "axisSoftMax": 60,
- "axisSoftMin": 0,
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "line"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 60
- }
- ]
- },
- "unit": "m"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "libera.chat "
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 49
- },
- "id": 243,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_received_pdu_time[10m]))) / 60",
- "instant": false,
- "legendFormat": "{{server_name}} ",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Age of last PDU received from nominated hosts",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Triangular growth may indicate a problem with federation senders on the monitored instance---but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "axisSoftMax": 60,
- "axisSoftMin": 0,
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "line"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 60
- }
- ]
- },
- "unit": "m"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "libera.chat"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 49
- },
- "id": 241,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_sent_pdu_time[10m]))) / 60",
- "instant": false,
- "legendFormat": "{{server_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Age of last PDU sent to nominated hosts",
- "type": "timeseries"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "The number of events in the in-memory queues ",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 57
- },
- "hiddenSeries": false,
- "id": 142,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "interval": "",
- "legendFormat": "pending PDUs {{job}}-{{index}}",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "interval": "",
- "legendFormat": "pending EDUs {{job}}-{{index}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "In-memory federation transmission queues",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:547",
- "format": "short",
- "label": "events",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:548",
- "format": "short",
- "label": "",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Number of events queued up on the master process for processing by the federation sender",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 57
- },
- "hiddenSeries": false,
- "id": 140,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "presence changed",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "presence map",
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "presence destinations",
- "refId": "E"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "keyed edus",
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "other edus",
- "refId": "D"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "stream positions",
- "refId": "F"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Outgoing EDU queues on master",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "none",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "cards": {
- "cardPadding": -1
- },
- "color": {
- "cardColor": "#b4ff00",
- "colorScale": "sqrt",
- "colorScheme": "interpolateInferno",
- "exponent": 0.5,
- "min": 0,
- "mode": "spectrum"
- },
- "dataFormat": "tsbuckets",
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 66
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 166,
- "legend": {
- "show": false
- },
- "links": [],
- "options": {
- "calculate": false,
- "calculation": {},
- "cellGap": -1,
- "cellValues": {
- "decimals": 2
- },
- "color": {
- "exponent": 0.5,
- "fill": "#b4ff00",
- "min": 0,
- "mode": "scheme",
- "reverse": false,
- "scale": "exponential",
- "scheme": "Inferno",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": false
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": true
- },
- "yAxis": {
- "axisPlacement": "left",
- "decimals": 0,
- "reverse": false,
- "unit": "s"
- }
- },
- "pluginVersion": "9.2.2",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{instance=\"$instance\",name=\"federation_sender\"}[$bucket_size])) by (le)",
- "format": "heatmap",
- "instant": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{ le }}",
- "refId": "A"
- }
- ],
- "title": "Federation send PDU lag",
- "tooltip": {
- "show": true,
- "showHistogram": true
- },
- "tooltipDecimals": 2,
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "yAxis": {
- "decimals": 0,
- "format": "s",
- "logBase": 1,
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 66
- },
- "hiddenSeries": false,
- "id": 162,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "rightSide": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "connected",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "Avg",
- "fill": 0,
- "linewidth": 3
- },
- {
- "alias": "99%",
- "color": "#C4162A",
- "fillBelowTo": "90%"
- },
- {
- "alias": "90%",
- "color": "#FF7383",
- "fillBelowTo": "75%"
- },
- {
- "alias": "75%",
- "color": "#FFEE52",
- "fillBelowTo": "50%"
- },
- {
- "alias": "50%",
- "color": "#73BF69",
- "fillBelowTo": "25%"
- },
- {
- "alias": "25%",
- "color": "#1F60C4",
- "fillBelowTo": "5%"
- },
- {
- "alias": "5%",
- "lines": false
- },
- {
- "alias": "Average",
- "color": "rgb(255, 255, 255)",
- "lines": true,
- "linewidth": 3
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "99%",
- "refId": "D"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "90%",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "75%",
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "50%",
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "interval": "",
- "legendFormat": "25%",
- "refId": "F"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "interval": "",
- "legendFormat": "5%",
- "refId": "G"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
- "interval": "",
- "legendFormat": "Average",
- "refId": "H"
- }
- ],
- "thresholds": [
- {
- "colorMode": "warning",
- "fill": false,
- "line": true,
- "op": "gt",
- "value": 0.25,
- "yaxis": "left"
- },
- {
- "colorMode": "critical",
- "fill": false,
- "line": true,
- "op": "gt",
- "value": 1,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "Federation send PDU lag quantiles",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "hertz",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "cards": {
- "cardPadding": -1
- },
- "color": {
- "cardColor": "#b4ff00",
- "colorScale": "sqrt",
- "colorScheme": "interpolateInferno",
- "exponent": 0.5,
- "min": 0,
- "mode": "spectrum"
- },
- "dataFormat": "tsbuckets",
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 75
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 164,
- "legend": {
- "show": false
- },
- "links": [],
- "options": {
- "calculate": false,
- "calculation": {},
- "cellGap": -1,
- "cellValues": {
- "decimals": 2
- },
- "color": {
- "exponent": 0.5,
- "fill": "#b4ff00",
- "min": 0,
- "mode": "scheme",
- "reverse": false,
- "scale": "exponential",
- "scheme": "Inferno",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": false
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": true
- },
- "yAxis": {
- "axisPlacement": "left",
- "decimals": 0,
- "reverse": false,
- "unit": "s"
- }
- },
- "pluginVersion": "9.2.2",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{instance=\"$instance\"}[$bucket_size])) by (le)",
- "format": "heatmap",
- "instant": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{ le }}",
- "refId": "A"
- }
- ],
- "title": "Handle inbound PDU time",
- "tooltip": {
- "show": true,
- "showHistogram": true
- },
- "tooltipDecimals": 2,
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "yAxis": {
- "decimals": 0,
- "format": "s",
- "logBase": 1,
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 75
- },
- "hiddenSeries": false,
- "id": 203,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}}",
- "range": true,
- "refId": "A",
- "step": 4
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Age of oldest event in staging area",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:92",
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "$$hashKey": "object:93",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 84
- },
- "hiddenSeries": false,
- "id": 202,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}}",
- "range": true,
- "refId": "A",
- "step": 4
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Number of events in federation staging area",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:92",
- "format": "none",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "$$hashKey": "object:93",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 84
- },
- "hiddenSeries": false,
- "id": 205,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))",
- "interval": "",
- "legendFormat": "soft-failed events",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Soft-failed event rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:131",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:132",
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Federation",
- "type": "row"
- },
- {
- "collapsed": true,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 31
- },
- "id": 227,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 154
- },
- "id": 239,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(synapse_rate_limit_reject_total{instance=\"$instance\"}[$bucket_size]))",
- "refId": "A"
- }
- ],
- "title": "Number of rate limit rejected requests",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 154
- },
- "id": 235,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))",
- "refId": "A"
- }
- ],
- "title": "Number of requests being slept by the rate limiter",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Why is the data zero (0)? https://github.com/matrix-org/synapse/pull/13541#discussion_r951926322",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 162
- },
- "id": 237,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "9.0.4",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(synapse_rate_limit_reject_affected_hosts{instance=\"$instance\"}[$bucket_size]))",
- "refId": "A"
- }
- ],
- "title": "Number of hosts being rejected by the rate limiter",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "https://github.com/matrix-org/synapse/pull/13541",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 162
- },
- "id": 233,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "9.0.4",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(synapse_rate_limit_sleep_affected_hosts{instance=\"$instance\"}[$bucket_size]))",
- "refId": "A"
- }
- ],
- "title": "Number of hosts being slept by the rate limiter",
- "type": "timeseries"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 170
- },
- "hiddenSeries": false,
- "id": 229,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "rightSide": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "connected",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "$$hashKey": "object:276",
- "alias": "Avg",
- "fill": 0,
- "linewidth": 3
- },
- {
- "$$hashKey": "object:277",
- "alias": "99%",
- "color": "#C4162A",
- "fillBelowTo": "90%"
- },
- {
- "$$hashKey": "object:278",
- "alias": "90%",
- "color": "#FF7383",
- "fillBelowTo": "75%"
- },
- {
- "$$hashKey": "object:279",
- "alias": "75%",
- "color": "#FFEE52",
- "fillBelowTo": "50%"
- },
- {
- "$$hashKey": "object:280",
- "alias": "50%",
- "color": "#73BF69",
- "fillBelowTo": "25%"
- },
- {
- "$$hashKey": "object:281",
- "alias": "25%",
- "color": "#1F60C4",
- "fillBelowTo": "5%"
- },
- {
- "$$hashKey": "object:282",
- "alias": "5%",
- "lines": false
- },
- {
- "$$hashKey": "object:283",
- "alias": "Average",
- "color": "rgb(255, 255, 255)",
- "lines": true,
- "linewidth": 3
- },
- {
- "$$hashKey": "object:284",
- "alias": ">99%",
- "color": "#B877D9",
- "fill": 3,
- "lines": true
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.9995, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "hide": false,
- "intervalFactor": 1,
- "legendFormat": ">99%",
- "range": true,
- "refId": "E"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "99%",
- "range": true,
- "refId": "D"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.9, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "90%",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.75, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "75%",
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.5, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "50%",
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.25, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "legendFormat": "25%",
- "refId": "F"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.05, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
- "legendFormat": "5%",
- "refId": "G"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_rate_limit_queue_wait_time_seconds_sum{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_rate_limit_queue_wait_time_seconds_count{index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
- "legendFormat": "Average",
- "refId": "H"
- }
- ],
- "thresholds": [
- {
- "$$hashKey": "object:283",
- "colorMode": "warning",
- "fill": false,
- "line": true,
- "op": "gt",
- "value": 1,
- "yaxis": "left"
- },
- {
- "$$hashKey": "object:284",
- "colorMode": "critical",
- "fill": false,
- "line": true,
- "op": "gt",
- "value": 2,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "Rate limit queue wait time Quantiles (all workers)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:255",
- "format": "s",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:256",
- "format": "hertz",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "line"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "hertz"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Default reject threshold (50 requests within a second)"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "red",
- "mode": "fixed"
- }
- },
- {
- "id": "custom.lineWidth",
- "value": 2
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 170
- },
- "id": 231,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "50",
- "hide": false,
- "legendFormat": "Default reject threshold (50 requests within a second)",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Rate of requests being slept by the rate limiter",
- "type": "timeseries"
- }
- ],
- "title": "Federation rate limiter",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 32
- },
- "id": 60,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "hertz"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 155
- },
- "id": 51,
- "links": [],
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.2.2",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_http_httppusher_http_pushes_processed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "processed {{job}}-{{index}}",
- "range": true,
- "refId": "A",
- "step": 20
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_http_httppusher_http_pushes_failed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed {{job}}-{{index}}",
- "range": true,
- "refId": "B",
- "step": 20
- }
- ],
- "title": "HTTP Push rate",
- "type": "timeseries"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 155
- },
- "hiddenSeries": false,
- "id": 134,
- "legend": {
- "avg": false,
- "current": false,
- "hideZero": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})",
- "legendFormat": "{{kind}} {{app_id}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Active pusher instances by app",
- "tooltip": {
- "shared": false,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Pushes",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 33
- },
- "id": 219,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "How many entries in current state that we are iterating over while calculating push rules.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 33
- },
- "hiddenSeries": false,
- "id": 209,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "8.4.3",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{index}}",
- "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total",
- "refId": "A",
- "step": 2
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Iterations over State",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Rate that the cached push rules for a room get invalidated due to underlying push rules being changed. ",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 33
- },
- "hiddenSeries": false,
- "id": 211,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "8.4.3",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{index}}",
- "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
- "refId": "A",
- "step": 2
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Push Rule Invalidations",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "How often the \"delta optimisation\" happens.\n\nThe delta optimisation is when we update the push rules for a room incrementally after a state change where we know the delta between the old state and the new state.\n\nThis can't happen if we don't the delta or we're calculating push rules from scratch.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 40
- },
- "hiddenSeries": false,
- "id": 213,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "8.4.3",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "Number of calls",
- "yaxis": 2
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Hit Rate",
- "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
- "refId": "A",
- "step": 2
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Number of calls",
- "refId": "B",
- "step": 2
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Delta Optimisation",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": "",
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "format": "hertz",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "How often we have the correct cached push rules for a room.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 40
- },
- "hiddenSeries": false,
- "id": 215,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "8.4.3",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "Number of calls",
- "yaxis": 2
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Hit Rate",
- "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
- "refId": "A",
- "step": 2
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Number of calls",
- "refId": "B",
- "step": 2
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "How often we reuse existing calculated push rules",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": "",
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "format": "hertz",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "How often we have existing cached push rules for the room. \n\nNote that these might be outdated and need to be recalculated if the state has changed.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 47
- },
- "hiddenSeries": false,
- "id": 217,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "8.4.3",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "Number of calls",
- "yaxis": 2
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Hit Rate",
- "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
- "refId": "A",
- "step": 2
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Number of calls",
- "refId": "B",
- "step": 2
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "How often we have the RulesForRoom cached",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": "",
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "format": "hertz",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Push Rule Cache",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 34
- },
- "id": 58,
- "panels": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 35
- },
- "id": 48,
- "links": [],
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "9.0.4",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}}",
- "refId": "A",
- "step": 20
- }
- ],
- "title": "Avg time waiting for db conn",
- "type": "timeseries"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 35
- },
- "hiddenSeries": false,
- "id": 104,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.0.4",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
- "format": "time_series",
- "hide": false,
- "intervalFactor": 1,
- "legendFormat": "{{job}} {{index}} 99%",
- "refId": "A",
- "step": 20
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}} {{index}} 95%",
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}} {{index}} 90%",
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}} {{index}} mean",
- "refId": "D"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Db scheduling time quantiles",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 42
- },
- "hiddenSeries": false,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.0.4",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "topk(10, rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} {{desc}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Top DB transactions by txn rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 42
- },
- "hiddenSeries": false,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.0.4",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "instant": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{desc}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "DB transactions by total txn time",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 49
- },
- "hiddenSeries": false,
- "id": 180,
- "legend": {
- "avg": false,
- "current": false,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": false
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.0.4",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "instant": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{desc}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Average DB txn time",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 6,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 49
- },
- "hiddenSeries": false,
- "id": 200,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.4",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "99%",
- "refId": "D"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "90%",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "75%",
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "50%",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Time waiting for DB connection quantiles",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:203",
- "format": "s",
- "label": "",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:204",
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Database",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 35
- },
- "id": 59,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 13,
- "w": 12,
- "x": 0,
- "y": 158
- },
- "hiddenSeries": false,
- "id": 12,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} {{block_name}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Total CPU Usage by Block",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 13,
- "w": 12,
- "x": 12,
- "y": 158
- },
- "hiddenSeries": false,
- "id": 26,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])) / rate(synapse_util_metrics_block_count_total[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} {{block_name}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Average CPU Time per Block",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 13,
- "w": 12,
- "x": 0,
- "y": 171
- },
- "hiddenSeries": false,
- "id": 13,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} {{block_name}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Total DB Usage by Block",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:196",
- "format": "percentunit",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "$$hashKey": "object:197",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "The time each database transaction takes to execute, on average, broken down by metrics block.",
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 13,
- "w": 12,
- "x": 12,
- "y": 171
- },
- "hiddenSeries": false,
- "id": 27,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} {{block_name}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Average Database Transaction time, by Block",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 13,
- "w": 12,
- "x": 0,
- "y": 184
- },
- "hiddenSeries": false,
- "id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} {{block_name}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Average Transactions per Block",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "none",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 13,
- "w": 12,
- "x": 12,
- "y": 184
- },
- "hiddenSeries": false,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_util_metrics_block_time_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} {{block_name}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Average Wallclock Time per Block",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:180",
- "format": "s",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:181",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 15,
- "w": 12,
- "x": 0,
- "y": 197
- },
- "hiddenSeries": false,
- "id": 154,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "{{job}}-{{index}} {{block_name}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Block count",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Per-block metrics",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 36
- },
- "id": 61,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "decimals": 2,
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 0,
- "y": 36
- },
- "hiddenSeries": false,
- "id": 1,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideEmpty": true,
- "hideZero": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.4",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{name}} {{job}}-{{index}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Cache Hit Ratio",
- "tooltip": {
- "msResolution": true,
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": "",
- "logBase": 1,
- "max": "1",
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 12,
- "y": 36
- },
- "hiddenSeries": false,
- "id": 8,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideZero": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.4",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_util_caches_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{name}} {{job}}-{{index}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Cache Size",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 0,
- "y": 46
- },
- "hiddenSeries": false,
- "id": 38,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideZero": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.4",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{name}} {{job}}-{{index}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Cache request rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "rps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 12,
- "y": 46
- },
- "hiddenSeries": false,
- "id": 39,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "hideEmpty": false,
- "hideZero": false,
- "max": true,
- "min": false,
- "rightSide": false,
- "show": true,
- "sort": "max",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.4",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "topk(10, rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{name}} {{job}}-{{index}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Top 10 cache misses",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:101",
- "format": "rps",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:102",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 56
- },
- "hiddenSeries": false,
- "id": 65,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.4",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_util_caches_cache_evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{name}} ({{reason}}) {{job}}-{{index}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Cache eviction rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "label": "entries / second",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Caches",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 37
- },
- "id": 148,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {},
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 29
- },
- "hiddenSeries": false,
- "id": 146,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_util_caches_response_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "interval": "",
- "legendFormat": "{{name}} {{job}}-{{index}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Response cache size",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {},
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 29
- },
- "hiddenSeries": false,
- "id": 150,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_util_caches_response_cache_hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "{{name}} {{job}}-{{index}}",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "",
- "interval": "",
- "legendFormat": "",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Response cache hit rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Response caches",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 38
- },
- "id": 62,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {},
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 30
- },
- "hiddenSeries": false,
- "id": 91,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])",
- "format": "time_series",
- "instant": false,
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} gen {{gen}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Total GC time by bucket (10m smoothing)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "decimals": 3,
- "editable": true,
- "error": false,
- "fieldConfig": {
- "defaults": {
- "custom": {},
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "grid": {},
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 30
- },
- "hiddenSeries": false,
- "id": 21,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{job}} {{index}} gen {{gen}} ",
- "refId": "A",
- "step": 20,
- "target": ""
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Average GC Time Per Collection",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.",
- "fieldConfig": {
- "defaults": {
- "custom": {},
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 39
- },
- "hiddenSeries": false,
- "id": 89,
- "legend": {
- "avg": false,
- "current": false,
- "hideEmpty": true,
- "hideZero": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/gen 0$/",
- "yaxis": 2
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} gen {{gen}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Allocation counts",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": "Gen N-1 GCs since last Gen N GC",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "label": "Objects since last Gen 0 GC",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {},
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 39
- },
- "hiddenSeries": false,
- "id": 93,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "connected",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} gen {{gen}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Object counts per collection",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {},
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 48
- },
- "hiddenSeries": false,
- "id": 95,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} gen {{gen}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "GC frequency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "cards": {
- "cardPadding": 0
- },
- "color": {
- "cardColor": "#b4ff00",
- "colorScale": "sqrt",
- "colorScheme": "interpolateSpectral",
- "exponent": 0.5,
- "min": 0,
- "mode": "spectrum"
- },
- "dataFormat": "tsbuckets",
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 48
- },
- "heatmap": {},
- "hideZeroBuckets": true,
- "highlightCards": true,
- "id": 87,
- "legend": {
- "show": true
- },
- "links": [],
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)",
- "format": "heatmap",
- "intervalFactor": 1,
- "legendFormat": "{{le}}",
- "refId": "A"
- }
- ],
- "title": "GC durations",
- "tooltip": {
- "show": true,
- "showHistogram": false
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "yAxis": {
- "format": "s",
- "logBase": 1,
- "show": true
- },
- "yBucketBound": "auto"
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "GC",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 39
- },
- "id": 63,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 162
- },
- "hiddenSeries": false,
- "id": 43,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} {{command}}",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Rate of outgoing commands",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:89",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:90",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "hertz"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 162
- },
- "id": 41,
- "links": [],
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "8.4.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "rate(synapse_replication_tcp_resource_stream_updates_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{stream_name}}",
- "refId": "A",
- "step": 20
- }
- ],
- "title": "Rate of outgoing RDATA commands, by stream",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "hertz"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 169
- },
- "id": 42,
- "links": [],
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "8.4.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} {{command}}",
- "refId": "A",
- "step": 20
- }
- ],
- "title": "Rate of incoming commands (including echoes)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "axisSoftMin": 1,
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "hertz"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 169
- },
- "id": 220,
- "links": [],
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "8.4.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "exemplar": true,
- "expr": "rate(synapse_replication_tcp_protocol_inbound_rdata_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{stream_name}}",
- "refId": "A",
- "step": 20
- }
- ],
- "title": "Rate of incoming RDATA commands (excluding echoes), by stream",
- "type": "timeseries"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 176
- },
- "hiddenSeries": false,
- "id": 144,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_replication_tcp_command_queue{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "interval": "",
- "legendFormat": "{{stream_name}} {{job}}-{{index}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Queued incoming RDATA commands, by stream",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:218",
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:219",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 176
- },
- "hiddenSeries": false,
- "id": 115,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_replication_tcp_protocol_close_reason_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{reason_type}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Replication connection close reasons",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:260",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:261",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 183
- },
- "hiddenSeries": false,
- "id": 113,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{stream_name}}",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Replication connections",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Replication",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 40
- },
- "id": 69,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 163
- },
- "hiddenSeries": false,
- "id": 67,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "connected",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - on() group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{name}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Event processing lag",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": "events",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 163
- },
- "hiddenSeries": false,
- "id": 71,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "connected",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{name}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Age of last processed event",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 172
- },
- "hiddenSeries": false,
- "id": 121,
- "interval": "",
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "connected",
- "options": {
- "alertThreshold": true
- },
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "deriv(synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{name}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Event processing catchup rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "none",
- "label": "fallbehind(-) / catchup(+): s/sec",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Event processing loop positions",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 41
- },
- "id": 126,
- "panels": [
- {
- "cards": {
- "cardPadding": 0
- },
- "color": {
- "cardColor": "#B877D9",
- "colorScale": "sqrt",
- "colorScheme": "interpolateInferno",
- "exponent": 0.5,
- "min": 0,
- "mode": "opacity"
- },
- "dataFormat": "tsbuckets",
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 42
- },
- "heatmap": {},
- "hideZeroBuckets": true,
- "highlightCards": true,
- "id": 122,
- "legend": {
- "show": true
- },
- "links": [],
- "options": {
- "calculate": false,
- "calculation": {},
- "cellGap": 1,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#B877D9",
- "min": 0,
- "mode": "opacity",
- "reverse": false,
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": true
- },
- "yAxis": {
- "axisPlacement": "left",
- "decimals": 0,
- "reverse": false,
- "unit": "short"
- }
- },
- "pluginVersion": "9.2.2",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
- "format": "heatmap",
- "intervalFactor": 1,
- "legendFormat": "{{le}}",
- "refId": "A"
- }
- ],
- "title": "Number of rooms, by number of forward extremities in room",
- "tooltip": {
- "show": true,
- "showHistogram": true
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "yAxis": {
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 42
- },
- "hiddenSeries": false,
- "id": 124,
- "interval": "",
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "connected",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0",
- "format": "heatmap",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{le}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Room counts, by number of extremities",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "none",
- "label": "Number of rooms",
- "logBase": 10,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "cards": {
- "cardPadding": 0
- },
- "color": {
- "cardColor": "#5794F2",
- "colorScale": "sqrt",
- "colorScheme": "interpolateInferno",
- "exponent": 0.5,
- "min": 0,
- "mode": "opacity"
- },
- "dataFormat": "tsbuckets",
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 50
- },
- "heatmap": {},
- "hideZeroBuckets": true,
- "highlightCards": true,
- "id": 127,
- "legend": {
- "show": true
- },
- "links": [],
- "options": {
- "calculate": false,
- "calculation": {},
- "cellGap": 1,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#5794F2",
- "min": 0,
- "mode": "opacity",
- "reverse": false,
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": true
- },
- "yAxis": {
- "axisPlacement": "left",
- "decimals": 0,
- "reverse": false,
- "unit": "short"
- }
- },
- "pluginVersion": "9.2.2",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
- "format": "heatmap",
- "intervalFactor": 1,
- "legendFormat": "{{le}}",
- "refId": "A"
- }
- ],
- "title": "Events persisted, by number of forward extremities in room (heatmap)",
- "tooltip": {
- "show": true,
- "showHistogram": true
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "yAxis": {
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 50
- },
- "hiddenSeries": false,
- "id": 128,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "50%",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "75%",
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "90%",
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "99%",
- "refId": "D"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Events persisted, by number of forward extremities in room (quantiles)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": "Number of extremities in room",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "cards": {
- "cardPadding": 0
- },
- "color": {
- "cardColor": "#FF9830",
- "colorScale": "sqrt",
- "colorScheme": "interpolateInferno",
- "exponent": 0.5,
- "min": 0,
- "mode": "opacity"
- },
- "dataFormat": "tsbuckets",
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 58
- },
- "heatmap": {},
- "hideZeroBuckets": true,
- "highlightCards": true,
- "id": 129,
- "legend": {
- "show": true
- },
- "links": [],
- "options": {
- "calculate": false,
- "calculation": {},
- "cellGap": 1,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#FF9830",
- "min": 0,
- "mode": "opacity",
- "reverse": false,
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": true
- },
- "yAxis": {
- "axisPlacement": "left",
- "decimals": 0,
- "reverse": false,
- "unit": "short"
- }
- },
- "pluginVersion": "9.2.2",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
- "format": "heatmap",
- "intervalFactor": 1,
- "legendFormat": "{{le}}",
- "refId": "A"
- }
- ],
- "title": "Events persisted, by number of stale forward extremities in room (heatmap)",
- "tooltip": {
- "show": true,
- "showHistogram": true
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "yAxis": {
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "For given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 58
- },
- "hiddenSeries": false,
- "id": 130,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "50%",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "75%",
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "90%",
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "99%",
- "refId": "D"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Events persisted, by number of stale forward extremities in room (quantiles)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": "Number of stale forward extremities in room",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "cards": {
- "cardPadding": 0
- },
- "color": {
- "cardColor": "#73BF69",
- "colorScale": "sqrt",
- "colorScheme": "interpolateInferno",
- "exponent": 0.5,
- "min": 0,
- "mode": "opacity"
- },
- "dataFormat": "tsbuckets",
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 66
- },
- "heatmap": {},
- "hideZeroBuckets": true,
- "highlightCards": true,
- "id": 131,
- "legend": {
- "show": true
- },
- "links": [],
- "options": {
- "calculate": false,
- "calculation": {},
- "cellGap": 1,
- "cellValues": {},
- "color": {
- "exponent": 0.5,
- "fill": "#73BF69",
- "min": 0,
- "mode": "opacity",
- "reverse": false,
- "scale": "exponential",
- "scheme": "Oranges",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": true
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": true
- },
- "yAxis": {
- "axisPlacement": "left",
- "decimals": 0,
- "reverse": false,
- "unit": "short"
- }
- },
- "pluginVersion": "9.2.2",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "heatmap",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{le}}",
- "refId": "A"
- }
- ],
- "title": "Number of state resolution performed, by number of state groups involved (heatmap)",
- "tooltip": {
- "show": true,
- "showHistogram": true
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "yAxis": {
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 66
- },
- "hiddenSeries": false,
- "id": 132,
- "interval": "",
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "50%",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "75%",
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "90%",
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "99%",
- "refId": "D"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Number of state resolutions performed, by number of state groups involved (quantiles)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": "Number of state groups",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 74
- },
- "hiddenSeries": false,
- "id": 179,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
- "interval": "",
- "legendFormat": "State res ",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
- "interval": "",
- "legendFormat": "Potential to prune",
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_storage_events_times_pruned_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
- "interval": "",
- "legendFormat": "Pruned",
- "refId": "C"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Stale extremity dropping",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Extremities",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 42
- },
- "id": 158,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 43
- },
- "hiddenSeries": false,
- "id": 156,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "$$hashKey": "object:632",
- "alias": "Max",
- "color": "#bf1b00",
- "fill": 0,
- "linewidth": 2
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "max(synapse_admin_mau_max{instance=\"$instance\"})",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "Max",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "max(synapse_admin_mau_current{instance=\"$instance\"})",
- "hide": false,
- "legendFormat": "Current",
- "range": true,
- "refId": "C"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "MAU Limits",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:176",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:177",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 43
- },
- "hiddenSeries": false,
- "id": 160,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "synapse_admin_mau_current_mau_by_service{instance=\"$instance\"}",
- "interval": "",
- "legendFormat": "{{ app_service }}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "MAU by Appservice",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "MAU",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 43
- },
- "id": 177,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 44
- },
- "hiddenSeries": false,
- "id": 173,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_notifier_users_woken_by_stream_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "format": "time_series",
- "hide": false,
- "intervalFactor": 2,
- "legendFormat": "{{stream}} {{index}}",
- "metric": "synapse_notifier",
- "range": true,
- "refId": "A",
- "step": 2
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Notifier Streams Woken",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:734",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:735",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 44
- },
- "hiddenSeries": false,
- "id": 175,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_handler_presence_get_updates_total{job=~\"$job\",instance=\"$instance\"}[$bucket_size])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{type}} {{index}}",
- "range": true,
- "refId": "A",
- "step": 2
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Presence Stream Fetch Type Rates",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:819",
- "format": "hertz",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:820",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Notifier",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 44
- },
- "id": 170,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 45
- },
- "hiddenSeries": false,
- "id": 168,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_appservice_api_sent_events_total{instance=\"$instance\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "{{service}}",
- "range": true,
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Sent Events rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:177",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:178",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 45
- },
- "hiddenSeries": false,
- "id": 171,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_appservice_api_sent_transactions_total{instance=\"$instance\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "{{exported_service }} {{ service }}",
- "range": true,
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Transactions rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:260",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:261",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Appservices",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 45
- },
- "id": 188,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 46
- },
- "hiddenSeries": false,
- "id": 182,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_handler_presence_notified_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "Notified",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_handler_presence_federation_presence_out_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "Remote ping",
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_handler_presence_presence_updates_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "Total updates",
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_handler_presence_federation_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "Remote updates",
- "refId": "D"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(synapse_handler_presence_bump_active_time_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "Bump active time",
- "refId": "E"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Presence",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 46
- },
- "hiddenSeries": false,
- "id": 184,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_handler_presence_state_transition_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "{{from}} -> {{to}}",
- "range": true,
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Presence state transitions",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:1090",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:1091",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 54
- },
- "hiddenSeries": false,
- "id": 186,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_handler_presence_notify_reason_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "{{reason}}",
- "range": true,
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Presence notify reason",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:165",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:166",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Presence",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 46
- },
- "id": 197,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "hertz"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 47
- },
- "id": 191,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.0.4",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_external_cache_set{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "External Cache Set Rate",
- "type": "timeseries"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 47
- },
- "hiddenSeries": false,
- "id": 193,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "sum without (hit) (rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size]))",
- "interval": "",
- "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
- "range": true,
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "External Cache Get Rate",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:390",
- "format": "hertz",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:391",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "cards": {
- "cardPadding": -1
- },
- "color": {
- "cardColor": "#b4ff00",
- "colorScale": "sqrt",
- "colorScheme": "interpolateInferno",
- "exponent": 0.5,
- "min": 0,
- "mode": "spectrum"
- },
- "dataFormat": "tsbuckets",
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "scaleDistribution": {
- "type": "linear"
- }
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 55
- },
- "heatmap": {},
- "hideZeroBuckets": false,
- "highlightCards": true,
- "id": 195,
- "legend": {
- "show": false
- },
- "links": [],
- "options": {
- "calculate": false,
- "calculation": {},
- "cellGap": -1,
- "cellValues": {
- "decimals": 2
- },
- "color": {
- "exponent": 0.5,
- "fill": "#b4ff00",
- "min": 0,
- "mode": "scheme",
- "reverse": false,
- "scale": "exponential",
- "scheme": "Inferno",
- "steps": 128
- },
- "exemplars": {
- "color": "rgba(255,0,255,0.7)"
- },
- "filterValues": {
- "le": 1e-9
- },
- "legend": {
- "show": false
- },
- "rowsFrame": {
- "layout": "auto"
- },
- "showValue": "never",
- "tooltip": {
- "show": true,
- "yHistogram": true
- },
- "yAxis": {
- "axisPlacement": "left",
- "decimals": 0,
- "reverse": false,
- "unit": "s"
- }
- },
- "pluginVersion": "9.2.2",
- "reverseYBuckets": false,
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])) by (le)",
- "format": "heatmap",
- "instant": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{le}}",
- "refId": "A"
- }
- ],
- "title": "External Cache Response Time",
- "tooltip": {
- "show": true,
- "showHistogram": true
- },
- "tooltipDecimals": 2,
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "yAxis": {
- "decimals": 0,
- "format": "s",
- "logBase": 1,
- "show": true
- },
- "yBucketBound": "auto"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "hertz"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 55
- },
- "id": 223,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.0.4",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "editorMode": "code",
- "expr": "rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\", hit=\"False\"}[$bucket_size])",
- "interval": "",
- "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "External Cache Miss Rate",
- "type": "timeseries"
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "External Cache",
- "type": "row"
- }
- ],
- "refresh": false,
- "schemaVersion": 37,
- "style": "dark",
- "tags": [
- "matrix"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": false,
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "includeAll": false,
- "multi": false,
- "name": "DS_PROMETHEUS",
- "label": "Datasource",
- "options": [],
- "query": "prometheus",
- "queryValue": "",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "allFormat": "glob",
- "auto": true,
- "auto_count": 100,
- "auto_min": "30s",
- "current": {
- "selected": false,
- "text": "auto",
- "value": "$__auto_interval_bucket_size"
- },
- "hide": 0,
- "includeAll": false,
- "label": "Bucket Size",
- "multi": false,
- "multiFormat": "glob",
- "name": "bucket_size",
- "options": [
- {
- "selected": true,
- "text": "auto",
- "value": "$__auto_interval_bucket_size"
- },
- {
- "selected": false,
- "text": "30s",
- "value": "30s"
- },
- {
- "selected": false,
- "text": "1m",
- "value": "1m"
- },
- {
- "selected": false,
- "text": "2m",
- "value": "2m"
- },
- {
- "selected": false,
- "text": "5m",
- "value": "5m"
- },
- {
- "selected": false,
- "text": "10m",
- "value": "10m"
- },
- {
- "selected": false,
- "text": "15m",
- "value": "15m"
- }
- ],
- "query": "30s,1m,2m,5m,10m,15m",
- "queryValue": "",
- "refresh": 2,
- "skipUrlSync": false,
- "type": "interval"
- },
- {
- "current": {},
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "definition": "",
- "hide": 0,
- "includeAll": false,
- "multi": false,
- "name": "instance",
- "options": [],
- "query": {
- "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, instance)",
- "refId": "Prometheus-instance-Variable-Query"
- },
- "refresh": 2,
- "regex": "",
- "skipUrlSync": false,
- "sort": 1,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allFormat": "regex wildcard",
- "allValue": "",
- "current": {},
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "definition": "",
- "hide": 0,
- "hideLabel": false,
- "includeAll": true,
- "label": "Job",
- "multi": true,
- "multiFormat": "regex values",
- "name": "job",
- "options": [],
- "query": {
- "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, job)",
- "refId": "Prometheus-job-Variable-Query"
- },
- "refresh": 2,
- "refresh_on_load": false,
- "regex": "",
- "skipUrlSync": false,
- "sort": 1,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allFormat": "regex wildcard",
- "allValue": ".*",
- "current": {},
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "definition": "",
- "hide": 0,
- "hideLabel": false,
- "includeAll": true,
- "label": "",
- "multi": true,
- "multiFormat": "regex values",
- "name": "index",
- "options": [],
- "query": {
- "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, index)",
- "refId": "Prometheus-index-Variable-Query"
- },
- "refresh": 2,
- "refresh_on_load": false,
- "regex": "",
- "skipUrlSync": false,
- "sort": 3,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-3h",
- "to": "now"
- },
- "timepicker": {
- "now": true,
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "",
- "title": "Synapse",
- "uid": "000000012",
- "version": 160,
- "weekStart": ""
- }
- draupnir4all-status-dashboard.json: |-
- {
- "__inputs": [
- {
- "name": "DS_LOKI",
- "label": "Loki",
- "description": "",
- "type": "datasource",
- "pluginId": "loki",
- "pluginName": "Loki"
- }
- ],
- "__elements": {},
- "__requires": [
- {
- "type": "grafana",
- "id": "grafana",
- "name": "Grafana",
- "version": "10.4.0"
- },
- {
- "type": "panel",
- "id": "logs",
- "name": "Logs",
- "version": ""
- },
- {
- "type": "datasource",
- "id": "loki",
- "name": "Loki",
- "version": "1.0.0"
- },
- {
- "type": "panel",
- "id": "stat",
- "name": "Stat",
- "version": ""
- }
- ],
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard"
- }
- ]
- },
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "id": null,
- "links": [],
- "panels": [
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 3,
- "panels": [],
- "title": "Status Overview",
- "type": "row"
- },
- {
- "datasource": "${DS_LOKI}",
- "description": "Green means everything is as expected",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 1
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Value #Failed"
- },
- "properties": [
- {
- "id": "displayName",
- "value": "Failed"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Value #BackOff"
- },
- "properties": [
- {
- "id": "displayName",
- "value": "BackOff"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 1
- },
- "id": 1,
- "options": {
- "colorMode": "background_solid",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "max"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "name",
- "wideLayout": true
- },
- "pluginVersion": "10.4.0",
- "targets": [
- {
- "datasource": "${DS_LOKI}",
- "editorMode": "builder",
- "expr": "max(count_over_time({app=\"kube-event-exporter\"} | json | namespace = `matrix` | reason = `Failed` | name =~ `.*draupnir-synapse.*` [5m]))",
- "intervalMs": 1000,
- "legendFormat": "Failed",
- "maxDataPoints": 43200,
- "queryType": "instant",
- "refId": "Failed"
- },
- {
- "datasource": "${DS_LOKI}",
- "editorMode": "builder",
- "expr": "count_over_time({app=\"kube-event-exporter\"} | json | namespace = `matrix` | reason = `BackOff` | name =~ `.*draupnir-synapse.*` [5m])",
- "hide": false,
- "intervalMs": 1000,
- "legendFormat": "BackOff",
- "maxDataPoints": 43200,
- "queryType": "instant",
- "refId": "BackOff"
- }
- ],
- "title": "Draupnir Synapse Status",
- "type": "stat"
- },
- {
- "datasource": "${DS_LOKI}",
- "description": "Green means everything is as expected",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 1
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Value #Failed"
- },
- "properties": [
- {
- "id": "displayName",
- "value": "Failed"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Value #BackOff"
- },
- "properties": [
- {
- "id": "displayName",
- "value": "BackOff"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 1
- },
- "id": 2,
- "options": {
- "colorMode": "background_solid",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "max"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "name",
- "wideLayout": true
- },
- "pluginVersion": "10.4.0",
- "targets": [
- {
- "datasource": "${DS_LOKI}",
- "editorMode": "builder",
- "expr": "max(count_over_time({app=\"kube-event-exporter\"} | json | namespace = `matrix` | reason = `Failed` | name =~ `.*draupnir4all.*` [5m]))",
- "intervalMs": 1000,
- "legendFormat": "Failed",
- "maxDataPoints": 43200,
- "queryType": "instant",
- "refId": "Failed"
- },
- {
- "datasource": "${DS_LOKI}",
- "editorMode": "builder",
- "expr": "count_over_time({app=\"kube-event-exporter\"} | json | namespace = `matrix` | reason = `BackOff` | name =~ `.*draupnir4all.*` [5m])",
- "hide": false,
- "intervalMs": 1000,
- "legendFormat": "BackOff",
- "maxDataPoints": 43200,
- "queryType": "instant",
- "refId": "BackOff"
- }
- ],
- "title": "Draupnir4All Status",
- "type": "stat"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 8
- },
- "id": 4,
- "panels": [],
- "title": "Logs",
- "type": "row"
- },
- {
- "datasource": "${DS_LOKI}",
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 9
- },
- "id": 5,
- "options": {
- "dedupStrategy": "exact",
- "enableLogDetails": true,
- "prettifyLogMessage": true,
- "showCommonLabels": false,
- "showLabels": false,
- "showTime": true,
- "sortOrder": "Descending",
- "wrapLogMessage": false
- },
- "targets": [
- {
- "datasource": "${DS_LOKI}",
- "editorMode": "builder",
- "expr": "{app=\"kube-event-exporter\"} | json | namespace = `matrix` | name =~ `.*draupnir-synapse.*`",
- "queryType": "range",
- "refId": "A"
- }
- ],
- "title": "Draupnir4All Synapse Events",
- "type": "logs"
- },
- {
- "datasource": "${DS_LOKI}",
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 17
- },
- "id": 6,
- "options": {
- "dedupStrategy": "exact",
- "enableLogDetails": true,
- "prettifyLogMessage": true,
- "showCommonLabels": false,
- "showLabels": false,
- "showTime": true,
- "sortOrder": "Descending",
- "wrapLogMessage": false
- },
- "targets": [
- {
- "datasource": "${DS_LOKI}",
- "editorMode": "builder",
- "expr": "{app=\"kube-event-exporter\"} | json | namespace = `matrix` | name =~ `.*draupnir4all.*`",
- "queryType": "range",
- "refId": "A"
- }
- ],
- "title": "Draupnir4All Events",
- "type": "logs"
- }
- ],
- "refresh": "10s",
- "schemaVersion": 39,
- "tags": [],
- "templating": {
- "list": []
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ]
- },
- "timezone": "browser",
- "title": "Draupnir4All Status and logs",
- "uid": "bdz3qba37rwg0e",
- "version": 4,
- "weekStart": ""
- }
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- labels:
- grafana_dashboard: "1"
- annotations:
- grafana_folder: Flux
- name: flux-dashboards
-data:
- flux-control-plane-dashboard.json: |-
- {
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
- "type": "dashboard"
- },
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "enable": true,
- "iconColor": "red",
- "name": "flux events",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [
- "flux"
- ],
- "type": "tags"
- }
- }
- ]
- },
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "id": 1,
- "links": [],
- "liveNow": false,
- "panels": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "blue",
- "value": null
- },
- {
- "color": "red",
- "value": 100
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 0,
- "y": 0
- },
- "id": 24,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "last"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "value"
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(go_info{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
- "interval": "",
- "legendFormat": "pods",
- "refId": "A"
- }
- ],
- "title": "Controllers",
- "type": "stat"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "blue",
- "value": null
- },
- {
- "color": "#EAB839",
- "value": 50
- },
- {
- "color": "red",
- "value": 100
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 6,
- "y": 0
- },
- "id": 23,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "max(workqueue_longest_running_processor_seconds{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
- "hide": false,
- "interval": "",
- "legendFormat": "seconds",
- "refId": "B"
- }
- ],
- "title": "Max Work Queue",
- "type": "stat"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "blue",
- "value": null
- },
- {
- "color": "#EAB839",
- "value": 500000000
- },
- {
- "color": "red",
- "value": 900000000
- }
- ]
- },
- "unit": "decbits"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 12,
- "y": 0
- },
- "id": 25,
- "options": {
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showThresholdLabels": false,
- "showThresholdMarkers": true,
- "text": {}
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(go_memstats_alloc_bytes{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
- "interval": "",
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Memory",
- "type": "gauge"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "blue",
- "value": null
- },
- {
- "color": "#EAB839",
- "value": 50
- },
- {
- "color": "red",
- "value": 100
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 18,
- "y": 0
- },
- "id": 26,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m]))",
- "interval": "",
- "legendFormat": "requests",
- "refId": "A"
- }
- ],
- "title": "API Requests",
- "type": "stat"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 5
- },
- "id": 21,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\"}[1m]))",
- "hide": false,
- "interval": "",
- "legendFormat": "total",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",code!~\"2..\"}[1m]))",
- "hide": false,
- "interval": "",
- "legendFormat": "errors",
- "refId": "B"
- }
- ],
- "title": "Kubernetes API Requests",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 13
- },
- "id": 15,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Resource Usage",
- "type": "row"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 0,
- "y": 14
- },
- "id": 11,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "rate(process_cpu_seconds_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m])",
- "interval": "",
- "legendFormat": "{{pod}}",
- "refId": "A"
- }
- ],
- "title": "CPU Usage",
- "type": "timeseries"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "decimals": 0,
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 12,
- "y": 14
- },
- "id": 13,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"POD\",container!=\"\",pod=~\".*-controller-.*\"}) by (pod)",
- "hide": false,
- "interval": "",
- "legendFormat": "{{pod}}",
- "refId": "A"
- }
- ],
- "title": "Memory Usage",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 25
- },
- "id": 17,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Reconciliation Stats",
- "type": "row"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 26
- },
- "id": 27,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "workqueue_longest_running_processor_seconds{name=\"kustomization\"}",
- "hide": false,
- "interval": "",
- "legendFormat": "kustomizations",
- "refId": "B"
- }
- ],
- "title": "Cluster Reconciliation Duration",
- "type": "timeseries"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "bars",
- "fillOpacity": 100,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "stepAfter",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "opm"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 34
- },
- "id": 2,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result!=\"error\"}[1m])) by (controller)",
- "format": "time_series",
- "interval": "",
- "legendFormat": "successful reconciliations ",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result=\"error\"}[1m])) by (controller)",
- "format": "time_series",
- "interval": "",
- "legendFormat": "failed reconciliations ",
- "refId": "B"
- }
- ],
- "title": "Cluster Reconciliations ops/min",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 42
- },
- "id": 29,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Sources Stats",
- "type": "row"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "bars",
- "fillOpacity": 100,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "stepAfter",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "opm"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 43
- },
- "id": 4,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result!=\"error\"}[1m]))",
- "format": "time_series",
- "interval": "",
- "legendFormat": "successful git pulls",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result=\"error\"}[1m]))",
- "format": "time_series",
- "interval": "",
- "legendFormat": "failed git pulls",
- "refId": "B"
- }
- ],
- "title": "Git Repos ops/min",
- "type": "timeseries"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "bars",
- "fillOpacity": 100,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "stepAfter",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "opm"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 43
- },
- "id": 30,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result!=\"error\"}[1m]))",
- "format": "time_series",
- "interval": "",
- "legendFormat": "successful oci pulls",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result=\"error\"}[1m]))",
- "format": "time_series",
- "interval": "",
- "legendFormat": "failed oci pulls",
- "refId": "B"
- }
- ],
- "title": "OCI Repos ops/min",
- "type": "timeseries"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "bars",
- "fillOpacity": 100,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "stepAfter",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "opm"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 52
- },
- "id": 31,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result!=\"error\"}[1m]))",
- "format": "time_series",
- "interval": "",
- "legendFormat": "successful helm pulls",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result=\"error\"}[1m]))",
- "format": "time_series",
- "interval": "",
- "legendFormat": "failed helm pulls",
- "refId": "B"
- }
- ],
- "title": "Helm Repos ops/min",
- "type": "timeseries"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "bars",
- "fillOpacity": 100,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "stepAfter",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "opm"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 52
- },
- "id": 32,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result!=\"error\"}[1m]))",
- "format": "time_series",
- "interval": "",
- "legendFormat": "successful bucket pulls",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result=\"error\"}[1m]))",
- "format": "time_series",
- "interval": "",
- "legendFormat": "failed bucket pulls",
- "refId": "B"
- }
- ],
- "title": "Buckets ops/min",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 61
- },
- "id": 19,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "refId": "A"
- }
- ],
- "title": "Helm Stats",
- "type": "row"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 62
- },
- "id": 9,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "list",
- "placement": "right",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.50, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
- "hide": true,
- "interval": "",
- "legendFormat": "P50",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.90, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
- "hide": true,
- "interval": "",
- "legendFormat": "P90",
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
- "hide": false,
- "interval": "",
- "legendFormat": "P99",
- "refId": "C"
- }
- ],
- "title": "Helm Release Duration",
- "type": "timeseries"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "bars",
- "fillOpacity": 100,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "stepAfter",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "opm"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 70
- },
- "id": 5,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result!=\"error\"}[1m])) by (controller)",
- "format": "time_series",
- "interval": "",
- "legendFormat": "successful reconciliations ",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result=\"error\"}[1m])) by (controller)",
- "format": "time_series",
- "interval": "",
- "legendFormat": "failed reconciliations ",
- "refId": "B"
- }
- ],
- "title": "Helm Releases ops/min",
- "type": "timeseries"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "bars",
- "fillOpacity": 100,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "stepAfter",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "opm"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 70
- },
- "id": 6,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result!=\"error\"}[1m])) by (controller)",
- "format": "time_series",
- "interval": "",
- "legendFormat": "successful chart pulls",
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "${DS_PROMETHEUS}"
- },
- "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result=\"error\"}[1m])) by (controller)",
- "format": "time_series",
- "interval": "",
- "legendFormat": "failed chart pulls",
- "refId": "B"
- }
- ],
- "title": "Helm Charts ops/min",
- "type": "timeseries"
- }
- ],
- "refresh": "10s",
- "schemaVersion": 38,
- "style": "light",
- "tags": [
- "flux"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": false,
- "text": "Prometheus",
- "value": "Prometheus"
- },
- "hide": 2,
- "includeAll": false,
- "multi": false,
- "name": "DS_PROMETHEUS",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "current": {
- "selected": false,
- "text": "flux-system",
- "value": "flux-system"
- },
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "definition": "workqueue_work_duration_seconds_count",
- "hide": 0,
- "includeAll": false,
- "multi": false,
- "name": "namespace",
- "options": [],
- "query": {
- "query": "workqueue_work_duration_seconds_count",
- "refId": "Prometheus-namespace-Variable-Query"
- },
- "refresh": 2,
- "regex": "/.*namespace=\"([^\"]*).*/",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-15m",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ]
- },
- "timezone": "",
- "title": "Flux Control Plane",
- "uid": "flux-control-plane",
- "version": 2,
- "weekStart": ""
- }
- flux-cluster-dashboard.json: |-
- {
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": "-- Grafana --",
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard"
- },
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "enable": true,
- "iconColor": "red",
- "name": "flux events",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [
- "flux"
- ],
- "type": "tags"
- }
- }
- ]
- },
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "id": 1,
- "links": [],
- "liveNow": false,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "mappings": [],
- "noValue": "0",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "blue",
- "value": null
- },
- {
- "color": "red",
- "value": 100
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 0,
- "y": 0
- },
- "id": 24,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "last"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "value"
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "prometheus"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\"})",
- "instant": true,
- "interval": "",
- "legendFormat": "",
- "range": false,
- "refId": "A"
- }
- ],
- "title": "Cluster Reconcilers",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "mappings": [],
- "noValue": "0",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "red",
- "value": null
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 6,
- "y": 0
- },
- "id": 28,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "last"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "value"
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "prometheus"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\", ready=\"False\"})",
- "instant": true,
- "interval": "",
- "legendFormat": "",
- "range": false,
- "refId": "A"
- }
- ],
- "title": "Failing Reconcilers",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "mappings": [],
- "noValue": "0",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "blue",
- "value": null
- },
- {
- "color": "red",
- "value": 100
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 12,
- "y": 0
- },
- "id": 29,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "last"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "value"
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "prometheus"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"})",
- "instant": true,
- "interval": "",
- "legendFormat": "",
- "range": false,
- "refId": "A"
- }
- ],
- "title": "Kubernetes Manifests Sources",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "decimals": 0,
- "mappings": [],
- "noValue": "0",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "red",
- "value": null
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 18,
- "y": 0
- },
- "id": 30,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "last"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "value"
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "prometheus"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\", ready=\"False\"})",
- "instant": true,
- "interval": "",
- "legendFormat": "",
- "range": false,
- "refId": "A"
- }
- ],
- "title": "Failing Sources",
- "type": "stat"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "#EAB839",
- "value": 1
- },
- {
- "color": "red",
- "value": 61
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 4,
- "w": 12,
- "x": 0,
- "y": 5
- },
- "id": 8,
- "options": {
- "displayMode": "gradient",
- "minVizHeight": 10,
- "minVizWidth": 0,
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "text": {},
- "valueMode": "color"
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "exemplar": true,
- "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind)",
- "interval": "",
- "legendFormat": "{{kind}}",
- "refId": "A"
- }
- ],
- "title": "Reconciler ops avg. duration",
- "type": "bargauge"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "#EAB839",
- "value": 1
- },
- {
- "color": "red",
- "value": 61
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 4,
- "w": 12,
- "x": 12,
- "y": 5
- },
- "id": 31,
- "options": {
- "displayMode": "gradient",
- "minVizHeight": 10,
- "minVizWidth": 0,
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "text": {},
- "valueMode": "color"
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "exemplar": true,
- "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind)",
- "interval": "",
- "legendFormat": "{{kind}}",
- "refId": "A"
- }
- ],
- "title": "Source ops avg. duration",
- "type": "bargauge"
- },
- {
- "collapsed": false,
- "datasource": "${DS_PROMETHEUS}",
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 9
- },
- "id": 15,
- "panels": [],
- "title": "Status",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "align": "auto",
- "cellOptions": {
- "type": "auto"
- },
- "filterable": true,
- "inspect": false
- },
- "mappings": [
- {
- "options": {
- "False": {
- "color": "red",
- "index": 1,
- "text": "Not Ready"
- },
- "True": {
- "color": "blue",
- "index": 0,
- "text": "Ready"
- }
- },
- "type": "value"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "transparent",
- "value": null
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byType",
- "options": "string"
- },
- "properties": [
- {
- "id": "custom.cellOptions",
- "value": {
- "mode": "basic",
- "type": "color-background"
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 22,
- "w": 12,
- "x": 0,
- "y": 10
- },
- "id": 33,
- "options": {
- "cellHeight": "sm",
- "footer": {
- "countRows": false,
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
- },
- "showHeader": true,
- "sortBy": [
- {
- "desc": true,
- "displayName": "Status"
- }
- ]
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "prometheus"
- },
- "editorMode": "code",
- "exemplar": true,
- "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\"}",
- "format": "table",
- "instant": true,
- "interval": "",
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Cluster reconciliation readiness ",
- "transformations": [
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Time": false,
- "Value": false,
- "__name__": false,
- "app": false,
- "chart_name": false,
- "chart_source_name": false,
- "container": false,
- "customresource_group": false,
- "customresource_kind": false,
- "customresource_version": false,
- "endpoint": false,
- "exported_namespace": false,
- "gotk_type": false,
- "instance": false,
- "job": false,
- "kubernetes_namespace": false,
- "kubernetes_pod_name": false,
- "namespace": false,
- "pod": false,
- "pod_template_hash": false,
- "revision": false,
- "service": false,
- "source_name": false,
- "status": false,
- "suspended": false,
- "type": false
- },
- "indexByName": {
- "Time": 0,
- "Value": 15,
- "__name__": 1,
- "container": 2,
- "customresource_group": 4,
- "customresource_kind": 5,
- "customresource_version": 6,
- "endpoint": 7,
- "exported_namespace": 3,
- "instance": 8,
- "job": 9,
- "name": 10,
- "namespace": 11,
- "pod": 12,
- "ready": 13,
- "service": 14
- },
- "renameByName": {
- "Value": "",
- "customresource_kind": "Kind",
- "exported_namespace": "Namespace",
- "kind": "Kind",
- "name": "Name",
- "namespace": "Operator Namespace",
- "ready": "Status"
- }
- }
- },
- {
- "id": "filterFieldsByName",
- "options": {
- "include": {
- "pattern": "(Namespace|Kind|Name|Status)"
- }
- }
- }
- ],
- "type": "table"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "align": "auto",
- "cellOptions": {
- "type": "auto"
- },
- "filterable": true,
- "inspect": false
- },
- "mappings": [
- {
- "options": {
- "False": {
- "color": "red",
- "index": 1,
- "text": "Not Ready"
- },
- "True": {
- "color": "blue",
- "index": 0,
- "text": "Ready"
- }
- },
- "type": "value"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "transparent",
- "value": null
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byType",
- "options": "string"
- },
- "properties": [
- {
- "id": "custom.cellOptions",
- "value": {
- "mode": "basic",
- "type": "color-background"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Status"
- },
- "properties": [
- {
- "id": "noValue",
- "value": "Ready"
- },
- {
- "id": "color",
- "value": {
- "fixedColor": "blue",
- "mode": "fixed"
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 12,
- "y": 10
- },
- "id": 34,
- "options": {
- "cellHeight": "sm",
- "footer": {
- "countRows": false,
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
- },
- "showHeader": true,
- "sortBy": [
- {
- "desc": true,
- "displayName": "Status"
- }
- ]
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "prometheus"
- },
- "editorMode": "code",
- "exemplar": true,
- "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}",
- "format": "table",
- "instant": true,
- "interval": "",
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Source acquisition readiness ",
- "transformations": [
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Time": false,
- "Value": false,
- "__name__": false,
- "app": false,
- "bucket_name": false,
- "container": false,
- "customresource_group": false,
- "customresource_kind": false,
- "customresource_version": false,
- "endpoint": false,
- "exported_namespace": false,
- "gotk_type": false,
- "instance": false,
- "job": false,
- "kubernetes_namespace": false,
- "kubernetes_pod_name": false,
- "namespace": false,
- "pod": false,
- "pod_template_hash": false,
- "ready": false,
- "revision": false,
- "service": false,
- "status": false,
- "suspended": false,
- "type": false,
- "url": false
- },
- "indexByName": {
- "Time": 0,
- "Value": 15,
- "__name__": 1,
- "container": 2,
- "customresource_group": 5,
- "customresource_kind": 6,
- "customresource_version": 7,
- "endpoint": 8,
- "exported_namespace": 4,
- "instance": 9,
- "job": 10,
- "name": 11,
- "namespace": 3,
- "pod": 12,
- "ready": 13,
- "service": 14
- },
- "renameByName": {
- "Value": "",
- "customresource_kind": "Kind",
- "exported_namespace": "Namespace",
- "kind": "Kind",
- "name": "Name",
- "namespace": "Operator Namespace",
- "ready": "Status"
- }
- }
- },
- {
- "id": "filterFieldsByName",
- "options": {
- "include": {
- "pattern": "(Namespace|Kind|Name|Status)"
- }
- }
- }
- ],
- "type": "table"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "align": "auto",
- "cellOptions": {
- "type": "auto"
- },
- "filterable": true,
- "inspect": false
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "transparent",
- "value": null
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byType",
- "options": "string"
- },
- "properties": [
- {
- "id": "custom.cellOptions",
- "value": {
- "mode": "basic",
- "type": "color-background"
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 12,
- "y": 21
- },
- "id": 36,
- "options": {
- "cellHeight": "sm",
- "footer": {
- "countRows": false,
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
- },
- "showHeader": true,
- "sortBy": [
- {
- "desc": true,
- "displayName": "Namespace"
- }
- ]
- },
- "pluginVersion": "10.1.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "prometheus"
- },
- "editorMode": "code",
- "exemplar": true,
- "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", suspended=\"true\"}",
- "format": "table",
- "instant": true,
- "interval": "",
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Suspended Objects",
- "transformations": [
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Time": false,
- "Value": false,
- "__name__": false,
- "app": false,
- "bucket_name": false,
- "container": false,
- "customresource_group": false,
- "customresource_kind": false,
- "customresource_version": false,
- "endpoint": false,
- "exported_namespace": false,
- "gotk_type": false,
- "instance": false,
- "job": false,
- "kubernetes_namespace": false,
- "kubernetes_pod_name": false,
- "namespace": false,
- "pod": false,
- "pod_template_hash": false,
- "ready": false,
- "revision": false,
- "service": false,
- "source_name": false,
- "status": false,
- "suspended": false,
- "type": false,
- "url": false
- },
- "indexByName": {
- "Time": 0,
- "Value": 15,
- "__name__": 1,
- "container": 2,
- "customresource_group": 5,
- "customresource_kind": 6,
- "customresource_version": 7,
- "endpoint": 8,
- "exported_namespace": 4,
- "instance": 9,
- "job": 10,
- "name": 11,
- "namespace": 3,
- "pod": 12,
- "ready": 13,
- "service": 14
- },
- "renameByName": {
- "customresource_kind": "Kind",
- "exported_namespace": "Namespace",
- "name": "Name"
- }
- }
- },
- {
- "id": "filterFieldsByName",
- "options": {
- "include": {
- "pattern": "(Namespace|Name|Kind)"
- }
- }
- }
- ],
- "type": "table"
- },
- {
- "collapsed": false,
- "datasource": "${DS_PROMETHEUS}",
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 32
- },
- "id": 17,
- "panels": [],
- "title": "Timing",
- "type": "row"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsNull",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 33
- },
- "id": 27,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "exemplar": true,
- "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind, name)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind, name)",
- "hide": false,
- "interval": "",
- "legendFormat": "{{kind}}/{{name}}",
- "refId": "B"
- }
- ],
- "title": "Cluster reconciliation duration",
- "type": "timeseries"
- },
- {
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsNull",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 41
- },
- "id": 35,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "exemplar": true,
- "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind, name)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind, name)",
- "hide": false,
- "interval": "",
- "legendFormat": "{{kind}}/{{name}}",
- "refId": "B"
- }
- ],
- "title": "Source acquisition duration",
- "type": "timeseries"
- }
- ],
- "refresh": "30s",
- "schemaVersion": 38,
- "style": "light",
- "tags": [
- "flux"
- ],
- "templating": {
- "list": [
- {
- "allValue": "",
- "current": {
- "selected": true,
- "text": [
- "All"
- ],
- "value": [
- "$__all"
- ]
- },
- "datasource": {
- "type": "prometheus",
- "uid": "$DS_PROMETHEUS"
- },
- "definition": "label_values(gotk_reconcile_condition, namespace)",
- "hide": 0,
- "includeAll": true,
- "multi": true,
- "name": "operator_namespace",
- "options": [],
- "query": {
- "query": "label_values(gotk_reconcile_condition, namespace)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 2,
- "regex": "",
- "skipUrlSync": false,
- "sort": 5,
- "tagValuesQuery": "",
- "tags": [],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {
- "selected": true,
- "text": [
- "All"
- ],
- "value": [
- "$__all"
- ]
- },
- "datasource": {
- "type": "prometheus",
- "uid": "$DS_PROMETHEUS"
- },
- "definition": "label_values(gotk_resource_info,exported_namespace)",
- "hide": 0,
- "includeAll": true,
- "multi": true,
- "name": "namespace",
- "options": [],
- "query": {
- "query": "label_values(gotk_resource_info,exported_namespace)",
- "refId": "PrometheusVariableQueryEditor-VariableQuery"
- },
- "refresh": 2,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tags": [],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {
- "selected": false,
- "text": "Prometheus",
- "value": "Prometheus"
- },
- "hide": 0,
- "includeAll": false,
- "label": "Datasource",
- "multi": false,
- "name": "DS_PROMETHEUS",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- }
- ]
- },
- "time": {
- "from": "now-15m",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ]
- },
- "timezone": "",
- "title": "Flux Cluster Stats",
- "uid": "flux-cluster",
- "version": 4,
- "weekStart": ""
- }
- flux-logs-dashboard.json: |-
- {
- "__inputs": [
- {
- "name": "DS_LOKI",
- "label": "Loki",
- "description": "",
- "type": "datasource",
- "pluginId": "loki",
- "pluginName": "Loki"
- }
- ],
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": "-- Grafana --",
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
- "type": "dashboard"
- },
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "enable": true,
- "iconColor": "red",
- "name": "flux events",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [
- "flux"
- ],
- "type": "tags"
- }
- }
- ]
- },
- "description": "Flux logs collected from Kubernetes, stored in Loki",
- "editable": true,
- "gnetId": null,
- "graphTooltip": 0,
- "id": 29,
- "iteration": 1653748775696,
- "links": [],
- "liveNow": false,
- "panels": [
- {
- "datasource": "${DS_LOKI}",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "bars",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 4,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 4,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "hidden",
- "placement": "bottom"
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": "${DS_LOKI}",
- "expr": "sum(count_over_time({namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\" [$__interval]))",
- "instant": false,
- "legendFormat": "Log count",
- "range": true,
- "refId": "A"
- }
- ],
- "type": "timeseries"
- },
- {
- "datasource": "${DS_LOKI}",
- "description": "Logs from services running in Kubernetes",
- "gridPos": {
- "h": 25,
- "w": 24,
- "x": 0,
- "y": 4
- },
- "id": 2,
- "options": {
- "dedupStrategy": "numbers",
- "enableLogDetails": false,
- "prettifyLogMessage": true,
- "showCommonLabels": false,
- "showLabels": false,
- "showTime": false,
- "sortOrder": "Descending",
- "wrapLogMessage": false
- },
- "targets": [
- {
- "datasource": "${DS_LOKI}",
- "expr": "{namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\"",
- "refId": "A"
- }
- ],
- "type": "logs"
- }
- ],
- "refresh": "10s",
- "schemaVersion": 36,
- "style": "light",
- "tags": [
- "flux"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": false,
- "text": "",
- "value": ""
- },
- "description": "String to search for",
- "hide": 0,
- "label": "Search Query",
- "name": "query",
- "options": [
- {
- "selected": true,
- "text": "",
- "value": ""
- }
- ],
- "query": "",
- "skipUrlSync": false,
- "type": "textbox"
- },
- {
- "allValue": "info|error",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "hide": 0,
- "includeAll": true,
- "multi": false,
- "name": "level",
- "options": [
- {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- {
- "selected": false,
- "text": "info",
- "value": "info"
- },
- {
- "selected": false,
- "text": "error",
- "value": "error"
- }
- ],
- "query": "info,error",
- "queryValue": "",
- "skipUrlSync": false,
- "type": "custom"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": [
- "All"
- ],
- "value": [
- "$__all"
- ]
- },
- "datasource": "${DS_LOKI}",
- "definition": "label_values(app)",
- "hide": 0,
- "includeAll": true,
- "multi": true,
- "name": "controller",
- "options": [],
- "query": "label_values(app)",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "type": "query"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": [
- "flux-system"
- ],
- "value": [
- "flux-system"
- ]
- },
- "datasource": "${DS_LOKI}",
- "definition": "label_values(namespace)",
- "hide": 0,
- "includeAll": true,
- "multi": true,
- "name": "namespace",
- "options": [],
- "query": "label_values(namespace)",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "type": "query"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "${DS_LOKI}",
- "definition": "label_values(stream)",
- "hide": 0,
- "includeAll": true,
- "multi": true,
- "name": "stream",
- "options": [],
- "query": "label_values(stream)",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "type": "query"
- },
- {
- "current": {
- "selected": false,
- "text": "Loki",
- "value": "Loki"
- },
- "hide": 0,
- "includeAll": false,
- "label": "Datasource",
- "multi": false,
- "name": "DS_LOKI",
- "options": [],
- "query": "loki",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- }
- ]
- },
- "time": {
- "from": "now-6h",
- "to": "now"
- },
- "timepicker": {},
- "timezone": "",
- "title": "Flux Logs",
- "uid": "flux-logs",
- "version": 2
- }
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- labels:
- grafana_dashboard: "1"
name: postgres-dashboard
annotations:
grafana_folder: Postgres
data:
- postgres.json: "{\n\"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana
+ postgres.json:
+ "{\n\"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana
--\",\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\":
\"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n},\n\"description\": \"Performance metrics
for Postgres\",\n\"editable\": true,\n\"gnetId\": 455,\n\"graphTooltip\": 0,\n\"id\": 1,\n\"iteration\": 1603191461722,\n\"links\":
@@ -25848,8255 +295,3 @@ data:
\ \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n
\ \"2d\",\n \"7d\",\n \"30d\"\n ]\n},\n\"timezone\": \"browser\",\n\"title\": \"Postgres Overview\",\n\"uid\":
\"wGgaPlciz\",\n\"version\": 5\n}"
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- labels:
- grafana_dashboard: "1"
- annotations:
- grafana_folder: Envoy
- name: envoy-clusters-dashboard
-data:
- envoy-clusters.json: |-
- {
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard"
- }
- ]
- },
- "description": "Envoy proxy monitoring Dashboard with cluster and service level templates. ",
- "editable": true,
- "fiscalYearStartMonth": 0,
- "gnetId": 11021,
- "graphTooltip": 0,
- "id": 2,
- "links": [],
- "liveNow": false,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "max": 3,
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#d44a3a",
- "value": null
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 1
- },
- {
- "color": "#299c46",
- "value": 2
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 5,
- "x": 0,
- "y": 0
- },
- "id": 9,
- "maxDataPoints": 100,
- "options": {
- "minVizHeight": 75,
- "minVizWidth": 75,
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showThresholdLabels": false,
- "showThresholdMarkers": true,
- "sizing": "auto"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum(envoy_server_live{})",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Live servers",
- "type": "gauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 4,
- "x": 5,
- "y": 0
- },
- "id": 12,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "avg(envoy_server_uptime)",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Avg uptime per node",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 5,
- "x": 9,
- "y": 0
- },
- "id": 11,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "sum(envoy_server_memory_allocated{})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Allocated Memory",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 4,
- "x": 14,
- "y": 0
- },
- "id": 13,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum(envoy_server_memory_heap_size)",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Heap Size",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 3,
- "x": 18,
- "y": 0
- },
- "id": 19,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "(sum(envoy_cluster_membership_healthy{envoy_cluster_name=~\"$cluster\"}) - sum(envoy_cluster_membership_total{envoy_cluster_name=~\"$cluster\"}))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Unhealthy Clusters",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "0": {
- "text": "NOT WELL"
- },
- "1": {
- "text": "OK"
- }
- },
- "type": "value"
- },
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#d44a3a",
- "value": null
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 0
- },
- {
- "color": "#299c46",
- "value": 1
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 3,
- "x": 21,
- "y": 0
- },
- "id": 20,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "(sum(envoy_cluster_membership_total{envoy_cluster_name=~\"$cluster\"})-sum(envoy_cluster_membership_healthy{envoy_cluster_name=~\"$cluster\"})) == bool 0",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Cluster State",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 5
- },
- "id": 2,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum(envoy_cluster_upstream_cx_active{envoy_cluster_name=~\"$cluster\"}) by (envoy_cluster_name)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{envoy_cluster_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Total active connections",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "httproute/default/backend/rule/0"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 5
- },
- "id": 4,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "editorMode": "code",
- "expr": "sum(irate(envoy_cluster_upstream_rq_total{envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{envoy_cluster_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Total requests",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 12
- },
- "id": 15,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum(irate(envoy_cluster_upstream_cx_rx_bytes_total{envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{envoy_cluster_name}} - in",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum(irate(envoy_cluster_upstream_cx_tx_bytes_total{envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{envoy_cluster_name}} - out",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Upstream Network Traffic",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 12
- },
- "id": 17,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "sum(irate(envoy_http_downstream_cx_rx_bytes_total{envoy_http_conn_manager_prefix=~\"http\"}[5m]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{service}} - in",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "sum(irate(envoy_http_downstream_cx_tx_bytes_total{envoy_http_conn_manager_prefix=~\"http\"}[5m]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{service}} - out",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Downstream Network Traffic",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 19
- },
- "id": 22,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$cluster\"}[5m])) by (le, envoy_cluster_name))",
- "instant": false,
- "legendFormat": "{{envoy_cluster_name}} 99%",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$cluster\"}[5m])) by (le, envoy_cluster_name))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{envoy_cluster_name}} 90%",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$cluster\"}[5m])) by (le, envoy_cluster_name))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{envoy_cluster_name}} 50%",
- "range": true,
- "refId": "C"
- }
- ],
- "title": "Upstream Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 12,
- "y": 19
- },
- "id": 24,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"2\", envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)",
- "instant": false,
- "legendFormat": "{{envoy_cluster_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream 2xx Responses",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 18,
- "y": 19
- },
- "id": 28,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_cluster_name=~\"$cluster\",envoy_response_code_class=~\"4\"}[1m])) by (envoy_cluster_name)",
- "instant": false,
- "legendFormat": "{{envoy_cluster_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream 4xx Responses",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 27
- },
- "id": 7,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum(envoy_cluster_membership_healthy{envoy_cluster_name=~\"$cluster\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "healthy",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "$datasource"
- },
- "expr": "sum(envoy_cluster_membership_total{envoy_cluster_name=~\"$cluster\",service=~\"$service\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "total",
- "refId": "B"
- }
- ],
- "title": "Downstream members",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 12,
- "y": 27
- },
- "id": 30,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_cluster_name=~\"$cluster\",envoy_response_code_class=~\"5\"}[5m])) by (envoy_cluster_name)",
- "instant": false,
- "legendFormat": "{{envoy_cluster_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream 5xx Responses",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 18,
- "y": 27
- },
- "id": 26,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_cluster_name=~\"$cluster\",envoy_response_code_class=~\"3\"}[5m])) by (envoy_cluster_name)",
- "instant": false,
- "legendFormat": "{{envoy_cluster_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream 3xx Responses",
- "type": "timeseries"
- }
- ],
- "refresh": "30s",
- "schemaVersion": 39,
- "tags": [
- "Data Plane"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": false,
- "text": "Prometheus",
- "value": "Prometheus"
- },
- "hide": 0,
- "includeAll": false,
- "multi": false,
- "name": "datasource",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "current": {
- "selected": false,
- "text": "httproute/default/backend/rule/0",
- "value": "httproute/default/backend/rule/0"
- },
- "datasource": {
- "uid": "$datasource"
- },
- "definition": "label_values(envoy_cluster_name)",
- "hide": 0,
- "includeAll": true,
- "label": "Cluster",
- "multi": false,
- "name": "cluster",
- "options": [],
- "query": {
- "query": "label_values(envoy_cluster_name)",
- "refId": "PrometheusVariableQueryEditor-VariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 1,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-15m",
- "to": "now"
- },
- "timeRangeUpdatedDuringEditOrView": false,
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "",
- "title": "Envoy Clusters",
- "uid": "8WkEOMnANKE6PW5hhpVv",
- "version": 1,
- "weekStart": ""
- }
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- labels:
- grafana_dashboard: "1"
- annotations:
- grafana_folder: Envoy
- name: envoy-gateway-global-dashboard
-data:
- envoy-gateway-global.json: |-
- {
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard"
- }
- ]
- },
- "description": "Envoy Gateway monitoring Dashboard with exported metrics.",
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "links": [],
- "panels": [
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 2,
- "panels": [],
- "title": "Watching Components",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "How long in seconds a subscribed watchable is handled.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 7,
- "x": 0,
- "y": 1
- },
- "id": 1,
- "maxPerRow": 3,
- "options": {
- "displayMode": "basic",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "repeat": "Runner",
- "repeatDirection": "v",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(le) (watchable_subscribe_duration_seconds_bucket{runner=~\"$Runner\", namespace=\"$Namespace\"})",
- "format": "heatmap",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{le}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Duration Bucket: $Runner",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 3,
- "x": 7,
- "y": 1
- },
- "id": 24,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "value_and_name",
- "wideLayout": false
- },
- "pluginVersion": "11.0.0",
- "repeat": "Runner",
- "repeatDirection": "v",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "exemplar": false,
- "expr": "avg by(runner) (watchable_subscribe_duration_seconds_sum{runner=~\"$Runner\", namespace=\"$Namespace\"})",
- "format": "time_series",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "Avg",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "max by(runner) (watchable_subscribe_duration_seconds_sum{runner=~\"$Runner\", namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "Max",
- "range": true,
- "refId": "B",
- "useBackend": false
- }
- ],
- "title": "Duration Status",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Current depth of watchable map.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "fixedColor": "super-light-blue",
- "mode": "shades"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 2,
- "x": 10,
- "y": 1
- },
- "id": 7,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "center",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "value",
- "wideLayout": false
- },
- "pluginVersion": "11.0.0",
- "repeat": "Runner",
- "repeatDirection": "v",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(runner) (watchable_depth{runner=~\"$Runner\", namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Depth",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "fixedColor": "light-blue",
- "mode": "shades"
- },
- "mappings": [],
- "noValue": "0",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Success"
- },
- "properties": [
- {
- "id": "displayName",
- "value": "Success"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 7,
- "w": 7,
- "x": 12,
- "y": 1
- },
- "id": 10,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "center",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "value_and_name",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "repeat": "Runner",
- "repeatDirection": "v",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(status) (watchable_subscribe_total{runner=\"$Runner\", namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": false,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum(watchable_subscribe_total{runner=\"$Runner\", namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "Total",
- "range": true,
- "refId": "B",
- "useBackend": false
- }
- ],
- "title": "Statistics",
- "type": "stat"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 36
- },
- "id": 35,
- "panels": [],
- "title": "Status Updater",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "How long a status update takes to finish for all Kind.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "fixedColor": "super-light-blue",
- "mode": "thresholds"
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 6,
- "x": 0,
- "y": 37
- },
- "id": 61,
- "options": {
- "displayMode": "basic",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(le) (status_update_duration_seconds_bucket{namespace=\"$Namespace\"})",
- "format": "heatmap",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{le}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Total Duration Bucket",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "#EAB839",
- "value": 0.2
- },
- {
- "color": "red",
- "value": 0.5
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 6,
- "x": 6,
- "y": 37
- },
- "id": 82,
- "options": {
- "displayMode": "gradient",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "vertical",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}",
- "format": "time_series",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{kind}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Avg Duration",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "continuous-GrYlRd"
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "#EAB839",
- "value": 0.1
- },
- {
- "color": "red",
- "value": 0.5
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 6,
- "x": 12,
- "y": 37
- },
- "id": 83,
- "options": {
- "displayMode": "gradient",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "vertical",
- "reduceOptions": {
- "calcs": [
- "max"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}",
- "format": "time_series",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{kind}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Max Duration",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "continuous-BlPu"
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "yellow",
- "value": 0.01
- },
- {
- "color": "red",
- "value": 0.1
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 6,
- "x": 18,
- "y": 37
- },
- "id": 84,
- "options": {
- "displayMode": "gradient",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "vertical",
- "reduceOptions": {
- "calcs": [
- "logmin"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}",
- "format": "time_series",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{kind}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Min Duration",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of status updates by object kind.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": true,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "left",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineStyle": {
- "fill": "solid"
- },
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "fieldMinMax": false,
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 10,
- "x": 0,
- "y": 46
- },
- "id": 56,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "10.4.1",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(kind) (status_update_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{kind}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Total",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": true,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "left",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 10,
- "x": 10,
- "y": 46
- },
- "id": 57,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(status) (status_update_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Status",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 54
- },
- "id": 126,
- "panels": [],
- "title": "xDS Server",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "fixedColor": "super-light-green",
- "mode": "shades"
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 10,
- "x": 0,
- "y": 55
- },
- "id": 127,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "center",
- "orientation": "vertical",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "value_and_name",
- "wideLayout": false
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum(xds_snapshot_create_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "Total",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(status) (xds_snapshot_create_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{status}}",
- "range": true,
- "refId": "B",
- "useBackend": false
- }
- ],
- "title": "Snapshot Creation Status",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "fixedColor": "orange",
- "mode": "shades"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 5,
- "x": 10,
- "y": 55
- },
- "id": 149,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "max(xds_stream_duration_seconds_bucket{namespace=\"$Namespace\", isDeltaStream=\"true\"})",
- "format": "heatmap",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Finished Stream",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Maximum duration seconds for finished xDS delta stream connection.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "fixedColor": "red",
- "mode": "shades"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 4,
- "w": 4,
- "x": 15,
- "y": 55
- },
- "id": 150,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "max"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "xds_stream_duration_seconds_sum{namespace=\"$Namespace\", isDeltaStream=\"true\"}",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Max Duration",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Minimum duration seconds for finished xDS delta stream connection.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "fixedColor": "light-green",
- "mode": "shades"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 4,
- "w": 4,
- "x": 15,
- "y": 59
- },
- "id": 151,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "min"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "xds_stream_duration_seconds_sum{namespace=\"$Namespace\", isDeltaStream=\"true\"}",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Min Duration",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of xds snapshot cache updates by node id.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic-by-name"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 20,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 3,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 10,
- "x": 0,
- "y": 63
- },
- "id": 152,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "10.4.1",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(nodeID) (xds_snapshot_update_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Update Total",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 5,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 9,
- "x": 10,
- "y": 63
- },
- "id": 153,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "10.4.1",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(status) (xds_snapshot_update_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{nodeID}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Update Status",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 71
- },
- "id": 156,
- "panels": [],
- "title": "Infrastructure Manager",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 0,
- "y": 72
- },
- "id": 199,
- "options": {
- "displayMode": "gradient",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(le) (resource_apply_duration_seconds_bucket{namespace=\"$Namespace\"})",
- "format": "heatmap",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{le}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Total Apply Duration Bucket",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 0.3
- },
- {
- "color": "red",
- "value": 0.5
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 6,
- "y": 72
- },
- "id": 220,
- "options": {
- "displayMode": "gradient",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Avg Apply Duration",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "continuous-GrYlRd"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 0.3
- },
- {
- "color": "red",
- "value": 0.5
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 12,
- "y": 72
- },
- "id": 221,
- "options": {
- "displayMode": "gradient",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "max"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Max Apply Duration",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "continuous-BlPu"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 0.3
- },
- {
- "color": "red",
- "value": 0.5
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 18,
- "y": 72
- },
- "id": 222,
- "options": {
- "displayMode": "gradient",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "logmin"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Min Apply Duration",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of applied resources sumed by kind.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 25,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 80
- },
- "id": 157,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "hidden",
- "placement": "right",
- "showLegend": false
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(kind) (resource_apply_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Total Applied Resources",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of applied resources that succeed sumed by kind.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 25,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 80
- },
- "id": 229,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "hidden",
- "placement": "right",
- "showLegend": false
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(status) (resource_apply_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Applied Resources Status",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of applied resources sumed by infra name.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 80
- },
- "id": 178,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "10.4.1",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(name) (resource_apply_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Total Applied Infrastructures",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 0,
- "y": 87
- },
- "id": 223,
- "options": {
- "displayMode": "gradient",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "sum by(le) (resource_delete_duration_seconds_bucket{namespace=\"$Namespace\"})",
- "format": "heatmap",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{le}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Total Delete Duration Bucket",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 0.1
- },
- {
- "color": "red",
- "value": 0.3
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 6,
- "y": 87
- },
- "id": 224,
- "options": {
- "displayMode": "gradient",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Avg Delete Duration",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "continuous-GrYlRd"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 0.1
- },
- {
- "color": "red",
- "value": 0.3
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 12,
- "y": 87
- },
- "id": 225,
- "options": {
- "displayMode": "gradient",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "max"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Max Delete Duration",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "continuous-BlPu"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 0.1
- },
- {
- "color": "red",
- "value": 0.3
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 18,
- "y": 87
- },
- "id": 226,
- "options": {
- "displayMode": "gradient",
- "maxVizHeight": 300,
- "minVizHeight": 16,
- "minVizWidth": 8,
- "namePlacement": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "logmin"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Min Delete Duration",
- "type": "bargauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of deleted resources sumed by kind.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 25,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 95
- },
- "id": 227,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "hidden",
- "placement": "right",
- "showLegend": false
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(kind) (resource_delete_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Total Deleted Resources",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of deleted resources that succeed sumed by kind.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 25,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 95
- },
- "id": 232,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "hidden",
- "placement": "right",
- "showLegend": false
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(status) (resource_delete_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Deleted Resources Status",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of deleted resources sumed by infra name.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 95
- },
- "id": 228,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "10.4.1",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(name) (resource_delete_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Total Deleted Infrastructures",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 102
- },
- "id": 249,
- "panels": [],
- "title": "Wasm",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 4,
- "x": 0,
- "y": 103
- },
- "id": 250,
- "options": {
- "minVizHeight": 75,
- "minVizWidth": 75,
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showThresholdLabels": false,
- "showThresholdMarkers": true,
- "sizing": "auto"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "wasm_cache_entries{namespace=\"$Namespace\"}",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Cache Entries",
- "type": "gauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of Wasm remote fetch cache lookups.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 7,
- "x": 4,
- "y": 103
- },
- "id": 251,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(hit) (wasm_cache_lookup_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "hit={{hit}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Cache Lookups",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of Wasm remote fetches and results.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 7,
- "x": 11,
- "y": 103
- },
- "id": 252,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum by(status) (wasm_remote_fetch_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{status}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "sum(wasm_remote_fetch_total{namespace=\"$Namespace\"})",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "Total",
- "range": true,
- "refId": "B",
- "useBackend": false
- }
- ],
- "title": "Cache Remote Fetches",
- "type": "stat"
- }
- ],
- "refresh": "",
- "schemaVersion": 39,
- "tags": [
- "Control Plane"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": false,
- "text": "Prometheus",
- "value": "PBFA97CFB590B2093"
- },
- "hide": 0,
- "includeAll": false,
- "multi": false,
- "name": "datasource",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "envoy-gateway-system",
- "value": "envoy-gateway-system"
- },
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "definition": "label_values(watchable_depth,namespace)",
- "hide": 0,
- "includeAll": false,
- "multi": false,
- "name": "Namespace",
- "options": [],
- "query": {
- "qryType": 1,
- "query": "label_values(watchable_depth,namespace)",
- "refId": "PrometheusVariableQueryEditor-VariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "type": "query"
- },
- {
- "allValue": ".*",
- "current": {
- "selected": true,
- "text": [
- "All"
- ],
- "value": [
- "$__all"
- ]
- },
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "definition": "label_values(watchable_depth,runner)",
- "hide": 0,
- "includeAll": true,
- "multi": true,
- "name": "Runner",
- "options": [],
- "query": {
- "qryType": 1,
- "query": "label_values(watchable_depth,runner)",
- "refId": "PrometheusVariableQueryEditor-VariableQuery"
- },
- "refresh": 2,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "type": "query"
- }
- ]
- },
- "time": {
- "from": "now-6h",
- "to": "now"
- },
- "timeRangeUpdatedDuringEditOrView": false,
- "timepicker": {},
- "timezone": "browser",
- "title": "Envoy Gateway Global",
- "uid": "bdn8lriao7myoa",
- "version": 1,
- "weekStart": ""
- }
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- labels:
- grafana_dashboard: "1"
- annotations:
- grafana_folder: Envoy
- name: envoy-proxy-global-dashboard
-data:
- envoy-proxy-global.json: |-
- {
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard"
- }
- ]
- },
- "description": "Envoy proxy monitoring Dashboard with service level templates.",
- "editable": true,
- "fiscalYearStartMonth": 0,
- "gnetId": 11022,
- "graphTooltip": 0,
- "id": 3,
- "links": [],
- "liveNow": false,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "max": 3,
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#d44a3a",
- "value": null
- },
- {
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 1
- },
- {
- "color": "#299c46",
- "value": 2
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 5,
- "x": 0,
- "y": 0
- },
- "id": 37,
- "maxDataPoints": 100,
- "options": {
- "minVizHeight": 75,
- "minVizWidth": 75,
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showThresholdLabels": false,
- "showThresholdMarkers": true,
- "sizing": "auto"
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "sum(envoy_server_live)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Live servers",
- "type": "gauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 5,
- "x": 5,
- "y": 0
- },
- "id": 39,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "avg by(pod) (envoy_server_uptime{namespace=~\"$Namespace\"})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Avg uptime per node",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 5,
- "x": 10,
- "y": 0
- },
- "id": 43,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum by(pod) (envoy_server_memory_heap_size{namespace=~\"$Namespace\"})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Heap Size",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 5,
- "x": 15,
- "y": 0
- },
- "id": 41,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "sum by(pod) (envoy_server_memory_allocated{namespace=~\"$Namespace\"})",
- "hide": false,
- "instant": false,
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Allocated Memory",
- "type": "stat"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 7
- },
- "id": 24,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "DownStream",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 0,
- "y": 8
- },
- "id": 3,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum(rate(envoy_http_downstream_rq_total[5m]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Envoy HTTP Downstream Rq total",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Downstream RPS",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 6,
- "y": 8
- },
- "id": 9,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "builder",
- "expr": "sum by(namespace) (rate(envoy_http_downstream_cx_total{namespace=~\"$Namespace\"}[5m]))",
- "instant": false,
- "legendFormat": "{{namespace}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Downstream CPS",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 12,
- "y": 8
- },
- "id": 16,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le) (rate(envoy_http_downstream_rq_time_bucket[5m])))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{service}} 90%",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(envoy_http_downstream_rq_time_bucket[5m])))",
- "format": "time_series",
- "hide": false,
- "intervalFactor": 1,
- "legendFormat": "{{service}} 50% ",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(envoy_http_downstream_rq_time_bucket[5m])))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{service}} 99%",
- "range": true,
- "refId": "C"
- }
- ],
- "title": "Downstream Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 18,
- "y": 8
- },
- "id": 4,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum by(namespace) (envoy_listener_downstream_cx_active{namespace=~\"$Namespace\"})",
- "format": "time_series",
- "instant": false,
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{namespace}}",
- "refId": "A"
- }
- ],
- "title": "Downstream Total Connections",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 0,
- "y": 16
- },
- "id": 32,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "builder",
- "expr": "sum by(namespace) (rate(envoy_tcp_downstream_cx_total{namespace=~\"$Namespace\"}[5m]))",
- "instant": false,
- "legendFormat": "{{namespace}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "TCP Downstream CPS",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 6,
- "y": 16
- },
- "id": 31,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "builder",
- "expr": "sum by(namespace) (rate(envoy_tcp_downstream_cx_rx_bytes_total{namespace=~\"$Namespace\"}[5m]))",
- "instant": false,
- "legendFormat": "{{namespace}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "TCP Downstream Bytes Rx/second",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 12,
- "y": 16
- },
- "id": 33,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "maxHeight": 600,
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "builder",
- "expr": "sum by(namespace) (rate(envoy_tcp_downstream_cx_tx_bytes_total{namespace=~\"$Namespace\"}[5m]))",
- "instant": false,
- "legendFormat": "{{namespace}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "TCP Downstream Bytes Tx/Second",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 24
- },
- "id": 22,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "UpStream",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Displays the number of Requests per Second being performed against each Upstream.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 0,
- "y": 25
- },
- "id": 2,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "sum by(namespace) (rate(envoy_cluster_upstream_rq_total{namespace=~\"$Namespace\"}[5m]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream RPS",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 6,
- "y": 25
- },
- "id": 14,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum(rate(envoy_cluster_upstream_cx_total{namespace=~\"$Namespace\",}[5m])) by (namespace)",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream CPS",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 12,
- "y": 25
- },
- "id": 10,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{namespace=~\"$Namespace\"}[5m])) by (le, namespace))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{namespace}} 99%",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum(rate(envoy_cluster_upstream_rq_time_bucket{namespace=~\"$Namespace\"}[5m])) by (le, namespace))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{namespace}} 90%",
- "range": true,
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum(rate(envoy_cluster_upstream_rq_time_bucket{namespace=~\"$Namespace\"}[5m])) by (le, namespace))",
- "format": "time_series",
- "hide": false,
- "intervalFactor": 1,
- "legendFormat": "{{namespace}} 50% ",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Upstream Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 18,
- "y": 25
- },
- "id": 15,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum(envoy_cluster_upstream_cx_active{namespace=~\"$Namespace\"}) by (namespace)",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{namespace}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream Total Connections",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 0,
- "y": 33
- },
- "id": 34,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "builder",
- "expr": "sum by(namespace) (rate(envoy_cluster_upstream_cx_rx_bytes_total{namespace=~\"$Namespace\"}[5m]))",
- "instant": false,
- "legendFormat": "{{namespace}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream Bytes Rx/Second",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 6,
- "y": 33
- },
- "id": 35,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "builder",
- "expr": "sum by(namespace) (rate(envoy_cluster_upstream_cx_tx_bytes_total{namespace=~\"$Namespace\"}[5m]))",
- "instant": false,
- "legendFormat": "{{namespace}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream Bytes Tx/Second",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 41
- },
- "id": 28,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Upstream Response Codes",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 0,
- "y": 42
- },
- "id": 5,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "builder",
- "exemplar": false,
- "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"2\"}[5m]))",
- "format": "time_series",
- "instant": false,
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Value",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream 2xx Responses",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 6,
- "y": 42
- },
- "id": 11,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"3\"}[5m]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "value",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream 3xx Responses",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 12,
- "y": 42
- },
- "id": 12,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=\"4\"}[5m]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Value",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream 4xx Responses",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 18,
- "y": 42
- },
- "id": 13,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=\"5\"}[5m]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Value",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Upstream 5xx Responses",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 50
- },
- "id": 26,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Total",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 0,
- "y": 51
- },
- "id": 17,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "avg(envoy_cluster_membership_healthy{namespace=~\"$Namespace\"}) by (namespace) / avg(envoy_cluster_membership_total{namespace=~\"$Namespace\"}) by (namespace)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{namespace}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Endpoint Percentage Health",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 6,
- "y": 51
- },
- "id": 19,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "sum by(namespace) (envoy_cluster_membership_total{namespace=~\"$Namespace\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{namespace}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Total Endpoints",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 12,
- "y": 51
- },
- "id": 18,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "builder",
- "expr": "sum by(namespace) (envoy_cluster_membership_healthy{namespace=~\"$Namespace\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{namespace}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Healthy Endpoints",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byValue",
- "options": {
- "op": "gte",
- "reducer": "allIsZero",
- "value": 0
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": true,
- "tooltip": true,
- "viz": false
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 18,
- "y": 51
- },
- "id": 20,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "10.0.2",
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "editorMode": "code",
- "expr": "sum by(namespace) (envoy_cluster_membership_total{namespace=~\"$Namespace\"}) - sum by(namespace) (envoy_cluster_membership_healthy{namespace=~\"$Namespace\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{namespace}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Unhealthy Endpoints",
- "type": "timeseries"
- }
- ],
- "refresh": "10s",
- "schemaVersion": 39,
- "tags": [
- "Data Plane"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": false,
- "text": "Prometheus",
- "value": "PBFA97CFB590B2093"
- },
- "hide": 0,
- "includeAll": false,
- "multi": false,
- "name": "datasource",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "selected": true,
- "text": [
- "All"
- ],
- "value": [
- "$__all"
- ]
- },
- "datasource": {
- "uid": "$datasource"
- },
- "definition": "",
- "hide": 0,
- "includeAll": true,
- "multi": true,
- "name": "Namespace",
- "options": [],
- "query": "label_values(envoy_cluster_upstream_rq_time_bucket,namespace)",
- "refresh": 2,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-5m",
- "to": "now"
- },
- "timeRangeUpdatedDuringEditOrView": false,
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "",
- "title": "Envoy Global",
- "uid": "heHhNSFf6Na8vIZWRs8H",
- "version": 1,
- "weekStart": ""
- }
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- labels:
- grafana_dashboard: "1"
- annotations:
- grafana_folder: Envoy
- name: global-ratelimit-dashboard
-data:
- global-ratelimit.json: |-
- {
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard"
- }
- ]
- },
- "description": "Envoy Gateway monitoring Dashboard with exported metrics.",
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "id": 37,
- "links": [],
- "panels": [
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 273,
- "panels": [],
- "title": "Global Ratelimit",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "description": "Resident memory size",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "decmbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 7,
- "x": 9,
- "y": 1
- },
- "id": 291,
- "options": {
- "minVizHeight": 75,
- "minVizWidth": 75,
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showThresholdLabels": false,
- "showThresholdMarkers": false,
- "sizing": "auto"
- },
- "pluginVersion": "11.3.1",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "exemplar": false,
- "expr": "(process_resident_memory_bytes{app_kubernetes_io_component=\"ratelimit\"})/1000000",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Resident Memory",
- "transparent": true,
- "type": "gauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "description": "Virtual memory size",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "percentage",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "decmbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 7,
- "x": 16,
- "y": 1
- },
- "id": 325,
- "options": {
- "minVizHeight": 75,
- "minVizWidth": 75,
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showThresholdLabels": false,
- "showThresholdMarkers": false,
- "sizing": "auto"
- },
- "pluginVersion": "11.3.1",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "exemplar": false,
- "expr": "(process_virtual_memory_bytes{app_kubernetes_io_component=\"ratelimit\"})/1000000",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Virtual Memory",
- "transparent": true,
- "type": "gauge"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "description": "Number of ratelimit rule hits in total",
- "fieldConfig": {
- "defaults": {
- "color": {
- "fixedColor": "light-blue",
- "mode": "shades"
- },
- "fieldMinMax": false,
- "mappings": [],
- "noValue": "0",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 8
- },
- "id": 308,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "center",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "value",
- "wideLayout": false
- },
- "pluginVersion": "11.3.1",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "exemplar": false,
- "expr": "ratelimit_service_rate_limit_total_hits{domain=\"$DefaultDomain\", key1=\"httproute/default/http-ratelimit/rule/0/match/0/ratelimit_example_httproute/default/http-ratelimit/rule/0/match/0/ratelimit_example\", key2=\"rule-0-match-0_rule-0-match-0\"}",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "interval": "",
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Total Hits",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "description": "Number of rule hits over the NearLimit ratio threshold (currently 80%) but under the threshold rate.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "fixedColor": "light-blue",
- "mode": "shades"
- },
- "fieldMinMax": false,
- "mappings": [],
- "noValue": "0",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 6,
- "y": 8
- },
- "id": 326,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "center",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "value",
- "wideLayout": false
- },
- "pluginVersion": "11.3.1",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "exemplar": false,
- "expr": "ratelimit_service_rate_limit_near_limit{domain=\"$DefaultDomain\", key1=\"httproute/default/http-ratelimit/rule/0/match/0/ratelimit_example_httproute/default/http-ratelimit/rule/0/match/0/ratelimit_example\", key2=\"rule-0-match-0_rule-0-match-0\"}",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "interval": "",
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Near Limit",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "description": "Number of rule hits exceeding the threshold rate",
- "fieldConfig": {
- "defaults": {
- "color": {
- "fixedColor": "light-blue",
- "mode": "shades"
- },
- "fieldMinMax": false,
- "mappings": [],
- "noValue": "0",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 12,
- "y": 8
- },
- "id": 327,
- "options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "center",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "value",
- "wideLayout": false
- },
- "pluginVersion": "11.3.1",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "exemplar": false,
- "expr": "ratelimit_service_rate_limit_over_limit{domain=\"$DefaultDomain\", key1=\"httproute/default/http-ratelimit/rule/0/match/0/ratelimit_example_httproute/default/http-ratelimit/rule/0/match/0/ratelimit_example\", key2=\"rule-0-match-0_rule-0-match-0\"}",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "interval": "",
- "legendFormat": "__auto",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Over Limit",
- "type": "stat"
- }
- ],
- "preload": false,
- "refresh": "",
- "schemaVersion": 40,
- "tags": [
- "Control Plane"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "Prometheus",
- "value": "PBFA97CFB590B2093"
- },
- "includeAll": false,
- "name": "datasource",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "envoy-gateway-system",
- "value": "envoy-gateway-system"
- },
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "definition": "label_values(watchable_depth,namespace)",
- "includeAll": false,
- "name": "Namespace",
- "options": [],
- "query": {
- "qryType": 1,
- "query": "label_values(watchable_depth,namespace)",
- "refId": "PrometheusVariableQueryEditor-VariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "type": "query"
- },
- {
- "allValue": ".*",
- "current": {
- "text": [
- "All"
- ],
- "value": [
- "$__all"
- ]
- },
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "definition": "label_values(watchable_depth,runner)",
- "includeAll": true,
- "multi": true,
- "name": "Runner",
- "options": [],
- "query": {
- "qryType": 1,
- "query": "label_values(watchable_depth,runner)",
- "refId": "PrometheusVariableQueryEditor-VariableQuery"
- },
- "refresh": 2,
- "regex": "",
- "type": "query"
- },
- {
- "current": {
- "text": "default/eg/http",
- "value": "default/eg/http"
- },
- "description": "DefaultDomain is set to default/eg/http",
- "includeAll": false,
- "name": "DefaultDomain",
- "options": [
- {
- "selected": true,
- "text": "default/eg/http",
- "value": "default/eg/http"
- }
- ],
- "query": "default/eg/http",
- "type": "custom"
- }
- ]
- },
- "time": {
- "from": "now-6h",
- "to": "now"
- },
- "timepicker": {},
- "timezone": "browser",
- "title": "Global Ratelimit",
- "uid": "R2xvYmFsIFJhdGVsaW1pdAo",
- "version": 1,
- "weekStart": ""
- }
diff --git a/apps/base/victoria-metrics-k8s-stack/dashboards/ceph-cluster-dashboard.json b/apps/base/victoria-metrics-k8s-stack/dashboards/ceph-cluster-dashboard.json
@@ -0,0 +1,5679 @@
+{
+ "__inputs": [],
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "5.3.2"
+ },
+ {
+ "id": "graph",
+ "name": "Graph",
+ "type": "panel",
+ "version": "5.0.0"
+ },
+ {
+ "id": "heatmap",
+ "name": "Heatmap",
+ "type": "panel",
+ "version": "5.0.0"
+ },
+ {
+ "id": "singlestat",
+ "name": "Singlestat",
+ "type": "panel",
+ "version": "5.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "showIn": 0,
+ "tags": [],
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Overview of your Ceph cluster.",
+ "editable": false,
+ "gnetId": 2842,
+ "graphTooltip": 0,
+ "hideControls": false,
+ "id": null,
+ "links": [],
+ "panels": [
+ {
+ "collapse": false,
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 2,
+ "panels": [],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "CLUSTER STATE",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "0": {
+ "text": "HEALTHY"
+ },
+ "1": {
+ "text": "WARNING"
+ },
+ "2": {
+ "text": "ERROR"
+ }
+ },
+ "type": "value"
+ },
+ {
+ "id": 1,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#9ac48a"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 2
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 0,
+ "y": 1
+ },
+ "id": 3,
+ "interval": "1m",
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_health_status{}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Ceph health status",
+ "transparent": true,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "Bps"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 3,
+ "y": 1
+ },
+ "id": 4,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_w_in_bytes{}[5m]))",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Write Throughput",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#d44a3a"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0
+ },
+ {
+ "color": "#9ac48a",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "Bps"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 6,
+ "y": 1
+ },
+ "id": 5,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_r_out_bytes{}[5m]))",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Read Throughput",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0.025
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "decbytes"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 9,
+ "y": 1
+ },
+ "id": 6,
+ "interval": "1m",
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_cluster_total_bytes{}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Cluster Capacity",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(245, 54, 54, 0.9)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0.1
+ },
+ {
+ "color": "rgba(50, 172, 45, 0.97)",
+ "value": 0.3
+ }
+ ]
+ },
+ "unit": "percentunit"
+ }
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 3,
+ "x": 12,
+ "y": 1
+ },
+ "id": 7,
+ "interval": "1m",
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_cluster_total_bytes{}-ceph_cluster_total_used_bytes{})/ceph_cluster_total_bytes{}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Available Capacity",
+ "transparent": false,
+ "type": "gauge"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 15,
+ "y": 1
+ },
+ "id": 8,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pool_objects{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of Objects",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 18,
+ "y": 1
+ },
+ "id": 9,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "delta"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_osd_op_w_in_bytes{})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Bytes Written",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 21,
+ "y": 1
+ },
+ "id": 10,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "delta"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_osd_op_r_out_bytes{})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Bytes Read",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#9ac48a"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "#e24d42",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 0,
+ "y": 4
+ },
+ "id": 11,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count(ALERTS{alertstate=\"firing\",alertname=~\"^Ceph.+\"}) OR vector(0)",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Alerts starting with Ceph",
+ "transparent": true,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "ops"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 3,
+ "y": 4
+ },
+ "id": 12,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_w{}[5m]))",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Write IOPS",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#d44a3a"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0
+ },
+ {
+ "color": "#9ac48a",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "ops"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 6,
+ "y": 4
+ },
+ "id": 13,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_r{}[5m]))",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Read IOPS",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0.025
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 0.1
+ }
+ ]
+ },
+ "unit": "decbytes"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 9,
+ "y": 4
+ },
+ "id": 14,
+ "interval": "1m",
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_cluster_total_used_bytes{}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Used Capacity",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#ea6460"
+ },
+ {
+ "color": "#052b51",
+ "value": 0
+ },
+ {
+ "color": "#508642",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 15,
+ "y": 4
+ },
+ "id": 15,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "diff"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pool_objects)",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Difference",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 128
+ }
+ ]
+ },
+ "unit": "short"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 18,
+ "y": 4
+ },
+ "id": 16,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_mon_num_sessions{})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Mon Session Num",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": null
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 2
+ },
+ {
+ "color": "green",
+ "value": 3
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 21,
+ "y": 4
+ },
+ "id": 17,
+ "interval": "1m",
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count(ceph_mon_quorum_status{}) or vector(0)",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Monitors In Quorum",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "collapse": false,
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 7
+ },
+ "id": 18,
+ "panels": [],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "OSD STATE",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#9ac48a"
+ },
+ {
+ "color": "rgba(237, 40, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 0,
+ "y": 8
+ },
+ "id": 19,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count(ceph_osd_up{}) - count(ceph_osd_in{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "OSDs OUT",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "#eab839",
+ "value": 1
+ },
+ {
+ "color": "#ea6460",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 2,
+ "y": 8
+ },
+ "id": 20,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count(ceph_osd_up{} == 0.0) OR vector(0)",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "OSDs DOWN",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 4,
+ "y": 8
+ },
+ "id": 21,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_osd_up{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "OSDs UP",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 6,
+ "y": 8
+ },
+ "id": 22,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_osd_in{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "OSDs IN",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 250
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 300
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 8,
+ "y": 8
+ },
+ "id": 23,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(ceph_osd_numpg{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Avg PGs",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 10
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 50
+ }
+ ]
+ },
+ "unit": "ms"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 10,
+ "y": 8
+ },
+ "id": 24,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(ceph_osd_apply_latency_ms{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Avg Apply Latency",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 10
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 50
+ }
+ ]
+ },
+ "unit": "ms"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 13,
+ "y": 8
+ },
+ "id": 25,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(ceph_osd_commit_latency_ms{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Avg Commit Latency",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 4,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "color": "#299c46",
+ "text": "0"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#299c46"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "#d44a3a",
+ "value": 2
+ }
+ ]
+ },
+ "unit": "ms"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 16,
+ "y": 8
+ },
+ "id": 26,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Avg Op Write Latency",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 6,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "color": "#299c46",
+ "text": "0"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#299c46"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "#d44a3a",
+ "value": 2
+ }
+ ]
+ },
+ "unit": "ms"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 19,
+ "y": 8
+ },
+ "id": 27,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(rate(ceph_osd_op_r_latency_sum{}[5m])/rate(ceph_osd_op_r_latency_count{}[5m]) >= 0)",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Avg Op Read Latency",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "collapse": false,
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 9
+ },
+ "id": 28,
+ "panels": [],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "CLUSTER STATS",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 40,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 0,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 2,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Available"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Total Capacity"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#447EBC",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Used"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#BF1B00",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Used"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#BF1B00",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_avail"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_space"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_used"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#890F02",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Total Capacity"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineWidth",
+ "value": 3
+ },
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 10
+ },
+ "id": 29,
+ "interval": "$interval",
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_cluster_total_bytes{}-ceph_cluster_total_used_bytes{}",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Available",
+ "refId": "A",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_cluster_total_used_bytes{}",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Used",
+ "refId": "B",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_cluster_total_bytes{}",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Total Capacity",
+ "refId": "C",
+ "step": 300
+ }
+ ],
+ "title": "Capacity",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Total Capacity"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Used"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#BF1B00",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_avail"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_space"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_used"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#890F02",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 10
+ },
+ "id": 30,
+ "interval": "$interval",
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_w{}[5m]))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Write",
+ "refId": "A",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_r{}[5m]))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Read",
+ "refId": "B",
+ "step": 300
+ }
+ ],
+ "title": "IOPS",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 10
+ },
+ "id": 31,
+ "interval": "$interval",
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_w_in_bytes{}[5m]))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Write",
+ "refId": "A",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_r_out_bytes{}[5m]))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Read",
+ "refId": "B",
+ "step": 300
+ }
+ ],
+ "title": "Cluster Throughput",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 18
+ },
+ "id": 32,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_bytes_used{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Pool Used Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 18
+ },
+ "id": 33,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_avail_raw{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}} Avail",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_stored_raw{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}} Stored",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Pool RAW Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 18
+ },
+ "id": 34,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_objects{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Objects Per Pool",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 26
+ },
+ "id": 35,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_quota_bytes{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Pool Quota Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 26
+ },
+ "id": 36,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_quota_objects{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Pool Objects Quota",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 26
+ },
+ "id": 37,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count(ceph_bluestore_kv_commit_lat_count{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "BlueStore",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Type Count",
+ "type": "timeseries"
+ },
+ {
+ "collapse": true,
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 27
+ },
+ "id": 38,
+ "panels": [
+ {
+ "columns": [],
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "inspect": false
+ },
+ "decimals": 2,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Time"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Time"
+ },
+ {
+ "id": "unit",
+ "value": "time: YYYY-MM-DD HH:mm:ss"
+ },
+ {
+ "id": "custom.align"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 28
+ },
+ "id": 39,
+ "links": [],
+ "options": {
+ "footer": {
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true
+ },
+ "pluginVersion": "9.1.3",
+ "styles": "",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ALERTS{alertstate=\"firing\",alertname=~\"^Ceph.+\"}",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Alerts starting with Ceph",
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducer": []
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "columns": [],
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "filterable": false,
+ "inspect": false
+ },
+ "decimals": 2,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Time"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Time"
+ },
+ {
+ "id": "unit",
+ "value": "time: YYYY-MM-DD HH:mm:ss"
+ },
+ {
+ "id": "custom.align"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 28
+ },
+ "id": 40,
+ "links": [],
+ "options": {
+ "footer": {
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true
+ },
+ "pluginVersion": "9.1.3",
+ "styles": "",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "topk(5,sort_desc(ceph_osd_apply_latency_ms{} + ceph_osd_commit_latency_ms{}))",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "_auto",
+ "refId": "A"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top Sluggish OSDs",
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducer": []
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "columns": [],
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "inspect": false
+ },
+ "decimals": 2,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Time"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Time"
+ },
+ {
+ "id": "unit",
+ "value": "time: YYYY-MM-DD HH:mm:ss"
+ },
+ {
+ "id": "custom.align"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 28
+ },
+ "id": 41,
+ "links": [],
+ "options": {
+ "footer": {
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true
+ },
+ "pluginVersion": "9.1.3",
+ "styles": "",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_osd_up{} == 0",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Down OSDs",
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducer": []
+ }
+ }
+ ],
+ "type": "table"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Alerts",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "collapse": true,
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 28
+ },
+ "id": 42,
+ "panels": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "scaleDistributionLog": 2,
+ "type": "log"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "id": 43,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "node_memory_Active_anon_bytes{}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(node_memory_Active_anon_bytes{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Cluster Memory Usage",
+ "refId": "B"
+ }
+ ],
+ "title": "Node Memory Usage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "scaleDistributionLog": 2,
+ "type": "log"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "id": 44,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg by(instance)(irate(node_cpu_seconds_total{job='node',mode!=\"idle\"}[$interval])) * 100",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Node CPU Usage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 35
+ },
+ "id": 45,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum by (instance)(irate(node_disk_read_bytes_total{}[$interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Node Out",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 35
+ },
+ "id": 46,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum by (instance)(irate(node_disk_written_bytes_total{}[$interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Node In",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 44
+ },
+ "id": 47,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(node_filesystem_free_bytes{ mountpoint=\"/\", device != \"rootfs\"})*100 / (node_filesystem_size_bytes{ mountpoint=\"/\", device != \"rootfs\"})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Free Space in root filesystem",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Node Statistics (NodeExporter)",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "collapse": false,
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 45
+ },
+ "id": 48,
+ "panels": [],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "OBJECTS",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Total.*$/"
+ },
+ "properties": [
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 12,
+ "w": 6,
+ "x": 0,
+ "y": 46
+ },
+ "id": 49,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "asc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pool_objects)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Total",
+ "range": true,
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "OSD Type Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Total.*$/"
+ },
+ "properties": [
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 12,
+ "w": 8,
+ "x": 6,
+ "y": 46
+ },
+ "id": 50,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "asc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_active{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Active",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_clean{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Clean",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_peering{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Peering",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_degraded{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Degraded",
+ "range": true,
+ "refId": "D",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_stale{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Stale",
+ "range": true,
+ "refId": "E",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_unclean_pgs{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Unclean",
+ "range": true,
+ "refId": "F",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_undersized{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Undersized",
+ "range": true,
+ "refId": "G",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_incomplete{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Incomplete",
+ "range": true,
+ "refId": "H"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_forced_backfill{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Forced Backfill",
+ "range": true,
+ "refId": "I"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_forced_recovery{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Forced Recovery",
+ "range": true,
+ "refId": "J"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_creating{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Creating",
+ "range": true,
+ "refId": "K"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_wait_backfill{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Wait Backfill",
+ "range": true,
+ "refId": "L"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_deep{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Deep",
+ "range": true,
+ "refId": "M"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_scrubbing{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Scrubbing",
+ "range": true,
+ "refId": "N"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_recovering{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Recovering",
+ "range": true,
+ "refId": "O"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_repair{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Repair",
+ "range": true,
+ "refId": "P"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_down{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Down",
+ "range": true,
+ "refId": "Q"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_peered{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Peered",
+ "range": true,
+ "refId": "R"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_backfill{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Backfill",
+ "range": true,
+ "refId": "S"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_remapped{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Remapped",
+ "range": true,
+ "refId": "T"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_backfill_toofull{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Backfill Toofull",
+ "range": true,
+ "refId": "U"
+ }
+ ],
+ "title": "PGs State",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Total.*$/"
+ },
+ "properties": [
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 10,
+ "x": 14,
+ "y": 46
+ },
+ "id": 51,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "asc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_degraded{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Degraded",
+ "range": true,
+ "refId": "A",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_stale{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Stale",
+ "range": true,
+ "refId": "B",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_undersized{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Undersized",
+ "range": true,
+ "refId": "C",
+ "step": 300
+ }
+ ],
+ "title": "Stuck PGs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 10,
+ "x": 14,
+ "y": 52
+ },
+ "id": 52,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_recovery_ops{}[$interval]))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "OPS",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Recovery Operations",
+ "type": "timeseries"
+ },
+ {
+ "collapse": false,
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 53
+ },
+ "id": 53,
+ "panels": [],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "LATENCY",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ }
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 54
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 54,
+ "legend": {
+ "show": true
+ },
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ },
+ "value": "1"
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_osd_apply_latency_ms{}",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Apply Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": null,
+ "format": "ms",
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true,
+ "splitFactor": 1
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": 10
+ },
+ {
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#65c5db",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ }
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 54
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 55,
+ "legend": {
+ "show": true
+ },
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#65c5db",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_osd_commit_latency_ms{}",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Commit Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": null,
+ "format": "ms",
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true,
+ "splitFactor": 1
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
+ },
+ {
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#806eb7",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ }
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 62
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 56,
+ "legend": {
+ "show": true
+ },
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#806eb7",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 2,
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "rate(ceph_osd_op_r_latency_sum{}[5m]) / rate(ceph_osd_op_r_latency_count{}[5m]) >= 0",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Read Op Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": null,
+ "format": "ms",
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true,
+ "splitFactor": 1
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
+ },
+ {
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#f9934e",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ }
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 62
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 57,
+ "legend": {
+ "show": true
+ },
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#f9934e",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 2,
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Write Op Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": 2,
+ "format": "ms",
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true,
+ "splitFactor": 1
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 70
+ },
+ "id": 58,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(rate(ceph_osd_op_r_latency_sum{}[5m]) / rate(ceph_osd_op_r_latency_count{}[5m]) >= 0)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Read",
+ "refId": "A"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Write",
+ "refId": "B"
+ }
+ ],
+ "title": "Recovery Operations",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 70
+ },
+ "id": 59,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(ceph_osd_apply_latency_ms{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "apply",
+ "metric": "ceph_osd_perf_apply_latency_seconds",
+ "refId": "A",
+ "step": 4
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(ceph_osd_commit_latency_ms{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "commit",
+ "metric": "ceph_osd_perf_commit_latency_seconds",
+ "refId": "B",
+ "step": 4
+ }
+ ],
+ "title": "AVG OSD Apply + Commit Latency",
+ "type": "timeseries"
+ },
+ {
+ "collapse": true,
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 71
+ },
+ "id": 60,
+ "panels": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 6,
+ "x": 0,
+ "y": 72
+ },
+ "id": 61,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count by (ceph_version)(ceph_osd_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Ceph OSD Versions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 6,
+ "x": 6,
+ "y": 72
+ },
+ "id": 62,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count by (ceph_version)(ceph_mon_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Ceph Mon Versions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 6,
+ "x": 12,
+ "y": 72
+ },
+ "id": 63,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count by (ceph_version)(ceph_mds_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Ceph MDS Versions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 6,
+ "x": 18,
+ "y": 72
+ },
+ "id": 64,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count by (ceph_version)(ceph_rgw_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Ceph RGW Versions",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ceph Versions",
+ "titleSize": "h6",
+ "type": "row"
+ }
+ ],
+ "refresh": "1m",
+ "rows": [],
+ "schemaVersion": 37,
+ "style": "dark",
+ "tags": [
+ "ceph-mixin"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 0,
+ "label": "Data Source",
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "auto": true,
+ "auto_count": 10,
+ "auto_min": "1m",
+ "current": {
+ "text": "$__auto_interval_interval",
+ "value": "$__auto_interval_interval"
+ },
+ "hide": 0,
+ "label": "Interval",
+ "name": "interval",
+ "options": [
+ {
+ "selected": true,
+ "text": "auto",
+ "value": "$__auto_interval_interval"
+ },
+ {
+ "selected": false,
+ "text": "5s",
+ "value": "5s"
+ },
+ {
+ "selected": false,
+ "text": "10s",
+ "value": "10s"
+ },
+ {
+ "selected": false,
+ "text": "30s",
+ "value": "30s"
+ },
+ {
+ "selected": false,
+ "text": "1m",
+ "value": "1m"
+ },
+ {
+ "selected": false,
+ "text": "10m",
+ "value": "10m"
+ },
+ {
+ "selected": false,
+ "text": "30m",
+ "value": "30m"
+ },
+ {
+ "selected": false,
+ "text": "1h",
+ "value": "1h"
+ },
+ {
+ "selected": false,
+ "text": "6h",
+ "value": "6h"
+ },
+ {
+ "selected": false,
+ "text": "12h",
+ "value": "12h"
+ },
+ {
+ "selected": false,
+ "text": "1d",
+ "value": "1d"
+ },
+ {
+ "selected": false,
+ "text": "7d",
+ "value": "7d"
+ },
+ {
+ "selected": false,
+ "text": "14d",
+ "value": "14d"
+ },
+ {
+ "selected": false,
+ "text": "30d",
+ "value": "30d"
+ }
+ ],
+ "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
+ "refresh": 2,
+ "type": "interval",
+ "valuelabels": {}
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Ceph Cluster",
+ "uid": "tbO9LAiZK",
+ "version": 0
+}
diff --git a/apps/base/victoria-metrics-k8s-stack/dashboards/ceph-osd-single-dashboard.json b/apps/base/victoria-metrics-k8s-stack/dashboards/ceph-osd-single-dashboard.json
@@ -0,0 +1,1615 @@
+{
+ "__inputs": [],
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "panel",
+ "id": "gauge",
+ "name": "Gauge",
+ "version": ""
+ },
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "9.0.5"
+ },
+ {
+ "type": "panel",
+ "id": "heatmap",
+ "name": "Heatmap",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "stat",
+ "name": "Stat",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "CEPH OSD Status.",
+ "editable": false,
+ "fiscalYearStartMonth": 0,
+ "gnetId": 5336,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 11,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Status / Total OSDs",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [
+ {
+ "options": {
+ "from": 0,
+ "result": {
+ "index": 0,
+ "text": "N/A"
+ },
+ "to": 0
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 0,
+ "result": {
+ "index": 1,
+ "text": "DOWN"
+ },
+ "to": 0.99
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 0.99,
+ "result": {
+ "index": 2,
+ "text": "UP"
+ },
+ "to": 1
+ },
+ "type": "range"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": null
+ },
+ {
+ "color": "rgba(237, 40, 40, 0.89)",
+ "value": 0
+ },
+ {
+ "color": "rgba(50, 172, 45, 0.97)",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 0,
+ "y": 1
+ },
+ "id": 6,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum without (instance) (ceph_osd_up{ceph_daemon=\"$osd\"})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "range": true,
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Status",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [
+ {
+ "options": {
+ "from": 0,
+ "result": {
+ "index": 0,
+ "text": "N/A"
+ },
+ "to": 0
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 0,
+ "result": {
+ "index": 1,
+ "text": "OUT"
+ },
+ "to": 0.99
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 0.99,
+ "result": {
+ "index": 2,
+ "text": "IN"
+ },
+ "to": 1
+ },
+ "type": "range"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": null
+ },
+ {
+ "color": "rgba(237, 40, 40, 0.89)",
+ "value": 0
+ },
+ {
+ "color": "rgba(50, 172, 45, 0.97)",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 2,
+ "y": 1
+ },
+ "id": 8,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance) (ceph_osd_in{ceph_daemon=\"$osd\"})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Available",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [
+ {
+ "options": {
+ "from": 0,
+ "result": {
+ "index": 0,
+ "text": "N/A"
+ },
+ "to": 0
+ },
+ "type": "range"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(255, 255, 255, 0.9)",
+ "value": null
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 4,
+ "y": 1
+ },
+ "id": 10,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "count without (instance, ceph_daemon) (ceph_osd_up)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Total OSDs",
+ "type": "stat"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 4
+ },
+ "id": 12,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "OSD: $osd",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ },
+ {
+ "color": "rgba(216, 200, 27, 0.27)",
+ "value": 250
+ },
+ {
+ "color": "rgba(234, 112, 112, 0.22)",
+ "value": 300
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Average.*/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 11,
+ "x": 0,
+ "y": 5
+ },
+ "id": 5,
+ "interval": "$interval",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance, ceph_daemon) (ceph_osd_numpg{ceph_daemon=~\"$osd\"})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Number of PGs",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "avg without (instance, ceph_daemon) (ceph_osd_numpg)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Average Number of PGs in the Cluster",
+ "refId": "B",
+ "step": 60
+ }
+ ],
+ "title": "PGs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 9,
+ "x": 11,
+ "y": 5
+ },
+ "id": 2,
+ "interval": "$interval",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance, ceph_daemon) (ceph_osd_stat_bytes{ceph_daemon=\"osd.0\"}-ceph_osd_stat_bytes_used{ceph_daemon=\"osd.0\"})",
+ "format": "time_series",
+ "hide": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Available",
+ "metric": "ceph_osd_avail_bytes",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance, ceph_daemon) (ceph_osd_stat_bytes_used{ceph_daemon=~\"$osd\"})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Used",
+ "metric": "ceph_osd_avail_bytes",
+ "refId": "B",
+ "step": 60
+ }
+ ],
+ "title": "OSD Storage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)",
+ "value": null
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 60
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 4,
+ "x": 20,
+ "y": 5
+ },
+ "id": 7,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "9.0.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance, ceph_daemon) (ceph_osd_stat_bytes_used{ceph_daemon=\"osd.0\"}/ceph_osd_stat_bytes{ceph_daemon=\"osd.0\"})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Utilization",
+ "type": "gauge"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 12
+ },
+ "id": 13,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Latency, Storage, Utilization Varience",
+ "type": "row"
+ },
+ {
+ "cards": {},
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 13
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 83,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ },
+ "value": "1"
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance) (ceph_osd_apply_latency_ms{ceph_daemon='$osd'})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Apply Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketSize": "",
+ "yAxis": {
+ "format": "ms",
+ "logBase": 2,
+ "min": "0",
+ "show": true,
+ "splitFactor": 1
+ },
+ "yBucketBound": "auto",
+ "yBucketSize": 10
+ },
+ {
+ "cards": {},
+ "color": {
+ "cardColor": "#65c5db",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 13
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 84,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#65c5db",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance) (ceph_osd_commit_latency_ms{ceph_daemon='$osd'})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Commit Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketSize": "",
+ "yAxis": {
+ "format": "ms",
+ "logBase": 2,
+ "min": "0",
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "cards": {},
+ "color": {
+ "cardColor": "#806eb7",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 21
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 85,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#806eb7",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 2,
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance) (rate(ceph_osd_op_r_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_r_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Read Op Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": 2,
+ "format": "ms",
+ "logBase": 2,
+ "min": "0",
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "cards": {},
+ "color": {
+ "cardColor": "#f9934e",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 21
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 86,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#f9934e",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 2,
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance) (rate(ceph_osd_op_w_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_w_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Write Op Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": 2,
+ "format": "ms",
+ "logBase": 2,
+ "min": "0",
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "id": 44,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "avg without (instance, ceph_daemon) (rate(ceph_osd_op_r_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_r_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "read",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "avg without (instance, ceph_daemon) (rate(ceph_osd_op_w_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_w_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "write",
+ "refId": "B"
+ }
+ ],
+ "title": "Avg OSD Op Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "id": 35,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "avg without (instance, ceph_daemon) (ceph_osd_apply_latency_ms{ceph_daemon='$osd'})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "apply",
+ "metric": "ceph_osd_perf_apply_latency_seconds",
+ "refId": "A",
+ "step": 4
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "avg without (instance, ceph_daemon)(ceph_osd_commit_latency_ms{ceph_daemon='$osd'})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "commit",
+ "metric": "ceph_osd_perf_commit_latency_seconds",
+ "refId": "B",
+ "step": 4
+ }
+ ],
+ "title": "AVG OSD Apply + Commit Latency",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "5m",
+ "schemaVersion": 36,
+ "style": "dark",
+ "tags": [
+ "ceph",
+ "osd"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Data source",
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "auto": true,
+ "auto_count": 10,
+ "auto_min": "1m",
+ "current": {
+ "selected": false,
+ "text": "30s",
+ "value": "30s"
+ },
+ "datasource": "Prometheus",
+ "hide": 0,
+ "includeAll": false,
+ "label": "Interval",
+ "multi": false,
+ "name": "interval",
+ "options": [
+ {
+ "selected": false,
+ "text": "auto",
+ "value": "$__auto_interval_interval"
+ },
+ {
+ "selected": false,
+ "text": "10s",
+ "value": "10s"
+ },
+ {
+ "selected": true,
+ "text": "30s",
+ "value": "30s"
+ },
+ {
+ "selected": false,
+ "text": "1m",
+ "value": "1m"
+ },
+ {
+ "selected": false,
+ "text": "10m",
+ "value": "10m"
+ },
+ {
+ "selected": false,
+ "text": "30m",
+ "value": "30m"
+ },
+ {
+ "selected": false,
+ "text": "1h",
+ "value": "1h"
+ },
+ {
+ "selected": false,
+ "text": "6h",
+ "value": "6h"
+ },
+ {
+ "selected": false,
+ "text": "12h",
+ "value": "12h"
+ },
+ {
+ "selected": false,
+ "text": "1d",
+ "value": "1d"
+ },
+ {
+ "selected": false,
+ "text": "7d",
+ "value": "7d"
+ },
+ {
+ "selected": false,
+ "text": "14d",
+ "value": "14d"
+ },
+ {
+ "selected": false,
+ "text": "30d",
+ "value": "30d"
+ }
+ ],
+ "query": "10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
+ "queryValue": "",
+ "refresh": 2,
+ "skipUrlSync": false,
+ "type": "interval"
+ },
+ {
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "label_values(ceph_osd_up, ceph_daemon)",
+ "hide": 0,
+ "includeAll": false,
+ "label": "OSD",
+ "multi": false,
+ "name": "osd",
+ "options": [],
+ "query": {
+ "query": "label_values(ceph_osd_up, ceph_daemon)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "Ceph - OSD (Single)",
+ "uid": "Fj5fAfzik",
+ "version": 2,
+ "weekStart": ""
+}
diff --git a/apps/base/victoria-metrics-k8s-stack/dashboards/ceph-pools-dashboard.json b/apps/base/victoria-metrics-k8s-stack/dashboards/ceph-pools-dashboard.json
@@ -0,0 +1,925 @@
+{
+ "__inputs": [],
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "panel",
+ "id": "gauge",
+ "name": "Gauge",
+ "version": ""
+ },
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "9.0.5"
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Ceph Pools dashboard.",
+ "editable": false,
+ "fiscalYearStartMonth": 0,
+ "gnetId": 5342,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 11,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Pool: $pool",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 30,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Total.*$/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineWidth",
+ "value": 4
+ },
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Raw.*$/"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#BF1B00",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineWidth",
+ "value": 4
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 20,
+ "x": 0,
+ "y": 1
+ },
+ "id": 2,
+ "interval": "$interval",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_max_avail) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Available - {{ name }}",
+ "metric": "ceph_pool_available_bytes",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_stored) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Stored - {{ name }}",
+ "metric": "ceph_pool",
+ "refId": "B",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_stored + ceph_pool_max_avail) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "hide": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Total - {{ name }}",
+ "metric": "ceph_pool",
+ "refId": "C",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_stored_raw) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Raw - {{ name }}",
+ "metric": "ceph_pool",
+ "refId": "D",
+ "step": 60
+ }
+ ],
+ "title": "Pool Storage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "percentage",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 75
+ },
+ {
+ "color": "red",
+ "value": 90
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 4,
+ "x": 20,
+ "y": 1
+ },
+ "id": 10,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "9.0.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance, pool_id, name) ((ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail)) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"}))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Usage",
+ "type": "gauge"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 8
+ },
+ "id": 12,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Pool Info",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 9
+ },
+ "id": 7,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_objects) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Objects - {{ name }}",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_dirty) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Dirty Objects - {{ name }}",
+ "refId": "B",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_quota_objects) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Quota Objects - {{ name }}",
+ "refId": "C"
+ }
+ ],
+ "title": "Objects in Pool",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "IOPS",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 9
+ },
+ "id": 4,
+ "interval": "$interval",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((irate(ceph_pool_rd[3m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Read - {{ name }}",
+ "refId": "B",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((irate(ceph_pool_wr[3m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Write - {{ name }}",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "IOPS",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 24,
+ "x": 0,
+ "y": 17
+ },
+ "id": 5,
+ "interval": "$interval",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((irate(ceph_pool_rd_bytes[5m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Read Bytes - {{ name }}",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((irate(ceph_pool_wr_bytes[5m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Written Bytes - {{ name }}",
+ "refId": "B",
+ "step": 60
+ }
+ ],
+ "title": "Throughput",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "1m",
+ "schemaVersion": 36,
+ "style": "dark",
+ "tags": [
+ "ceph",
+ "pools"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Data source",
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "auto": true,
+ "auto_count": 10,
+ "auto_min": "1m",
+ "current": {
+ "selected": false,
+ "text": "30s",
+ "value": "30s"
+ },
+ "datasource": "Prometheus",
+ "hide": 0,
+ "includeAll": false,
+ "label": "Interval",
+ "multi": false,
+ "name": "interval",
+ "options": [
+ {
+ "selected": false,
+ "text": "auto",
+ "value": "$__auto_interval_interval"
+ },
+ {
+ "selected": false,
+ "text": "10s",
+ "value": "10s"
+ },
+ {
+ "selected": true,
+ "text": "30s",
+ "value": "30s"
+ },
+ {
+ "selected": false,
+ "text": "1m",
+ "value": "1m"
+ },
+ {
+ "selected": false,
+ "text": "10m",
+ "value": "10m"
+ },
+ {
+ "selected": false,
+ "text": "30m",
+ "value": "30m"
+ },
+ {
+ "selected": false,
+ "text": "1h",
+ "value": "1h"
+ },
+ {
+ "selected": false,
+ "text": "6h",
+ "value": "6h"
+ },
+ {
+ "selected": false,
+ "text": "12h",
+ "value": "12h"
+ },
+ {
+ "selected": false,
+ "text": "1d",
+ "value": "1d"
+ },
+ {
+ "selected": false,
+ "text": "7d",
+ "value": "7d"
+ },
+ {
+ "selected": false,
+ "text": "14d",
+ "value": "14d"
+ },
+ {
+ "selected": false,
+ "text": "30d",
+ "value": "30d"
+ }
+ ],
+ "query": "10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
+ "queryValue": "",
+ "refresh": 2,
+ "skipUrlSync": false,
+ "type": "interval"
+ },
+ {
+ "allValue": ".*",
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "label_values(ceph_pool_metadata, name)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Pool",
+ "multi": true,
+ "name": "pool",
+ "options": [],
+ "query": {
+ "query": "label_values(ceph_pool_metadata, name)",
+ "refId": "Prometheus-pool-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 3,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "Ceph - Pools",
+ "uid": "-gtf0Bzik",
+ "version": 2,
+ "weekStart": ""
+}
diff --git a/apps/base/victoria-metrics-k8s-stack/dashboards/flagger.json b/apps/base/victoria-metrics-k8s-stack/dashboards/flagger.json
@@ -0,0 +1,1226 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": 3,
+ "iteration": 1553160305729,
+ "links": [],
+ "panels": [
+ {
+ "content": "<div class=\"dashboard-header text-center\">\n<span>RED: $target.$namespace</span>\n</div>",
+ "gridPos": {
+ "h": 3,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 89,
+ "links": [],
+ "mode": "html",
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "prometheus",
+ "format": "ops",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 0,
+ "y": 3
+ },
+ "id": 90,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": true,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": true
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "round(sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\"}[30s])), 0.001)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": "",
+ "title": "Primary: Incoming Request Volume",
+ "transparent": false,
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(50, 172, 45, 0.97)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(245, 54, 54, 0.9)"
+ ],
+ "datasource": "prometheus",
+ "decimals": null,
+ "format": "percentunit",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 80,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": false
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 6,
+ "y": 3
+ },
+ "id": 98,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": true,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": true
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\",envoy_response_code!~\"5.*\"}[30s])) / sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\"}[30s]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "B"
+ }
+ ],
+ "thresholds": "95, 99, 99.5",
+ "title": "Incoming Success Rate",
+ "transparent": false,
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "prometheus",
+ "format": "ops",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 12,
+ "y": 3
+ },
+ "id": 97,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(40, 224, 65, 0.18)",
+ "full": true,
+ "lineColor": "#7eb26d",
+ "show": true
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "round(sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\"}[30s])), 0.001)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": "",
+ "title": "Canary: Incoming Request Volume",
+ "transparent": false,
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(50, 172, 45, 0.97)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(245, 54, 54, 0.9)"
+ ],
+ "datasource": "prometheus",
+ "decimals": null,
+ "format": "percentunit",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 80,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": false
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 18,
+ "y": 3
+ },
+ "id": 99,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(40, 224, 65, 0.18)",
+ "full": true,
+ "lineColor": "#7eb26d",
+ "show": true
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\",envoy_response_code!~\"5.*\"}[30s])) / sum(irate(envoy_cluster_upstream_rq{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\"}[30s]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "B"
+ }
+ ],
+ "thresholds": "95, 99, 99.5",
+ "title": "Incoming Success Rate",
+ "transparent": false,
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "current"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus",
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 0,
+ "y": 7
+ },
+ "id": 96,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "(sum(rate(envoy_cluster_upstream_cx_rx_bytes_total{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-primary\"}[30s])))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "traffic",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Primary: Incoming Traffic",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "Bps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus",
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 12,
+ "y": 7
+ },
+ "id": 91,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "(sum(rate(envoy_cluster_upstream_cx_rx_bytes_total{kubernetes_namespace=~\"$namespace\",envoy_cluster_name=~\"$target-canary\"}[30s])))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "traffic",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Canary: Incoming Traffic",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "Bps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "content": "<div class=\"dashboard-header text-center\">\n<span>USE: $target.$namespace</span>\n</div>",
+ "gridPos": {
+ "h": 3,
+ "w": 24,
+ "x": 0,
+ "y": 11
+ },
+ "id": 101,
+ "links": [],
+ "mode": "html",
+ "title": "",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus",
+ "fill": 1,
+ "gridPos": {
+ "h": 6,
+ "w": 12,
+ "x": 0,
+ "y": 14
+ },
+ "id": 100,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(container_cpu_usage_seconds_total{cpu=\"total\",namespace=\"$namespace\",pod=~\"$target-primary.*\", container!~\"POD|istio-proxy\"}[1m])) by (pod)",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{ pod }}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Primary: CPU Usage by Pod",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": "CPU seconds / second",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus",
+ "fill": 1,
+ "gridPos": {
+ "h": 6,
+ "w": 12,
+ "x": 12,
+ "y": 14
+ },
+ "id": 102,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(container_cpu_usage_seconds_total{cpu=\"total\",namespace=\"$namespace\",pod=~\"$target.*\", pod!~\"$target-primary.*\", container!~\"POD|istio-proxy\"}[1m])) by (pod)",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{ pod }}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Canary: CPU Usage by Pod",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": "CPU seconds / second",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus",
+ "fill": 1,
+ "gridPos": {
+ "h": 6,
+ "w": 12,
+ "x": 0,
+ "y": 20
+ },
+ "id": 103,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod=~\"$target-primary.*\", container!~\"POD|istio-proxy\"}) by (pod)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{ pod }}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Primary: Memory Usage by Pod",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "bytes",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus",
+ "fill": 1,
+ "gridPos": {
+ "h": 6,
+ "w": 12,
+ "x": 12,
+ "y": 20
+ },
+ "id": 104,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod=~\"$target.*\", pod!~\"$target-primary.*\", container!~\"POD|istio-proxy\"}) by (pod)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{ pod }}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Canary: Memory Usage by Pod",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "bytes",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus",
+ "fill": 1,
+ "gridPos": {
+ "h": 6,
+ "w": 12,
+ "x": 0,
+ "y": 26
+ },
+ "id": 105,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "received",
+ "color": "#f9d9f9"
+ },
+ {
+ "alias": "transmited",
+ "color": "#f29191"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate (container_network_receive_bytes_total{namespace=\"$namespace\",pod=~\"$target-primary.*\"}[1m])) ",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "received",
+ "refId": "A"
+ },
+ {
+ "expr": "-sum (rate (container_network_transmit_bytes_total{namespace=\"$namespace\",pod=~\"$target-primary.*\"}[1m]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "transmited",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Primary: Network I/O",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "Bps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "prometheus",
+ "fill": 1,
+ "gridPos": {
+ "h": 6,
+ "w": 12,
+ "x": 12,
+ "y": 26
+ },
+ "id": 106,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "received",
+ "color": "#f9d9f9"
+ },
+ {
+ "alias": "transmited",
+ "color": "#f29191"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate (container_network_receive_bytes_total{namespace=\"$namespace\",pod=~\"$target.*\",pod!~\"$target-primary.*\"}[1m])) ",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "received",
+ "refId": "A"
+ },
+ {
+ "expr": "-sum (rate (container_network_transmit_bytes_total{namespace=\"$namespace\",pod=~\"$target.*\",pod!~\"$target-primary.*\"}[1m]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "transmited",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Canary: Network I/O",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "Bps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "refresh": "10s",
+ "schemaVersion": 16,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "allValue": null,
+ "current": null,
+ "datasource": "prometheus",
+ "definition": "query_result(sum(envoy_cluster_upstream_rq) by (kubernetes_namespace))",
+ "hide": 0,
+ "includeAll": false,
+ "label": "Namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [],
+ "query": "query_result(sum(envoy_cluster_upstream_rq) by (kubernetes_namespace))",
+ "refresh": 1,
+ "regex": "/.*_namespace=\"([^\"]*).*/",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": null,
+ "datasource": "prometheus",
+ "definition": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",envoy_cluster_name=~\".*-primary\"}) by (envoy_cluster_name))",
+ "hide": 0,
+ "includeAll": false,
+ "label": "Target",
+ "multi": false,
+ "name": "target",
+ "options": [],
+ "query": "query_result(sum(envoy_cluster_upstream_rq{kubernetes_namespace=\"$namespace\",envoy_cluster_name=~\".*-primary\"}) by (envoy_cluster_name))",
+ "refresh": 1,
+ "regex": "/.*envoy_cluster_name=\"(.*)-primary\"/",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-5m",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Envoy Canary",
+ "uid": "flagger-envoy",
+ "version": 6
+}
diff --git a/apps/base/victoria-metrics-k8s-stack/dashboards/flux-cluster-dashboard.json b/apps/base/victoria-metrics-k8s-stack/dashboards/flux-cluster-dashboard.json
@@ -0,0 +1,1383 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "iconColor": "red",
+ "name": "flux events",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [
+ "flux"
+ ],
+ "type": "tags"
+ }
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 1,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "mappings": [],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 0
+ },
+ "id": 24,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "value"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\"})",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "Cluster Reconcilers",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "mappings": [],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 0
+ },
+ "id": 28,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "value"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\", ready=\"False\"})",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "Failing Reconcilers",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "mappings": [],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 12,
+ "y": 0
+ },
+ "id": 29,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "value"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"})",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "Kubernetes Manifests Sources",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "mappings": [],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 18,
+ "y": 0
+ },
+ "id": 30,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "value"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\", ready=\"False\"})",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "Failing Sources",
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 1
+ },
+ {
+ "color": "red",
+ "value": 61
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 0,
+ "y": 5
+ },
+ "id": 8,
+ "options": {
+ "displayMode": "gradient",
+ "minVizHeight": 10,
+ "minVizWidth": 0,
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "text": {},
+ "valueMode": "color"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind)",
+ "interval": "",
+ "legendFormat": "{{kind}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Reconciler ops avg. duration",
+ "type": "bargauge"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 1
+ },
+ {
+ "color": "red",
+ "value": 61
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 12,
+ "y": 5
+ },
+ "id": 31,
+ "options": {
+ "displayMode": "gradient",
+ "minVizHeight": 10,
+ "minVizWidth": 0,
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "text": {},
+ "valueMode": "color"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind)",
+ "interval": "",
+ "legendFormat": "{{kind}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Source ops avg. duration",
+ "type": "bargauge"
+ },
+ {
+ "collapsed": false,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 9
+ },
+ "id": 15,
+ "panels": [],
+ "title": "Status",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "filterable": true,
+ "inspect": false
+ },
+ "mappings": [
+ {
+ "options": {
+ "False": {
+ "color": "red",
+ "index": 1,
+ "text": "Not Ready"
+ },
+ "True": {
+ "color": "blue",
+ "index": 0,
+ "text": "Ready"
+ }
+ },
+ "type": "value"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byType",
+ "options": "string"
+ },
+ "properties": [
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "mode": "basic",
+ "type": "color-background"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 22,
+ "w": 12,
+ "x": 0,
+ "y": 10
+ },
+ "id": 33,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "Status"
+ }
+ ]
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\"}",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Cluster reconciliation readiness ",
+ "transformations": [
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": false,
+ "Value": false,
+ "__name__": false,
+ "app": false,
+ "chart_name": false,
+ "chart_source_name": false,
+ "container": false,
+ "customresource_group": false,
+ "customresource_kind": false,
+ "customresource_version": false,
+ "endpoint": false,
+ "exported_namespace": false,
+ "gotk_type": false,
+ "instance": false,
+ "job": false,
+ "kubernetes_namespace": false,
+ "kubernetes_pod_name": false,
+ "namespace": false,
+ "pod": false,
+ "pod_template_hash": false,
+ "revision": false,
+ "service": false,
+ "source_name": false,
+ "status": false,
+ "suspended": false,
+ "type": false
+ },
+ "indexByName": {
+ "Time": 0,
+ "Value": 15,
+ "__name__": 1,
+ "container": 2,
+ "customresource_group": 4,
+ "customresource_kind": 5,
+ "customresource_version": 6,
+ "endpoint": 7,
+ "exported_namespace": 3,
+ "instance": 8,
+ "job": 9,
+ "name": 10,
+ "namespace": 11,
+ "pod": 12,
+ "ready": 13,
+ "service": 14
+ },
+ "renameByName": {
+ "Value": "",
+ "customresource_kind": "Kind",
+ "exported_namespace": "Namespace",
+ "kind": "Kind",
+ "name": "Name",
+ "namespace": "Operator Namespace",
+ "ready": "Status"
+ }
+ }
+ },
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "(Namespace|Kind|Name|Status)"
+ }
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "filterable": true,
+ "inspect": false
+ },
+ "mappings": [
+ {
+ "options": {
+ "False": {
+ "color": "red",
+ "index": 1,
+ "text": "Not Ready"
+ },
+ "True": {
+ "color": "blue",
+ "index": 0,
+ "text": "Ready"
+ }
+ },
+ "type": "value"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byType",
+ "options": "string"
+ },
+ "properties": [
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "mode": "basic",
+ "type": "color-background"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Status"
+ },
+ "properties": [
+ {
+ "id": "noValue",
+ "value": "Ready"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "blue",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 12,
+ "y": 10
+ },
+ "id": 34,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "Status"
+ }
+ ]
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Source acquisition readiness ",
+ "transformations": [
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": false,
+ "Value": false,
+ "__name__": false,
+ "app": false,
+ "bucket_name": false,
+ "container": false,
+ "customresource_group": false,
+ "customresource_kind": false,
+ "customresource_version": false,
+ "endpoint": false,
+ "exported_namespace": false,
+ "gotk_type": false,
+ "instance": false,
+ "job": false,
+ "kubernetes_namespace": false,
+ "kubernetes_pod_name": false,
+ "namespace": false,
+ "pod": false,
+ "pod_template_hash": false,
+ "ready": false,
+ "revision": false,
+ "service": false,
+ "status": false,
+ "suspended": false,
+ "type": false,
+ "url": false
+ },
+ "indexByName": {
+ "Time": 0,
+ "Value": 15,
+ "__name__": 1,
+ "container": 2,
+ "customresource_group": 5,
+ "customresource_kind": 6,
+ "customresource_version": 7,
+ "endpoint": 8,
+ "exported_namespace": 4,
+ "instance": 9,
+ "job": 10,
+ "name": 11,
+ "namespace": 3,
+ "pod": 12,
+ "ready": 13,
+ "service": 14
+ },
+ "renameByName": {
+ "Value": "",
+ "customresource_kind": "Kind",
+ "exported_namespace": "Namespace",
+ "kind": "Kind",
+ "name": "Name",
+ "namespace": "Operator Namespace",
+ "ready": "Status"
+ }
+ }
+ },
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "(Namespace|Kind|Name|Status)"
+ }
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "filterable": true,
+ "inspect": false
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byType",
+ "options": "string"
+ },
+ "properties": [
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "mode": "basic",
+ "type": "color-background"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 12,
+ "y": 21
+ },
+ "id": 36,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "Namespace"
+ }
+ ]
+ },
+ "pluginVersion": "10.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", suspended=\"true\"}",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Suspended Objects",
+ "transformations": [
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": false,
+ "Value": false,
+ "__name__": false,
+ "app": false,
+ "bucket_name": false,
+ "container": false,
+ "customresource_group": false,
+ "customresource_kind": false,
+ "customresource_version": false,
+ "endpoint": false,
+ "exported_namespace": false,
+ "gotk_type": false,
+ "instance": false,
+ "job": false,
+ "kubernetes_namespace": false,
+ "kubernetes_pod_name": false,
+ "namespace": false,
+ "pod": false,
+ "pod_template_hash": false,
+ "ready": false,
+ "revision": false,
+ "service": false,
+ "source_name": false,
+ "status": false,
+ "suspended": false,
+ "type": false,
+ "url": false
+ },
+ "indexByName": {
+ "Time": 0,
+ "Value": 15,
+ "__name__": 1,
+ "container": 2,
+ "customresource_group": 5,
+ "customresource_kind": 6,
+ "customresource_version": 7,
+ "endpoint": 8,
+ "exported_namespace": 4,
+ "instance": 9,
+ "job": 10,
+ "name": 11,
+ "namespace": 3,
+ "pod": 12,
+ "ready": 13,
+ "service": 14
+ },
+ "renameByName": {
+ "customresource_kind": "Kind",
+ "exported_namespace": "Namespace",
+ "name": "Name"
+ }
+ }
+ },
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "(Namespace|Name|Kind)"
+ }
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "collapsed": false,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 32
+ },
+ "id": 17,
+ "panels": [],
+ "title": "Timing",
+ "type": "row"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byValue",
+ "options": {
+ "op": "gte",
+ "reducer": "allIsZero",
+ "value": 0
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": true,
+ "tooltip": true,
+ "viz": false
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byValue",
+ "options": {
+ "op": "gte",
+ "reducer": "allIsNull",
+ "value": 0
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": true,
+ "tooltip": true,
+ "viz": false
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 33
+ },
+ "id": 27,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind, name)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind, name)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{kind}}/{{name}}",
+ "refId": "B"
+ }
+ ],
+ "title": "Cluster reconciliation duration",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byValue",
+ "options": {
+ "op": "gte",
+ "reducer": "allIsZero",
+ "value": 0
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": true,
+ "tooltip": true,
+ "viz": false
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byValue",
+ "options": {
+ "op": "gte",
+ "reducer": "allIsNull",
+ "value": 0
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": true,
+ "tooltip": true,
+ "viz": false
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 41
+ },
+ "id": 35,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind, name)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind, name)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{kind}}/{{name}}",
+ "refId": "B"
+ }
+ ],
+ "title": "Source acquisition duration",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "30s",
+ "schemaVersion": 38,
+ "style": "light",
+ "tags": [
+ "flux"
+ ],
+ "templating": {
+ "list": [
+ {
+ "allValue": "",
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$DS_PROMETHEUS"
+ },
+ "definition": "label_values(gotk_reconcile_condition, namespace)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "operator_namespace",
+ "options": [],
+ "query": {
+ "query": "label_values(gotk_reconcile_condition, namespace)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 5,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$DS_PROMETHEUS"
+ },
+ "definition": "label_values(gotk_resource_info,exported_namespace)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "namespace",
+ "options": [],
+ "query": {
+ "query": "label_values(gotk_resource_info,exported_namespace)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Datasource",
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-15m",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "",
+ "title": "Flux Cluster Stats",
+ "uid": "flux-cluster",
+ "version": 4,
+ "weekStart": ""
+}
diff --git a/apps/base/victoria-metrics-k8s-stack/dashboards/flux-control-plane-dashboard.json b/apps/base/victoria-metrics-k8s-stack/dashboards/flux-control-plane-dashboard.json
@@ -0,0 +1,1730 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "iconColor": "red",
+ "name": "flux events",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [
+ "flux"
+ ],
+ "type": "tags"
+ }
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 1,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 0
+ },
+ "id": 24,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "value"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(go_info{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
+ "interval": "",
+ "legendFormat": "pods",
+ "refId": "A"
+ }
+ ],
+ "title": "Controllers",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 50
+ },
+ {
+ "color": "red",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 0
+ },
+ "id": 23,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "max(workqueue_longest_running_processor_seconds{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "seconds",
+ "refId": "B"
+ }
+ ],
+ "title": "Max Work Queue",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 500000000
+ },
+ {
+ "color": "red",
+ "value": 900000000
+ }
+ ]
+ },
+ "unit": "decbits"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 12,
+ "y": 0
+ },
+ "id": 25,
+ "options": {
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true,
+ "text": {}
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(go_memstats_alloc_bytes{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Memory",
+ "type": "gauge"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 50
+ },
+ {
+ "color": "red",
+ "value": 100
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 18,
+ "y": 0
+ },
+ "id": 26,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m]))",
+ "interval": "",
+ "legendFormat": "requests",
+ "refId": "A"
+ }
+ ],
+ "title": "API Requests",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 5
+ },
+ "id": 21,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\"}[1m]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "total",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",code!~\"2..\"}[1m]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "errors",
+ "refId": "B"
+ }
+ ],
+ "title": "Kubernetes API Requests",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 13
+ },
+ "id": 15,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Resource Usage",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 0,
+ "y": 14
+ },
+ "id": 11,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(process_cpu_seconds_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m])",
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "CPU Usage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 0,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 12,
+ "y": 14
+ },
+ "id": 13,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"POD\",container!=\"\",pod=~\".*-controller-.*\"}) by (pod)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Memory Usage",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 25
+ },
+ "id": 17,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Reconciliation Stats",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 26
+ },
+ "id": 27,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "workqueue_longest_running_processor_seconds{name=\"kustomization\"}",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "kustomizations",
+ "refId": "B"
+ }
+ ],
+ "title": "Cluster Reconciliation Duration",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 34
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result!=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful reconciliations ",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed reconciliations ",
+ "refId": "B"
+ }
+ ],
+ "title": "Cluster Reconciliations ops/min",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 42
+ },
+ "id": 29,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Sources Stats",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 43
+ },
+ "id": 4,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result!=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful git pulls",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed git pulls",
+ "refId": "B"
+ }
+ ],
+ "title": "Git Repos ops/min",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 43
+ },
+ "id": 30,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result!=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful oci pulls",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed oci pulls",
+ "refId": "B"
+ }
+ ],
+ "title": "OCI Repos ops/min",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 52
+ },
+ "id": 31,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result!=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful helm pulls",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed helm pulls",
+ "refId": "B"
+ }
+ ],
+ "title": "Helm Repos ops/min",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 52
+ },
+ "id": 32,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result!=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful bucket pulls",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed bucket pulls",
+ "refId": "B"
+ }
+ ],
+ "title": "Buckets ops/min",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 61
+ },
+ "id": 19,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Helm Stats",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 62
+ },
+ "id": 9,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
+ "hide": true,
+ "interval": "",
+ "legendFormat": "P50",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
+ "hide": true,
+ "interval": "",
+ "legendFormat": "P90",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "P99",
+ "refId": "C"
+ }
+ ],
+ "title": "Helm Release Duration",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 70
+ },
+ "id": 5,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result!=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful reconciliations ",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed reconciliations ",
+ "refId": "B"
+ }
+ ],
+ "title": "Helm Releases ops/min",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 70
+ },
+ "id": 6,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result!=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful chart pulls",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed chart pulls",
+ "refId": "B"
+ }
+ ],
+ "title": "Helm Charts ops/min",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "10s",
+ "schemaVersion": 38,
+ "style": "light",
+ "tags": [
+ "flux"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 2,
+ "includeAll": false,
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "flux-system",
+ "value": "flux-system"
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "workqueue_work_duration_seconds_count",
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "namespace",
+ "options": [],
+ "query": {
+ "query": "workqueue_work_duration_seconds_count",
+ "refId": "Prometheus-namespace-Variable-Query"
+ },
+ "refresh": 2,
+ "regex": "/.*namespace=\"([^\"]*).*/",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-15m",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "",
+ "title": "Flux Control Plane",
+ "uid": "flux-control-plane",
+ "version": 2,
+ "weekStart": ""
+}
diff --git a/apps/base/victoria-metrics-k8s-stack/dashboards/flux-logs-dashboard.json b/apps/base/victoria-metrics-k8s-stack/dashboards/flux-logs-dashboard.json
@@ -0,0 +1,332 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_LOKI",
+ "label": "Loki",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "loki",
+ "pluginName": "Loki"
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "iconColor": "red",
+ "name": "flux events",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [
+ "flux"
+ ],
+ "type": "tags"
+ }
+ }
+ ]
+ },
+ "description": "Flux logs collected from Kubernetes, stored in Loki",
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": 29,
+ "iteration": 1653748775696,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": "${DS_LOKI}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 4,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": "${DS_LOKI}",
+ "expr": "sum(count_over_time({namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\" [$__interval]))",
+ "instant": false,
+ "legendFormat": "Log count",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_LOKI}",
+ "description": "Logs from services running in Kubernetes",
+ "gridPos": {
+ "h": 25,
+ "w": 24,
+ "x": 0,
+ "y": 4
+ },
+ "id": 2,
+ "options": {
+ "dedupStrategy": "numbers",
+ "enableLogDetails": false,
+ "prettifyLogMessage": true,
+ "showCommonLabels": false,
+ "showLabels": false,
+ "showTime": false,
+ "sortOrder": "Descending",
+ "wrapLogMessage": false
+ },
+ "targets": [
+ {
+ "datasource": "${DS_LOKI}",
+ "expr": "{namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\"",
+ "refId": "A"
+ }
+ ],
+ "type": "logs"
+ }
+ ],
+ "refresh": "10s",
+ "schemaVersion": 36,
+ "style": "light",
+ "tags": [
+ "flux"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "",
+ "value": ""
+ },
+ "description": "String to search for",
+ "hide": 0,
+ "label": "Search Query",
+ "name": "query",
+ "options": [
+ {
+ "selected": true,
+ "text": "",
+ "value": ""
+ }
+ ],
+ "query": "",
+ "skipUrlSync": false,
+ "type": "textbox"
+ },
+ {
+ "allValue": "info|error",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "hide": 0,
+ "includeAll": true,
+ "multi": false,
+ "name": "level",
+ "options": [
+ {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ {
+ "selected": false,
+ "text": "info",
+ "value": "info"
+ },
+ {
+ "selected": false,
+ "text": "error",
+ "value": "error"
+ }
+ ],
+ "query": "info,error",
+ "queryValue": "",
+ "skipUrlSync": false,
+ "type": "custom"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": "${DS_LOKI}",
+ "definition": "label_values(app)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "controller",
+ "options": [],
+ "query": "label_values(app)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "type": "query"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": [
+ "flux-system"
+ ],
+ "value": [
+ "flux-system"
+ ]
+ },
+ "datasource": "${DS_LOKI}",
+ "definition": "label_values(namespace)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "namespace",
+ "options": [],
+ "query": "label_values(namespace)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "type": "query"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "${DS_LOKI}",
+ "definition": "label_values(stream)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "stream",
+ "options": [],
+ "query": "label_values(stream)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "type": "query"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "Loki",
+ "value": "Loki"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Datasource",
+ "multi": false,
+ "name": "DS_LOKI",
+ "options": [],
+ "query": "loki",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "Flux Logs",
+ "uid": "flux-logs",
+ "version": 2
+}
diff --git a/apps/base/victoria-metrics-k8s-stack/dashboards/synapse.json b/apps/base/victoria-metrics-k8s-stack/dashboards/synapse.json
@@ -0,0 +1,13473 @@
+{
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "9.2.2"
+ },
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph (old)",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "heatmap",
+ "name": "Heatmap",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "enable": false,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "limit": 100,
+ "name": "Annotations & Alerts",
+ "showIn": 0,
+ "type": "dashboard"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "enable": true,
+ "expr": "changes(process_start_time_seconds{instance=\"$instance\",job=~\"synapse\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"$instance\",job=\"synapse\"}",
+ "iconColor": "purple",
+ "name": "deploys",
+ "titleFormat": "Deployed {{version}}"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [
+ {
+ "asDropdown": false,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "matrix"
+ ],
+ "title": "Dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 73,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Overview",
+ "type": "row"
+ },
+ {
+ "cards": {
+ "cardPadding": -1,
+ "cardRound": 0
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 1
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 189,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellRadius": 0,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Event Send Time (excluding errors, all workers)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "format": "s",
+ "logBase": 2,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 152,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Avg",
+ "fill": 0,
+ "linewidth": 3
+ },
+ {
+ "alias": "99%",
+ "color": "#C4162A",
+ "fillBelowTo": "90%"
+ },
+ {
+ "alias": "90%",
+ "color": "#FF7383",
+ "fillBelowTo": "75%"
+ },
+ {
+ "alias": "75%",
+ "color": "#FFEE52",
+ "fillBelowTo": "50%"
+ },
+ {
+ "alias": "50%",
+ "color": "#73BF69",
+ "fillBelowTo": "25%"
+ },
+ {
+ "alias": "25%",
+ "color": "#1F60C4",
+ "fillBelowTo": "5%"
+ },
+ {
+ "alias": "5%",
+ "lines": false
+ },
+ {
+ "alias": "Average",
+ "color": "rgb(255, 255, 255)",
+ "lines": true,
+ "linewidth": 3
+ },
+ {
+ "alias": "Events",
+ "color": "#B877D9",
+ "hideTooltip": true,
+ "points": true,
+ "yaxis": 2,
+ "zindex": -3
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "legendFormat": "25%",
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "legendFormat": "5%",
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))",
+ "legendFormat": "Average",
+ "refId": "H"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "Events",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:283",
+ "colorMode": "warning",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:284",
+ "colorMode": "critical",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 2,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Event Send Time Quantiles (excluding errors, all workers)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:255",
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:256",
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 10
+ },
+ "hiddenSeries": false,
+ "id": 75,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 3,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} ",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:566",
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "CPU usage",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:538",
+ "format": "percentunit",
+ "logBase": 1,
+ "max": "1.5",
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:539",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 10
+ },
+ "hiddenSeries": false,
+ "id": 198,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 3,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}} {{index}}",
+ "refId": "A",
+ "step": 20,
+ "target": ""
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+ "hide": true,
+ "interval": "",
+ "legendFormat": "total",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memory",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "transformations": [],
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1560",
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1561",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "axisSoftMax": 1,
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 10,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 19
+ },
+ "id": 245,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1",
+ "legendFormat": "version {{version}}",
+ "range": true,
+ "refId": "deployed_synapse_versions"
+ }
+ ],
+ "title": "Deployed Synapse versions over time",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 19
+ },
+ "hiddenSeries": false,
+ "id": 37,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:639",
+ "alias": "/max$/",
+ "color": "#890F02",
+ "fill": 0,
+ "legend": false
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": true,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} max",
+ "refId": "B",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Open FDs",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:650",
+ "format": "none",
+ "label": "",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:651",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 26
+ },
+ "id": 54,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 27
+ },
+ "hiddenSeries": false,
+ "id": 5,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 3,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:1240",
+ "alias": "/user/"
+ },
+ {
+ "$$hashKey": "object:1241",
+ "alias": "/system/"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} system ",
+ "metric": "",
+ "range": true,
+ "refId": "B",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} user",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:1278",
+ "colorMode": "custom",
+ "fillColor": "rgba(255, 255, 255, 1)",
+ "line": true,
+ "lineColor": "rgba(216, 200, 27, 0.27)",
+ "op": "gt",
+ "value": 0.5,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:1279",
+ "colorMode": "custom",
+ "fillColor": "rgba(255, 255, 255, 1)",
+ "line": true,
+ "lineColor": "rgb(87, 6, 16)",
+ "op": "gt",
+ "value": 0.8,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:1498",
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "CPU",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1250",
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1.2",
+ "min": 0,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1251",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 27
+ },
+ "id": 105,
+ "interval": "",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.3.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.999, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{job}}-{{index}} 99.9%",
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} 99%",
+ "refId": "A",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 95%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} mean",
+ "refId": "D"
+ }
+ ],
+ "title": "Reactor tick quantiles",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 34
+ },
+ "hiddenSeries": false,
+ "id": 34,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}} {{index}}",
+ "refId": "A",
+ "step": 20,
+ "target": ""
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+ "interval": "",
+ "legendFormat": "total",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memory",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "transformations": [],
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 34
+ },
+ "hiddenSeries": false,
+ "id": 49,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/^up/",
+ "legend": false,
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Prometheus scrape time",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "decimals": 0,
+ "format": "none",
+ "label": "",
+ "logBase": 1,
+ "max": "0",
+ "min": "-1",
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 41
+ },
+ "hiddenSeries": false,
+ "id": 53,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:116",
+ "alias": "/^version .*/",
+ "lines": true,
+ "linewidth": 6,
+ "points": false
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1",
+ "hide": false,
+ "legendFormat": "version {{version}}",
+ "range": true,
+ "refId": "deployed_synapse_versions"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Up",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 41
+ },
+ "hiddenSeries": false,
+ "id": 120,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [
+ {
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Stacked CPU usage",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:572",
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:573",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 48
+ },
+ "hiddenSeries": false,
+ "id": 136,
+ "interval": "",
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_http_client_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "legendFormat": "{{job}}-{{index}} {{method}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_http_matrixfederationclient_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "legendFormat": "{{job}}-{{index}} {{method}} (federation)",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Outgoing HTTP request rate",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:123",
+ "format": "reqps",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:124",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "active threads",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 48
+ },
+ "id": 207,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "synapse_threadpool_working_threads{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Threadpool activity",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Process info",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 27
+ },
+ "id": 56,
+ "panels": [
+ {
+ "cards": {
+ "cardPadding": -1,
+ "cardRound": 0
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 28
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 85,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellRadius": 0,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Event Send Time (Including errors, across all workers)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "format": "s",
+ "logBase": 2,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 28
+ },
+ "hiddenSeries": false,
+ "id": 33,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size])) without (job,index)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 20,
+ "target": ""
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events Persisted (all workers)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:102",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:103",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 1,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 40,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_events_persisted_by_source_type{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{type}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events/s Local vs Remote",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 1,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 46,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_events_persisted_by_event_type{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 2,
+ "legendFormat": "{{type}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events/s by Type",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {
+ "irc-freenode (local)": "#EAB839"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 1,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 44,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_events_persisted_by_origin{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{origin_entity}} ({{origin_type}})",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events/s by Origin",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 1,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 45,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{origin_entity}} ({{origin_type}})",
+ "range": true,
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memberships/s by Origin",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:232",
+ "format": "hertz",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:233",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 51
+ },
+ "hiddenSeries": false,
+ "id": 118,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeatDirection": "h",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:316",
+ "alias": "mean",
+ "linewidth": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 99%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 95%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 50%",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} mean",
+ "range": true,
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Event send time quantiles by worker",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:263",
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:264",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "CPU and DB time spent on most expensive state resolution in a room, summed over all workers. This is a very rough proxy for \"how fast is state res\", but it doesn't accurately represent the system load (e.g. it completely ignores cheap state resolutions).\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 30,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s/s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 51
+ },
+ "id": 222,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": false,
+ "expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "legendFormat": "DB time",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": false,
+ "expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "legendFormat": "CPU time",
+ "refId": "C"
+ }
+ ],
+ "title": "Stateres worst-case",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Event persistence",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 28
+ },
+ "id": 57,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 2,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 4,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:234",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(216, 200, 27, 0.27)",
+ "op": "gt",
+ "value": 100,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:235",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(234, 112, 112, 0.22)",
+ "op": "gt",
+ "value": 250,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Request Count by arrival time",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:206",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:207",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 32,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{method}} {{servlet}} {{job}}-{{index}}",
+ "refId": "A",
+ "step": 20,
+ "target": ""
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Top 10 Request Counts",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:305",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:306",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 2,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 139,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:135",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(216, 200, 27, 0.27)",
+ "op": "gt",
+ "value": 100,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:136",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(234, 112, 112, 0.22)",
+ "op": "gt",
+ "value": 250,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Total CPU Usage by Endpoint",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:107",
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:108",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 52,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:417",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(216, 200, 27, 0.27)",
+ "op": "gt",
+ "value": 100,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:418",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(234, 112, 112, 0.22)",
+ "op": "gt",
+ "value": 250,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Average CPU Usage by Endpoint",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:389",
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:390",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 7,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "DB Usage by endpoint",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:488",
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:489",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 2,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 47,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": true,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "(sum(rate(synapse_http_server_response_time_seconds_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))/(sum(rate(synapse_http_server_response_time_seconds_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Non-sync avg response time",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 53
+ },
+ "hiddenSeries": false,
+ "id": 103,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Total",
+ "color": "rgb(255, 255, 255)",
+ "fill": 0,
+ "linewidth": 3
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "topk(10,synapse_http_server_in_flight_requests_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Total",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Requests in flight",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Requests",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 29
+ },
+ "id": 97,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 99,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "CPU usage by background jobs",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 101,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_background_process_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "DB usage by background jobs (including scheduling time)",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 138,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Background jobs in flight",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Background jobs",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 30
+ },
+ "id": 81,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 79,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "successful txn rate",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_util_metrics_block_count_total{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))",
+ "legendFormat": "failed txn rate",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Outgoing federation transaction rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 83,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_server_received_pdus_total{instance=~\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "pdus",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_server_received_edus_total{instance=~\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "edus",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Incoming PDU/EDU rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 40
+ },
+ "hiddenSeries": false,
+ "id": 109,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations_count_total{instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "pdus",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_client_sent_edus_total{instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "edus",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Outgoing PDU/EDU rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 40
+ },
+ "hiddenSeries": false,
+ "id": 111,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_federation_client_sent_edus_by_type_total{instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Outgoing EDUs by type",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:462",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:463",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Triangular growth may indicate a problem with federation sending from the remote host --- but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "axisSoftMax": 60,
+ "axisSoftMin": 0,
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 60
+ }
+ ]
+ },
+ "unit": "m"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "libera.chat "
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 49
+ },
+ "id": 243,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_received_pdu_time[10m]))) / 60",
+ "instant": false,
+ "legendFormat": "{{server_name}} ",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Age of last PDU received from nominated hosts",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Triangular growth may indicate a problem with federation senders on the monitored instance---but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "axisSoftMax": 60,
+ "axisSoftMin": 0,
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 60
+ }
+ ]
+ },
+ "unit": "m"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "libera.chat"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 49
+ },
+ "id": 241,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_sent_pdu_time[10m]))) / 60",
+ "instant": false,
+ "legendFormat": "{{server_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Age of last PDU sent to nominated hosts",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "The number of events in the in-memory queues ",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 57
+ },
+ "hiddenSeries": false,
+ "id": 142,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "pending PDUs {{job}}-{{index}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "pending EDUs {{job}}-{{index}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "In-memory federation transmission queues",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:547",
+ "format": "short",
+ "label": "events",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:548",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Number of events queued up on the master process for processing by the federation sender",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 57
+ },
+ "hiddenSeries": false,
+ "id": 140,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "presence changed",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "presence map",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "presence destinations",
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "keyed edus",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "other edus",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "stream positions",
+ "refId": "F"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Outgoing EDU queues on master",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": -1
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 66
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 166,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellValues": {
+ "decimals": 2
+ },
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "min": 0,
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{instance=\"$instance\",name=\"federation_sender\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{ le }}",
+ "refId": "A"
+ }
+ ],
+ "title": "Federation send PDU lag",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "tooltipDecimals": 2,
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 66
+ },
+ "hiddenSeries": false,
+ "id": 162,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Avg",
+ "fill": 0,
+ "linewidth": 3
+ },
+ {
+ "alias": "99%",
+ "color": "#C4162A",
+ "fillBelowTo": "90%"
+ },
+ {
+ "alias": "90%",
+ "color": "#FF7383",
+ "fillBelowTo": "75%"
+ },
+ {
+ "alias": "75%",
+ "color": "#FFEE52",
+ "fillBelowTo": "50%"
+ },
+ {
+ "alias": "50%",
+ "color": "#73BF69",
+ "fillBelowTo": "25%"
+ },
+ {
+ "alias": "25%",
+ "color": "#1F60C4",
+ "fillBelowTo": "5%"
+ },
+ {
+ "alias": "5%",
+ "lines": false
+ },
+ {
+ "alias": "Average",
+ "color": "rgb(255, 255, 255)",
+ "lines": true,
+ "linewidth": 3
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "interval": "",
+ "legendFormat": "25%",
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "interval": "",
+ "legendFormat": "5%",
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Average",
+ "refId": "H"
+ }
+ ],
+ "thresholds": [
+ {
+ "colorMode": "warning",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 0.25,
+ "yaxis": "left"
+ },
+ {
+ "colorMode": "critical",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Federation send PDU lag quantiles",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": -1
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 75
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 164,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellValues": {
+ "decimals": 2
+ },
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "min": 0,
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{instance=\"$instance\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{ le }}",
+ "refId": "A"
+ }
+ ],
+ "title": "Handle inbound PDU time",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "tooltipDecimals": 2,
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 75
+ },
+ "hiddenSeries": false,
+ "id": 203,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}}",
+ "range": true,
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Age of oldest event in staging area",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:92",
+ "format": "ms",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:93",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 84
+ },
+ "hiddenSeries": false,
+ "id": 202,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}}",
+ "range": true,
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Number of events in federation staging area",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:92",
+ "format": "none",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:93",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 84
+ },
+ "hiddenSeries": false,
+ "id": 205,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "soft-failed events",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Soft-failed event rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:131",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:132",
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Federation",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 31
+ },
+ "id": 227,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 154
+ },
+ "id": 239,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_reject_total{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of rate limit rejected requests",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 154
+ },
+ "id": 235,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of requests being slept by the rate limiter",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Why is the data zero (0)? https://github.com/matrix-org/synapse/pull/13541#discussion_r951926322",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 162
+ },
+ "id": 237,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_reject_affected_hosts{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of hosts being rejected by the rate limiter",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "https://github.com/matrix-org/synapse/pull/13541",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 162
+ },
+ "id": 233,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_sleep_affected_hosts{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of hosts being slept by the rate limiter",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 170
+ },
+ "hiddenSeries": false,
+ "id": 229,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:276",
+ "alias": "Avg",
+ "fill": 0,
+ "linewidth": 3
+ },
+ {
+ "$$hashKey": "object:277",
+ "alias": "99%",
+ "color": "#C4162A",
+ "fillBelowTo": "90%"
+ },
+ {
+ "$$hashKey": "object:278",
+ "alias": "90%",
+ "color": "#FF7383",
+ "fillBelowTo": "75%"
+ },
+ {
+ "$$hashKey": "object:279",
+ "alias": "75%",
+ "color": "#FFEE52",
+ "fillBelowTo": "50%"
+ },
+ {
+ "$$hashKey": "object:280",
+ "alias": "50%",
+ "color": "#73BF69",
+ "fillBelowTo": "25%"
+ },
+ {
+ "$$hashKey": "object:281",
+ "alias": "25%",
+ "color": "#1F60C4",
+ "fillBelowTo": "5%"
+ },
+ {
+ "$$hashKey": "object:282",
+ "alias": "5%",
+ "lines": false
+ },
+ {
+ "$$hashKey": "object:283",
+ "alias": "Average",
+ "color": "rgb(255, 255, 255)",
+ "lines": true,
+ "linewidth": 3
+ },
+ {
+ "$$hashKey": "object:284",
+ "alias": ">99%",
+ "color": "#B877D9",
+ "fill": 3,
+ "lines": true
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9995, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": ">99%",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.9, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.25, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "legendFormat": "25%",
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.05, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "legendFormat": "5%",
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_rate_limit_queue_wait_time_seconds_sum{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_rate_limit_queue_wait_time_seconds_count{index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "legendFormat": "Average",
+ "refId": "H"
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:283",
+ "colorMode": "warning",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:284",
+ "colorMode": "critical",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 2,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Rate limit queue wait time Quantiles (all workers)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:255",
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:256",
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Default reject threshold (50 requests within a second)"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "red",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.lineWidth",
+ "value": 2
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 170
+ },
+ "id": 231,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "50",
+ "hide": false,
+ "legendFormat": "Default reject threshold (50 requests within a second)",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Rate of requests being slept by the rate limiter",
+ "type": "timeseries"
+ }
+ ],
+ "title": "Federation rate limiter",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 32
+ },
+ "id": 60,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 155
+ },
+ "id": 51,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_http_httppusher_http_pushes_processed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "processed {{job}}-{{index}}",
+ "range": true,
+ "refId": "A",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_http_httppusher_http_pushes_failed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "failed {{job}}-{{index}}",
+ "range": true,
+ "refId": "B",
+ "step": 20
+ }
+ ],
+ "title": "HTTP Push rate",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 155
+ },
+ "hiddenSeries": false,
+ "id": 134,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})",
+ "legendFormat": "{{kind}} {{app_id}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Active pusher instances by app",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Pushes",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 33
+ },
+ "id": 219,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How many entries in current state that we are iterating over while calculating push rules.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 33
+ },
+ "hiddenSeries": false,
+ "id": 209,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{index}}",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Iterations over State",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Rate that the cached push rules for a room get invalidated due to underlying push rules being changed. ",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 33
+ },
+ "hiddenSeries": false,
+ "id": 211,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{index}}",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Push Rule Invalidations",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How often the \"delta optimisation\" happens.\n\nThe delta optimisation is when we update the push rules for a room incrementally after a state change where we know the delta between the old state and the new state.\n\nThis can't happen if we don't the delta or we're calculating push rules from scratch.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 40
+ },
+ "hiddenSeries": false,
+ "id": 213,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Number of calls",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Hit Rate",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
+ "refId": "A",
+ "step": 2
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Number of calls",
+ "refId": "B",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Delta Optimisation",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How often we have the correct cached push rules for a room.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 40
+ },
+ "hiddenSeries": false,
+ "id": 215,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Number of calls",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Hit Rate",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
+ "refId": "A",
+ "step": 2
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Number of calls",
+ "refId": "B",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "How often we reuse existing calculated push rules",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How often we have existing cached push rules for the room. \n\nNote that these might be outdated and need to be recalculated if the state has changed.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 47
+ },
+ "hiddenSeries": false,
+ "id": 217,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Number of calls",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Hit Rate",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
+ "refId": "A",
+ "step": 2
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Number of calls",
+ "refId": "B",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "How often we have the RulesForRoom cached",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Push Rule Cache",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 34
+ },
+ "id": 58,
+ "panels": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 35
+ },
+ "id": 48,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "title": "Avg time waiting for db conn",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 104,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{job}} {{index}} 99%",
+ "refId": "A",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}} {{index}} 95%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}} {{index}} 90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}} {{index}} mean",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Db scheduling time quantiles",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 42
+ },
+ "hiddenSeries": false,
+ "id": 10,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "topk(10, rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{desc}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Top DB transactions by txn rate",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 42
+ },
+ "hiddenSeries": false,
+ "id": 11,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{desc}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "DB transactions by total txn time",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 49
+ },
+ "hiddenSeries": false,
+ "id": 180,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": false
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{desc}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average DB txn time",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 6,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 49
+ },
+ "hiddenSeries": false,
+ "id": 200,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Time waiting for DB connection quantiles",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:203",
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:204",
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Database",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 35
+ },
+ "id": 59,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 0,
+ "y": 158
+ },
+ "hiddenSeries": false,
+ "id": 12,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Total CPU Usage by Block",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 12,
+ "y": 158
+ },
+ "hiddenSeries": false,
+ "id": 26,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])) / rate(synapse_util_metrics_block_count_total[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average CPU Time per Block",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 0,
+ "y": 171
+ },
+ "hiddenSeries": false,
+ "id": 13,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Total DB Usage by Block",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:196",
+ "format": "percentunit",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:197",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "The time each database transaction takes to execute, on average, broken down by metrics block.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 12,
+ "y": 171
+ },
+ "hiddenSeries": false,
+ "id": 27,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average Database Transaction time, by Block",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 0,
+ "y": 184
+ },
+ "hiddenSeries": false,
+ "id": 28,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average Transactions per Block",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 12,
+ "y": 184
+ },
+ "hiddenSeries": false,
+ "id": 25,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_metrics_block_time_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average Wallclock Time per Block",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:180",
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:181",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 15,
+ "w": 12,
+ "x": 0,
+ "y": 197
+ },
+ "hiddenSeries": false,
+ "id": 154,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Block count",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Per-block metrics",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 36
+ },
+ "id": 61,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 0,
+ "y": 36
+ },
+ "hiddenSeries": false,
+ "id": 1,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache Hit Ratio",
+ "tooltip": {
+ "msResolution": true,
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1",
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 12,
+ "y": 36
+ },
+ "hiddenSeries": false,
+ "id": 8,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_util_caches_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache Size",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 0,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 38,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache request rate",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "rps",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 12,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 39,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "max",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "topk(10, rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Top 10 cache misses",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:101",
+ "format": "rps",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:102",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 56
+ },
+ "hiddenSeries": false,
+ "id": 65,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_caches_cache_evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}} ({{reason}}) {{job}}-{{index}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache eviction rate",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": "entries / second",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Caches",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 37
+ },
+ "id": 148,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 146,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_util_caches_response_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Response cache size",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 150,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_caches_response_cache_hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Response cache hit rate",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Response caches",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 38
+ },
+ "id": 62,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 91,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Total GC time by bucket (10m smoothing)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 3,
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 21,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}} {{index}} gen {{gen}} ",
+ "refId": "A",
+ "step": 20,
+ "target": ""
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average GC Time Per Collection",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 89,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/gen 0$/",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Allocation counts",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Gen N-1 GCs since last Gen N GC",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": "Objects since last Gen 0 GC",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 93,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Object counts per collection",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 48
+ },
+ "hiddenSeries": false,
+ "id": 95,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "GC frequency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": 0
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateSpectral",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 48
+ },
+ "heatmap": {},
+ "hideZeroBuckets": true,
+ "highlightCards": true,
+ "id": 87,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "GC durations",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "GC",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 39
+ },
+ "id": 63,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 162
+ },
+ "hiddenSeries": false,
+ "id": 43,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{command}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Rate of outgoing commands",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:89",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:90",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 162
+ },
+ "id": 41,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.4.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "rate(synapse_replication_tcp_resource_stream_updates_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{stream_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "title": "Rate of outgoing RDATA commands, by stream",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 169
+ },
+ "id": 42,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.4.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{command}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "title": "Rate of incoming commands (including echoes)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "axisSoftMin": 1,
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 169
+ },
+ "id": 220,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.4.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "rate(synapse_replication_tcp_protocol_inbound_rdata_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{stream_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "title": "Rate of incoming RDATA commands (excluding echoes), by stream",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 176
+ },
+ "hiddenSeries": false,
+ "id": 144,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_replication_tcp_command_queue{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "{{stream_name}} {{job}}-{{index}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Queued incoming RDATA commands, by stream",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:218",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:219",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 176
+ },
+ "hiddenSeries": false,
+ "id": 115,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_replication_tcp_protocol_close_reason_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{reason_type}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Replication connection close reasons",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:260",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:261",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 183
+ },
+ "hiddenSeries": false,
+ "id": 113,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{stream_name}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Replication connections",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Replication",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 40
+ },
+ "id": 69,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 163
+ },
+ "hiddenSeries": false,
+ "id": 67,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - on() group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Event processing lag",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "events",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 163
+ },
+ "hiddenSeries": false,
+ "id": 71,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Age of last processed event",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 172
+ },
+ "hiddenSeries": false,
+ "id": 121,
+ "interval": "",
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "deriv(synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Event processing catchup rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": "fallbehind(-) / catchup(+): s/sec",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Event processing loop positions",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 41
+ },
+ "id": 126,
+ "panels": [
+ {
+ "cards": {
+ "cardPadding": 0
+ },
+ "color": {
+ "cardColor": "#B877D9",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "opacity"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 42
+ },
+ "heatmap": {},
+ "hideZeroBuckets": true,
+ "highlightCards": true,
+ "id": 122,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": 1,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#B877D9",
+ "min": 0,
+ "mode": "opacity",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "short"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of rooms, by number of forward extremities in room",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 42
+ },
+ "hiddenSeries": false,
+ "id": 124,
+ "interval": "",
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0",
+ "format": "heatmap",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Room counts, by number of extremities",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": "Number of rooms",
+ "logBase": 10,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": 0
+ },
+ "color": {
+ "cardColor": "#5794F2",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "opacity"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 50
+ },
+ "heatmap": {},
+ "hideZeroBuckets": true,
+ "highlightCards": true,
+ "id": 127,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": 1,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#5794F2",
+ "min": 0,
+ "mode": "opacity",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "short"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Events persisted, by number of forward extremities in room (heatmap)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 50
+ },
+ "hiddenSeries": false,
+ "id": 128,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events persisted, by number of forward extremities in room (quantiles)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Number of extremities in room",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": 0
+ },
+ "color": {
+ "cardColor": "#FF9830",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "opacity"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 58
+ },
+ "heatmap": {},
+ "hideZeroBuckets": true,
+ "highlightCards": true,
+ "id": 129,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": 1,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#FF9830",
+ "min": 0,
+ "mode": "opacity",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "short"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Events persisted, by number of stale forward extremities in room (heatmap)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "For given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 58
+ },
+ "hiddenSeries": false,
+ "id": 130,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events persisted, by number of stale forward extremities in room (quantiles)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Number of stale forward extremities in room",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": 0
+ },
+ "color": {
+ "cardColor": "#73BF69",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "opacity"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 66
+ },
+ "heatmap": {},
+ "hideZeroBuckets": true,
+ "highlightCards": true,
+ "id": 131,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": 1,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#73BF69",
+ "min": 0,
+ "mode": "opacity",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "short"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "heatmap",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of state resolution performed, by number of state groups involved (heatmap)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 66
+ },
+ "hiddenSeries": false,
+ "id": 132,
+ "interval": "",
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Number of state resolutions performed, by number of state groups involved (quantiles)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Number of state groups",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 74
+ },
+ "hiddenSeries": false,
+ "id": 179,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "State res ",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Potential to prune",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_storage_events_times_pruned_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Pruned",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Stale extremity dropping",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Extremities",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 42
+ },
+ "id": 158,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 156,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:632",
+ "alias": "Max",
+ "color": "#bf1b00",
+ "fill": 0,
+ "linewidth": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "max(synapse_admin_mau_max{instance=\"$instance\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Max",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "max(synapse_admin_mau_current{instance=\"$instance\"})",
+ "hide": false,
+ "legendFormat": "Current",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "MAU Limits",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 160,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_admin_mau_current_mau_by_service{instance=\"$instance\"}",
+ "interval": "",
+ "legendFormat": "{{ app_service }}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "MAU by Appservice",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "MAU",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 43
+ },
+ "id": 177,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 173,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_notifier_users_woken_by_stream_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "{{stream}} {{index}}",
+ "metric": "synapse_notifier",
+ "range": true,
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Notifier Streams Woken",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:734",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:735",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 175,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_handler_presence_get_updates_total{job=~\"$job\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{type}} {{index}}",
+ "range": true,
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Presence Stream Fetch Type Rates",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:819",
+ "format": "hertz",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:820",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Notifier",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 44
+ },
+ "id": 170,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 168,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_appservice_api_sent_events_total{instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{service}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Sent Events rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:177",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:178",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 171,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_appservice_api_sent_transactions_total{instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{exported_service }} {{ service }}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Transactions rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:260",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:261",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Appservices",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 45
+ },
+ "id": 188,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 182,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_handler_presence_notified_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Notified",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_handler_presence_federation_presence_out_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Remote ping",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_handler_presence_presence_updates_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Total updates",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_handler_presence_federation_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Remote updates",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_handler_presence_bump_active_time_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Bump active time",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Presence",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 184,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_handler_presence_state_transition_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{from}} -> {{to}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Presence state transitions",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1090",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1091",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 54
+ },
+ "hiddenSeries": false,
+ "id": 186,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_handler_presence_notify_reason_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{reason}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Presence notify reason",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:165",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:166",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Presence",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 46
+ },
+ "id": 197,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 47
+ },
+ "id": 191,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_external_cache_set{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "External Cache Set Rate",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 47
+ },
+ "hiddenSeries": false,
+ "id": 193,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum without (hit) (rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "External Cache Get Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:390",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:391",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": -1
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 55
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 195,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellValues": {
+ "decimals": 2
+ },
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "min": 0,
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "External Cache Response Time",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "tooltipDecimals": 2,
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 55
+ },
+ "id": 223,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\", hit=\"False\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "External Cache Miss Rate",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "External Cache",
+ "type": "row"
+ }
+ ],
+ "refresh": false,
+ "schemaVersion": 37,
+ "style": "dark",
+ "tags": [
+ "matrix"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "label": "Datasource",
+ "options": [],
+ "query": "prometheus",
+ "queryValue": "",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "allFormat": "glob",
+ "auto": true,
+ "auto_count": 100,
+ "auto_min": "30s",
+ "current": {
+ "selected": false,
+ "text": "auto",
+ "value": "$__auto_interval_bucket_size"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Bucket Size",
+ "multi": false,
+ "multiFormat": "glob",
+ "name": "bucket_size",
+ "options": [
+ {
+ "selected": true,
+ "text": "auto",
+ "value": "$__auto_interval_bucket_size"
+ },
+ {
+ "selected": false,
+ "text": "30s",
+ "value": "30s"
+ },
+ {
+ "selected": false,
+ "text": "1m",
+ "value": "1m"
+ },
+ {
+ "selected": false,
+ "text": "2m",
+ "value": "2m"
+ },
+ {
+ "selected": false,
+ "text": "5m",
+ "value": "5m"
+ },
+ {
+ "selected": false,
+ "text": "10m",
+ "value": "10m"
+ },
+ {
+ "selected": false,
+ "text": "15m",
+ "value": "15m"
+ }
+ ],
+ "query": "30s,1m,2m,5m,10m,15m",
+ "queryValue": "",
+ "refresh": 2,
+ "skipUrlSync": false,
+ "type": "interval"
+ },
+ {
+ "current": {},
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "",
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "instance",
+ "options": [],
+ "query": {
+ "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, instance)",
+ "refId": "Prometheus-instance-Variable-Query"
+ },
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allFormat": "regex wildcard",
+ "allValue": "",
+ "current": {},
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "",
+ "hide": 0,
+ "hideLabel": false,
+ "includeAll": true,
+ "label": "Job",
+ "multi": true,
+ "multiFormat": "regex values",
+ "name": "job",
+ "options": [],
+ "query": {
+ "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, job)",
+ "refId": "Prometheus-job-Variable-Query"
+ },
+ "refresh": 2,
+ "refresh_on_load": false,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allFormat": "regex wildcard",
+ "allValue": ".*",
+ "current": {},
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "",
+ "hide": 0,
+ "hideLabel": false,
+ "includeAll": true,
+ "label": "",
+ "multi": true,
+ "multiFormat": "regex values",
+ "name": "index",
+ "options": [],
+ "query": {
+ "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, index)",
+ "refId": "Prometheus-index-Variable-Query"
+ },
+ "refresh": 2,
+ "refresh_on_load": false,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 3,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {
+ "now": true,
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Synapse",
+ "uid": "000000012",
+ "version": 160,
+ "weekStart": ""
+}
diff --git a/apps/base/victoria-metrics-k8s-stack/kustomization.yaml b/apps/base/victoria-metrics-k8s-stack/kustomization.yaml
@@ -8,3 +8,49 @@ resources:
- release.yaml
- minio_loki_metrics.yaml
- prometheus-adapter.yaml
+configMapGenerator:
+ - name: grafana-dashboards-matrix
+ namespace: monitoring
+ files:
+ - draupnir4all_status.json=dashboards/draupnir4all_status.json
+ - synapse.json=dashboards/synapse.json
+ options:
+ labels:
+ grafana_dashboard: "1"
+ annotations:
+ grafana_folder: Matrix
+ - name: grafana-dashboards-flux
+ namespace: monitoring
+ files:
+ - flux-control-plane-dashboard.json=dashboards/flux-control-plane-dashboard.json
+ - flux-cluster-dashboard.json=dashboards/flux-cluster-dashboard.json
+ - flux-logs-dashboard.json=dashboards/flux-logs-dashboard.json
+ - flagger.json:dashboards/flagger.json
+ options:
+ labels:
+ grafana_dashboard: "1"
+ annotations:
+ grafana_folder: Flux
+ - name: grafana-dashboards-ceph
+ namespace: monitoring
+ files:
+ - ceph-cluster-dashboard.json=dashboards/ceph-cluster-dashboard.json
+ - ceph-osd-single-dashboard.json=dashboards/ceph-osd-single-dashboard.json
+ - ceph-pools-dashboard.json=dashboards/ceph-pools-dashboard.json
+ options:
+ labels:
+ grafana_dashboard: "1"
+ annotations:
+ grafana_folder: Ceph
+ - name: grafana-dashboards-envoy
+ namespace: monitoring
+ files:
+ - envoy-clusters.json=dashboards/envoy_clusters.json
+ - envoy-gateway-global.json:dashboards/envoy-gateway-global.json
+ - envoy-proxy-global.json:dashboards/envoy-proxy-global.json
+ - global-ratelimit.json:dashboards/global-ratelimit.json
+ options:
+ labels:
+ grafana_dashboard: "1"
+ annotations:
+ grafana_folder: Envoy
