commit cdc078c551f104e516a5c4c959dd58288e680a3f
parent b782ec94343a0af7fce8ef36435030dd914844aa
Author: MTRNord <mtrnord1@gmail.com>
Date: Wed, 6 Nov 2024 19:49:23 +0100
Sidecar
Diffstat:
3 files changed, 26957 insertions(+), 1042 deletions(-)
diff --git a/apps/base/victoria-metrics-k8s-stack/dashboards.yaml b/apps/base/victoria-metrics-k8s-stack/dashboards.yaml
@@ -0,0 +1,25912 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ labels:
+ grafana_dashboard: "1"
+ name: ceph-cluster-dashboard
+data:
+ dashboard.json: |-
+ {
+ "__inputs": [],
+ "__requires": [
+ {
+ "id": "grafana",
+ "name": "Grafana",
+ "type": "grafana",
+ "version": "5.3.2"
+ },
+ {
+ "id": "graph",
+ "name": "Graph",
+ "type": "panel",
+ "version": "5.0.0"
+ },
+ {
+ "id": "heatmap",
+ "name": "Heatmap",
+ "type": "panel",
+ "version": "5.0.0"
+ },
+ {
+ "id": "singlestat",
+ "name": "Singlestat",
+ "type": "panel",
+ "version": "5.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "showIn": 0,
+ "tags": [],
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Overview of your Ceph cluster.",
+ "editable": false,
+ "gnetId": 2842,
+ "graphTooltip": 0,
+ "hideControls": false,
+ "id": null,
+ "links": [],
+ "panels": [
+ {
+ "collapse": false,
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 2,
+ "panels": [],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "CLUSTER STATE",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "0": {
+ "text": "HEALTHY"
+ },
+ "1": {
+ "text": "WARNING"
+ },
+ "2": {
+ "text": "ERROR"
+ }
+ },
+ "type": "value"
+ },
+ {
+ "id": 1,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#9ac48a"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 2
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 0,
+ "y": 1
+ },
+ "id": 3,
+ "interval": "1m",
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_health_status{}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Ceph health status",
+ "transparent": true,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "Bps"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 3,
+ "y": 1
+ },
+ "id": 4,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_w_in_bytes{}[5m]))",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Write Throughput",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#d44a3a"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0
+ },
+ {
+ "color": "#9ac48a",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "Bps"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 6,
+ "y": 1
+ },
+ "id": 5,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_r_out_bytes{}[5m]))",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Read Throughput",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0.025
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "decbytes"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 9,
+ "y": 1
+ },
+ "id": 6,
+ "interval": "1m",
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_cluster_total_bytes{}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Cluster Capacity",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(245, 54, 54, 0.9)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0.1
+ },
+ {
+ "color": "rgba(50, 172, 45, 0.97)",
+ "value": 0.3
+ }
+ ]
+ },
+ "unit": "percentunit"
+ }
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 3,
+ "x": 12,
+ "y": 1
+ },
+ "id": 7,
+ "interval": "1m",
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_cluster_total_bytes{}-ceph_cluster_total_used_bytes{})/ceph_cluster_total_bytes{}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Available Capacity",
+ "transparent": false,
+ "type": "gauge"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 15,
+ "y": 1
+ },
+ "id": 8,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pool_objects{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of Objects",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 18,
+ "y": 1
+ },
+ "id": 9,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "delta"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_osd_op_w_in_bytes{})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Bytes Written",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 21,
+ "y": 1
+ },
+ "id": 10,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "delta"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_osd_op_r_out_bytes{})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Bytes Read",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#9ac48a"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "#e24d42",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 0,
+ "y": 4
+ },
+ "id": 11,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count(ALERTS{alertstate=\"firing\",alertname=~\"^Ceph.+\"}) OR vector(0)",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Alerts starting with Ceph",
+ "transparent": true,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "ops"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 3,
+ "y": 4
+ },
+ "id": 12,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_w{}[5m]))",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Write IOPS",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#d44a3a"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0
+ },
+ {
+ "color": "#9ac48a",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "ops"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 6,
+ "y": 4
+ },
+ "id": 13,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_r{}[5m]))",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Read IOPS",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0.025
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 0.1
+ }
+ ]
+ },
+ "unit": "decbytes"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 9,
+ "y": 4
+ },
+ "id": 14,
+ "interval": "1m",
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_cluster_total_used_bytes{}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Used Capacity",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#ea6460"
+ },
+ {
+ "color": "#052b51",
+ "value": 0
+ },
+ {
+ "color": "#508642",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 15,
+ "y": 4
+ },
+ "id": 15,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "diff"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pool_objects)",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Difference",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 128
+ }
+ ]
+ },
+ "unit": "short"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 18,
+ "y": 4
+ },
+ "id": 16,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_mon_num_sessions{})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Mon Session Num",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": null
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 2
+ },
+ {
+ "color": "green",
+ "value": 3
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 21,
+ "y": 4
+ },
+ "id": 17,
+ "interval": "1m",
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count(ceph_mon_quorum_status{}) or vector(0)",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Monitors In Quorum",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "collapse": false,
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 7
+ },
+ "id": 18,
+ "panels": [],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "OSD STATE",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#9ac48a"
+ },
+ {
+ "color": "rgba(237, 40, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 0,
+ "y": 8
+ },
+ "id": 19,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count(ceph_osd_up{}) - count(ceph_osd_in{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "OSDs OUT",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "#eab839",
+ "value": 1
+ },
+ {
+ "color": "#ea6460",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 2,
+ "y": 8
+ },
+ "id": 20,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count(ceph_osd_up{} == 0.0) OR vector(0)",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "OSDs DOWN",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 4,
+ "y": 8
+ },
+ "id": 21,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_osd_up{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "OSDs UP",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 6,
+ "y": 8
+ },
+ "id": 22,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_osd_in{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "OSDs IN",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 250
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 300
+ }
+ ]
+ },
+ "unit": "none"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 8,
+ "y": 8
+ },
+ "id": 23,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(ceph_osd_numpg{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Avg PGs",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 10
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 50
+ }
+ ]
+ },
+ "unit": "ms"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 10,
+ "y": 8
+ },
+ "id": 24,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(ceph_osd_apply_latency_ms{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Avg Apply Latency",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 10
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 50
+ }
+ ]
+ },
+ "unit": "ms"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 13,
+ "y": 8
+ },
+ "id": 25,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(ceph_osd_commit_latency_ms{})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Avg Commit Latency",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 4,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "color": "#299c46",
+ "text": "0"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#299c46"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "#d44a3a",
+ "value": 2
+ }
+ ]
+ },
+ "unit": "ms"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 16,
+ "y": 8
+ },
+ "id": 26,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Avg Op Write Latency",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 6,
+ "links": [],
+ "mappings": [
+ {
+ "id": 0,
+ "options": {
+ "match": null,
+ "result": {
+ "color": "#299c46",
+ "text": "0"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#299c46"
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "#d44a3a",
+ "value": 2
+ }
+ ]
+ },
+ "unit": "ms"
+ }
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 19,
+ "y": 8
+ },
+ "id": 27,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(rate(ceph_osd_op_r_latency_sum{}[5m])/rate(ceph_osd_op_r_latency_count{}[5m]) >= 0)",
+ "format": "time_series",
+ "instant": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Avg Op Read Latency",
+ "transparent": false,
+ "type": "stat"
+ },
+ {
+ "collapse": false,
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 9
+ },
+ "id": 28,
+ "panels": [],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "CLUSTER STATS",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 40,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 0,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 2,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Available"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Total Capacity"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#447EBC",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Used"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#BF1B00",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Used"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#BF1B00",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_avail"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_space"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_used"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#890F02",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Total Capacity"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineWidth",
+ "value": 3
+ },
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 10
+ },
+ "id": 29,
+ "interval": "$interval",
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_cluster_total_bytes{}-ceph_cluster_total_used_bytes{}",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Available",
+ "refId": "A",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_cluster_total_used_bytes{}",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Used",
+ "refId": "B",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_cluster_total_bytes{}",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Total Capacity",
+ "refId": "C",
+ "step": 300
+ }
+ ],
+ "title": "Capacity",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Total Capacity"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Used"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#BF1B00",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_avail"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_space"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "total_used"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#890F02",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 10
+ },
+ "id": 30,
+ "interval": "$interval",
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_w{}[5m]))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Write",
+ "refId": "A",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_r{}[5m]))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Read",
+ "refId": "B",
+ "step": 300
+ }
+ ],
+ "title": "IOPS",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 10
+ },
+ "id": 31,
+ "interval": "$interval",
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_w_in_bytes{}[5m]))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Write",
+ "refId": "A",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_op_r_out_bytes{}[5m]))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Read",
+ "refId": "B",
+ "step": 300
+ }
+ ],
+ "title": "Cluster Throughput",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 18
+ },
+ "id": 32,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_bytes_used{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}}",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Pool Used Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 18
+ },
+ "id": 33,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_avail_raw{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}} Avail",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_stored_raw{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}} Stored",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Pool RAW Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 18
+ },
+ "id": 34,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_objects{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Objects Per Pool",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 26
+ },
+ "id": 35,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_quota_bytes{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Pool Quota Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 26
+ },
+ "id": 36,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(ceph_pool_quota_objects{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Pool Objects Quota",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 26
+ },
+ "id": 37,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count(ceph_bluestore_kv_commit_lat_count{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "BlueStore",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Type Count",
+ "type": "timeseries"
+ },
+ {
+ "collapse": true,
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 27
+ },
+ "id": 38,
+ "panels": [
+ {
+ "columns": [],
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "inspect": false
+ },
+ "decimals": 2,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Time"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Time"
+ },
+ {
+ "id": "unit",
+ "value": "time: YYYY-MM-DD HH:mm:ss"
+ },
+ {
+ "id": "custom.align"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 28
+ },
+ "id": 39,
+ "links": [],
+ "options": {
+ "footer": {
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true
+ },
+ "pluginVersion": "9.1.3",
+ "styles": "",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ALERTS{alertstate=\"firing\",alertname=~\"^Ceph.+\"}",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Alerts starting with Ceph",
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducer": []
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "columns": [],
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "filterable": false,
+ "inspect": false
+ },
+ "decimals": 2,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Time"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Time"
+ },
+ {
+ "id": "unit",
+ "value": "time: YYYY-MM-DD HH:mm:ss"
+ },
+ {
+ "id": "custom.align"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 28
+ },
+ "id": 40,
+ "links": [],
+ "options": {
+ "footer": {
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true
+ },
+ "pluginVersion": "9.1.3",
+ "styles": "",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "topk(5,sort_desc(ceph_osd_apply_latency_ms{} + ceph_osd_commit_latency_ms{}))",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "_auto",
+ "refId": "A"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Top Sluggish OSDs",
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducer": []
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "columns": [],
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "inspect": false
+ },
+ "decimals": 2,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Time"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Time"
+ },
+ {
+ "id": "unit",
+ "value": "time: YYYY-MM-DD HH:mm:ss"
+ },
+ {
+ "id": "custom.align"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 28
+ },
+ "id": 41,
+ "links": [],
+ "options": {
+ "footer": {
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true
+ },
+ "pluginVersion": "9.1.3",
+ "styles": "",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_osd_up{} == 0",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Down OSDs",
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducer": []
+ }
+ }
+ ],
+ "type": "table"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Alerts",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "collapse": true,
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 28
+ },
+ "id": 42,
+ "panels": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "scaleDistributionLog": 2,
+ "type": "log"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "id": 43,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "node_memory_Active_anon_bytes{}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(node_memory_Active_anon_bytes{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Cluster Memory Usage",
+ "refId": "B"
+ }
+ ],
+ "title": "Node Memory Usage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "scaleDistributionLog": 2,
+ "type": "log"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "id": 44,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg by(instance)(irate(node_cpu_seconds_total{job='node',mode!=\"idle\"}[$interval])) * 100",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Node CPU Usage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 35
+ },
+ "id": 45,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum by (instance)(irate(node_disk_read_bytes_total{}[$interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Node Out",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 35
+ },
+ "id": 46,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum by (instance)(irate(node_disk_written_bytes_total{}[$interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Node In",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 44
+ },
+ "id": 47,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "(node_filesystem_free_bytes{ mountpoint=\"/\", device != \"rootfs\"})*100 / (node_filesystem_size_bytes{ mountpoint=\"/\", device != \"rootfs\"})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Free Space in root filesystem",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Node Statistics (NodeExporter)",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "collapse": false,
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 45
+ },
+ "id": 48,
+ "panels": [],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "OBJECTS",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Total.*$/"
+ },
+ "properties": [
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 12,
+ "w": 6,
+ "x": 0,
+ "y": 46
+ },
+ "id": 49,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "asc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pool_objects)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Total",
+ "range": true,
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "OSD Type Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Total.*$/"
+ },
+ "properties": [
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 12,
+ "w": 8,
+ "x": 6,
+ "y": 46
+ },
+ "id": 50,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "asc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_active{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Active",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_clean{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Clean",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_peering{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Peering",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_degraded{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Degraded",
+ "range": true,
+ "refId": "D",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_stale{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Stale",
+ "range": true,
+ "refId": "E",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_unclean_pgs{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Unclean",
+ "range": true,
+ "refId": "F",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_undersized{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Undersized",
+ "range": true,
+ "refId": "G",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_incomplete{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Incomplete",
+ "range": true,
+ "refId": "H"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_forced_backfill{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Forced Backfill",
+ "range": true,
+ "refId": "I"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_forced_recovery{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Forced Recovery",
+ "range": true,
+ "refId": "J"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_creating{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Creating",
+ "range": true,
+ "refId": "K"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_wait_backfill{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Wait Backfill",
+ "range": true,
+ "refId": "L"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_deep{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Deep",
+ "range": true,
+ "refId": "M"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_scrubbing{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Scrubbing",
+ "range": true,
+ "refId": "N"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_recovering{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Recovering",
+ "range": true,
+ "refId": "O"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_repair{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Repair",
+ "range": true,
+ "refId": "P"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_down{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Down",
+ "range": true,
+ "refId": "Q"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_peered{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Peered",
+ "range": true,
+ "refId": "R"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_backfill{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Backfill",
+ "range": true,
+ "refId": "S"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_remapped{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Remapped",
+ "range": true,
+ "refId": "T"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_backfill_toofull{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Backfill Toofull",
+ "range": true,
+ "refId": "U"
+ }
+ ],
+ "title": "PGs State",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Total.*$/"
+ },
+ "properties": [
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 10,
+ "x": 14,
+ "y": 46
+ },
+ "id": 51,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "asc"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_degraded{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Degraded",
+ "range": true,
+ "refId": "A",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_stale{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Stale",
+ "range": true,
+ "refId": "B",
+ "step": 300
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(ceph_pg_undersized{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Undersized",
+ "range": true,
+ "refId": "C",
+ "step": 300
+ }
+ ],
+ "title": "Stuck PGs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 10,
+ "x": 14,
+ "y": 52
+ },
+ "id": 52,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "sum(irate(ceph_osd_recovery_ops{}[$interval]))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "OPS",
+ "refId": "A",
+ "step": 300
+ }
+ ],
+ "title": "Recovery Operations",
+ "type": "timeseries"
+ },
+ {
+ "collapse": false,
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 53
+ },
+ "id": 53,
+ "panels": [],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "LATENCY",
+ "titleSize": "h6",
+ "type": "row"
+ },
+ {
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ }
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 54
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 54,
+ "legend": {
+ "show": true
+ },
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ },
+ "value": "1"
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_osd_apply_latency_ms{}",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Apply Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": null,
+ "format": "ms",
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true,
+ "splitFactor": 1
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": 10
+ },
+ {
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#65c5db",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ }
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 54
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 55,
+ "legend": {
+ "show": true
+ },
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#65c5db",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "ceph_osd_commit_latency_ms{}",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Commit Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": null,
+ "format": "ms",
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true,
+ "splitFactor": 1
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
+ },
+ {
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#806eb7",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ }
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 62
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 56,
+ "legend": {
+ "show": true
+ },
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#806eb7",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 2,
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "rate(ceph_osd_op_r_latency_sum{}[5m]) / rate(ceph_osd_op_r_latency_count{}[5m]) >= 0",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Read Op Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": null,
+ "format": "ms",
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true,
+ "splitFactor": 1
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
+ },
+ {
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#f9934e",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ }
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 62
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 57,
+ "legend": {
+ "show": true
+ },
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#f9934e",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 2,
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Write Op Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": 2,
+ "format": "ms",
+ "logBase": 2,
+ "max": null,
+ "min": "0",
+ "show": true,
+ "splitFactor": 1
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 70
+ },
+ "id": 58,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(rate(ceph_osd_op_r_latency_sum{}[5m]) / rate(ceph_osd_op_r_latency_count{}[5m]) >= 0)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Read",
+ "refId": "A"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Write",
+ "refId": "B"
+ }
+ ],
+ "title": "Recovery Operations",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 70
+ },
+ "id": 59,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(ceph_osd_apply_latency_ms{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "apply",
+ "metric": "ceph_osd_perf_apply_latency_seconds",
+ "refId": "A",
+ "step": 4
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "avg(ceph_osd_commit_latency_ms{})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "commit",
+ "metric": "ceph_osd_perf_commit_latency_seconds",
+ "refId": "B",
+ "step": 4
+ }
+ ],
+ "title": "AVG OSD Apply + Commit Latency",
+ "type": "timeseries"
+ },
+ {
+ "collapse": true,
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 71
+ },
+ "id": 60,
+ "panels": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 6,
+ "x": 0,
+ "y": 72
+ },
+ "id": 61,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count by (ceph_version)(ceph_osd_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Ceph OSD Versions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 6,
+ "x": 6,
+ "y": 72
+ },
+ "id": 62,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count by (ceph_version)(ceph_mon_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Ceph Mon Versions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 6,
+ "x": 12,
+ "y": 72
+ },
+ "id": 63,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count by (ceph_version)(ceph_mds_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Ceph MDS Versions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 6,
+ "x": 18,
+ "y": 72
+ },
+ "id": 64,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "expr": "count by (ceph_version)(ceph_rgw_metadata{})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Ceph RGW Versions",
+ "type": "timeseries"
+ }
+ ],
+ "repeat": null,
+ "repeatIteration": null,
+ "repeatRowId": null,
+ "showTitle": true,
+ "title": "Ceph Versions",
+ "titleSize": "h6",
+ "type": "row"
+ }
+ ],
+ "refresh": "1m",
+ "rows": [],
+ "schemaVersion": 37,
+ "style": "dark",
+ "tags": [
+ "ceph-mixin"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 0,
+ "label": "Data Source",
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "auto": true,
+ "auto_count": 10,
+ "auto_min": "1m",
+ "current": {
+ "text": "$__auto_interval_interval",
+ "value": "$__auto_interval_interval"
+ },
+ "hide": 0,
+ "label": "Interval",
+ "name": "interval",
+ "options": [
+ {
+ "selected": true,
+ "text": "auto",
+ "value": "$__auto_interval_interval"
+ },
+ {
+ "selected": false,
+ "text": "5s",
+ "value": "5s"
+ },
+ {
+ "selected": false,
+ "text": "10s",
+ "value": "10s"
+ },
+ {
+ "selected": false,
+ "text": "30s",
+ "value": "30s"
+ },
+ {
+ "selected": false,
+ "text": "1m",
+ "value": "1m"
+ },
+ {
+ "selected": false,
+ "text": "10m",
+ "value": "10m"
+ },
+ {
+ "selected": false,
+ "text": "30m",
+ "value": "30m"
+ },
+ {
+ "selected": false,
+ "text": "1h",
+ "value": "1h"
+ },
+ {
+ "selected": false,
+ "text": "6h",
+ "value": "6h"
+ },
+ {
+ "selected": false,
+ "text": "12h",
+ "value": "12h"
+ },
+ {
+ "selected": false,
+ "text": "1d",
+ "value": "1d"
+ },
+ {
+ "selected": false,
+ "text": "7d",
+ "value": "7d"
+ },
+ {
+ "selected": false,
+ "text": "14d",
+ "value": "14d"
+ },
+ {
+ "selected": false,
+ "text": "30d",
+ "value": "30d"
+ }
+ ],
+ "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
+ "refresh": 2,
+ "type": "interval",
+ "valuelabels": {}
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Ceph Cluster",
+ "uid": "tbO9LAiZK",
+ "version": 0
+ }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ labels:
+ grafana_dashboard: "1"
+ name: ceph-osd-single-dashboard
+data:
+ dashboard.json: |-
+ {
+ "__inputs": [],
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "panel",
+ "id": "gauge",
+ "name": "Gauge",
+ "version": ""
+ },
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "9.0.5"
+ },
+ {
+ "type": "panel",
+ "id": "heatmap",
+ "name": "Heatmap",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "stat",
+ "name": "Stat",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "CEPH OSD Status.",
+ "editable": false,
+ "fiscalYearStartMonth": 0,
+ "gnetId": 5336,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 11,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Status / Total OSDs",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [
+ {
+ "options": {
+ "from": 0,
+ "result": {
+ "index": 0,
+ "text": "N/A"
+ },
+ "to": 0
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 0,
+ "result": {
+ "index": 1,
+ "text": "DOWN"
+ },
+ "to": 0.99
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 0.99,
+ "result": {
+ "index": 2,
+ "text": "UP"
+ },
+ "to": 1
+ },
+ "type": "range"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": null
+ },
+ {
+ "color": "rgba(237, 40, 40, 0.89)",
+ "value": 0
+ },
+ {
+ "color": "rgba(50, 172, 45, 0.97)",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 0,
+ "y": 1
+ },
+ "id": 6,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum without (instance) (ceph_osd_up{ceph_daemon=\"$osd\"})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "range": true,
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Status",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [
+ {
+ "options": {
+ "from": 0,
+ "result": {
+ "index": 0,
+ "text": "N/A"
+ },
+ "to": 0
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 0,
+ "result": {
+ "index": 1,
+ "text": "OUT"
+ },
+ "to": 0.99
+ },
+ "type": "range"
+ },
+ {
+ "options": {
+ "from": 0.99,
+ "result": {
+ "index": 2,
+ "text": "IN"
+ },
+ "to": 1
+ },
+ "type": "range"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": null
+ },
+ {
+ "color": "rgba(237, 40, 40, 0.89)",
+ "value": 0
+ },
+ {
+ "color": "rgba(50, 172, 45, 0.97)",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 2,
+ "y": 1
+ },
+ "id": 8,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance) (ceph_osd_in{ceph_daemon=\"$osd\"})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Available",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [
+ {
+ "options": {
+ "from": 0,
+ "result": {
+ "index": 0,
+ "text": "N/A"
+ },
+ "to": 0
+ },
+ "type": "range"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(255, 255, 255, 0.9)",
+ "value": null
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 2,
+ "x": 4,
+ "y": 1
+ },
+ "id": 10,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "count without (instance, ceph_daemon) (ceph_osd_up)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Total OSDs",
+ "type": "stat"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 4
+ },
+ "id": 12,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "OSD: $osd",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ },
+ {
+ "color": "rgba(216, 200, 27, 0.27)",
+ "value": 250
+ },
+ {
+ "color": "rgba(234, 112, 112, 0.22)",
+ "value": 300
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Average.*/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 11,
+ "x": 0,
+ "y": 5
+ },
+ "id": 5,
+ "interval": "$interval",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance, ceph_daemon) (ceph_osd_numpg{ceph_daemon=~\"$osd\"})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Number of PGs",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "avg without (instance, ceph_daemon) (ceph_osd_numpg)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Average Number of PGs in the Cluster",
+ "refId": "B",
+ "step": 60
+ }
+ ],
+ "title": "PGs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 9,
+ "x": 11,
+ "y": 5
+ },
+ "id": 2,
+ "interval": "$interval",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance, ceph_daemon) (ceph_osd_stat_bytes{ceph_daemon=\"osd.0\"}-ceph_osd_stat_bytes_used{ceph_daemon=\"osd.0\"})",
+ "format": "time_series",
+ "hide": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Available",
+ "metric": "ceph_osd_avail_bytes",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance, ceph_daemon) (ceph_osd_stat_bytes_used{ceph_daemon=~\"$osd\"})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Used",
+ "metric": "ceph_osd_avail_bytes",
+ "refId": "B",
+ "step": 60
+ }
+ ],
+ "title": "OSD Storage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)",
+ "value": null
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 60
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 4,
+ "x": 20,
+ "y": 5
+ },
+ "id": 7,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "9.0.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance, ceph_daemon) (ceph_osd_stat_bytes_used{ceph_daemon=\"osd.0\"}/ceph_osd_stat_bytes{ceph_daemon=\"osd.0\"})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Utilization",
+ "type": "gauge"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 12
+ },
+ "id": 13,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Latency, Storage, Utilization Varience",
+ "type": "row"
+ },
+ {
+ "cards": {},
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 13
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 83,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ },
+ "value": "1"
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance) (ceph_osd_apply_latency_ms{ceph_daemon='$osd'})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Apply Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketSize": "",
+ "yAxis": {
+ "format": "ms",
+ "logBase": 2,
+ "min": "0",
+ "show": true,
+ "splitFactor": 1
+ },
+ "yBucketBound": "auto",
+ "yBucketSize": 10
+ },
+ {
+ "cards": {},
+ "color": {
+ "cardColor": "#65c5db",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 13
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 84,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#65c5db",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance) (ceph_osd_commit_latency_ms{ceph_daemon='$osd'})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Commit Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketSize": "",
+ "yAxis": {
+ "format": "ms",
+ "logBase": 2,
+ "min": "0",
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "cards": {},
+ "color": {
+ "cardColor": "#806eb7",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 21
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 85,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#806eb7",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 2,
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance) (rate(ceph_osd_op_r_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_r_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Read Op Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": 2,
+ "format": "ms",
+ "logBase": 2,
+ "min": "0",
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "cards": {},
+ "color": {
+ "cardColor": "#f9934e",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "opacity"
+ },
+ "dataFormat": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 21
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 86,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": true,
+ "calculation": {
+ "yBuckets": {
+ "mode": "count",
+ "scale": {
+ "log": 2,
+ "type": "log"
+ }
+ }
+ },
+ "cellGap": 2,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#f9934e",
+ "mode": "opacity",
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 2,
+ "min": "0",
+ "reverse": false,
+ "unit": "ms"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance) (rate(ceph_osd_op_w_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_w_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
+ "format": "time_series",
+ "instant": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "OSD Write Op Latency Distribution",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketSize": "",
+ "yAxis": {
+ "decimals": 2,
+ "format": "ms",
+ "logBase": 2,
+ "min": "0",
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "id": 44,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "avg without (instance, ceph_daemon) (rate(ceph_osd_op_r_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_r_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "read",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "avg without (instance, ceph_daemon) (rate(ceph_osd_op_w_latency_sum{ceph_daemon='$osd'}[5m]) / rate(ceph_osd_op_w_latency_count{ceph_daemon='$osd'}[5m]) >= 0)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "write",
+ "refId": "B"
+ }
+ ],
+ "title": "Avg OSD Op Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "id": 35,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "avg without (instance, ceph_daemon) (ceph_osd_apply_latency_ms{ceph_daemon='$osd'})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "apply",
+ "metric": "ceph_osd_perf_apply_latency_seconds",
+ "refId": "A",
+ "step": 4
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "avg without (instance, ceph_daemon)(ceph_osd_commit_latency_ms{ceph_daemon='$osd'})",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "commit",
+ "metric": "ceph_osd_perf_commit_latency_seconds",
+ "refId": "B",
+ "step": 4
+ }
+ ],
+ "title": "AVG OSD Apply + Commit Latency",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "5m",
+ "schemaVersion": 36,
+ "style": "dark",
+ "tags": [
+ "ceph",
+ "osd"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Data source",
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "auto": true,
+ "auto_count": 10,
+ "auto_min": "1m",
+ "current": {
+ "selected": false,
+ "text": "30s",
+ "value": "30s"
+ },
+ "datasource": "Prometheus",
+ "hide": 0,
+ "includeAll": false,
+ "label": "Interval",
+ "multi": false,
+ "name": "interval",
+ "options": [
+ {
+ "selected": false,
+ "text": "auto",
+ "value": "$__auto_interval_interval"
+ },
+ {
+ "selected": false,
+ "text": "10s",
+ "value": "10s"
+ },
+ {
+ "selected": true,
+ "text": "30s",
+ "value": "30s"
+ },
+ {
+ "selected": false,
+ "text": "1m",
+ "value": "1m"
+ },
+ {
+ "selected": false,
+ "text": "10m",
+ "value": "10m"
+ },
+ {
+ "selected": false,
+ "text": "30m",
+ "value": "30m"
+ },
+ {
+ "selected": false,
+ "text": "1h",
+ "value": "1h"
+ },
+ {
+ "selected": false,
+ "text": "6h",
+ "value": "6h"
+ },
+ {
+ "selected": false,
+ "text": "12h",
+ "value": "12h"
+ },
+ {
+ "selected": false,
+ "text": "1d",
+ "value": "1d"
+ },
+ {
+ "selected": false,
+ "text": "7d",
+ "value": "7d"
+ },
+ {
+ "selected": false,
+ "text": "14d",
+ "value": "14d"
+ },
+ {
+ "selected": false,
+ "text": "30d",
+ "value": "30d"
+ }
+ ],
+ "query": "10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
+ "queryValue": "",
+ "refresh": 2,
+ "skipUrlSync": false,
+ "type": "interval"
+ },
+ {
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "label_values(ceph_osd_up, ceph_daemon)",
+ "hide": 0,
+ "includeAll": false,
+ "label": "OSD",
+ "multi": false,
+ "name": "osd",
+ "options": [],
+ "query": {
+ "query": "label_values(ceph_osd_up, ceph_daemon)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "Ceph - OSD (Single)",
+ "uid": "Fj5fAfzik",
+ "version": 2,
+ "weekStart": ""
+ }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ labels:
+ grafana_dashboard: "1"
+ name: ceph-pools-dashboard
+data:
+ dashboard.json: |-
+ {
+ "__inputs": [],
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "panel",
+ "id": "gauge",
+ "name": "Gauge",
+ "version": ""
+ },
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "9.0.5"
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Ceph Pools dashboard.",
+ "editable": false,
+ "fiscalYearStartMonth": 0,
+ "gnetId": 5342,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 11,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Pool: $pool",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 30,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Total.*$/"
+ },
+ "properties": [
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineWidth",
+ "value": 4
+ },
+ {
+ "id": "custom.stacking",
+ "value": {
+ "group": false,
+ "mode": "normal"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/^Raw.*$/"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#BF1B00",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineWidth",
+ "value": 4
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 20,
+ "x": 0,
+ "y": 1
+ },
+ "id": 2,
+ "interval": "$interval",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_max_avail) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Available - {{ name }}",
+ "metric": "ceph_pool_available_bytes",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_stored) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Stored - {{ name }}",
+ "metric": "ceph_pool",
+ "refId": "B",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_stored + ceph_pool_max_avail) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "hide": true,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Total - {{ name }}",
+ "metric": "ceph_pool",
+ "refId": "C",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_stored_raw) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Raw - {{ name }}",
+ "metric": "ceph_pool",
+ "refId": "D",
+ "step": 60
+ }
+ ],
+ "title": "Pool Storage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 2,
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "percentage",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 75
+ },
+ {
+ "color": "red",
+ "value": 90
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 4,
+ "x": 20,
+ "y": 1
+ },
+ "id": 10,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "9.0.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum without (instance, pool_id, name) ((ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail)) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"}))",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Usage",
+ "type": "gauge"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 8
+ },
+ "id": 12,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Pool Info",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 9
+ },
+ "id": 7,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_objects) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Objects - {{ name }}",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_dirty) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Dirty Objects - {{ name }}",
+ "refId": "B",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((ceph_pool_quota_objects) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Quota Objects - {{ name }}",
+ "refId": "C"
+ }
+ ],
+ "title": "Objects in Pool",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "IOPS",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 9
+ },
+ "id": 4,
+ "interval": "$interval",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((irate(ceph_pool_rd[3m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Read - {{ name }}",
+ "refId": "B",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((irate(ceph_pool_wr[3m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Write - {{ name }}",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "IOPS",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 24,
+ "x": 0,
+ "y": 17
+ },
+ "id": 5,
+ "interval": "$interval",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((irate(ceph_pool_rd_bytes[5m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Read Bytes - {{ name }}",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum((irate(ceph_pool_wr_bytes[5m])) *on (pool_id) group_left(name)(ceph_pool_metadata{name=~\"^$pool$\"})) by (name)",
+ "format": "time_series",
+ "interval": "$interval",
+ "intervalFactor": 1,
+ "legendFormat": "Written Bytes - {{ name }}",
+ "refId": "B",
+ "step": 60
+ }
+ ],
+ "title": "Throughput",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "1m",
+ "schemaVersion": 36,
+ "style": "dark",
+ "tags": [
+ "ceph",
+ "pools"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Data source",
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "auto": true,
+ "auto_count": 10,
+ "auto_min": "1m",
+ "current": {
+ "selected": false,
+ "text": "30s",
+ "value": "30s"
+ },
+ "datasource": "Prometheus",
+ "hide": 0,
+ "includeAll": false,
+ "label": "Interval",
+ "multi": false,
+ "name": "interval",
+ "options": [
+ {
+ "selected": false,
+ "text": "auto",
+ "value": "$__auto_interval_interval"
+ },
+ {
+ "selected": false,
+ "text": "10s",
+ "value": "10s"
+ },
+ {
+ "selected": true,
+ "text": "30s",
+ "value": "30s"
+ },
+ {
+ "selected": false,
+ "text": "1m",
+ "value": "1m"
+ },
+ {
+ "selected": false,
+ "text": "10m",
+ "value": "10m"
+ },
+ {
+ "selected": false,
+ "text": "30m",
+ "value": "30m"
+ },
+ {
+ "selected": false,
+ "text": "1h",
+ "value": "1h"
+ },
+ {
+ "selected": false,
+ "text": "6h",
+ "value": "6h"
+ },
+ {
+ "selected": false,
+ "text": "12h",
+ "value": "12h"
+ },
+ {
+ "selected": false,
+ "text": "1d",
+ "value": "1d"
+ },
+ {
+ "selected": false,
+ "text": "7d",
+ "value": "7d"
+ },
+ {
+ "selected": false,
+ "text": "14d",
+ "value": "14d"
+ },
+ {
+ "selected": false,
+ "text": "30d",
+ "value": "30d"
+ }
+ ],
+ "query": "10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
+ "queryValue": "",
+ "refresh": 2,
+ "skipUrlSync": false,
+ "type": "interval"
+ },
+ {
+ "allValue": ".*",
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "label_values(ceph_pool_metadata, name)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Pool",
+ "multi": true,
+ "name": "pool",
+ "options": [],
+ "query": {
+ "query": "label_values(ceph_pool_metadata, name)",
+ "refId": "Prometheus-pool-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 3,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "Ceph - Pools",
+ "uid": "-gtf0Bzik",
+ "version": 2,
+ "weekStart": ""
+ }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ labels:
+ grafana_dashboard: "1"
+ name: synapse-dashboard
+data:
+ dashboard.json: |-
+ {
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "9.2.2"
+ },
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph (old)",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "heatmap",
+ "name": "Heatmap",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "enable": false,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "limit": 100,
+ "name": "Annotations & Alerts",
+ "showIn": 0,
+ "type": "dashboard"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "enable": true,
+ "expr": "changes(process_start_time_seconds{instance=\"$instance\",job=~\"synapse\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"$instance\",job=\"synapse\"}",
+ "iconColor": "purple",
+ "name": "deploys",
+ "titleFormat": "Deployed {{version}}"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [
+ {
+ "asDropdown": false,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "matrix"
+ ],
+ "title": "Dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 73,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Overview",
+ "type": "row"
+ },
+ {
+ "cards": {
+ "cardPadding": -1,
+ "cardRound": 0
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 1
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 189,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellRadius": 0,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Event Send Time (excluding errors, all workers)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "format": "s",
+ "logBase": 2,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 152,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Avg",
+ "fill": 0,
+ "linewidth": 3
+ },
+ {
+ "alias": "99%",
+ "color": "#C4162A",
+ "fillBelowTo": "90%"
+ },
+ {
+ "alias": "90%",
+ "color": "#FF7383",
+ "fillBelowTo": "75%"
+ },
+ {
+ "alias": "75%",
+ "color": "#FFEE52",
+ "fillBelowTo": "50%"
+ },
+ {
+ "alias": "50%",
+ "color": "#73BF69",
+ "fillBelowTo": "25%"
+ },
+ {
+ "alias": "25%",
+ "color": "#1F60C4",
+ "fillBelowTo": "5%"
+ },
+ {
+ "alias": "5%",
+ "lines": false
+ },
+ {
+ "alias": "Average",
+ "color": "rgb(255, 255, 255)",
+ "lines": true,
+ "linewidth": 3
+ },
+ {
+ "alias": "Events",
+ "color": "#B877D9",
+ "hideTooltip": true,
+ "points": true,
+ "yaxis": 2,
+ "zindex": -3
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "legendFormat": "25%",
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "legendFormat": "5%",
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))",
+ "legendFormat": "Average",
+ "refId": "H"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "Events",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:283",
+ "colorMode": "warning",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:284",
+ "colorMode": "critical",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 2,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Event Send Time Quantiles (excluding errors, all workers)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:255",
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:256",
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 10
+ },
+ "hiddenSeries": false,
+ "id": 75,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 3,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} ",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:566",
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "CPU usage",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:538",
+ "format": "percentunit",
+ "logBase": 1,
+ "max": "1.5",
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:539",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 10
+ },
+ "hiddenSeries": false,
+ "id": 198,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 3,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}} {{index}}",
+ "refId": "A",
+ "step": 20,
+ "target": ""
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+ "hide": true,
+ "interval": "",
+ "legendFormat": "total",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memory",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "transformations": [],
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1560",
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1561",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "axisSoftMax": 1,
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 10,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 19
+ },
+ "id": 245,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1",
+ "legendFormat": "version {{version}}",
+ "range": true,
+ "refId": "deployed_synapse_versions"
+ }
+ ],
+ "title": "Deployed Synapse versions over time",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 19
+ },
+ "hiddenSeries": false,
+ "id": 37,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:639",
+ "alias": "/max$/",
+ "color": "#890F02",
+ "fill": 0,
+ "legend": false
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": true,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} max",
+ "refId": "B",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Open FDs",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:650",
+ "format": "none",
+ "label": "",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:651",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 26
+ },
+ "id": 54,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 27
+ },
+ "hiddenSeries": false,
+ "id": 5,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 3,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:1240",
+ "alias": "/user/"
+ },
+ {
+ "$$hashKey": "object:1241",
+ "alias": "/system/"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} system ",
+ "metric": "",
+ "range": true,
+ "refId": "B",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} user",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:1278",
+ "colorMode": "custom",
+ "fillColor": "rgba(255, 255, 255, 1)",
+ "line": true,
+ "lineColor": "rgba(216, 200, 27, 0.27)",
+ "op": "gt",
+ "value": 0.5,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:1279",
+ "colorMode": "custom",
+ "fillColor": "rgba(255, 255, 255, 1)",
+ "line": true,
+ "lineColor": "rgb(87, 6, 16)",
+ "op": "gt",
+ "value": 0.8,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:1498",
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "CPU",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1250",
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1.2",
+ "min": 0,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1251",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 27
+ },
+ "id": 105,
+ "interval": "",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.3.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.999, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{job}}-{{index}} 99.9%",
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} 99%",
+ "refId": "A",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 95%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} mean",
+ "refId": "D"
+ }
+ ],
+ "title": "Reactor tick quantiles",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 34
+ },
+ "hiddenSeries": false,
+ "id": 34,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}} {{index}}",
+ "refId": "A",
+ "step": 20,
+ "target": ""
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+ "interval": "",
+ "legendFormat": "total",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memory",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "transformations": [],
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 34
+ },
+ "hiddenSeries": false,
+ "id": 49,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/^up/",
+ "legend": false,
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Prometheus scrape time",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "decimals": 0,
+ "format": "none",
+ "label": "",
+ "logBase": 1,
+ "max": "0",
+ "min": "-1",
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 41
+ },
+ "hiddenSeries": false,
+ "id": 53,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:116",
+ "alias": "/^version .*/",
+ "lines": true,
+ "linewidth": 6,
+ "points": false
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1",
+ "hide": false,
+ "legendFormat": "version {{version}}",
+ "range": true,
+ "refId": "deployed_synapse_versions"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Up",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 41
+ },
+ "hiddenSeries": false,
+ "id": 120,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [
+ {
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Stacked CPU usage",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:572",
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:573",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 48
+ },
+ "hiddenSeries": false,
+ "id": 136,
+ "interval": "",
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_http_client_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "legendFormat": "{{job}}-{{index}} {{method}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_http_matrixfederationclient_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "legendFormat": "{{job}}-{{index}} {{method}} (federation)",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Outgoing HTTP request rate",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:123",
+ "format": "reqps",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:124",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "active threads",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 48
+ },
+ "id": 207,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "synapse_threadpool_working_threads{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Threadpool activity",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Process info",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 27
+ },
+ "id": 56,
+ "panels": [
+ {
+ "cards": {
+ "cardPadding": -1,
+ "cardRound": 0
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 28
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 85,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellRadius": 0,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Event Send Time (Including errors, across all workers)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "format": "s",
+ "logBase": 2,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 28
+ },
+ "hiddenSeries": false,
+ "id": 33,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size])) without (job,index)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 20,
+ "target": ""
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events Persisted (all workers)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:102",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:103",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 1,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 40,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_events_persisted_by_source_type{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{type}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events/s Local vs Remote",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 1,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 46,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_events_persisted_by_event_type{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 2,
+ "legendFormat": "{{type}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events/s by Type",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {
+ "irc-freenode (local)": "#EAB839"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 1,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 44,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_events_persisted_by_origin{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{origin_entity}} ({{origin_type}})",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events/s by Origin",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 1,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 45,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{origin_entity}} ({{origin_type}})",
+ "range": true,
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memberships/s by Origin",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:232",
+ "format": "hertz",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:233",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 51
+ },
+ "hiddenSeries": false,
+ "id": 118,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeatDirection": "h",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:316",
+ "alias": "mean",
+ "linewidth": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 99%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 95%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 50%",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} mean",
+ "range": true,
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Event send time quantiles by worker",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:263",
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:264",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "CPU and DB time spent on most expensive state resolution in a room, summed over all workers. This is a very rough proxy for \"how fast is state res\", but it doesn't accurately represent the system load (e.g. it completely ignores cheap state resolutions).\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 30,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s/s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 51
+ },
+ "id": 222,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": false,
+ "expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "legendFormat": "DB time",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": false,
+ "expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "legendFormat": "CPU time",
+ "refId": "C"
+ }
+ ],
+ "title": "Stateres worst-case",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Event persistence",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 28
+ },
+ "id": 57,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 2,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 4,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:234",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(216, 200, 27, 0.27)",
+ "op": "gt",
+ "value": 100,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:235",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(234, 112, 112, 0.22)",
+ "op": "gt",
+ "value": 250,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Request Count by arrival time",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:206",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:207",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 32,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{method}} {{servlet}} {{job}}-{{index}}",
+ "refId": "A",
+ "step": 20,
+ "target": ""
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Top 10 Request Counts",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:305",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:306",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 2,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 139,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:135",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(216, 200, 27, 0.27)",
+ "op": "gt",
+ "value": 100,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:136",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(234, 112, 112, 0.22)",
+ "op": "gt",
+ "value": 250,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Total CPU Usage by Endpoint",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:107",
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:108",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 52,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:417",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(216, 200, 27, 0.27)",
+ "op": "gt",
+ "value": 100,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:418",
+ "colorMode": "custom",
+ "fill": true,
+ "fillColor": "rgba(234, 112, 112, 0.22)",
+ "op": "gt",
+ "value": 250,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Average CPU Usage by Endpoint",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:389",
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:390",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 7,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "DB Usage by endpoint",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:488",
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:489",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 2,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 47,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": true,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "(sum(rate(synapse_http_server_response_time_seconds_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))/(sum(rate(synapse_http_server_response_time_seconds_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Non-sync avg response time",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 53
+ },
+ "hiddenSeries": false,
+ "id": 103,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Total",
+ "color": "rgb(255, 255, 255)",
+ "fill": 0,
+ "linewidth": 3
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "topk(10,synapse_http_server_in_flight_requests_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Total",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Requests in flight",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Requests",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 29
+ },
+ "id": 97,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 99,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "CPU usage by background jobs",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 101,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_background_process_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "DB usage by background jobs (including scheduling time)",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 138,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Background jobs in flight",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Background jobs",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 30
+ },
+ "id": 81,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 79,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "successful txn rate",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_util_metrics_block_count_total{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))",
+ "legendFormat": "failed txn rate",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Outgoing federation transaction rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 83,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_server_received_pdus_total{instance=~\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "pdus",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_server_received_edus_total{instance=~\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "edus",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Incoming PDU/EDU rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 40
+ },
+ "hiddenSeries": false,
+ "id": 109,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations_count_total{instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "pdus",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_client_sent_edus_total{instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "edus",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Outgoing PDU/EDU rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 40
+ },
+ "hiddenSeries": false,
+ "id": 111,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_federation_client_sent_edus_by_type_total{instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Outgoing EDUs by type",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:462",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:463",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Triangular growth may indicate a problem with federation sending from the remote host --- but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "axisSoftMax": 60,
+ "axisSoftMin": 0,
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 60
+ }
+ ]
+ },
+ "unit": "m"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "libera.chat "
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 49
+ },
+ "id": 243,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_received_pdu_time[10m]))) / 60",
+ "instant": false,
+ "legendFormat": "{{server_name}} ",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Age of last PDU received from nominated hosts",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Triangular growth may indicate a problem with federation senders on the monitored instance---but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "axisSoftMax": 60,
+ "axisSoftMin": 0,
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 60
+ }
+ ]
+ },
+ "unit": "m"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "libera.chat"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 49
+ },
+ "id": 241,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_sent_pdu_time[10m]))) / 60",
+ "instant": false,
+ "legendFormat": "{{server_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Age of last PDU sent to nominated hosts",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "The number of events in the in-memory queues ",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 57
+ },
+ "hiddenSeries": false,
+ "id": 142,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "pending PDUs {{job}}-{{index}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "pending EDUs {{job}}-{{index}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "In-memory federation transmission queues",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:547",
+ "format": "short",
+ "label": "events",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:548",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Number of events queued up on the master process for processing by the federation sender",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 57
+ },
+ "hiddenSeries": false,
+ "id": 140,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "presence changed",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "presence map",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "presence destinations",
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "keyed edus",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "other edus",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "stream positions",
+ "refId": "F"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Outgoing EDU queues on master",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": -1
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 66
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 166,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellValues": {
+ "decimals": 2
+ },
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "min": 0,
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{instance=\"$instance\",name=\"federation_sender\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{ le }}",
+ "refId": "A"
+ }
+ ],
+ "title": "Federation send PDU lag",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "tooltipDecimals": 2,
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 66
+ },
+ "hiddenSeries": false,
+ "id": 162,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Avg",
+ "fill": 0,
+ "linewidth": 3
+ },
+ {
+ "alias": "99%",
+ "color": "#C4162A",
+ "fillBelowTo": "90%"
+ },
+ {
+ "alias": "90%",
+ "color": "#FF7383",
+ "fillBelowTo": "75%"
+ },
+ {
+ "alias": "75%",
+ "color": "#FFEE52",
+ "fillBelowTo": "50%"
+ },
+ {
+ "alias": "50%",
+ "color": "#73BF69",
+ "fillBelowTo": "25%"
+ },
+ {
+ "alias": "25%",
+ "color": "#1F60C4",
+ "fillBelowTo": "5%"
+ },
+ {
+ "alias": "5%",
+ "lines": false
+ },
+ {
+ "alias": "Average",
+ "color": "rgb(255, 255, 255)",
+ "lines": true,
+ "linewidth": 3
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "interval": "",
+ "legendFormat": "25%",
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "interval": "",
+ "legendFormat": "5%",
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Average",
+ "refId": "H"
+ }
+ ],
+ "thresholds": [
+ {
+ "colorMode": "warning",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 0.25,
+ "yaxis": "left"
+ },
+ {
+ "colorMode": "critical",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Federation send PDU lag quantiles",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": -1
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 75
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 164,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellValues": {
+ "decimals": 2
+ },
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "min": 0,
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{instance=\"$instance\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{ le }}",
+ "refId": "A"
+ }
+ ],
+ "title": "Handle inbound PDU time",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "tooltipDecimals": 2,
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 75
+ },
+ "hiddenSeries": false,
+ "id": 203,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}}",
+ "range": true,
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Age of oldest event in staging area",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:92",
+ "format": "ms",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:93",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 84
+ },
+ "hiddenSeries": false,
+ "id": 202,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}}",
+ "range": true,
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Number of events in federation staging area",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:92",
+ "format": "none",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:93",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 84
+ },
+ "hiddenSeries": false,
+ "id": 205,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "soft-failed events",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Soft-failed event rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:131",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:132",
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Federation",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 31
+ },
+ "id": 227,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 154
+ },
+ "id": 239,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_reject_total{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of rate limit rejected requests",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 154
+ },
+ "id": 235,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of requests being slept by the rate limiter",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Why is the data zero (0)? https://github.com/matrix-org/synapse/pull/13541#discussion_r951926322",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 162
+ },
+ "id": 237,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_reject_affected_hosts{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of hosts being rejected by the rate limiter",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "https://github.com/matrix-org/synapse/pull/13541",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 162
+ },
+ "id": 233,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_sleep_affected_hosts{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of hosts being slept by the rate limiter",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 170
+ },
+ "hiddenSeries": false,
+ "id": 229,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:276",
+ "alias": "Avg",
+ "fill": 0,
+ "linewidth": 3
+ },
+ {
+ "$$hashKey": "object:277",
+ "alias": "99%",
+ "color": "#C4162A",
+ "fillBelowTo": "90%"
+ },
+ {
+ "$$hashKey": "object:278",
+ "alias": "90%",
+ "color": "#FF7383",
+ "fillBelowTo": "75%"
+ },
+ {
+ "$$hashKey": "object:279",
+ "alias": "75%",
+ "color": "#FFEE52",
+ "fillBelowTo": "50%"
+ },
+ {
+ "$$hashKey": "object:280",
+ "alias": "50%",
+ "color": "#73BF69",
+ "fillBelowTo": "25%"
+ },
+ {
+ "$$hashKey": "object:281",
+ "alias": "25%",
+ "color": "#1F60C4",
+ "fillBelowTo": "5%"
+ },
+ {
+ "$$hashKey": "object:282",
+ "alias": "5%",
+ "lines": false
+ },
+ {
+ "$$hashKey": "object:283",
+ "alias": "Average",
+ "color": "rgb(255, 255, 255)",
+ "lines": true,
+ "linewidth": 3
+ },
+ {
+ "$$hashKey": "object:284",
+ "alias": ">99%",
+ "color": "#B877D9",
+ "fill": 3,
+ "lines": true
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9995, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": ">99%",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.9, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.25, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "legendFormat": "25%",
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.05, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "legendFormat": "5%",
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_rate_limit_queue_wait_time_seconds_sum{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_rate_limit_queue_wait_time_seconds_count{index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "legendFormat": "Average",
+ "refId": "H"
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:283",
+ "colorMode": "warning",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:284",
+ "colorMode": "critical",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 2,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Rate limit queue wait time Quantiles (all workers)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:255",
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:256",
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Default reject threshold (50 requests within a second)"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "red",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.lineWidth",
+ "value": 2
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 170
+ },
+ "id": 231,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "50",
+ "hide": false,
+ "legendFormat": "Default reject threshold (50 requests within a second)",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Rate of requests being slept by the rate limiter",
+ "type": "timeseries"
+ }
+ ],
+ "title": "Federation rate limiter",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 32
+ },
+ "id": 60,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 155
+ },
+ "id": 51,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_http_httppusher_http_pushes_processed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "processed {{job}}-{{index}}",
+ "range": true,
+ "refId": "A",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_http_httppusher_http_pushes_failed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "failed {{job}}-{{index}}",
+ "range": true,
+ "refId": "B",
+ "step": 20
+ }
+ ],
+ "title": "HTTP Push rate",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 155
+ },
+ "hiddenSeries": false,
+ "id": 134,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})",
+ "legendFormat": "{{kind}} {{app_id}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Active pusher instances by app",
+ "tooltip": {
+ "shared": false,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Pushes",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 33
+ },
+ "id": 219,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How many entries in current state that we are iterating over while calculating push rules.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 33
+ },
+ "hiddenSeries": false,
+ "id": 209,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{index}}",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Iterations over State",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Rate that the cached push rules for a room get invalidated due to underlying push rules being changed. ",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 33
+ },
+ "hiddenSeries": false,
+ "id": 211,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{index}}",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Push Rule Invalidations",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How often the \"delta optimisation\" happens.\n\nThe delta optimisation is when we update the push rules for a room incrementally after a state change where we know the delta between the old state and the new state.\n\nThis can't happen if we don't the delta or we're calculating push rules from scratch.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 40
+ },
+ "hiddenSeries": false,
+ "id": 213,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Number of calls",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Hit Rate",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
+ "refId": "A",
+ "step": 2
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Number of calls",
+ "refId": "B",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Delta Optimisation",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How often we have the correct cached push rules for a room.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 40
+ },
+ "hiddenSeries": false,
+ "id": 215,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Number of calls",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Hit Rate",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
+ "refId": "A",
+ "step": 2
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Number of calls",
+ "refId": "B",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "How often we reuse existing calculated push rules",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How often we have existing cached push rules for the room. \n\nNote that these might be outdated and need to be recalculated if the state has changed.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 47
+ },
+ "hiddenSeries": false,
+ "id": 217,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Number of calls",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Hit Rate",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
+ "refId": "A",
+ "step": 2
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Number of calls",
+ "refId": "B",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "How often we have the RulesForRoom cached",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Push Rule Cache",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 34
+ },
+ "id": 58,
+ "panels": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 35
+ },
+ "id": 48,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "title": "Avg time waiting for db conn",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 104,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{job}} {{index}} 99%",
+ "refId": "A",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}} {{index}} 95%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}} {{index}} 90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}} {{index}} mean",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Db scheduling time quantiles",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 42
+ },
+ "hiddenSeries": false,
+ "id": 10,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "topk(10, rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{desc}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Top DB transactions by txn rate",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 42
+ },
+ "hiddenSeries": false,
+ "id": 11,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{desc}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "DB transactions by total txn time",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 49
+ },
+ "hiddenSeries": false,
+ "id": 180,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": false
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{desc}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average DB txn time",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 6,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 49
+ },
+ "hiddenSeries": false,
+ "id": 200,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Time waiting for DB connection quantiles",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:203",
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:204",
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Database",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 35
+ },
+ "id": 59,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 0,
+ "y": 158
+ },
+ "hiddenSeries": false,
+ "id": 12,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Total CPU Usage by Block",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 12,
+ "y": 158
+ },
+ "hiddenSeries": false,
+ "id": 26,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])) / rate(synapse_util_metrics_block_count_total[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average CPU Time per Block",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 0,
+ "y": 171
+ },
+ "hiddenSeries": false,
+ "id": 13,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Total DB Usage by Block",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:196",
+ "format": "percentunit",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:197",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "The time each database transaction takes to execute, on average, broken down by metrics block.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 12,
+ "y": 171
+ },
+ "hiddenSeries": false,
+ "id": 27,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average Database Transaction time, by Block",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 0,
+ "y": 184
+ },
+ "hiddenSeries": false,
+ "id": 28,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average Transactions per Block",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 13,
+ "w": 12,
+ "x": 12,
+ "y": 184
+ },
+ "hiddenSeries": false,
+ "id": 25,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_metrics_block_time_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average Wallclock Time per Block",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:180",
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:181",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 15,
+ "w": 12,
+ "x": 0,
+ "y": 197
+ },
+ "hiddenSeries": false,
+ "id": 154,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Block count",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Per-block metrics",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 36
+ },
+ "id": 61,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 0,
+ "y": 36
+ },
+ "hiddenSeries": false,
+ "id": 1,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache Hit Ratio",
+ "tooltip": {
+ "msResolution": true,
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1",
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 12,
+ "y": 36
+ },
+ "hiddenSeries": false,
+ "id": 8,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_util_caches_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache Size",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 0,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 38,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache request rate",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "rps",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 12,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 39,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "max",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "topk(10, rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Top 10 cache misses",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:101",
+ "format": "rps",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:102",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 56
+ },
+ "hiddenSeries": false,
+ "id": 65,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_caches_cache_evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}} ({{reason}}) {{job}}-{{index}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache eviction rate",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": "entries / second",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Caches",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 37
+ },
+ "id": 148,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 146,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_util_caches_response_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Response cache size",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 150,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_util_caches_response_cache_hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Response cache hit rate",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Response caches",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 38
+ },
+ "id": 62,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 91,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Total GC time by bucket (10m smoothing)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "decimals": 3,
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 30
+ },
+ "hiddenSeries": false,
+ "id": 21,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}} {{index}} gen {{gen}} ",
+ "refId": "A",
+ "step": 20,
+ "target": ""
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Average GC Time Per Collection",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 89,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/gen 0$/",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Allocation counts",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Gen N-1 GCs since last Gen N GC",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": "Objects since last Gen 0 GC",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 93,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Object counts per collection",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 48
+ },
+ "hiddenSeries": false,
+ "id": 95,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "GC frequency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": 0
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateSpectral",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 48
+ },
+ "heatmap": {},
+ "hideZeroBuckets": true,
+ "highlightCards": true,
+ "id": 87,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "GC durations",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "GC",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 39
+ },
+ "id": 63,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 162
+ },
+ "hiddenSeries": false,
+ "id": 43,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{command}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Rate of outgoing commands",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:89",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:90",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 162
+ },
+ "id": 41,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.4.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "rate(synapse_replication_tcp_resource_stream_updates_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{stream_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "title": "Rate of outgoing RDATA commands, by stream",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 169
+ },
+ "id": 42,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.4.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} {{command}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "title": "Rate of incoming commands (including echoes)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "axisSoftMin": 1,
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 169
+ },
+ "id": 220,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.4.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "rate(synapse_replication_tcp_protocol_inbound_rdata_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{stream_name}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "title": "Rate of incoming RDATA commands (excluding echoes), by stream",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 176
+ },
+ "hiddenSeries": false,
+ "id": 144,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_replication_tcp_command_queue{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "{{stream_name}} {{job}}-{{index}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Queued incoming RDATA commands, by stream",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:218",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:219",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 176
+ },
+ "hiddenSeries": false,
+ "id": 115,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_replication_tcp_protocol_close_reason_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{reason_type}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Replication connection close reasons",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:260",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:261",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 183
+ },
+ "hiddenSeries": false,
+ "id": 113,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{stream_name}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Replication connections",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Replication",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 40
+ },
+ "id": 69,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 163
+ },
+ "hiddenSeries": false,
+ "id": 67,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - on() group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Event processing lag",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "events",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 163
+ },
+ "hiddenSeries": false,
+ "id": 71,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Age of last processed event",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 172
+ },
+ "hiddenSeries": false,
+ "id": 121,
+ "interval": "",
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "deriv(synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Event processing catchup rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": "fallbehind(-) / catchup(+): s/sec",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Event processing loop positions",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 41
+ },
+ "id": 126,
+ "panels": [
+ {
+ "cards": {
+ "cardPadding": 0
+ },
+ "color": {
+ "cardColor": "#B877D9",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "opacity"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 42
+ },
+ "heatmap": {},
+ "hideZeroBuckets": true,
+ "highlightCards": true,
+ "id": 122,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": 1,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#B877D9",
+ "min": 0,
+ "mode": "opacity",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "short"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of rooms, by number of forward extremities in room",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 42
+ },
+ "hiddenSeries": false,
+ "id": 124,
+ "interval": "",
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0",
+ "format": "heatmap",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Room counts, by number of extremities",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": "Number of rooms",
+ "logBase": 10,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": 0
+ },
+ "color": {
+ "cardColor": "#5794F2",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "opacity"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 50
+ },
+ "heatmap": {},
+ "hideZeroBuckets": true,
+ "highlightCards": true,
+ "id": 127,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": 1,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#5794F2",
+ "min": 0,
+ "mode": "opacity",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "short"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Events persisted, by number of forward extremities in room (heatmap)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 50
+ },
+ "hiddenSeries": false,
+ "id": 128,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events persisted, by number of forward extremities in room (quantiles)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Number of extremities in room",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": 0
+ },
+ "color": {
+ "cardColor": "#FF9830",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "opacity"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 58
+ },
+ "heatmap": {},
+ "hideZeroBuckets": true,
+ "highlightCards": true,
+ "id": 129,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": 1,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#FF9830",
+ "min": 0,
+ "mode": "opacity",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "short"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Events persisted, by number of stale forward extremities in room (heatmap)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "For given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 58
+ },
+ "hiddenSeries": false,
+ "id": 130,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Events persisted, by number of stale forward extremities in room (quantiles)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Number of stale forward extremities in room",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": 0
+ },
+ "color": {
+ "cardColor": "#73BF69",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "opacity"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 66
+ },
+ "heatmap": {},
+ "hideZeroBuckets": true,
+ "highlightCards": true,
+ "id": 131,
+ "legend": {
+ "show": true
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": 1,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#73BF69",
+ "min": 0,
+ "mode": "opacity",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "short"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "heatmap",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of state resolution performed, by number of state groups involved (heatmap)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 66
+ },
+ "hiddenSeries": false,
+ "id": 132,
+ "interval": "",
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Number of state resolutions performed, by number of state groups involved (quantiles)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Number of state groups",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 74
+ },
+ "hiddenSeries": false,
+ "id": 179,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "State res ",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Potential to prune",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_storage_events_times_pruned_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Pruned",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Stale extremity dropping",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Extremities",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 42
+ },
+ "id": 158,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 156,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:632",
+ "alias": "Max",
+ "color": "#bf1b00",
+ "fill": 0,
+ "linewidth": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "max(synapse_admin_mau_max{instance=\"$instance\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Max",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "max(synapse_admin_mau_current{instance=\"$instance\"})",
+ "hide": false,
+ "legendFormat": "Current",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "MAU Limits",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:176",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:177",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 160,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_admin_mau_current_mau_by_service{instance=\"$instance\"}",
+ "interval": "",
+ "legendFormat": "{{ app_service }}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "MAU by Appservice",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "MAU",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 43
+ },
+ "id": 177,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 173,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_notifier_users_woken_by_stream_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "{{stream}} {{index}}",
+ "metric": "synapse_notifier",
+ "range": true,
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Notifier Streams Woken",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:734",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:735",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 175,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_handler_presence_get_updates_total{job=~\"$job\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{type}} {{index}}",
+ "range": true,
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Presence Stream Fetch Type Rates",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:819",
+ "format": "hertz",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:820",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Notifier",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 44
+ },
+ "id": 170,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 168,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_appservice_api_sent_events_total{instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{service}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Sent Events rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:177",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:178",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 45
+ },
+ "hiddenSeries": false,
+ "id": 171,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_appservice_api_sent_transactions_total{instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{exported_service }} {{ service }}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Transactions rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:260",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:261",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Appservices",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 45
+ },
+ "id": 188,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 182,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_handler_presence_notified_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Notified",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_handler_presence_federation_presence_out_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Remote ping",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_handler_presence_presence_updates_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Total updates",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_handler_presence_federation_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Remote updates",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(synapse_handler_presence_bump_active_time_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Bump active time",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Presence",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 184,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_handler_presence_state_transition_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{from}} -> {{to}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Presence state transitions",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1090",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1091",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 54
+ },
+ "hiddenSeries": false,
+ "id": 186,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_handler_presence_notify_reason_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{reason}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Presence notify reason",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:165",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:166",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Presence",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 46
+ },
+ "id": 197,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 47
+ },
+ "id": 191,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_external_cache_set{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "External Cache Set Rate",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 47
+ },
+ "hiddenSeries": false,
+ "id": 193,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.2.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "sum without (hit) (rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "External Cache Get Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:390",
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:391",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": -1
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 55
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 195,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellValues": {
+ "decimals": 2
+ },
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "min": 0,
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "External Cache Response Time",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "tooltipDecimals": 2,
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "yAxis": {
+ "decimals": 0,
+ "format": "s",
+ "logBase": 1,
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 55
+ },
+ "id": 223,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\", hit=\"False\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "External Cache Miss Rate",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "External Cache",
+ "type": "row"
+ }
+ ],
+ "refresh": false,
+ "schemaVersion": 37,
+ "style": "dark",
+ "tags": [
+ "matrix"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "label": "Datasource",
+ "options": [],
+ "query": "prometheus",
+ "queryValue": "",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "allFormat": "glob",
+ "auto": true,
+ "auto_count": 100,
+ "auto_min": "30s",
+ "current": {
+ "selected": false,
+ "text": "auto",
+ "value": "$__auto_interval_bucket_size"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Bucket Size",
+ "multi": false,
+ "multiFormat": "glob",
+ "name": "bucket_size",
+ "options": [
+ {
+ "selected": true,
+ "text": "auto",
+ "value": "$__auto_interval_bucket_size"
+ },
+ {
+ "selected": false,
+ "text": "30s",
+ "value": "30s"
+ },
+ {
+ "selected": false,
+ "text": "1m",
+ "value": "1m"
+ },
+ {
+ "selected": false,
+ "text": "2m",
+ "value": "2m"
+ },
+ {
+ "selected": false,
+ "text": "5m",
+ "value": "5m"
+ },
+ {
+ "selected": false,
+ "text": "10m",
+ "value": "10m"
+ },
+ {
+ "selected": false,
+ "text": "15m",
+ "value": "15m"
+ }
+ ],
+ "query": "30s,1m,2m,5m,10m,15m",
+ "queryValue": "",
+ "refresh": 2,
+ "skipUrlSync": false,
+ "type": "interval"
+ },
+ {
+ "current": {},
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "",
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "instance",
+ "options": [],
+ "query": {
+ "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, instance)",
+ "refId": "Prometheus-instance-Variable-Query"
+ },
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allFormat": "regex wildcard",
+ "allValue": "",
+ "current": {},
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "",
+ "hide": 0,
+ "hideLabel": false,
+ "includeAll": true,
+ "label": "Job",
+ "multi": true,
+ "multiFormat": "regex values",
+ "name": "job",
+ "options": [],
+ "query": {
+ "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, job)",
+ "refId": "Prometheus-job-Variable-Query"
+ },
+ "refresh": 2,
+ "refresh_on_load": false,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allFormat": "regex wildcard",
+ "allValue": ".*",
+ "current": {},
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "",
+ "hide": 0,
+ "hideLabel": false,
+ "includeAll": true,
+ "label": "",
+ "multi": true,
+ "multiFormat": "regex values",
+ "name": "index",
+ "options": [],
+ "query": {
+ "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, index)",
+ "refId": "Prometheus-index-Variable-Query"
+ },
+ "refresh": 2,
+ "refresh_on_load": false,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 3,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {
+ "now": true,
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Synapse",
+ "uid": "000000012",
+ "version": 160,
+ "weekStart": ""
+ }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ labels:
+ grafana_dashboard: "1"
+ name: flux-cluster-dashboard
+data:
+ dashboard.json: |-
+ {
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "iconColor": "red",
+ "name": "flux events",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [
+ "flux"
+ ],
+ "type": "tags"
+ }
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 1,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "mappings": [],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 0
+ },
+ "id": 24,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "value"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\"})",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "Cluster Reconcilers",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "mappings": [],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 0
+ },
+ "id": 28,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "value"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\", ready=\"False\"})",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "Failing Reconcilers",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "mappings": [],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 12,
+ "y": 0
+ },
+ "id": 29,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "value"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"})",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "Kubernetes Manifests Sources",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "mappings": [],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 18,
+ "y": 0
+ },
+ "id": 30,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "value"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\", ready=\"False\"})",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "Failing Sources",
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 1
+ },
+ {
+ "color": "red",
+ "value": 61
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 0,
+ "y": 5
+ },
+ "id": 8,
+ "options": {
+ "displayMode": "gradient",
+ "minVizHeight": 10,
+ "minVizWidth": 0,
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "text": {},
+ "valueMode": "color"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind)",
+ "interval": "",
+ "legendFormat": "{{kind}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Reconciler ops avg. duration",
+ "type": "bargauge"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 1
+ },
+ {
+ "color": "red",
+ "value": 61
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 12,
+ "y": 5
+ },
+ "id": 31,
+ "options": {
+ "displayMode": "gradient",
+ "minVizHeight": 10,
+ "minVizWidth": 0,
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "text": {},
+ "valueMode": "color"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind)",
+ "interval": "",
+ "legendFormat": "{{kind}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Source ops avg. duration",
+ "type": "bargauge"
+ },
+ {
+ "collapsed": false,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 9
+ },
+ "id": 15,
+ "panels": [],
+ "title": "Status",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "filterable": true,
+ "inspect": false
+ },
+ "mappings": [
+ {
+ "options": {
+ "False": {
+ "color": "red",
+ "index": 1,
+ "text": "Not Ready"
+ },
+ "True": {
+ "color": "blue",
+ "index": 0,
+ "text": "Ready"
+ }
+ },
+ "type": "value"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byType",
+ "options": "string"
+ },
+ "properties": [
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "mode": "basic",
+ "type": "color-background"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 22,
+ "w": 12,
+ "x": 0,
+ "y": 10
+ },
+ "id": 33,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "Status"
+ }
+ ]
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\"}",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Cluster reconciliation readiness ",
+ "transformations": [
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": false,
+ "Value": false,
+ "__name__": false,
+ "app": false,
+ "chart_name": false,
+ "chart_source_name": false,
+ "container": false,
+ "customresource_group": false,
+ "customresource_kind": false,
+ "customresource_version": false,
+ "endpoint": false,
+ "exported_namespace": false,
+ "gotk_type": false,
+ "instance": false,
+ "job": false,
+ "kubernetes_namespace": false,
+ "kubernetes_pod_name": false,
+ "namespace": false,
+ "pod": false,
+ "pod_template_hash": false,
+ "revision": false,
+ "service": false,
+ "source_name": false,
+ "status": false,
+ "suspended": false,
+ "type": false
+ },
+ "indexByName": {
+ "Time": 0,
+ "Value": 15,
+ "__name__": 1,
+ "container": 2,
+ "customresource_group": 4,
+ "customresource_kind": 5,
+ "customresource_version": 6,
+ "endpoint": 7,
+ "exported_namespace": 3,
+ "instance": 8,
+ "job": 9,
+ "name": 10,
+ "namespace": 11,
+ "pod": 12,
+ "ready": 13,
+ "service": 14
+ },
+ "renameByName": {
+ "Value": "",
+ "customresource_kind": "Kind",
+ "exported_namespace": "Namespace",
+ "kind": "Kind",
+ "name": "Name",
+ "namespace": "Operator Namespace",
+ "ready": "Status"
+ }
+ }
+ },
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "(Namespace|Kind|Name|Status)"
+ }
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "filterable": true,
+ "inspect": false
+ },
+ "mappings": [
+ {
+ "options": {
+ "False": {
+ "color": "red",
+ "index": 1,
+ "text": "Not Ready"
+ },
+ "True": {
+ "color": "blue",
+ "index": 0,
+ "text": "Ready"
+ }
+ },
+ "type": "value"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byType",
+ "options": "string"
+ },
+ "properties": [
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "mode": "basic",
+ "type": "color-background"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Status"
+ },
+ "properties": [
+ {
+ "id": "noValue",
+ "value": "Ready"
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "blue",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 12,
+ "y": 10
+ },
+ "id": 34,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "Status"
+ }
+ ]
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Source acquisition readiness ",
+ "transformations": [
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": false,
+ "Value": false,
+ "__name__": false,
+ "app": false,
+ "bucket_name": false,
+ "container": false,
+ "customresource_group": false,
+ "customresource_kind": false,
+ "customresource_version": false,
+ "endpoint": false,
+ "exported_namespace": false,
+ "gotk_type": false,
+ "instance": false,
+ "job": false,
+ "kubernetes_namespace": false,
+ "kubernetes_pod_name": false,
+ "namespace": false,
+ "pod": false,
+ "pod_template_hash": false,
+ "ready": false,
+ "revision": false,
+ "service": false,
+ "status": false,
+ "suspended": false,
+ "type": false,
+ "url": false
+ },
+ "indexByName": {
+ "Time": 0,
+ "Value": 15,
+ "__name__": 1,
+ "container": 2,
+ "customresource_group": 5,
+ "customresource_kind": 6,
+ "customresource_version": 7,
+ "endpoint": 8,
+ "exported_namespace": 4,
+ "instance": 9,
+ "job": 10,
+ "name": 11,
+ "namespace": 3,
+ "pod": 12,
+ "ready": 13,
+ "service": 14
+ },
+ "renameByName": {
+ "Value": "",
+ "customresource_kind": "Kind",
+ "exported_namespace": "Namespace",
+ "kind": "Kind",
+ "name": "Name",
+ "namespace": "Operator Namespace",
+ "ready": "Status"
+ }
+ }
+ },
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "(Namespace|Kind|Name|Status)"
+ }
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "filterable": true,
+ "inspect": false
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byType",
+ "options": "string"
+ },
+ "properties": [
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "mode": "basic",
+ "type": "color-background"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 12,
+ "y": 21
+ },
+ "id": 36,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "Namespace"
+ }
+ ]
+ },
+ "pluginVersion": "10.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", suspended=\"true\"}",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Suspended Objects",
+ "transformations": [
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": false,
+ "Value": false,
+ "__name__": false,
+ "app": false,
+ "bucket_name": false,
+ "container": false,
+ "customresource_group": false,
+ "customresource_kind": false,
+ "customresource_version": false,
+ "endpoint": false,
+ "exported_namespace": false,
+ "gotk_type": false,
+ "instance": false,
+ "job": false,
+ "kubernetes_namespace": false,
+ "kubernetes_pod_name": false,
+ "namespace": false,
+ "pod": false,
+ "pod_template_hash": false,
+ "ready": false,
+ "revision": false,
+ "service": false,
+ "source_name": false,
+ "status": false,
+ "suspended": false,
+ "type": false,
+ "url": false
+ },
+ "indexByName": {
+ "Time": 0,
+ "Value": 15,
+ "__name__": 1,
+ "container": 2,
+ "customresource_group": 5,
+ "customresource_kind": 6,
+ "customresource_version": 7,
+ "endpoint": 8,
+ "exported_namespace": 4,
+ "instance": 9,
+ "job": 10,
+ "name": 11,
+ "namespace": 3,
+ "pod": 12,
+ "ready": 13,
+ "service": 14
+ },
+ "renameByName": {
+ "customresource_kind": "Kind",
+ "exported_namespace": "Namespace",
+ "name": "Name"
+ }
+ }
+ },
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "(Namespace|Name|Kind)"
+ }
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "collapsed": false,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 32
+ },
+ "id": 17,
+ "panels": [],
+ "title": "Timing",
+ "type": "row"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byValue",
+ "options": {
+ "op": "gte",
+ "reducer": "allIsZero",
+ "value": 0
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": true,
+ "tooltip": true,
+ "viz": false
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byValue",
+ "options": {
+ "op": "gte",
+ "reducer": "allIsNull",
+ "value": 0
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": true,
+ "tooltip": true,
+ "viz": false
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 33
+ },
+ "id": 27,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind, name)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"Kustomization|HelmRelease\"}[5m])) by (kind, name)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{kind}}/{{name}}",
+ "refId": "B"
+ }
+ ],
+ "title": "Cluster reconciliation duration",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byValue",
+ "options": {
+ "op": "gte",
+ "reducer": "allIsZero",
+ "value": 0
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": true,
+ "tooltip": true,
+ "viz": false
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byValue",
+ "options": {
+ "op": "gte",
+ "reducer": "allIsNull",
+ "value": 0
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": true,
+ "tooltip": true,
+ "viz": false
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 41
+ },
+ "id": 35,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind, name)\n/\n sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind, name)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{kind}}/{{name}}",
+ "refId": "B"
+ }
+ ],
+ "title": "Source acquisition duration",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "30s",
+ "schemaVersion": 38,
+ "style": "light",
+ "tags": [
+ "flux"
+ ],
+ "templating": {
+ "list": [
+ {
+ "allValue": "",
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$DS_PROMETHEUS"
+ },
+ "definition": "label_values(gotk_reconcile_condition, namespace)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "operator_namespace",
+ "options": [],
+ "query": {
+ "query": "label_values(gotk_reconcile_condition, namespace)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 5,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$DS_PROMETHEUS"
+ },
+ "definition": "label_values(gotk_resource_info,exported_namespace)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "namespace",
+ "options": [],
+ "query": {
+ "query": "label_values(gotk_resource_info,exported_namespace)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Datasource",
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-15m",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "",
+ "title": "Flux Cluster Stats",
+ "uid": "flux-cluster",
+ "version": 4,
+ "weekStart": ""
+ }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ labels:
+ grafana_dashboard: "1"
+ name: draupnir4all-status-dashboard
+data:
+ dashboard.json: |-
+ {
+ "__inputs": [
+ {
+ "name": "DS_LOKI",
+ "label": "Loki",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "loki",
+ "pluginName": "Loki"
+ }
+ ],
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "10.4.0"
+ },
+ {
+ "type": "panel",
+ "id": "logs",
+ "name": "Logs",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "loki",
+ "name": "Loki",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "stat",
+ "name": "Stat",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "panels": [
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 3,
+ "panels": [],
+ "title": "Status Overview",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "${DS_LOKI}"
+ },
+ "description": "Green means everything is as expected",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Value #Failed"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Failed"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Value #BackOff"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "BackOff"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 1
+ },
+ "id": 1,
+ "options": {
+ "colorMode": "background_solid",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "max"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "name",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "${DS_LOKI}"
+ },
+ "editorMode": "builder",
+ "expr": "max(count_over_time({app=\"kube-event-exporter\"} | json | namespace = `matrix` | reason = `Failed` | name =~ `.*draupnir-synapse.*` [5m]))",
+ "intervalMs": 1000,
+ "legendFormat": "Failed",
+ "maxDataPoints": 43200,
+ "queryType": "instant",
+ "refId": "Failed"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "${DS_LOKI}"
+ },
+ "editorMode": "builder",
+ "expr": "count_over_time({app=\"kube-event-exporter\"} | json | namespace = `matrix` | reason = `BackOff` | name =~ `.*draupnir-synapse.*` [5m])",
+ "hide": false,
+ "intervalMs": 1000,
+ "legendFormat": "BackOff",
+ "maxDataPoints": 43200,
+ "queryType": "instant",
+ "refId": "BackOff"
+ }
+ ],
+ "title": "Draupnir Synapse Status",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "${DS_LOKI}"
+ },
+ "description": "Green means everything is as expected",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Value #Failed"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Failed"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Value #BackOff"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "BackOff"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 1
+ },
+ "id": 2,
+ "options": {
+ "colorMode": "background_solid",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "max"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "name",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "${DS_LOKI}"
+ },
+ "editorMode": "builder",
+ "expr": "max(count_over_time({app=\"kube-event-exporter\"} | json | namespace = `matrix` | reason = `Failed` | name =~ `.*draupnir4all.*` [5m]))",
+ "intervalMs": 1000,
+ "legendFormat": "Failed",
+ "maxDataPoints": 43200,
+ "queryType": "instant",
+ "refId": "Failed"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "${DS_LOKI}"
+ },
+ "editorMode": "builder",
+ "expr": "count_over_time({app=\"kube-event-exporter\"} | json | namespace = `matrix` | reason = `BackOff` | name =~ `.*draupnir4all.*` [5m])",
+ "hide": false,
+ "intervalMs": 1000,
+ "legendFormat": "BackOff",
+ "maxDataPoints": 43200,
+ "queryType": "instant",
+ "refId": "BackOff"
+ }
+ ],
+ "title": "Draupnir4All Status",
+ "type": "stat"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 8
+ },
+ "id": 4,
+ "panels": [],
+ "title": "Logs",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "${DS_LOKI}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 9
+ },
+ "id": 5,
+ "options": {
+ "dedupStrategy": "exact",
+ "enableLogDetails": true,
+ "prettifyLogMessage": true,
+ "showCommonLabels": false,
+ "showLabels": false,
+ "showTime": true,
+ "sortOrder": "Descending",
+ "wrapLogMessage": false
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "${DS_LOKI}"
+ },
+ "editorMode": "builder",
+ "expr": "{app=\"kube-event-exporter\"} | json | namespace = `matrix` | name =~ `.*draupnir-synapse.*`",
+ "queryType": "range",
+ "refId": "A"
+ }
+ ],
+ "title": "Draupnir4All Synapse Events",
+ "type": "logs"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "${DS_LOKI}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 17
+ },
+ "id": 6,
+ "options": {
+ "dedupStrategy": "exact",
+ "enableLogDetails": true,
+ "prettifyLogMessage": true,
+ "showCommonLabels": false,
+ "showLabels": false,
+ "showTime": true,
+ "sortOrder": "Descending",
+ "wrapLogMessage": false
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "${DS_LOKI}"
+ },
+ "editorMode": "builder",
+ "expr": "{app=\"kube-event-exporter\"} | json | namespace = `matrix` | name =~ `.*draupnir4all.*`",
+ "queryType": "range",
+ "refId": "A"
+ }
+ ],
+ "title": "Draupnir4All Events",
+ "type": "logs"
+ }
+ ],
+ "refresh": "10s",
+ "schemaVersion": 39,
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "Draupnir4All Status and logs",
+ "uid": "bdz3qba37rwg0e",
+ "version": 4,
+ "weekStart": ""
+ }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ labels:
+ grafana_dashboard: "1"
+ name: flux-control-plane-dashboard
+data:
+ dashboard.json: |-
+ {
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "iconColor": "red",
+ "name": "flux events",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [
+ "flux"
+ ],
+ "type": "tags"
+ }
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 1,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 0
+ },
+ "id": 24,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "value"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(go_info{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
+ "interval": "",
+ "legendFormat": "pods",
+ "refId": "A"
+ }
+ ],
+ "title": "Controllers",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 50
+ },
+ {
+ "color": "red",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 0
+ },
+ "id": 23,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "max(workqueue_longest_running_processor_seconds{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "seconds",
+ "refId": "B"
+ }
+ ],
+ "title": "Max Work Queue",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 500000000
+ },
+ {
+ "color": "red",
+ "value": 900000000
+ }
+ ]
+ },
+ "unit": "decbits"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 12,
+ "y": 0
+ },
+ "id": 25,
+ "options": {
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true,
+ "text": {}
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(go_memstats_alloc_bytes{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Memory",
+ "type": "gauge"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 50
+ },
+ {
+ "color": "red",
+ "value": 100
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 18,
+ "y": 0
+ },
+ "id": 26,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m]))",
+ "interval": "",
+ "legendFormat": "requests",
+ "refId": "A"
+ }
+ ],
+ "title": "API Requests",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 5
+ },
+ "id": 21,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\"}[1m]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "total",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",code!~\"2..\"}[1m]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "errors",
+ "refId": "B"
+ }
+ ],
+ "title": "Kubernetes API Requests",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 13
+ },
+ "id": 15,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Resource Usage",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 0,
+ "y": 14
+ },
+ "id": 11,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "rate(process_cpu_seconds_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m])",
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "CPU Usage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 0,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 12,
+ "y": 14
+ },
+ "id": 13,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"POD\",container!=\"\",pod=~\".*-controller-.*\"}) by (pod)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Memory Usage",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 25
+ },
+ "id": 17,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Reconciliation Stats",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 26
+ },
+ "id": 27,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "workqueue_longest_running_processor_seconds{name=\"kustomization\"}",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "kustomizations",
+ "refId": "B"
+ }
+ ],
+ "title": "Cluster Reconciliation Duration",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 34
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result!=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful reconciliations ",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed reconciliations ",
+ "refId": "B"
+ }
+ ],
+ "title": "Cluster Reconciliations ops/min",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 42
+ },
+ "id": 29,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Sources Stats",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 43
+ },
+ "id": 4,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result!=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful git pulls",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed git pulls",
+ "refId": "B"
+ }
+ ],
+ "title": "Git Repos ops/min",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 43
+ },
+ "id": 30,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result!=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful oci pulls",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed oci pulls",
+ "refId": "B"
+ }
+ ],
+ "title": "OCI Repos ops/min",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 52
+ },
+ "id": 31,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result!=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful helm pulls",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed helm pulls",
+ "refId": "B"
+ }
+ ],
+ "title": "Helm Repos ops/min",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 52
+ },
+ "id": 32,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result!=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful bucket pulls",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result=\"error\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed bucket pulls",
+ "refId": "B"
+ }
+ ],
+ "title": "Buckets ops/min",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 61
+ },
+ "id": 19,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Helm Stats",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 62
+ },
+ "id": 9,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
+ "hide": true,
+ "interval": "",
+ "legendFormat": "P50",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.90, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
+ "hide": true,
+ "interval": "",
+ "legendFormat": "P90",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "P99",
+ "refId": "C"
+ }
+ ],
+ "title": "Helm Release Duration",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 70
+ },
+ "id": 5,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result!=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful reconciliations ",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed reconciliations ",
+ "refId": "B"
+ }
+ ],
+ "title": "Helm Releases ops/min",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "opm"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 70
+ },
+ "id": 6,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result!=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "successful chart pulls",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result=\"error\"}[1m])) by (controller)",
+ "format": "time_series",
+ "interval": "",
+ "legendFormat": "failed chart pulls",
+ "refId": "B"
+ }
+ ],
+ "title": "Helm Charts ops/min",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "10s",
+ "schemaVersion": 38,
+ "style": "light",
+ "tags": [
+ "flux"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 2,
+ "includeAll": false,
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "flux-system",
+ "value": "flux-system"
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "definition": "workqueue_work_duration_seconds_count",
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "namespace",
+ "options": [],
+ "query": {
+ "query": "workqueue_work_duration_seconds_count",
+ "refId": "Prometheus-namespace-Variable-Query"
+ },
+ "refresh": 2,
+ "regex": "/.*namespace=\"([^\"]*).*/",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-15m",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "",
+ "title": "Flux Control Plane",
+ "uid": "flux-control-plane",
+ "version": 2,
+ "weekStart": ""
+ }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ labels:
+ grafana_dashboard: "1"
+ name: flux-logs-dashboard
+data:
+ dashboard.json: |-
+ {
+ "__inputs": [
+ {
+ "name": "DS_LOKI",
+ "label": "Loki",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "loki",
+ "pluginName": "Loki"
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "iconColor": "red",
+ "name": "flux events",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [
+ "flux"
+ ],
+ "type": "tags"
+ }
+ }
+ ]
+ },
+ "description": "Flux logs collected from Kubernetes, stored in Loki",
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": 29,
+ "iteration": 1653748775696,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": "${DS_LOKI}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 4,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": "${DS_LOKI}",
+ "expr": "sum(count_over_time({namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\" [$__interval]))",
+ "instant": false,
+ "legendFormat": "Log count",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "type": "timeseries"
+ },
+ {
+ "datasource": "${DS_LOKI}",
+ "description": "Logs from services running in Kubernetes",
+ "gridPos": {
+ "h": 25,
+ "w": 24,
+ "x": 0,
+ "y": 4
+ },
+ "id": 2,
+ "options": {
+ "dedupStrategy": "numbers",
+ "enableLogDetails": false,
+ "prettifyLogMessage": true,
+ "showCommonLabels": false,
+ "showLabels": false,
+ "showTime": false,
+ "sortOrder": "Descending",
+ "wrapLogMessage": false
+ },
+ "targets": [
+ {
+ "datasource": "${DS_LOKI}",
+ "expr": "{namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\"",
+ "refId": "A"
+ }
+ ],
+ "type": "logs"
+ }
+ ],
+ "refresh": "10s",
+ "schemaVersion": 36,
+ "style": "light",
+ "tags": [
+ "flux"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "",
+ "value": ""
+ },
+ "description": "String to search for",
+ "hide": 0,
+ "label": "Search Query",
+ "name": "query",
+ "options": [
+ {
+ "selected": true,
+ "text": "",
+ "value": ""
+ }
+ ],
+ "query": "",
+ "skipUrlSync": false,
+ "type": "textbox"
+ },
+ {
+ "allValue": "info|error",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "hide": 0,
+ "includeAll": true,
+ "multi": false,
+ "name": "level",
+ "options": [
+ {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ {
+ "selected": false,
+ "text": "info",
+ "value": "info"
+ },
+ {
+ "selected": false,
+ "text": "error",
+ "value": "error"
+ }
+ ],
+ "query": "info,error",
+ "queryValue": "",
+ "skipUrlSync": false,
+ "type": "custom"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": "${DS_LOKI}",
+ "definition": "label_values(app)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "controller",
+ "options": [],
+ "query": "label_values(app)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "type": "query"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": true,
+ "text": [
+ "flux-system"
+ ],
+ "value": [
+ "flux-system"
+ ]
+ },
+ "datasource": "${DS_LOKI}",
+ "definition": "label_values(namespace)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "namespace",
+ "options": [],
+ "query": "label_values(namespace)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "type": "query"
+ },
+ {
+ "allValue": ".+",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": "${DS_LOKI}",
+ "definition": "label_values(stream)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "stream",
+ "options": [],
+ "query": "label_values(stream)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "type": "query"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "Loki",
+ "value": "Loki"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Datasource",
+ "multi": false,
+ "name": "DS_LOKI",
+ "options": [],
+ "query": "loki",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "Flux Logs",
+ "uid": "flux-logs",
+ "version": 2
+ }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ labels:
+ grafana_dashboard: "1"
+ name: postgres-dashboard
+data:
+ dashboard.json: "{\n\"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"--
+ Grafana --\",\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n
+ \ \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n},\n\"description\": \"Performance
+ metrics for Postgres\",\n\"editable\": true,\n\"gnetId\": 455,\n\"graphTooltip\": 0,\n\"id\": 1,\n\"iteration\": 1603191461722,\n\"links\":
+ [],\n\"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\":
+ false,\n \"datasource\": \"$datasource\",\n \"editable\": true,\n \"error\": false,\n \"fieldConfig\":
+ {\n \"defaults\": {\n \"custom\": {}\n },\n \"overrides\": []\n },\n \"fill\": 1,\n
+ \ \"fillGradient\": 0,\n \"grid\": {},\n \"gridPos\": {\n \"h\": 7,\n \"w\": 20,\n \"x\":
+ 0,\n \"y\": 0\n },\n \"hiddenSeries\": false,\n \"id\": 1,\n \"isNew\": true,\n \"legend\":
+ {\n \"alignAsTable\": true,\n \"avg\": true,\n \"current\": false,\n \"max\": true,\n
+ \ \"min\": true,\n \"rightSide\": true,\n \"show\": true,\n \"total\": false,\n \"values\":
+ true\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n
+ \ \"options\": {\n \"alertThreshold\": true\n },\n \"percentage\": false,\n \"pluginVersion\": \"7.2.1\",\n
+ \ \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\":
+ 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"alias\": \"fetched\",\n
+ \ \"dsType\": \"prometheus\",\n \"expr\": \"sum(irate(pg_stat_database_tup_fetched{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval]))\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"fetched\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\": \"A\",\n
+ \ \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"tup_fetched\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [\n \"10s\"\n ],\n \"type\": \"non_negative_derivative\"\n }\n
+ \ ]\n ],\n \"step\": 120,\n \"tags\": [\n {\n \"key\": \"instance\",\n
+ \ \"operator\": \"=~\",\n \"value\": \"/^$instance$/\"\n }\n ]\n },\n
+ \ {\n \"alias\": \"fetched\",\n \"dsType\": \"prometheus\",\n \"expr\": \"sum(irate(pg_stat_database_tup_returned{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval]))\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"returned\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\": \"B\",\n
+ \ \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"tup_fetched\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [\n \"10s\"\n ],\n \"type\": \"non_negative_derivative\"\n }\n
+ \ ]\n ],\n \"step\": 120,\n \"tags\": [\n {\n \"key\": \"instance\",\n
+ \ \"operator\": \"=~\",\n \"value\": \"/^$instance$/\"\n }\n ]\n },\n
+ \ {\n \"alias\": \"fetched\",\n \"dsType\": \"prometheus\",\n \"expr\": \"sum(irate(pg_stat_database_tup_inserted{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval]))\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"inserted\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\": \"C\",\n
+ \ \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"tup_fetched\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [\n \"10s\"\n ],\n \"type\": \"non_negative_derivative\"\n }\n
+ \ ]\n ],\n \"step\": 120,\n \"tags\": [\n {\n \"key\": \"instance\",\n
+ \ \"operator\": \"=~\",\n \"value\": \"/^$instance$/\"\n }\n ]\n },\n
+ \ {\n \"alias\": \"fetched\",\n \"dsType\": \"prometheus\",\n \"expr\": \"sum(irate(pg_stat_database_tup_updated{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval]))\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"updated\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\": \"D\",\n
+ \ \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"tup_fetched\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [\n \"10s\"\n ],\n \"type\": \"non_negative_derivative\"\n }\n
+ \ ]\n ],\n \"step\": 120,\n \"tags\": [\n {\n \"key\": \"instance\",\n
+ \ \"operator\": \"=~\",\n \"value\": \"/^$instance$/\"\n }\n ]\n },\n
+ \ {\n \"alias\": \"fetched\",\n \"dsType\": \"prometheus\",\n \"expr\": \"sum(irate(pg_stat_database_tup_deleted{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval]))\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"deleted\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\": \"E\",\n
+ \ \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"tup_fetched\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [\n \"10s\"\n ],\n \"type\": \"non_negative_derivative\"\n }\n
+ \ ]\n ],\n \"step\": 120,\n \"tags\": [\n {\n \"key\": \"instance\",\n
+ \ \"operator\": \"=~\",\n \"value\": \"/^$instance$/\"\n }\n ]\n }\n
+ \ ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeRegions\": [],\n \"timeShift\": null,\n \"title\":
+ \"Rows\",\n \"tooltip\": {\n \"msResolution\": true,\n \"shared\": true,\n \"sort\": 0,\n
+ \ \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n
+ \ \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\":
+ [\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\":
+ null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\":
+ null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n
+ \ ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"cacheTimeout\":
+ null,\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54,
+ 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"$datasource\",\n
+ \ \"decimals\": 0,\n \"editable\": true,\n \"error\": false,\n \"fieldConfig\": {\n \"defaults\":
+ {\n \"custom\": {}\n },\n \"overrides\": []\n },\n \"format\": \"none\",\n \"gauge\":
+ {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n
+ \ \"thresholdMarkers\": true\n },\n \"gridPos\": {\n \"h\": 3,\n \"w\": 4,\n \"x\":
+ 20,\n \"y\": 0\n },\n \"height\": \"55px\",\n \"id\": 11,\n \"interval\": null,\n \"isNew\":
+ true,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value
+ to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\":
+ 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"nullText\": null,\n
+ \ \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n
+ \ \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n
+ \ }\n ],\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": true,\n
+ \ \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": true\n },\n \"tableColumn\": \"\",\n \"targets\":
+ [\n {\n \"dsType\": \"prometheus\",\n \"expr\": \"sum(irate(pg_stat_database_xact_commit{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval]))
+ + sum(irate(pg_stat_database_xact_rollback{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval]))\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"measurement\":
+ \"postgresql\",\n \"policy\": \"default\",\n \"refId\": \"A\",\n \"resultFormat\": \"time_series\",\n
+ \ \"select\": [\n [\n {\n \"params\": [\n \"xact_commit\"\n
+ \ ],\n \"type\": \"field\"\n },\n {\n \"params\":
+ [],\n \"type\": \"mean\"\n },\n {\n \"params\": [\n \"10s\"\n
+ \ ],\n \"type\": \"non_negative_derivative\"\n }\n ]\n ],\n
+ \ \"step\": 1800,\n \"tags\": [\n {\n \"key\": \"instance\",\n \"operator\":
+ \"=~\",\n \"value\": \"/^$instance$/\"\n }\n ]\n }\n ],\n \"thresholds\":
+ \"\",\n \"title\": \"QPS\",\n \"transparent\": true,\n \"type\": \"singlestat\",\n \"valueFontSize\":
+ \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\":
+ \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n
+ \ \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"$datasource\",\n \"decimals\": 1,\n \"editable\":
+ true,\n \"error\": false,\n \"fieldConfig\": {\n \"defaults\": {\n \"custom\": {}\n },\n
+ \ \"overrides\": []\n },\n \"fill\": 1,\n \"fillGradient\": 0,\n \"grid\": {},\n \"gridPos\":
+ {\n \"h\": 7,\n \"w\": 12,\n \"x\": 0,\n \"y\": 7\n },\n \"hiddenSeries\": false,\n
+ \ \"id\": 2,\n \"isNew\": true,\n \"legend\": {\n \"alignAsTable\": true,\n \"avg\": true,\n
+ \ \"current\": false,\n \"hideZero\": true,\n \"max\": true,\n \"min\": true,\n \"rightSide\":
+ false,\n \"show\": true,\n \"total\": false,\n \"values\": true\n },\n \"lines\": true,\n
+ \ \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"options\": {\n \"alertThreshold\":
+ true\n },\n \"percentage\": false,\n \"pluginVersion\": \"7.2.1\",\n \"pointradius\": 5,\n \"points\":
+ false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n
+ \ \"steppedLine\": false,\n \"targets\": [\n {\n \"alias\": \"Buffers Allocated\",\n \"dsType\":
+ \"prometheus\",\n \"expr\": \"irate(pg_stat_bgwriter_buffers_alloc{job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval])\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"buffers_alloc\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\":
+ \"A\",\n \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"buffers_alloc\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [],\n \"type\": \"difference\"\n }\n ]\n ],\n \"step\": 240,\n
+ \ \"tags\": [\n {\n \"key\": \"instance\",\n \"operator\": \"=~\",\n \"value\":
+ \"/^$instance$/\"\n }\n ]\n },\n {\n \"alias\": \"Buffers Allocated\",\n \"dsType\":
+ \"prometheus\",\n \"expr\": \"irate(pg_stat_bgwriter_buffers_backend_fsync{job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval])\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"buffers_backend_fsync\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\":
+ \"B\",\n \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"buffers_alloc\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [],\n \"type\": \"difference\"\n }\n ]\n ],\n \"step\": 240,\n
+ \ \"tags\": [\n {\n \"key\": \"instance\",\n \"operator\": \"=~\",\n \"value\":
+ \"/^$instance$/\"\n }\n ]\n },\n {\n \"alias\": \"Buffers Allocated\",\n \"dsType\":
+ \"prometheus\",\n \"expr\": \"irate(pg_stat_bgwriter_buffers_backend{job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval])\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"buffers_backend\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\":
+ \"C\",\n \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"buffers_alloc\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [],\n \"type\": \"difference\"\n }\n ]\n ],\n \"step\": 240,\n
+ \ \"tags\": [\n {\n \"key\": \"instance\",\n \"operator\": \"=~\",\n \"value\":
+ \"/^$instance$/\"\n }\n ]\n },\n {\n \"alias\": \"Buffers Allocated\",\n \"dsType\":
+ \"prometheus\",\n \"expr\": \"irate(pg_stat_bgwriter_buffers_clean{job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval])\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"buffers_clean\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\":
+ \"D\",\n \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"buffers_alloc\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [],\n \"type\": \"difference\"\n }\n ]\n ],\n \"step\": 240,\n
+ \ \"tags\": [\n {\n \"key\": \"instance\",\n \"operator\": \"=~\",\n \"value\":
+ \"/^$instance$/\"\n }\n ]\n },\n {\n \"alias\": \"Buffers Allocated\",\n \"dsType\":
+ \"prometheus\",\n \"expr\": \"irate(pg_stat_bgwriter_buffers_checkpoint{job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval])\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"buffers_checkpoint\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\":
+ \"E\",\n \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"buffers_alloc\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [],\n \"type\": \"difference\"\n }\n ]\n ],\n \"step\": 240,\n
+ \ \"tags\": [\n {\n \"key\": \"instance\",\n \"operator\": \"=~\",\n \"value\":
+ \"/^$instance$/\"\n }\n ]\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n
+ \ \"timeRegions\": [],\n \"timeShift\": null,\n \"title\": \"Buffers\",\n \"tooltip\": {\n \"msResolution\":
+ false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\":
+ \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\":
+ true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"label\":
+ null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": 0,\n \"show\": true\n },\n
+ \ {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n
+ \ \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\":
+ null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\":
+ false,\n \"datasource\": \"$datasource\",\n \"editable\": true,\n \"error\": false,\n \"fieldConfig\":
+ {\n \"defaults\": {\n \"custom\": {}\n },\n \"overrides\": []\n },\n \"fill\": 1,\n
+ \ \"fillGradient\": 0,\n \"grid\": {},\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\":
+ 12,\n \"y\": 7\n },\n \"hiddenSeries\": false,\n \"id\": 3,\n \"isNew\": true,\n \"legend\":
+ {\n \"avg\": false,\n \"current\": false,\n \"max\": false,\n \"min\": false,\n \"show\":
+ true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n
+ \ \"links\": [],\n \"nullPointMode\": \"connected\",\n \"options\": {\n \"alertThreshold\": true\n
+ \ },\n \"percentage\": false,\n \"pluginVersion\": \"7.2.1\",\n \"pointradius\": 5,\n \"points\": false,\n
+ \ \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\":
+ false,\n \"targets\": [\n {\n \"alias\": \"conflicts\",\n \"dsType\": \"prometheus\",\n \"expr\":
+ \"sum(rate(pg_stat_database_deadlocks{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval]))\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"deadlocks\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\": \"A\",\n
+ \ \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"conflicts\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [],\n \"type\": \"difference\"\n }\n ]\n ],\n \"step\": 240,\n
+ \ \"tags\": [\n {\n \"key\": \"instance\",\n \"operator\": \"=~\",\n \"value\":
+ \"/^$instance$/\"\n }\n ]\n },\n {\n \"alias\": \"deadlocks\",\n \"dsType\":
+ \"prometheus\",\n \"expr\": \"sum(rate(pg_stat_database_conflicts{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval]))\",\n
+ \ \"format\": \"time_series\",\n \"groupBy\": [\n {\n \"params\": [\n \"$interval\"\n
+ \ ],\n \"type\": \"time\"\n },\n {\n \"params\": [\n \"null\"\n
+ \ ],\n \"type\": \"fill\"\n }\n ],\n \"intervalFactor\": 2,\n \"legendFormat\":
+ \"conflicts\",\n \"measurement\": \"postgresql\",\n \"policy\": \"default\",\n \"refId\": \"B\",\n
+ \ \"resultFormat\": \"time_series\",\n \"select\": [\n [\n {\n \"params\":
+ [\n \"deadlocks\"\n ],\n \"type\": \"field\"\n },\n {\n
+ \ \"params\": [],\n \"type\": \"mean\"\n },\n {\n \"params\":
+ [],\n \"type\": \"difference\"\n }\n ]\n ],\n \"step\": 240,\n
+ \ \"tags\": [\n {\n \"key\": \"instance\",\n \"operator\": \"=~\",\n \"value\":
+ \"/^$instance$/\"\n }\n ]\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n
+ \ \"timeRegions\": [],\n \"timeShift\": null,\n \"title\": \"Conflicts/Deadlocks\",\n \"tooltip\": {\n
+ \ \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n
+ \ },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\":
+ null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n
+ \ \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": 0,\n \"show\":
+ true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\":
+ null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n
+ \ \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\":
+ 10,\n \"dashes\": false,\n \"datasource\": \"$datasource\",\n \"editable\": true,\n \"error\": false,\n
+ \ \"fieldConfig\": {\n \"defaults\": {\n \"custom\": {}\n },\n \"overrides\": []\n },\n
+ \ \"fill\": 1,\n \"fillGradient\": 0,\n \"grid\": {},\n \"gridPos\": {\n \"h\": 7,\n \"w\":
+ 12,\n \"x\": 0,\n \"y\": 14\n },\n \"hiddenSeries\": false,\n \"id\": 12,\n \"isNew\": true,\n
+ \ \"legend\": {\n \"avg\": false,\n \"current\": false,\n \"max\": false,\n \"min\":
+ false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n
+ \ \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"options\": {\n \"alertThreshold\":
+ true\n },\n \"percentage\": true,\n \"pluginVersion\": \"7.2.1\",\n \"pointradius\": 1,\n \"points\":
+ false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n
+ \ \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"sum by (datname) (rate(pg_stat_database_blks_hit{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval]))
+ / (sum by (datname)(rate(pg_stat_database_blks_hit{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval]))
+ + sum by (datname)(rate(pg_stat_database_blks_read{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[$__rate_interval])))\",\n
+ \ \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"{{datname}} - cache
+ hit rate\",\n \"refId\": \"A\",\n \"step\": 240\n }\n ],\n \"thresholds\": [],\n \"timeFrom\":
+ null,\n \"timeRegions\": [],\n \"timeShift\": null,\n \"title\": \"Cache hit ratio\",\n \"tooltip\": {\n
+ \ \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n
+ \ },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\":
+ null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"percentunit\",\n
+ \ \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\":
+ true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\":
+ null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n
+ \ \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\":
+ 10,\n \"dashes\": false,\n \"datasource\": \"$datasource\",\n \"editable\": true,\n \"error\": false,\n
+ \ \"fieldConfig\": {\n \"defaults\": {\n \"custom\": {}\n },\n \"overrides\": []\n },\n
+ \ \"fill\": 1,\n \"fillGradient\": 0,\n \"grid\": {},\n \"gridPos\": {\n \"h\": 7,\n \"w\":
+ 12,\n \"x\": 12,\n \"y\": 14\n },\n \"hiddenSeries\": false,\n \"id\": 13,\n \"isNew\":
+ true,\n \"legend\": {\n \"avg\": false,\n \"current\": false,\n \"max\": false,\n \"min\":
+ false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n
+ \ \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"options\": {\n \"alertThreshold\":
+ true\n },\n \"percentage\": false,\n \"pluginVersion\": \"7.2.1\",\n \"pointradius\": 5,\n \"points\":
+ false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n
+ \ \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"pg_stat_database_numbackends{datname=~\\\"$db\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}\",\n
+ \ \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"{{datname}} - {{__name__}}\",\n
+ \ \"refId\": \"A\",\n \"step\": 240\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n
+ \ \"timeRegions\": [],\n \"timeShift\": null,\n \"title\": \"Number of active connections\",\n \"tooltip\":
+ {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n
+ \ },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\":
+ null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n
+ \ \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\":
+ true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\":
+ null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n
+ \ \"alignLevel\": null\n }\n }\n],\n\"refresh\": false,\n\"schemaVersion\": 26,\n\"style\": \"dark\",\n\"tags\":
+ [\n \"postgres\"\n],\n\"templating\": {\n \"list\": [\n {\n \"hide\": 0,\n \"includeAll\":
+ false,\n \"label\": \"Data Source\",\n \"multi\": false,\n \"name\": \"datasource\",\n \"options\":
+ [],\n \"query\": \"prometheus\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"skipUrlSync\":
+ false,\n \"type\": \"datasource\"\n }, \n {\n \"allValue\": \".+\",\n \"datasource\":
+ \"$datasource\",\n \"definition\": \"label_values(pg_up, job)\",\n \"hide\": 0,\n \"includeAll\":
+ true,\n \"label\": \"job\",\n \"multi\": true,\n \"name\": \"job\",\n \"options\": [],\n
+ \ \"query\": \"label_values(pg_up, job)\",\n \"refresh\": 0,\n \"regex\": \"\",\n \"skipUrlSync\":
+ false,\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n
+ \ \"type\": \"query\",\n \"useTags\": false\n },\n {\n \"allValue\": \".+\",\n \"datasource\":
+ \"$datasource\",\n \"definition\": \"\",\n \"hide\": 0,\n \"includeAll\": true,\n \"label\":
+ \"instance\",\n \"multi\": true,\n \"name\": \"instance\",\n \"options\": [],\n \"query\":
+ \"label_values(up{job=~\\\"$job\\\"},instance)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"skipUrlSync\":
+ false,\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n
+ \ \"type\": \"query\",\n \"useTags\": false\n },\n {\n \"allValue\": \".+\",\n \"datasource\":
+ \"$datasource\",\n \"definition\": \"label_values(pg_stat_database_tup_fetched{instance=~\\\"$instance\\\",datname!~\\\"template.*|postgres\\\"},datname)\",\n
+ \ \"hide\": 0,\n \"includeAll\": true,\n \"label\": \"db\",\n \"multi\": false,\n \"name\":
+ \"db\",\n \"options\": [],\n \"query\": \"label_values(pg_stat_database_tup_fetched{instance=~\\\"$instance\\\",datname!~\\\"template.*|postgres\\\"},datname)\",\n
+ \ \"refresh\": 1,\n \"regex\": \"\",\n \"skipUrlSync\": false,\n \"sort\": 0,\n \"tagValuesQuery\":
+ \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n
+ \ }\n ]\n},\n\"time\": {\n \"from\": \"now-1h\",\n \"to\": \"now\"\n},\n\"timepicker\": {\n \"refresh_intervals\":
+ [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n
+ \ \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n
+ \ \"2d\",\n \"7d\",\n \"30d\"\n ]\n},\n\"timezone\": \"browser\",\n\"title\": \"Postgres Overview\",\n\"uid\":
+ \"wGgaPlciz\",\n\"version\": 5\n}"
diff --git a/apps/base/victoria-metrics-k8s-stack/kustomization.yaml b/apps/base/victoria-metrics-k8s-stack/kustomization.yaml
@@ -3,6 +3,7 @@ kind: Kustomization
namespace: monitoring
resources:
- ./crds
+ - dashboards.json
- repository.yaml
- release.yaml
- minio_loki_metrics.yaml
diff --git a/apps/base/victoria-metrics-k8s-stack/release.yaml b/apps/base/victoria-metrics-k8s-stack/release.yaml
@@ -65,11 +65,11 @@ spec:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
- matchExpressions:
- - key: ENC[AES256_GCM,data:1DK2,iv:BdONcnpjAcPGSMBu9FjC7OKJhY+cgywAT/ogo3amaxA=,tag:BbDcrbQJRVPeIux175Xfkw==,type:str]
- operator: In
- values:
- - vminsert
+ matchExpressions:
+ - key: ENC[AES256_GCM,data:1DK2,iv:BdONcnpjAcPGSMBu9FjC7OKJhY+cgywAT/ogo3amaxA=,tag:BbDcrbQJRVPeIux175Xfkw==,type:str]
+ operator: In
+ values:
+ - vminsert
topologyKey: kubernetes.io/hostname
vmselect:
extraArgs:
@@ -93,11 +93,11 @@ spec:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
- matchExpressions:
- - key: ENC[AES256_GCM,data:LhBR,iv:dm+w77ND4Cl89Qs8qvEYt1sxCRLD0w1kPMX6fG/rTIs=,tag:ejVPqyvNn4dLB5lERCKFBQ==,type:str]
- operator: In
- values:
- - vmselect
+ matchExpressions:
+ - key: ENC[AES256_GCM,data:LhBR,iv:dm+w77ND4Cl89Qs8qvEYt1sxCRLD0w1kPMX6fG/rTIs=,tag:ejVPqyvNn4dLB5lERCKFBQ==,type:str]
+ operator: In
+ values:
+ - vmselect
topologyKey: kubernetes.io/hostname
vmstorage:
extraArgs:
@@ -110,11 +110,11 @@ spec:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
- matchExpressions:
- - key: ENC[AES256_GCM,data:jm2f,iv:6ey3oJ/pfPmgx0/y/+9BmNM+UnlMPAnuJNO8pFrjXFg=,tag:LbdconZMqw3GtFYLXYH96A==,type:str]
- operator: In
- values:
- - vmstorage
+ matchExpressions:
+ - key: ENC[AES256_GCM,data:jm2f,iv:6ey3oJ/pfPmgx0/y/+9BmNM+UnlMPAnuJNO8pFrjXFg=,tag:LbdconZMqw3GtFYLXYH96A==,type:str]
+ operator: In
+ values:
+ - vmstorage
topologyKey: kubernetes.io/hostname
storage:
volumeClaimTemplate:
@@ -132,7 +132,7 @@ spec:
dashboards:
logLevel: WARN
watchServerTimeout: 3600
- enabled: false
+ enabled: true
adminPassword: ENC[AES256_GCM,data:0RAePf48YM29I+mPXQAB1XCvbujgJx/KGuEsgZCu4/EuMMqLS80BeqqpcZFtw3HlXs9gz79/Nnu9OnC7wJFBAA==,iv:GzvGCghNsDMmz67MWGCvExxzqkE/XYl49BGjwwlJ3ws=,tag:xcjd3MuJBXfye6DvFB50Zg==,type:str]
alerting:
rules.yaml:
@@ -143,1044 +143,1045 @@ spec:
folder: Cluster
interval: 5m
rules:
- - uid: ddz3zj028wb28b
- title: CertManagerHittingRateLimits
- condition: B
- data:
- - refId: ENC[AES256_GCM,data:OQ==,iv:7rX1LGyPZwto+Q+OSLbWUx3sXpPVnoOh09lbsIfrFZk=,tag:HvFVZC1UaKtNSWJlA31dfw==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:8Hep,iv:+nFAVrzAIA2aRlDHsf5x8OBfJ24G/89WWfC/YlXKxrs=,tag:LZOjPKhMxXm1CaoRB016Ug==,type:int]
- to: ENC[AES256_GCM,data:lw==,iv:xaCkH0FH3C3k2Dl2sTw//dcOlmQ4W1CB/exCuQPfdso=,tag:YyamoCCcegYxPQmLaiA5Kw==,type:int]
- datasourceUid: ENC[AES256_GCM,data:BjQYk0xg8JTnIQ==,iv:M6nzO83cjfR5KVbuedL1tCC6JOeYEL6ZC2mykSH2KdA=,tag:SGswBh4EFLR7A2IYj9rUAg==,type:str]
- model:
- editorMode: ENC[AES256_GCM,data:TdLl+g==,iv:GVnS0hStuDUMBFucsV58D01J+wM07HBV8sjtv6lBSaM=,tag:o7gR0Zjy+63ouXVBXHXZBQ==,type:str]
- expr: ENC[AES256_GCM,data:lfamyin/QDJD/kxXZDfsvdLeHJYz0/zYanShD8jSafP8OTmCl9Tp1Dqbq+krr70sVa6AOXEygz2S2wRN0Hnubxxi2pDtK42LJQk+TB25krRbZ2sXavRbzQ+qYZj0YbxwmaQ+ExXgY9sj/g6yoklXQd9y,iv:dpJdmAh4x8IuueTHFwgYVEWT4KyJnV48Vf8/ZhKVbUs=,tag:fg3c1VRyLQljLFZ0NYyRMA==,type:str]
- instant: ENC[AES256_GCM,data:YYgJSw==,iv:shi4bE0+Ekxnzuds873Qw/y863jJqo4sPxWb5cLVXfI=,tag:G8oP3io6Nr7tERWEQXAcrg==,type:bool]
- intervalMs: ENC[AES256_GCM,data:FRh8OA==,iv:IPTymRgaOWA80N1I6iYwI5UQbOpBt6dcMK/wecxAvf4=,tag:9rvaw8ETvLwWQQLppdEG+w==,type:int]
- legendFormat: ENC[AES256_GCM,data:SNwNkNuQ,iv:9AhGAS2JtbEArO6mw1cfweiylOeWsroA2yFA94covcg=,tag:oX/T3ZzMltthkwEr4iFrag==,type:str]
- maxDataPoints: ENC[AES256_GCM,data:N9o4pLA=,iv:PrcXa9xEE3tuxA0UzXdViwMGu+y/FoJwEDrbkWAnEAU=,tag:XOAm/wplBRFhPiQMqLfHRg==,type:int]
- range: ENC[AES256_GCM,data:ACQShno=,iv:zfN/sQWSkvD5cKNQl9gMUvGisF7jst5VdtX7kkIC3hg=,tag:JTaHBRFxfaNZjPstcm+zlQ==,type:bool]
- refId: ENC[AES256_GCM,data:XA==,iv:RCDQpeTdojGSnVKEzz860GLT/D9tzYDpVpownicKUW8=,tag:N24ISyE066ERkcjJ3okVIQ==,type:str]
- - refId: ENC[AES256_GCM,data:cQ==,iv:Af8z+rJ6+fwwI0HYff49IXz8yNFUNE2RJR6wIyZv0Io=,tag:GTuwYtGmUojfgOQntWtLDg==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:6LoX,iv:39cJgTJAWH065LRLsQt1FEGIvfRG33CEDX2Uj6tL52g=,tag:7836/qvDNDqWRnukm5UnLg==,type:int]
- to: ENC[AES256_GCM,data:NQ==,iv:RmVyxnEcWHt4EcRDqcaVnTuegdaGqAPU+BF0ptFbtCM=,tag:n/09FcdfsyrRRdKqbOyRDg==,type:int]
- datasourceUid: ENC[AES256_GCM,data:4Dl9yXzc4IE=,iv:EKPDweX6y8qYngApWY+FdUDuRyGi5D5tLJbvbP1r5go=,tag:ztBOh/4f9UyAG0Ad6Ol84w==,type:str]
- model:
- conditions:
- - evaluator:
- params: []
- type: ENC[AES256_GCM,data:knk=,iv:ipTDdrQ0/NnNiAqORxx7Tjg00pX6PFx++T4puPaKsOo=,tag:/TGUK1qh8g28Pm8d7dN2jQ==,type:str]
- operator:
- type: ENC[AES256_GCM,data:nKwx,iv:eYpiKlhS1bD7WBUlmWDNjReL4940kES6Iww2MlPW/ZA=,tag:7AfKXbZAuy/9Ri0h931LdA==,type:str]
- query:
- params:
- - ENC[AES256_GCM,data:Aw==,iv:HXMEpwV87xHbDty4mNV2Zo+IA4i4bVpQec2jHlPj3+w=,tag:Q1O6aT7xAEClZYNYr3296g==,type:str]
- reducer:
- params: []
- type: ENC[AES256_GCM,data:4OonHQ==,iv:RdDGlAxUYwMEdzwCrwvvOUSmCI1DtRrzSCxNWvxNzhQ=,tag:xtzYYrNm2XvRP86dGKRYLg==,type:str]
- type: ENC[AES256_GCM,data:dvTvmZU=,iv:JjSZo1zsMbM0wUdO5ASh86B3djyL3vIVFRpkDb0yYqc=,tag:2My5Yn0+8NKedW8ertG/gA==,type:str]
- datasource:
- type: ENC[AES256_GCM,data:JKqpg28o6P4=,iv:kad96vkl7JZ/W6+66CCNoy3bZEMaUmql4mFa/b47gq8=,tag:o0ZOxzQca0imXONHvZ78MQ==,type:str]
- uid: ENC[AES256_GCM,data:ua25patPAV0=,iv:Q+ZO2F7KfTR0UMLnDrrC/5Fn2reBU0rlFuqQ+51ltpM=,tag:jnTZIUqwojtc5WahDMbyig==,type:str]
- expression: ENC[AES256_GCM,data:CA==,iv:J2QD05D/1rUWmS9m25NL+K6Pof+WnnXz55xtIaLMaRs=,tag:j4oL5ZtdQj7srz6WjTeyUA==,type:str]
- intervalMs: ENC[AES256_GCM,data:5YOyhA==,iv:jM77S9D9cGgsRWnV/Y3RvbDyw7eY2vH+UlU/nsp9aeM=,tag:E2Sj5fv7TIfk1ozgKMh0ew==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:VA77DZU=,iv:dhrK9yVaqFF8UZiWaISqpH1E/AFXZ4GClHKpecDVFa8=,tag:wS2HSme4z1Mr7cT6fLq1qA==,type:int]
- reducer: ENC[AES256_GCM,data:f7nvIA==,iv:otFz2kg5v3I05AgH4JDsbLSehsM+HrMwboGaObqKBcs=,tag:BD9zFS+BhzL75rikcmuhiQ==,type:str]
- refId: ENC[AES256_GCM,data:Og==,iv:34R9tcvbUoT9u7ZxcZHGy58UHXCzskosgVqsciEmNUk=,tag:9903Wu1WM6uHkzqus+6awQ==,type:str]
- type: ENC[AES256_GCM,data:BYFLsaX+,iv:JjWu7fwRaaa3zrQ++upSDkyO5fuPspfeFJnNzR8N3kk=,tag:oU7FDd75J+jO66DpNXJmuA==,type:str]
- noDataState: OK
- execErrState: Error
- for: 5m
- annotations:
- description: Depending on the rate limit, cert-manager may be unable to generate certificates for up to a week.
- summary: Cert manager hitting LetsEncrypt rate limits.
- labels: {}
- isPaused: false
- notification_settings:
- receiver: PagerDuty Warning
+ - uid: ddz3zj028wb28b
+ title: CertManagerHittingRateLimits
+ condition: B
+ data:
+ - refId: ENC[AES256_GCM,data:OQ==,iv:7rX1LGyPZwto+Q+OSLbWUx3sXpPVnoOh09lbsIfrFZk=,tag:HvFVZC1UaKtNSWJlA31dfw==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:8Hep,iv:+nFAVrzAIA2aRlDHsf5x8OBfJ24G/89WWfC/YlXKxrs=,tag:LZOjPKhMxXm1CaoRB016Ug==,type:int]
+ to: ENC[AES256_GCM,data:lw==,iv:xaCkH0FH3C3k2Dl2sTw//dcOlmQ4W1CB/exCuQPfdso=,tag:YyamoCCcegYxPQmLaiA5Kw==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:BjQYk0xg8JTnIQ==,iv:M6nzO83cjfR5KVbuedL1tCC6JOeYEL6ZC2mykSH2KdA=,tag:SGswBh4EFLR7A2IYj9rUAg==,type:str]
+ model:
+ editorMode: ENC[AES256_GCM,data:TdLl+g==,iv:GVnS0hStuDUMBFucsV58D01J+wM07HBV8sjtv6lBSaM=,tag:o7gR0Zjy+63ouXVBXHXZBQ==,type:str]
+ expr: ENC[AES256_GCM,data:lfamyin/QDJD/kxXZDfsvdLeHJYz0/zYanShD8jSafP8OTmCl9Tp1Dqbq+krr70sVa6AOXEygz2S2wRN0Hnubxxi2pDtK42LJQk+TB25krRbZ2sXavRbzQ+qYZj0YbxwmaQ+ExXgY9sj/g6yoklXQd9y,iv:dpJdmAh4x8IuueTHFwgYVEWT4KyJnV48Vf8/ZhKVbUs=,tag:fg3c1VRyLQljLFZ0NYyRMA==,type:str]
+ instant: ENC[AES256_GCM,data:YYgJSw==,iv:shi4bE0+Ekxnzuds873Qw/y863jJqo4sPxWb5cLVXfI=,tag:G8oP3io6Nr7tERWEQXAcrg==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:FRh8OA==,iv:IPTymRgaOWA80N1I6iYwI5UQbOpBt6dcMK/wecxAvf4=,tag:9rvaw8ETvLwWQQLppdEG+w==,type:int]
+ legendFormat: ENC[AES256_GCM,data:SNwNkNuQ,iv:9AhGAS2JtbEArO6mw1cfweiylOeWsroA2yFA94covcg=,tag:oX/T3ZzMltthkwEr4iFrag==,type:str]
+ maxDataPoints: ENC[AES256_GCM,data:N9o4pLA=,iv:PrcXa9xEE3tuxA0UzXdViwMGu+y/FoJwEDrbkWAnEAU=,tag:XOAm/wplBRFhPiQMqLfHRg==,type:int]
+ range: ENC[AES256_GCM,data:ACQShno=,iv:zfN/sQWSkvD5cKNQl9gMUvGisF7jst5VdtX7kkIC3hg=,tag:JTaHBRFxfaNZjPstcm+zlQ==,type:bool]
+ refId: ENC[AES256_GCM,data:XA==,iv:RCDQpeTdojGSnVKEzz860GLT/D9tzYDpVpownicKUW8=,tag:N24ISyE066ERkcjJ3okVIQ==,type:str]
+ - refId: ENC[AES256_GCM,data:cQ==,iv:Af8z+rJ6+fwwI0HYff49IXz8yNFUNE2RJR6wIyZv0Io=,tag:GTuwYtGmUojfgOQntWtLDg==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:6LoX,iv:39cJgTJAWH065LRLsQt1FEGIvfRG33CEDX2Uj6tL52g=,tag:7836/qvDNDqWRnukm5UnLg==,type:int]
+ to: ENC[AES256_GCM,data:NQ==,iv:RmVyxnEcWHt4EcRDqcaVnTuegdaGqAPU+BF0ptFbtCM=,tag:n/09FcdfsyrRRdKqbOyRDg==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:4Dl9yXzc4IE=,iv:EKPDweX6y8qYngApWY+FdUDuRyGi5D5tLJbvbP1r5go=,tag:ztBOh/4f9UyAG0Ad6Ol84w==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params: []
+ type: ENC[AES256_GCM,data:knk=,iv:ipTDdrQ0/NnNiAqORxx7Tjg00pX6PFx++T4puPaKsOo=,tag:/TGUK1qh8g28Pm8d7dN2jQ==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:nKwx,iv:eYpiKlhS1bD7WBUlmWDNjReL4940kES6Iww2MlPW/ZA=,tag:7AfKXbZAuy/9Ri0h931LdA==,type:str]
+ query:
+ params:
+ - ENC[AES256_GCM,data:Aw==,iv:HXMEpwV87xHbDty4mNV2Zo+IA4i4bVpQec2jHlPj3+w=,tag:Q1O6aT7xAEClZYNYr3296g==,type:str]
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:4OonHQ==,iv:RdDGlAxUYwMEdzwCrwvvOUSmCI1DtRrzSCxNWvxNzhQ=,tag:xtzYYrNm2XvRP86dGKRYLg==,type:str]
+ type: ENC[AES256_GCM,data:dvTvmZU=,iv:JjSZo1zsMbM0wUdO5ASh86B3djyL3vIVFRpkDb0yYqc=,tag:2My5Yn0+8NKedW8ertG/gA==,type:str]
+ datasource:
+ type: ENC[AES256_GCM,data:JKqpg28o6P4=,iv:kad96vkl7JZ/W6+66CCNoy3bZEMaUmql4mFa/b47gq8=,tag:o0ZOxzQca0imXONHvZ78MQ==,type:str]
+ uid: ENC[AES256_GCM,data:ua25patPAV0=,iv:Q+ZO2F7KfTR0UMLnDrrC/5Fn2reBU0rlFuqQ+51ltpM=,tag:jnTZIUqwojtc5WahDMbyig==,type:str]
+ expression: ENC[AES256_GCM,data:CA==,iv:J2QD05D/1rUWmS9m25NL+K6Pof+WnnXz55xtIaLMaRs=,tag:j4oL5ZtdQj7srz6WjTeyUA==,type:str]
+ intervalMs: ENC[AES256_GCM,data:5YOyhA==,iv:jM77S9D9cGgsRWnV/Y3RvbDyw7eY2vH+UlU/nsp9aeM=,tag:E2Sj5fv7TIfk1ozgKMh0ew==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:VA77DZU=,iv:dhrK9yVaqFF8UZiWaISqpH1E/AFXZ4GClHKpecDVFa8=,tag:wS2HSme4z1Mr7cT6fLq1qA==,type:int]
+ reducer: ENC[AES256_GCM,data:f7nvIA==,iv:otFz2kg5v3I05AgH4JDsbLSehsM+HrMwboGaObqKBcs=,tag:BD9zFS+BhzL75rikcmuhiQ==,type:str]
+ refId: ENC[AES256_GCM,data:Og==,iv:34R9tcvbUoT9u7ZxcZHGy58UHXCzskosgVqsciEmNUk=,tag:9903Wu1WM6uHkzqus+6awQ==,type:str]
+ type: ENC[AES256_GCM,data:BYFLsaX+,iv:JjWu7fwRaaa3zrQ++upSDkyO5fuPspfeFJnNzR8N3kk=,tag:oU7FDd75J+jO66DpNXJmuA==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 5m
+ annotations:
+ description: Depending on the rate limit, cert-manager may be unable to generate certificates
+ for up to a week.
+ summary: Cert manager hitting LetsEncrypt rate limits.
+ labels: {}
+ isPaused: false
+ notification_settings:
+ receiver: PagerDuty Warning
- orgId: 1
name: Resources
folder: Cluster
interval: 15m
rules:
- - uid: bdz3y4mow1o1se
- title: KubePersistentVolumeFillingUp
- condition: B
- data:
- - refId: ENC[AES256_GCM,data:ow==,iv:VQh2Qd+AuMx8CbGvO5YxBHf5OEyHGsQr392AGYa33VM=,tag:iIUsX2XB/NTtlmhey6spkQ==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:SL9A,iv:/JPczBmZnnPgebcE/mMLB2/8no+DiNuV9N2ZebViGZk=,tag:vbfwrEzdPRG8+opPt+u5yg==,type:int]
- to: ENC[AES256_GCM,data:wQ==,iv:ZIDRMgftW3AwdqXnrF5obkjeqoPMdMIRy0C9haDnu5A=,tag:Ht5XNqLCitZLdY/QKfSFRg==,type:int]
- datasourceUid: ENC[AES256_GCM,data:N6j3K7j+4sOvGA==,iv:KaH3EhpLpS1qwUVYCXuu4RI3DR+nKbqfz/RE6Be8Mwk=,tag:lPxv/j8bp/xdOa7ACo/yJw==,type:str]
- model:
- datasource:
- type: ENC[AES256_GCM,data:9VWlykVP/xufnQ==,iv:jDWnkXrLtNwuX6fU/mGXw3+h5wFeTkmWlC3AABSmM1Y=,tag:asvDBMSxOn7X2J1nC0jXHA==,type:str]
- uid: ENC[AES256_GCM,data:U3njfD+8BWTqYg==,iv:Wcan/Jo8YErS0v6Ry7y6YZfkpS0GIF3ksNpD6zBKhNI=,tag:C7o5H4lXoOYxRxKZlsPKLA==,type:str]
- editorMode: ENC[AES256_GCM,data:XPjWlA==,iv:yBVMnzSZLXpFB1utVxCNDjh0qslmXqM4zJOBaemaH9M=,tag:kDlaavmk3v6W5a50EXUr0w==,type:str]
- expr: ENC[AES256_GCM,data:UFA/gWW/VIRMw11B1O2WCwjMRdN8CBFlB0IAN3VJc5ulLc/+XagBHdvI3IGfdYWVnF66ZMSgKmFiTse7nSDvig5c1K0OJJiprEUcNzcL4UNK0jLcd/edMbhwjo+iAR25tsUKv1qyOx1gzSd9H7K3oA4lR8HS8o+zFjZ2GCYuPkBRZVvS0wPrsadWdUQzO4VmbEzSc+ww8NUuWGZWPATvyQfZu/ogadcgVdnf1sHovB8i3pkCFDWm4ucZEY5ZxLHDHHgahykvuY5RG6y8fVQg2e701DbYKHIaCbLbYsUm/gczP3e6P4Din2MMVlIO8e6vjvAZpd/j6KDPrtxnzhE0+hRFBzXIpE+OMYaO2mDEjQrhczESUcPsrSpzMY1byhw7eZbcMmdQVRcpYFrMVc+CDX/kN0FKQSoxfSi2XFunkh2j729ERoODujmm0woRnghbWERouwYl6AhodhLczcsAxvYxQa+qhzqD33tFGWjnS2QMrMReUBdTa/WSvmmg6pO8vBMY68Hqpr5FXObFCNA0YeOLN3phrxXnPI7l+PQG5SW9UgrU6tB9rA14Hp0tEAd2v4LR2DdESUr6QbyEAS0+0vEOw+RrzJoYh60cTMTSjML+8jUbASpWkROFZzwly3gFuAH8+e7pNvbN696AXwLFBFhDrJ6OlOhSmdL+Is3LSV+PAzEa7fhLUDAbW1YKYVn7P9FS4n/4dPUhKwWKM3HQMMdpBIZ5PYEVSNDuYRo=,iv:ee2zvyzubDZCZ9WQZTvh94knkCJwkDOUKTt44IuBYCk=,tag:i0qOYQJCt6f0M0Tm/OZwAw==,type:str]
- instant: ENC[AES256_GCM,data:kMcxrw==,iv:Pv1ncpnddHAEg9jBCq4x7zrEP1vTyNSYVT449ey6qJs=,tag:wZh1bRSM+QfDBXi9M46WpQ==,type:bool]
- intervalMs: ENC[AES256_GCM,data:ofPowQ==,iv:vofLhtCJQQo8O2VH7GewKaBXYpOMvx6L9yvqh+eH2UM=,tag:okqulpZchT2SIhVuetaCTg==,type:int]
- legendFormat: ENC[AES256_GCM,data:wvJLyvwB,iv:fhiVOyb0yJqLhCo34o4GsaN7SRIuDFc86Yyb17plnk4=,tag:QnRQwceVUB8lxu8SwawKEw==,type:str]
- maxDataPoints: ENC[AES256_GCM,data:VigbDpc=,iv:xQXEkXP72/ehKrIAjUtFKdvsP0ZxDFDvTZ3Ii8DUBGU=,tag:iQ7rDcpc3lMJqqFf7KrWmw==,type:int]
- range: ENC[AES256_GCM,data:m0oTVGk=,iv:qqcYqKzSvMQEqRpEWPxaebh4+j5NNeOSJbjqWraALJE=,tag:3fiCQYpicju7m4kwV9TiHg==,type:bool]
- refId: ENC[AES256_GCM,data:4Q==,iv:FyvR3c7L91hFB9nXzNzVNfLfUGclBhUeDuU1PSi5uyM=,tag:hyBHTlU3QBseLkMvHdMJUQ==,type:str]
- - refId: ENC[AES256_GCM,data:+w==,iv:AVd/ywg8+BsPuDPzwXutfPap3oSQzgB/uxMXM2is7nU=,tag:N9YGkK9Cby23iwrubAdHdQ==,type:str]
- datasourceUid: ENC[AES256_GCM,data:D0wxdUzyD/M=,iv:1w6wxjXxNKjFEcRgwdBtXRfRbjLGkTJ0RWOasgrG4YE=,tag:DT5X97EWgJFn9jPr3xJQCg==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:rw==,iv:K7qyE4+pjCdm1feqvDVBIoDclNx4qSt1B+azQdEIPFo=,tag:L+QtbV1X+QKEcc5IQoEzpA==,type:int]
- - ENC[AES256_GCM,data:SA==,iv:PwApG90rFRmERs/gUDsUKodGX5UWWu6AsWS6EQQfWRA=,tag:dEB7d6RCJGjF5koVzEn6iQ==,type:int]
- type: ENC[AES256_GCM,data:VfU=,iv:xX/hHvGCXRhPM0GJ08nln4LofbVzTCs3ZHsQ40kppPo=,tag:y1ulttUmW/OO5x05KeaD9g==,type:str]
- operator:
- type: ENC[AES256_GCM,data:PV8p,iv:p9+bq60kikQCqo+kAhSuRNlPgG8BBFqy2sN+5GUhuXs=,tag:B05Y4EiqoJvMx0bTkepbWQ==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:OZvT,iv:ZKKHbzWZ5PlufFpQGRcRnjtHvs7bj593TQ8mhEd5LlI=,tag:D46mTzC+I9iCK7WzJm3WyQ==,type:str]
- type: ENC[AES256_GCM,data:HJKlF34=,iv:VhUISc0WxrV16DlEo1Zf6FVvbzeU3VYFe1bT0tpsdUs=,tag:8rcwWW4TyslxGk7+k3mVkg==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:9pdO1ec82GjUgQ==,iv:EiNArXEwiUGRKUNnC33QHNxUa7JO/zzaVb4ISY5pkBo=,tag:Z6ke4r1bl43pWyjskpJkUQ==,type:str]
- type: ENC[AES256_GCM,data:Z5wFctXpgHg=,iv:XaDaC6ew5ImfUDnDZ1IaUoWrQR9zHotEJzp7iJ9l55g=,tag:RE+qBTmWbaJFRbl6QFPgag==,type:str]
- uid: ENC[AES256_GCM,data:8m1/YQQmJ4A=,iv:ydl6t946ksVeNvUcybAfd527lHTFHtCaXz54qj8ZuFo=,tag:/PdVixkalX4XiXpZIwFAEw==,type:str]
- expression: ENC[AES256_GCM,data:RQ==,iv:wDZJ2CRxP2h1floMO+8amle6v2KW8NJv/0uJN5PupRI=,tag:vvoMg70GpTUiVsq/oePSig==,type:str]
- intervalMs: ENC[AES256_GCM,data:Kon5tA==,iv:mxjhQsu43jFSIS8vbBYiegjZnKONHs78+aoDTKunyBs=,tag:YQx4IUujsNoK6fSXUZddzg==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:nkwhQ9I=,iv:Q4NDIpqKF9FBo2Wpq6thCYYkoYEO7MBwEvBhDdFixiM=,tag:M1hLwFZPrt9k+3UnLJ+SvA==,type:int]
- reducer: ENC[AES256_GCM,data:7utATw==,iv:KsTWq1kkfPAKO1u/6VUji/tahxjH30vxxbtp9fhEfQs=,tag:GvjF2BqU6HRkW5jc8ysBLg==,type:str]
- refId: ENC[AES256_GCM,data:EA==,iv:t93BEr0rOhootGAccSrDsoiQr+4QhbEQJXTO1RwHv0E=,tag:/Y1GHqu0WNJRRrRP54XChA==,type:str]
- type: ENC[AES256_GCM,data:5doSPAyN,iv:zDkAajCsN22A/Awq5jxyhAMxPiykldekPUC+8thrRPM=,tag:5HJQQmHJJGKXrBRjt421Xg==,type:str]
- noDataState: OK
- execErrState: Error
- for: 20m
- annotations:
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
- summary: |-
- {{`The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }}
- in Namespace {{ $labels.namespace }}
- {{ with $labels.cluster - }}
- on Cluster {{ . }}
- {{ - end }}
- is only {{ $value | humanizePercentage }}
- free.`}}
- labels: {}
- isPaused: false
- notification_settings:
- receiver: PagerDuty Warning
- - uid: cdz3yt59a0miob
- title: KubePersistentVolumeErrors
- condition: B
- data:
- - refId: ENC[AES256_GCM,data:zQ==,iv:c1CXoGMJMsclp49nL+IY6z24SLXpb9lIJoE1rmjJCgU=,tag:U+oxTWpcQTGh0+4bj9M/Eg==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:sg4h,iv:tvs+qBXddO3BlIul1QXckwsbcqVx0kX9Nmql8XWXAS8=,tag:JyMKNSnJdWoGH5828XYTIg==,type:int]
- to: ENC[AES256_GCM,data:Nw==,iv:Aw32aHA4fQ39Xjv9S7EraQLmR2xM2/sqFm5QsU2Wm3c=,tag:kZRhmtqrSscqH0eh2z4EeQ==,type:int]
- datasourceUid: ENC[AES256_GCM,data:DXATji4fuDj+2g==,iv:HAQp03YxpYHoqHR4FFVv6xMEj6qgkICYoLPH7nvFhQY=,tag:ncV2f7BPoI6xekRu1ObPFQ==,type:str]
- model:
- editorMode: ENC[AES256_GCM,data:XxY4gg==,iv:yAXW8xLJzI/SdsYbGv6C3HQn8xZkOSC7iuVFwDObwQU=,tag:BjRjT7/DOnIzQCSa9LGgNw==,type:str]
- expr: ENC[AES256_GCM,data:UUU7j/Z2eXn7o9BHyTWqEeDDQO36FU9w+zdCrGkKBaMrznXyQ973tyepG0JNcgwtOwSmT9csWAflMRH/6lLxOwyGFlfteSe8kl8bXtV2/AiHZPicoXajsjBr,iv:Q2AHaMwGXh9L11ntNJtxMrLo2gPiQ8phmMrlE1K4Qyo=,tag:/LjzcnoRT+SZKguW6/Op0A==,type:str]
- instant: ENC[AES256_GCM,data:PjrEHw==,iv:HmIIq6m4g7d9v32mGyyK2/oKrV2BTxRIAGMZSROEXiA=,tag:EW8ghtnptXBJfaDt2QOoYg==,type:bool]
- intervalMs: ENC[AES256_GCM,data:bBnq7w==,iv:Lkt1FeWvOGDF4T/EpJf5S1RTVbf74izq5zUghhR1FmE=,tag:B6faZPXI9s3yusnSg6/aBg==,type:int]
- legendFormat: ENC[AES256_GCM,data:hHbaPzSQ,iv:OulPDN8dkaQMOEhgakdKtbye9UMjL/4SX9x65xU0RyQ=,tag:joSMgJX3v0SkzHYYa+EKyw==,type:str]
- maxDataPoints: ENC[AES256_GCM,data:so82lS0=,iv:lkCngsBpGp9FZkOM7JB8WRZXfgtaLv3UAenYxU4Jiow=,tag:Ym3Or3g8IjWlZchEjM/gwQ==,type:int]
- range: ENC[AES256_GCM,data:xBRXW1M=,iv:1wXv+nafWpz2poZNVxYrVzKX345FMkaU1h7f4qjkIEY=,tag:p3ZvJtK+IkDzZ6qWDf7flA==,type:bool]
- refId: ENC[AES256_GCM,data:ug==,iv:jifQgO3L7gHjQzDo+N+EqsABh4lg4Iq5efSpQL3mJIs=,tag:QPYvh+GNC5fOScj/BFCQNA==,type:str]
- - refId: ENC[AES256_GCM,data:qQ==,iv:RMoavAMbzQtBcF6xyyZNz9fBUvvPYg08mmhDW2pGjdQ=,tag:vwQ8q1Gl5b5Xq8zD/Yw7WQ==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:7P4z,iv:63bh+YvEqfZBp0VH2itVT5n9rs/uqNP4LW8iFhch5A4=,tag:L7xdBQnQy+pkU3nRszG/eA==,type:int]
- to: ENC[AES256_GCM,data:Xw==,iv:eU7B5elJtwGe67OJ/wGWKmBYQR2VdEkwg2s2BSIUBBQ=,tag:bys7gjdan7NLRy6MNpruyA==,type:int]
- datasourceUid: ENC[AES256_GCM,data:L/0hLajj3Mw=,iv:uC7RKrS0HmQzE7bFkL2wVA1bXCq5HGvojil18KbUIYU=,tag:apy2p32SOJwRJjlcjtvHwg==,type:str]
- model:
- conditions:
- - evaluator:
- params: []
- type: ENC[AES256_GCM,data:iEw=,iv:K+pvaBtW4dX3FUkwz2AEVOq/zp6yrKc6U3xjlLpNOsU=,tag:x61G+q5OILuSJSjbhcl1ow==,type:str]
- operator:
- type: ENC[AES256_GCM,data:atzA,iv:ZWT7a5r45zRj5pCes9FdVmhg1gnEtXAU2jJFVuJz410=,tag:AyEHuwctm0mGyDar5rzhtg==,type:str]
- query:
- params:
- - ENC[AES256_GCM,data:MQ==,iv:mcY/eNW+wPOjosfm/x9jdQYSqsTTMPDVfnowDEhO7Eo=,tag:JFluNfjbuVJ3Nl57Q5X8rQ==,type:str]
- reducer:
- params: []
- type: ENC[AES256_GCM,data:a4Em+g==,iv:0MDGY9VBLavotxuRvN9epF2W9jbzg+bzyMn08GcfCrM=,tag:fdxZIZhzQuC81qfE4GWnzg==,type:str]
- type: ENC[AES256_GCM,data:g/QhdxA=,iv:fCigQof2aQK9EAONeWIxuNTl4DvmeR2kIuGiSR4o5iA=,tag:33jRSMDBVKxH6ezLmC6TAg==,type:str]
- datasource:
- type: ENC[AES256_GCM,data:V5QvOCbG1ag=,iv:A8uOZpJ31BUQsIOWo6Q+ZWBKge6hTKZfvnW1mcUjMb8=,tag:Rw9abHAKWMPS0uFQhMY2Cg==,type:str]
- uid: ENC[AES256_GCM,data:4wkHUyFrMgg=,iv:mbxO6WX97bhNPVlrMNBIY5xckBp54pK0fqUcrCoUttU=,tag:cEwoJNgp1SKpU6CKbPp1Cw==,type:str]
- expression: ENC[AES256_GCM,data:Jw==,iv:APk679cPC40yEJSchUAVHl+8sl5fKY9rW8blgQzTIuY=,tag:yof2ENlOGU1iMv7uY2xR2Q==,type:str]
- intervalMs: ENC[AES256_GCM,data:LcRtGw==,iv:imEDPLXuG19WaMwjyEQXqpllDEM15jSuwCdYLQ4MlV0=,tag:RS9j0ZVPgPkIEm+U9epX0g==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:rFEGmI0=,iv:a5VD9YSI2NsIkU0CUkoZD35D1Fsm7QY30FPY+RvkkFA=,tag:7yzjiuCxlCcqtnymBv4bEw==,type:int]
- reducer: ENC[AES256_GCM,data:/S7zMQ==,iv:ZCzUduTcQKn87m8Eyt+Qzj4aFMq9QTQ4cnpY8LMNFNA=,tag:p/Tls+QvSluVbAUBT9041g==,type:str]
- refId: ENC[AES256_GCM,data:AQ==,iv:gAk/prn94j4mCK61E3aetAwLLFB/Qtk8L4uvom6BS8c=,tag:SOIzDAZLr0JERNGaepmnDQ==,type:str]
- type: ENC[AES256_GCM,data:Tl3158xM,iv:+GTtR4Cm+jb/g+RL3ydIE9YxyPtHmjVFFOEVVLIsxLI=,tag:FGKyAOPNcjbk8o2mnbevdg==,type:str]
- noDataState: OK
- execErrState: Error
- for: 20m
- annotations:
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeerrors
- summary: |-
- {{`The persistent volume {{ $labels.persistentvolume }}
- {{ with $labels.cluster - }}
- on Cluster {{ . }}
- {{ - end }}
- has status {{ $labels.phase }}`}}
- .
- isPaused: false
- - uid: cdz3ywkoiwgzka
- title: KubePersistentVolumeInodesFillingUp
- condition: B
- data:
- - refId: ENC[AES256_GCM,data:4w==,iv:pqshYc7GIb7zIJ4yXgoVjLODAWOjdmsQYXxVB0jwoz4=,tag:MUOWION0FPJGqviEIDRf3A==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:sSI5,iv:dkUg4hcXZtNjLNnrn9rV59CxwAM2vPZCxqvL3aFx5Ck=,tag:QogfsoAm5t9d6rl8SSwn4g==,type:int]
- to: ENC[AES256_GCM,data:iQ==,iv:ULkKmLfGi/K1VS70tF9cu4mthNbbNKvS7CQFR4VW1Ms=,tag:D9XlISE/8Y/dJQQB+TwtYw==,type:int]
- datasourceUid: ENC[AES256_GCM,data:7WrBz2JFqIvFVQ==,iv:0plHO2BbmlfAHJUfxEhhjH3PIlhjhW+8kp6oVeAl13w=,tag:vu0IUTozZmfjRxu5yGtlBg==,type:str]
- model:
- editorMode: ENC[AES256_GCM,data:xCJmgA==,iv:lBcCAbJMf6BACO/Os4+M5Jt0f71GK8j353fXN0FdPoI=,tag:SKqAk62nsEPjqvQtapNlYQ==,type:str]
- expr: ENC[AES256_GCM,data:VhjUd9vZlalEEFYW564qeF0X8bdR5+M0KrUCH6xUzvyxcQdWVZzwKX2WRWj1hBqOQ/a/5LH1b+h0ejNjE1ElcKyuHmhISZgmgf/FKtDPsnmQ1RyiA08l6ctRYP+9I8ldSCwI/6qj3NvCMdNOM4Kddl8TAAUogrhgrZhWn5GlAMp3SOG+hLKbQPx0l0DmQnq4CrVCygW33WAg47JGiqRyW0t/0TBDqZIdbZRxS0wPzY5OTEvvJ0d4v6nAwd7ucLsw787g/d6UiYzc6jmZNVpJxYh9dboVivKV3WMsiDE7L0ek9qVLCE2oL1ijPaFThVzahz5S/wnUd0GmcC4ztGLXeigvWLViy+6jrLhordem+/YofhFoJoE1ztolHApVcyWJkLQtm+0fG9H0fuEoUdpj0Wedk7qtNom/cOTzOCaM0cUKDjZmptW5e5JUBAKv9O81sgQbbh0MEtQO2yghhCfWI6mpai/GlhKzhQ4OTcll9G1snsb2Vvi4ft0vSJHXZLUD4TySEDfQTZCkaDqbAvLQ1fDyn84oO/QtKYGc1gZhnl0V4w4Ext2TUNcT8NgRlpDeh/O4miaKtX6Z+pWxVjF0csaYFEFYR2PEo59DEuqAr8kPhu0w0oyUXDRTKPJmff9LIe0OPkYeZ5eTNOgvaVbgV3E0miq9DXmEJNWCriJBZsPE+Q540AECJ2YNGorcy0eswCd0eR/HV/XTySAcyr+Nj6PPtH+jtWRdws7SRtD5I7hT+JFNywGv3pQWtdGhPEoFIzEsWor2cH1loo4EH2fh76ve0rhkrKR6tfjIut2NYUNsg1EfiAbcqRyvIF4vFH29LnXk1DkJ1mdD1PdKXRJ1jDZfb1IzGly8euA6cAEaxo9ECLCXoGsrNbvinnDvnyE+4RGaR7NL2g==,iv:JGZsTzrMZq8hDzbNKO8SwVcFcgug5YjI0lsBq0S9bhY=,tag:mQ14UPlyY10K4hPq0lv4vA==,type:str]
- instant: ENC[AES256_GCM,data:Lbda7g==,iv:0SLAF80nzwCtWysFoymhKX0Gqw/6nwOYsDMZOjQDLHs=,tag:zlkh5r9Vikg8GWGg+Bm6mQ==,type:bool]
- intervalMs: ENC[AES256_GCM,data:GvhVPQ==,iv:KsymeUMH8u6KBgew1u3vQmqSZ8H/BtLwTOvY8Q1yQKc=,tag:nyKDxvXRAyeWYt0vcuZijw==,type:int]
- legendFormat: ENC[AES256_GCM,data:fd2PcIRx,iv:L6sASuJHMfWeT6ZdKuHC0Pcw5SsXbJE0hGJZ9dlSydk=,tag:FPqVg/9CsI+ivz88wS5Zig==,type:str]
- maxDataPoints: ENC[AES256_GCM,data:JIupsGI=,iv:Lw/6kbOOtJae0rINilyXjrC0lGh9lAwUIfa386iN6GY=,tag:9ZarrIjSkE9NjjBZJLN/nw==,type:int]
- range: ENC[AES256_GCM,data:WF4JjA8=,iv:tkKZxMpXVvGEetrtfZ9bpKQ/37YWs5C12tUu463DJIE=,tag:htF6pY0dNDlrPCJeFgugwQ==,type:bool]
- refId: ENC[AES256_GCM,data:Lg==,iv:rWMOkNnOwml+JdPDnSk84T/raY/OGjVzfFaJWrtbNc8=,tag:fk5wajgSi/eWyMo31hX2UA==,type:str]
- - refId: ENC[AES256_GCM,data:+w==,iv:tFCwg1BnKMVsXIOILO5ZS0/A63aV91i0HoK9jp4ohPs=,tag:nGYHJA0onIYfTcCim3ywaQ==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:Dgz3,iv:qXCutwmkXT/QH0Ooa8xntDLqK+CXCdYwuCNpOXVRHSQ=,tag:WwKah4zevFVLLLqLj1Uo+Q==,type:int]
- to: ENC[AES256_GCM,data:cg==,iv:IrSCZrce9xtb8JB/FzwbKCgIb4frTEREGWA/W5Z1nf8=,tag:qYWmLF5/KWaPpU5+SMJb+Q==,type:int]
- datasourceUid: ENC[AES256_GCM,data:RN+t6DO1he4=,iv:/nic/uZtlBMEgoW/lPWPXZm9NK3WhUkLs+T4oyQmetY=,tag:0zSOvm+urRa02pbzHL+I0w==,type:str]
- model:
- conditions:
- - evaluator:
- params: []
- type: ENC[AES256_GCM,data:qMY=,iv:3HXrE/+vFjNQhd2JzHkao3A5j8HwDwsbDVbmd51CHZI=,tag:CoIfsqtEthQVbkofhWLs/Q==,type:str]
- operator:
- type: ENC[AES256_GCM,data:zVfp,iv:xy5G+ss9ISOVLfYMyqgjfLEQPDOAwptDzt1krDNXT6U=,tag:fi74oUW51VW4xjjcLQSNjQ==,type:str]
- query:
- params:
- - ENC[AES256_GCM,data:PA==,iv:GOLgJ94vU/5+VSn6peC0m7kU9XH1qCaz1TRqEBhU7Ig=,tag:ouVL3qZfoGfdqKaop1Zh5g==,type:str]
- reducer:
- params: []
- type: ENC[AES256_GCM,data:upYYaw==,iv:eYcW9MAESfxfUwm1enLuh+OynZWNZZt1ZjdAY3xWoYo=,tag:Tz9bNlBHPrXICjBX+LAjMA==,type:str]
- type: ENC[AES256_GCM,data:wvAScqQ=,iv:KYcA+QZx9ve+W8DES3T4qgGyWkJHvoHxT3TmmDY/HNA=,tag:atn/tQcIl+SxbevFhbF7ZA==,type:str]
- datasource:
- type: ENC[AES256_GCM,data:GkT6Os3jLFM=,iv:O0FB4AuOLgov9TsKCXVoLk1IrjacviJHNAUShkTgg6A=,tag:kGbOU/Jb32QMzuNifRcZzA==,type:str]
- uid: ENC[AES256_GCM,data:oCfsFmqqKfE=,iv:PfmstWIbdazTnbYdHK51hpQaN96zmO+U9XlMUXfZ6FI=,tag:g+b956ntyUpem9VUn7Ziaw==,type:str]
- expression: ENC[AES256_GCM,data:Qg==,iv:Mok+QCAdrbw5BKhsPzNK/OpiFs0hMGx3LQM3nqTsk1c=,tag:++SY6isgUrO1YKno9vEzrw==,type:str]
- intervalMs: ENC[AES256_GCM,data:/amoxw==,iv:3bv1IHEqUns395S7SSlHtkLLYESElsIEwLFj8SAa4ac=,tag:2Eguw0EUqMjRgW+3svNSiA==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:KHrx6p8=,iv:0g5cLHl9XwDFdOKjQWAwIsCMpjybZin8uJyUYO5eGyQ=,tag:tOkRRoazg8j8rSmDUQqL7A==,type:int]
- reducer: ENC[AES256_GCM,data:aLqBJA==,iv:jY2cb7lAP/JuX5ERXYYPHDCqTyjW9C/NlBXHeg+I3Zw=,tag:/XvFiYYH9qoyBBttvz7rww==,type:str]
- refId: ENC[AES256_GCM,data:lw==,iv:9hA6kzdlPaTimd3C1CZRE4X41ClenhE1xYOFOlswKNg=,tag:4svprWHRnw0XKswhQmbYcw==,type:str]
- type: ENC[AES256_GCM,data:vI5lOMRQ,iv:WUpR1QscIMRdZUYT+kqdxXohQMN9vH2gd/Pf+ICI7Nk=,tag:XoMn1Li4EtkgfEO0Aduong==,type:str]
- noDataState: OK
- execErrState: Error
- for: 20m
- annotations:
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup
- summary: |-
- {{`Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }}
- in Namespace {{ $labels.namespace }}
- {{ with $labels.cluster - }}
- on Cluster {{ . }}
- {{ - end }}
- is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }}
- of its inodes are free.`}}
- isPaused: false
- notification_settings:
- receiver: PagerDuty Warning
+ - uid: bdz3y4mow1o1se
+ title: KubePersistentVolumeFillingUp
+ condition: B
+ data:
+ - refId: ENC[AES256_GCM,data:ow==,iv:VQh2Qd+AuMx8CbGvO5YxBHf5OEyHGsQr392AGYa33VM=,tag:iIUsX2XB/NTtlmhey6spkQ==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:SL9A,iv:/JPczBmZnnPgebcE/mMLB2/8no+DiNuV9N2ZebViGZk=,tag:vbfwrEzdPRG8+opPt+u5yg==,type:int]
+ to: ENC[AES256_GCM,data:wQ==,iv:ZIDRMgftW3AwdqXnrF5obkjeqoPMdMIRy0C9haDnu5A=,tag:Ht5XNqLCitZLdY/QKfSFRg==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:N6j3K7j+4sOvGA==,iv:KaH3EhpLpS1qwUVYCXuu4RI3DR+nKbqfz/RE6Be8Mwk=,tag:lPxv/j8bp/xdOa7ACo/yJw==,type:str]
+ model:
+ datasource:
+ type: ENC[AES256_GCM,data:9VWlykVP/xufnQ==,iv:jDWnkXrLtNwuX6fU/mGXw3+h5wFeTkmWlC3AABSmM1Y=,tag:asvDBMSxOn7X2J1nC0jXHA==,type:str]
+ uid: ENC[AES256_GCM,data:U3njfD+8BWTqYg==,iv:Wcan/Jo8YErS0v6Ry7y6YZfkpS0GIF3ksNpD6zBKhNI=,tag:C7o5H4lXoOYxRxKZlsPKLA==,type:str]
+ editorMode: ENC[AES256_GCM,data:XPjWlA==,iv:yBVMnzSZLXpFB1utVxCNDjh0qslmXqM4zJOBaemaH9M=,tag:kDlaavmk3v6W5a50EXUr0w==,type:str]
+ expr: ENC[AES256_GCM,data:UFA/gWW/VIRMw11B1O2WCwjMRdN8CBFlB0IAN3VJc5ulLc/+XagBHdvI3IGfdYWVnF66ZMSgKmFiTse7nSDvig5c1K0OJJiprEUcNzcL4UNK0jLcd/edMbhwjo+iAR25tsUKv1qyOx1gzSd9H7K3oA4lR8HS8o+zFjZ2GCYuPkBRZVvS0wPrsadWdUQzO4VmbEzSc+ww8NUuWGZWPATvyQfZu/ogadcgVdnf1sHovB8i3pkCFDWm4ucZEY5ZxLHDHHgahykvuY5RG6y8fVQg2e701DbYKHIaCbLbYsUm/gczP3e6P4Din2MMVlIO8e6vjvAZpd/j6KDPrtxnzhE0+hRFBzXIpE+OMYaO2mDEjQrhczESUcPsrSpzMY1byhw7eZbcMmdQVRcpYFrMVc+CDX/kN0FKQSoxfSi2XFunkh2j729ERoODujmm0woRnghbWERouwYl6AhodhLczcsAxvYxQa+qhzqD33tFGWjnS2QMrMReUBdTa/WSvmmg6pO8vBMY68Hqpr5FXObFCNA0YeOLN3phrxXnPI7l+PQG5SW9UgrU6tB9rA14Hp0tEAd2v4LR2DdESUr6QbyEAS0+0vEOw+RrzJoYh60cTMTSjML+8jUbASpWkROFZzwly3gFuAH8+e7pNvbN696AXwLFBFhDrJ6OlOhSmdL+Is3LSV+PAzEa7fhLUDAbW1YKYVn7P9FS4n/4dPUhKwWKM3HQMMdpBIZ5PYEVSNDuYRo=,iv:ee2zvyzubDZCZ9WQZTvh94knkCJwkDOUKTt44IuBYCk=,tag:i0qOYQJCt6f0M0Tm/OZwAw==,type:str]
+ instant: ENC[AES256_GCM,data:kMcxrw==,iv:Pv1ncpnddHAEg9jBCq4x7zrEP1vTyNSYVT449ey6qJs=,tag:wZh1bRSM+QfDBXi9M46WpQ==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:ofPowQ==,iv:vofLhtCJQQo8O2VH7GewKaBXYpOMvx6L9yvqh+eH2UM=,tag:okqulpZchT2SIhVuetaCTg==,type:int]
+ legendFormat: ENC[AES256_GCM,data:wvJLyvwB,iv:fhiVOyb0yJqLhCo34o4GsaN7SRIuDFc86Yyb17plnk4=,tag:QnRQwceVUB8lxu8SwawKEw==,type:str]
+ maxDataPoints: ENC[AES256_GCM,data:VigbDpc=,iv:xQXEkXP72/ehKrIAjUtFKdvsP0ZxDFDvTZ3Ii8DUBGU=,tag:iQ7rDcpc3lMJqqFf7KrWmw==,type:int]
+ range: ENC[AES256_GCM,data:m0oTVGk=,iv:qqcYqKzSvMQEqRpEWPxaebh4+j5NNeOSJbjqWraALJE=,tag:3fiCQYpicju7m4kwV9TiHg==,type:bool]
+ refId: ENC[AES256_GCM,data:4Q==,iv:FyvR3c7L91hFB9nXzNzVNfLfUGclBhUeDuU1PSi5uyM=,tag:hyBHTlU3QBseLkMvHdMJUQ==,type:str]
+ - refId: ENC[AES256_GCM,data:+w==,iv:AVd/ywg8+BsPuDPzwXutfPap3oSQzgB/uxMXM2is7nU=,tag:N9YGkK9Cby23iwrubAdHdQ==,type:str]
+ datasourceUid: ENC[AES256_GCM,data:D0wxdUzyD/M=,iv:1w6wxjXxNKjFEcRgwdBtXRfRbjLGkTJ0RWOasgrG4YE=,tag:DT5X97EWgJFn9jPr3xJQCg==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:rw==,iv:K7qyE4+pjCdm1feqvDVBIoDclNx4qSt1B+azQdEIPFo=,tag:L+QtbV1X+QKEcc5IQoEzpA==,type:int]
+ - ENC[AES256_GCM,data:SA==,iv:PwApG90rFRmERs/gUDsUKodGX5UWWu6AsWS6EQQfWRA=,tag:dEB7d6RCJGjF5koVzEn6iQ==,type:int]
+ type: ENC[AES256_GCM,data:VfU=,iv:xX/hHvGCXRhPM0GJ08nln4LofbVzTCs3ZHsQ40kppPo=,tag:y1ulttUmW/OO5x05KeaD9g==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:PV8p,iv:p9+bq60kikQCqo+kAhSuRNlPgG8BBFqy2sN+5GUhuXs=,tag:B05Y4EiqoJvMx0bTkepbWQ==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:OZvT,iv:ZKKHbzWZ5PlufFpQGRcRnjtHvs7bj593TQ8mhEd5LlI=,tag:D46mTzC+I9iCK7WzJm3WyQ==,type:str]
+ type: ENC[AES256_GCM,data:HJKlF34=,iv:VhUISc0WxrV16DlEo1Zf6FVvbzeU3VYFe1bT0tpsdUs=,tag:8rcwWW4TyslxGk7+k3mVkg==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:9pdO1ec82GjUgQ==,iv:EiNArXEwiUGRKUNnC33QHNxUa7JO/zzaVb4ISY5pkBo=,tag:Z6ke4r1bl43pWyjskpJkUQ==,type:str]
+ type: ENC[AES256_GCM,data:Z5wFctXpgHg=,iv:XaDaC6ew5ImfUDnDZ1IaUoWrQR9zHotEJzp7iJ9l55g=,tag:RE+qBTmWbaJFRbl6QFPgag==,type:str]
+ uid: ENC[AES256_GCM,data:8m1/YQQmJ4A=,iv:ydl6t946ksVeNvUcybAfd527lHTFHtCaXz54qj8ZuFo=,tag:/PdVixkalX4XiXpZIwFAEw==,type:str]
+ expression: ENC[AES256_GCM,data:RQ==,iv:wDZJ2CRxP2h1floMO+8amle6v2KW8NJv/0uJN5PupRI=,tag:vvoMg70GpTUiVsq/oePSig==,type:str]
+ intervalMs: ENC[AES256_GCM,data:Kon5tA==,iv:mxjhQsu43jFSIS8vbBYiegjZnKONHs78+aoDTKunyBs=,tag:YQx4IUujsNoK6fSXUZddzg==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:nkwhQ9I=,iv:Q4NDIpqKF9FBo2Wpq6thCYYkoYEO7MBwEvBhDdFixiM=,tag:M1hLwFZPrt9k+3UnLJ+SvA==,type:int]
+ reducer: ENC[AES256_GCM,data:7utATw==,iv:KsTWq1kkfPAKO1u/6VUji/tahxjH30vxxbtp9fhEfQs=,tag:GvjF2BqU6HRkW5jc8ysBLg==,type:str]
+ refId: ENC[AES256_GCM,data:EA==,iv:t93BEr0rOhootGAccSrDsoiQr+4QhbEQJXTO1RwHv0E=,tag:/Y1GHqu0WNJRRrRP54XChA==,type:str]
+ type: ENC[AES256_GCM,data:5doSPAyN,iv:zDkAajCsN22A/Awq5jxyhAMxPiykldekPUC+8thrRPM=,tag:5HJQQmHJJGKXrBRjt421Xg==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 20m
+ annotations:
+ runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
+ summary: |-
+ {{`The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }}
+ in Namespace {{ $labels.namespace }}
+ {{ with $labels.cluster - }}
+ on Cluster {{ . }}
+ {{ - end }}
+ is only {{ $value | humanizePercentage }}
+ free.`}}
+ labels: {}
+ isPaused: false
+ notification_settings:
+ receiver: PagerDuty Warning
+ - uid: cdz3yt59a0miob
+ title: KubePersistentVolumeErrors
+ condition: B
+ data:
+ - refId: ENC[AES256_GCM,data:zQ==,iv:c1CXoGMJMsclp49nL+IY6z24SLXpb9lIJoE1rmjJCgU=,tag:U+oxTWpcQTGh0+4bj9M/Eg==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:sg4h,iv:tvs+qBXddO3BlIul1QXckwsbcqVx0kX9Nmql8XWXAS8=,tag:JyMKNSnJdWoGH5828XYTIg==,type:int]
+ to: ENC[AES256_GCM,data:Nw==,iv:Aw32aHA4fQ39Xjv9S7EraQLmR2xM2/sqFm5QsU2Wm3c=,tag:kZRhmtqrSscqH0eh2z4EeQ==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:DXATji4fuDj+2g==,iv:HAQp03YxpYHoqHR4FFVv6xMEj6qgkICYoLPH7nvFhQY=,tag:ncV2f7BPoI6xekRu1ObPFQ==,type:str]
+ model:
+ editorMode: ENC[AES256_GCM,data:XxY4gg==,iv:yAXW8xLJzI/SdsYbGv6C3HQn8xZkOSC7iuVFwDObwQU=,tag:BjRjT7/DOnIzQCSa9LGgNw==,type:str]
+ expr: ENC[AES256_GCM,data:UUU7j/Z2eXn7o9BHyTWqEeDDQO36FU9w+zdCrGkKBaMrznXyQ973tyepG0JNcgwtOwSmT9csWAflMRH/6lLxOwyGFlfteSe8kl8bXtV2/AiHZPicoXajsjBr,iv:Q2AHaMwGXh9L11ntNJtxMrLo2gPiQ8phmMrlE1K4Qyo=,tag:/LjzcnoRT+SZKguW6/Op0A==,type:str]
+ instant: ENC[AES256_GCM,data:PjrEHw==,iv:HmIIq6m4g7d9v32mGyyK2/oKrV2BTxRIAGMZSROEXiA=,tag:EW8ghtnptXBJfaDt2QOoYg==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:bBnq7w==,iv:Lkt1FeWvOGDF4T/EpJf5S1RTVbf74izq5zUghhR1FmE=,tag:B6faZPXI9s3yusnSg6/aBg==,type:int]
+ legendFormat: ENC[AES256_GCM,data:hHbaPzSQ,iv:OulPDN8dkaQMOEhgakdKtbye9UMjL/4SX9x65xU0RyQ=,tag:joSMgJX3v0SkzHYYa+EKyw==,type:str]
+ maxDataPoints: ENC[AES256_GCM,data:so82lS0=,iv:lkCngsBpGp9FZkOM7JB8WRZXfgtaLv3UAenYxU4Jiow=,tag:Ym3Or3g8IjWlZchEjM/gwQ==,type:int]
+ range: ENC[AES256_GCM,data:xBRXW1M=,iv:1wXv+nafWpz2poZNVxYrVzKX345FMkaU1h7f4qjkIEY=,tag:p3ZvJtK+IkDzZ6qWDf7flA==,type:bool]
+ refId: ENC[AES256_GCM,data:ug==,iv:jifQgO3L7gHjQzDo+N+EqsABh4lg4Iq5efSpQL3mJIs=,tag:QPYvh+GNC5fOScj/BFCQNA==,type:str]
+ - refId: ENC[AES256_GCM,data:qQ==,iv:RMoavAMbzQtBcF6xyyZNz9fBUvvPYg08mmhDW2pGjdQ=,tag:vwQ8q1Gl5b5Xq8zD/Yw7WQ==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:7P4z,iv:63bh+YvEqfZBp0VH2itVT5n9rs/uqNP4LW8iFhch5A4=,tag:L7xdBQnQy+pkU3nRszG/eA==,type:int]
+ to: ENC[AES256_GCM,data:Xw==,iv:eU7B5elJtwGe67OJ/wGWKmBYQR2VdEkwg2s2BSIUBBQ=,tag:bys7gjdan7NLRy6MNpruyA==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:L/0hLajj3Mw=,iv:uC7RKrS0HmQzE7bFkL2wVA1bXCq5HGvojil18KbUIYU=,tag:apy2p32SOJwRJjlcjtvHwg==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params: []
+ type: ENC[AES256_GCM,data:iEw=,iv:K+pvaBtW4dX3FUkwz2AEVOq/zp6yrKc6U3xjlLpNOsU=,tag:x61G+q5OILuSJSjbhcl1ow==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:atzA,iv:ZWT7a5r45zRj5pCes9FdVmhg1gnEtXAU2jJFVuJz410=,tag:AyEHuwctm0mGyDar5rzhtg==,type:str]
+ query:
+ params:
+ - ENC[AES256_GCM,data:MQ==,iv:mcY/eNW+wPOjosfm/x9jdQYSqsTTMPDVfnowDEhO7Eo=,tag:JFluNfjbuVJ3Nl57Q5X8rQ==,type:str]
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:a4Em+g==,iv:0MDGY9VBLavotxuRvN9epF2W9jbzg+bzyMn08GcfCrM=,tag:fdxZIZhzQuC81qfE4GWnzg==,type:str]
+ type: ENC[AES256_GCM,data:g/QhdxA=,iv:fCigQof2aQK9EAONeWIxuNTl4DvmeR2kIuGiSR4o5iA=,tag:33jRSMDBVKxH6ezLmC6TAg==,type:str]
+ datasource:
+ type: ENC[AES256_GCM,data:V5QvOCbG1ag=,iv:A8uOZpJ31BUQsIOWo6Q+ZWBKge6hTKZfvnW1mcUjMb8=,tag:Rw9abHAKWMPS0uFQhMY2Cg==,type:str]
+ uid: ENC[AES256_GCM,data:4wkHUyFrMgg=,iv:mbxO6WX97bhNPVlrMNBIY5xckBp54pK0fqUcrCoUttU=,tag:cEwoJNgp1SKpU6CKbPp1Cw==,type:str]
+ expression: ENC[AES256_GCM,data:Jw==,iv:APk679cPC40yEJSchUAVHl+8sl5fKY9rW8blgQzTIuY=,tag:yof2ENlOGU1iMv7uY2xR2Q==,type:str]
+ intervalMs: ENC[AES256_GCM,data:LcRtGw==,iv:imEDPLXuG19WaMwjyEQXqpllDEM15jSuwCdYLQ4MlV0=,tag:RS9j0ZVPgPkIEm+U9epX0g==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:rFEGmI0=,iv:a5VD9YSI2NsIkU0CUkoZD35D1Fsm7QY30FPY+RvkkFA=,tag:7yzjiuCxlCcqtnymBv4bEw==,type:int]
+ reducer: ENC[AES256_GCM,data:/S7zMQ==,iv:ZCzUduTcQKn87m8Eyt+Qzj4aFMq9QTQ4cnpY8LMNFNA=,tag:p/Tls+QvSluVbAUBT9041g==,type:str]
+ refId: ENC[AES256_GCM,data:AQ==,iv:gAk/prn94j4mCK61E3aetAwLLFB/Qtk8L4uvom6BS8c=,tag:SOIzDAZLr0JERNGaepmnDQ==,type:str]
+ type: ENC[AES256_GCM,data:Tl3158xM,iv:+GTtR4Cm+jb/g+RL3ydIE9YxyPtHmjVFFOEVVLIsxLI=,tag:FGKyAOPNcjbk8o2mnbevdg==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 20m
+ annotations:
+ runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeerrors
+ summary: |-
+ {{`The persistent volume {{ $labels.persistentvolume }}
+ {{ with $labels.cluster - }}
+ on Cluster {{ . }}
+ {{ - end }}
+ has status {{ $labels.phase }}`}}
+ .
+ isPaused: false
+ - uid: cdz3ywkoiwgzka
+ title: KubePersistentVolumeInodesFillingUp
+ condition: B
+ data:
+ - refId: ENC[AES256_GCM,data:4w==,iv:pqshYc7GIb7zIJ4yXgoVjLODAWOjdmsQYXxVB0jwoz4=,tag:MUOWION0FPJGqviEIDRf3A==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:sSI5,iv:dkUg4hcXZtNjLNnrn9rV59CxwAM2vPZCxqvL3aFx5Ck=,tag:QogfsoAm5t9d6rl8SSwn4g==,type:int]
+ to: ENC[AES256_GCM,data:iQ==,iv:ULkKmLfGi/K1VS70tF9cu4mthNbbNKvS7CQFR4VW1Ms=,tag:D9XlISE/8Y/dJQQB+TwtYw==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:7WrBz2JFqIvFVQ==,iv:0plHO2BbmlfAHJUfxEhhjH3PIlhjhW+8kp6oVeAl13w=,tag:vu0IUTozZmfjRxu5yGtlBg==,type:str]
+ model:
+ editorMode: ENC[AES256_GCM,data:xCJmgA==,iv:lBcCAbJMf6BACO/Os4+M5Jt0f71GK8j353fXN0FdPoI=,tag:SKqAk62nsEPjqvQtapNlYQ==,type:str]
+ expr: ENC[AES256_GCM,data:VhjUd9vZlalEEFYW564qeF0X8bdR5+M0KrUCH6xUzvyxcQdWVZzwKX2WRWj1hBqOQ/a/5LH1b+h0ejNjE1ElcKyuHmhISZgmgf/FKtDPsnmQ1RyiA08l6ctRYP+9I8ldSCwI/6qj3NvCMdNOM4Kddl8TAAUogrhgrZhWn5GlAMp3SOG+hLKbQPx0l0DmQnq4CrVCygW33WAg47JGiqRyW0t/0TBDqZIdbZRxS0wPzY5OTEvvJ0d4v6nAwd7ucLsw787g/d6UiYzc6jmZNVpJxYh9dboVivKV3WMsiDE7L0ek9qVLCE2oL1ijPaFThVzahz5S/wnUd0GmcC4ztGLXeigvWLViy+6jrLhordem+/YofhFoJoE1ztolHApVcyWJkLQtm+0fG9H0fuEoUdpj0Wedk7qtNom/cOTzOCaM0cUKDjZmptW5e5JUBAKv9O81sgQbbh0MEtQO2yghhCfWI6mpai/GlhKzhQ4OTcll9G1snsb2Vvi4ft0vSJHXZLUD4TySEDfQTZCkaDqbAvLQ1fDyn84oO/QtKYGc1gZhnl0V4w4Ext2TUNcT8NgRlpDeh/O4miaKtX6Z+pWxVjF0csaYFEFYR2PEo59DEuqAr8kPhu0w0oyUXDRTKPJmff9LIe0OPkYeZ5eTNOgvaVbgV3E0miq9DXmEJNWCriJBZsPE+Q540AECJ2YNGorcy0eswCd0eR/HV/XTySAcyr+Nj6PPtH+jtWRdws7SRtD5I7hT+JFNywGv3pQWtdGhPEoFIzEsWor2cH1loo4EH2fh76ve0rhkrKR6tfjIut2NYUNsg1EfiAbcqRyvIF4vFH29LnXk1DkJ1mdD1PdKXRJ1jDZfb1IzGly8euA6cAEaxo9ECLCXoGsrNbvinnDvnyE+4RGaR7NL2g==,iv:JGZsTzrMZq8hDzbNKO8SwVcFcgug5YjI0lsBq0S9bhY=,tag:mQ14UPlyY10K4hPq0lv4vA==,type:str]
+ instant: ENC[AES256_GCM,data:Lbda7g==,iv:0SLAF80nzwCtWysFoymhKX0Gqw/6nwOYsDMZOjQDLHs=,tag:zlkh5r9Vikg8GWGg+Bm6mQ==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:GvhVPQ==,iv:KsymeUMH8u6KBgew1u3vQmqSZ8H/BtLwTOvY8Q1yQKc=,tag:nyKDxvXRAyeWYt0vcuZijw==,type:int]
+ legendFormat: ENC[AES256_GCM,data:fd2PcIRx,iv:L6sASuJHMfWeT6ZdKuHC0Pcw5SsXbJE0hGJZ9dlSydk=,tag:FPqVg/9CsI+ivz88wS5Zig==,type:str]
+ maxDataPoints: ENC[AES256_GCM,data:JIupsGI=,iv:Lw/6kbOOtJae0rINilyXjrC0lGh9lAwUIfa386iN6GY=,tag:9ZarrIjSkE9NjjBZJLN/nw==,type:int]
+ range: ENC[AES256_GCM,data:WF4JjA8=,iv:tkKZxMpXVvGEetrtfZ9bpKQ/37YWs5C12tUu463DJIE=,tag:htF6pY0dNDlrPCJeFgugwQ==,type:bool]
+ refId: ENC[AES256_GCM,data:Lg==,iv:rWMOkNnOwml+JdPDnSk84T/raY/OGjVzfFaJWrtbNc8=,tag:fk5wajgSi/eWyMo31hX2UA==,type:str]
+ - refId: ENC[AES256_GCM,data:+w==,iv:tFCwg1BnKMVsXIOILO5ZS0/A63aV91i0HoK9jp4ohPs=,tag:nGYHJA0onIYfTcCim3ywaQ==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:Dgz3,iv:qXCutwmkXT/QH0Ooa8xntDLqK+CXCdYwuCNpOXVRHSQ=,tag:WwKah4zevFVLLLqLj1Uo+Q==,type:int]
+ to: ENC[AES256_GCM,data:cg==,iv:IrSCZrce9xtb8JB/FzwbKCgIb4frTEREGWA/W5Z1nf8=,tag:qYWmLF5/KWaPpU5+SMJb+Q==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:RN+t6DO1he4=,iv:/nic/uZtlBMEgoW/lPWPXZm9NK3WhUkLs+T4oyQmetY=,tag:0zSOvm+urRa02pbzHL+I0w==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params: []
+ type: ENC[AES256_GCM,data:qMY=,iv:3HXrE/+vFjNQhd2JzHkao3A5j8HwDwsbDVbmd51CHZI=,tag:CoIfsqtEthQVbkofhWLs/Q==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:zVfp,iv:xy5G+ss9ISOVLfYMyqgjfLEQPDOAwptDzt1krDNXT6U=,tag:fi74oUW51VW4xjjcLQSNjQ==,type:str]
+ query:
+ params:
+ - ENC[AES256_GCM,data:PA==,iv:GOLgJ94vU/5+VSn6peC0m7kU9XH1qCaz1TRqEBhU7Ig=,tag:ouVL3qZfoGfdqKaop1Zh5g==,type:str]
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:upYYaw==,iv:eYcW9MAESfxfUwm1enLuh+OynZWNZZt1ZjdAY3xWoYo=,tag:Tz9bNlBHPrXICjBX+LAjMA==,type:str]
+ type: ENC[AES256_GCM,data:wvAScqQ=,iv:KYcA+QZx9ve+W8DES3T4qgGyWkJHvoHxT3TmmDY/HNA=,tag:atn/tQcIl+SxbevFhbF7ZA==,type:str]
+ datasource:
+ type: ENC[AES256_GCM,data:GkT6Os3jLFM=,iv:O0FB4AuOLgov9TsKCXVoLk1IrjacviJHNAUShkTgg6A=,tag:kGbOU/Jb32QMzuNifRcZzA==,type:str]
+ uid: ENC[AES256_GCM,data:oCfsFmqqKfE=,iv:PfmstWIbdazTnbYdHK51hpQaN96zmO+U9XlMUXfZ6FI=,tag:g+b956ntyUpem9VUn7Ziaw==,type:str]
+ expression: ENC[AES256_GCM,data:Qg==,iv:Mok+QCAdrbw5BKhsPzNK/OpiFs0hMGx3LQM3nqTsk1c=,tag:++SY6isgUrO1YKno9vEzrw==,type:str]
+ intervalMs: ENC[AES256_GCM,data:/amoxw==,iv:3bv1IHEqUns395S7SSlHtkLLYESElsIEwLFj8SAa4ac=,tag:2Eguw0EUqMjRgW+3svNSiA==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:KHrx6p8=,iv:0g5cLHl9XwDFdOKjQWAwIsCMpjybZin8uJyUYO5eGyQ=,tag:tOkRRoazg8j8rSmDUQqL7A==,type:int]
+ reducer: ENC[AES256_GCM,data:aLqBJA==,iv:jY2cb7lAP/JuX5ERXYYPHDCqTyjW9C/NlBXHeg+I3Zw=,tag:/XvFiYYH9qoyBBttvz7rww==,type:str]
+ refId: ENC[AES256_GCM,data:lw==,iv:9hA6kzdlPaTimd3C1CZRE4X41ClenhE1xYOFOlswKNg=,tag:4svprWHRnw0XKswhQmbYcw==,type:str]
+ type: ENC[AES256_GCM,data:vI5lOMRQ,iv:WUpR1QscIMRdZUYT+kqdxXohQMN9vH2gd/Pf+ICI7Nk=,tag:XoMn1Li4EtkgfEO0Aduong==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 20m
+ annotations:
+ runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup
+ summary: |-
+ {{`Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }}
+ in Namespace {{ $labels.namespace }}
+ {{ with $labels.cluster - }}
+ on Cluster {{ . }}
+ {{ - end }}
+ is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }}
+ of its inodes are free.`}}
+ isPaused: false
+ notification_settings:
+ receiver: PagerDuty Warning
- orgId: 1
name: Draupnir4All
folder: Matrix
interval: 1m
rules:
- - uid: adz3nhdki7d34d
- title: Draupnir4All Synapse "Failed"
- condition: B
- data:
- - refId: ENC[AES256_GCM,data:9Q==,iv:xdJFG2iVuFSsgUC7cnPlOAZndQggOhEI46LkegCWYYc=,tag:Uud6d+Xk+8ZnfY2SiTrMYw==,type:str]
- queryType: ENC[AES256_GCM,data:e0j1O2Ipwg==,iv:B1ORvnkE6Eokm7vw8cHYOwE4Da+LdVO1tpwQb3e8b5c=,tag:44J1LYDQP5bbcy+a+xcUhA==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:0q6s,iv:6XxPR/sVrPnJpBfnFahWqq7Jg0vnqDT1gtBPWIlzLt4=,tag:guSvkDH8FopYHiMETmOmAw==,type:int]
- to: ENC[AES256_GCM,data:jA==,iv:f4+nsp7rPa72YMYPWtwyRVoRt0J1iaeeow2Ub0MFz7w=,tag:Rp5cmSfWlV4CWk8EkIS1Ig==,type:int]
- datasourceUid: ENC[AES256_GCM,data:h6a1zg==,iv:5eZ7wV7t3Lk9AxWb0SSL41duM5cKjQuHKH+DsTRtbsQ=,tag:cBar0O22COv4e5xAtZl5eA==,type:str]
- model:
- datasource:
- type: ENC[AES256_GCM,data:EiP3/g==,iv:zDb85K38fifDIhzQyLWcxBtpFgSiyf1C5018flXmInE=,tag:RGKgtLqgc4lvz3dOqQP+Hw==,type:str]
- uid: ENC[AES256_GCM,data:jAT/4g==,iv:jtSdwujeAa9H1vBaSkE62VawrEDJfrhx0kNupHWj6VM=,tag:+afVOmfD9yCfW33JFaKlpg==,type:str]
- editorMode: ENC[AES256_GCM,data:FYfFro9sdw==,iv:osh3hDRuz7u/ye4SCCdOUHVCl7nB7HK5mE+1GSE7tY8=,tag:GeHsAMFz11QktrazAnIWAQ==,type:str]
- expr: ENC[AES256_GCM,data:blTfuXBtz77BGtSFQWMJl2zKPfLGuNGYmw34TvS6lwJxmZvcSiI3EiwZXEXvBFwtwJnAxeti7l7m5ATLNnpY7cKvOzXeVHJZOg3FuHZB3KoKejKUyUutHBvdldvCnlR5N4P4STJgiZ8Ag9hO8tD00HBDIe2yN0nrLweZsmT2iXjv5dow,iv:okwpC1bd3IM73F+8kZXrSWrzNG6mWllegPASIBCvMUk=,tag:dyyX9fkrppUghgS3cZ9pBQ==,type:str]
- intervalMs: ENC[AES256_GCM,data:rSK9ig==,iv:iryVu8itZffFMjrP8ZQOnHfPQNZNwWTYLm1P3Mt0T1Y=,tag:4Fz3uIt1M6vWei6ZmVl5AA==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:U/wCy50=,iv:kcllCxGeJwsM6znY0jR8Uo8zg4YSbhvaF3s4RwQQ830=,tag:QVcRAyr86sj1ZE1/zRlu8w==,type:int]
- queryType: ENC[AES256_GCM,data:I74jRX8edQ==,iv:ChQOlvHyy/r36j3gtocSx6wppRQ2+f4U8ZcErvqertE=,tag:5ofEjGgB2mvirG2OM1kgIA==,type:str]
- refId: ENC[AES256_GCM,data:gg==,iv:0bCRley0puEk+mK4Ynkpqf+y9s6h7lcrDQ0TD04oGOQ=,tag:DdWKhEdOYmgdQSfRLHYg9Q==,type:str]
- - refId: ENC[AES256_GCM,data:dA==,iv:Or7NDPZmNyHiejhoNd+4IkBw+wwKPueevgzBarRXc1w=,tag:Qe6N/G0CK5Z7aHuH8ZcpBw==,type:str]
- datasourceUid: ENC[AES256_GCM,data:sfeWUmqdego=,iv:Efph9pMPEn/p2C/Imsdr4AhtxPlB4b6yoPSQhAdn3/Y=,tag:IOFFhlPqCguGDYNKtneOUQ==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:NA==,iv:sq+zQFGtd0mHbWeYmAUV1FWly+yd183q6tEpoOgz6RY=,tag:Ag7WSEOXnwCzogtwMGoq0A==,type:int]
- - ENC[AES256_GCM,data:ug==,iv:Y8bZ3vz3YNqN3M58E5pHKSrfmbos9Fw6V2RG5klevXA=,tag:2OCWsDeRQw0t09Xk3PRpig==,type:int]
- type: ENC[AES256_GCM,data:hgA=,iv:Cc7g8dg3s/KkuL1lTBjSgLOQE2AEqmYoLTHvGNagsJM=,tag:Aq+fNswt80Qs2htdTAD05g==,type:str]
- operator:
- type: ENC[AES256_GCM,data:D6vG,iv:lzXR0IGn9RAVRpJPhknV5xcf+NK1WQm5bUG5nc4jZkA=,tag:EPI5Gt/3h3q8gEh8pmET9Q==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:pSrU,iv:GuUqZrinLOlutVgeW2SdXN8ojcQmLNfX9t1RWBm1LDE=,tag:dbddvKgvdpGSLBdOITW/ag==,type:str]
- type: ENC[AES256_GCM,data:1oaHsDI=,iv:Wk8sOg4J0GfSWauTm2BhBF4C5r3taWdMQF1NGa44u+I=,tag:kj2u8n1abA+6ej6D13bmEw==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:Miv1hoV5Kb0vQw==,iv:UYGYmqwM6lujGdr73vrdX0FzlVlpJ19Q11G9mt496s4=,tag:pMv6+G7sPQA/sWPpIN5ADg==,type:str]
- type: ENC[AES256_GCM,data:lBV/3PdqsRk=,iv:A3prmO+bpb1dRpqznI0tlPkNOsxaijghfJyk9mWEWl8=,tag:nosdTIiw9nfFNKKhZCSfog==,type:str]
- uid: ENC[AES256_GCM,data:33IuU/CH0RM=,iv:PJ/hT88LuthC9HZ1axspkssv0mKm4+YdhnMTElARJrg=,tag:xS8by6CMnK7DZmy7i5TE/w==,type:str]
- expression: ENC[AES256_GCM,data:ng==,iv:KoVeGGcbEoMOJl3naQTRWulnpoYUHnipflq6nVEA36I=,tag:KWoeM4MLLiK41fiq3uxAmQ==,type:str]
- intervalMs: ENC[AES256_GCM,data:0ZNOKA==,iv:OijhhW2Cm1jQpwTsr1kwRN61Pnz0hUQGvEWlRf9h+Vc=,tag:pgtIaUYBVuyLQQ87yMY6oQ==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:zmKcBcA=,iv:15l4UoJmmuhHLNJpcEnC+EqMQxcTupTryqc7EvyGCag=,tag:0xaqBiL0WA5rXg6+uNJT5Q==,type:int]
- reducer: ENC[AES256_GCM,data:ul66,iv:Cwn8raRtVFrGBhmuDgxreheskTplnt9fyyxjzd0pnXU=,tag:Cx7VaU8vfszGgbIkhj5Buw==,type:str]
- refId: ENC[AES256_GCM,data:Gw==,iv:Oaffv2YG8ksl5C9NBhHxNcFv4s5raAzeJ+j9BzFuHes=,tag:rCftuRGrcEpcO2cSGsZT7Q==,type:str]
- type: ENC[AES256_GCM,data:L/8Fwak5,iv:Z3+kPThmsuJihctnAyf4l7OPV8P5A+hMCvHX9OLwI6I=,tag:Ha1z2qJBwpNqkOlpVCoRZg==,type:str]
- - refId: ENC[AES256_GCM,data:yA==,iv:d1EB15qeJ65J3jGIRnZ8fCGUiVV3s0La5OasdYdYvbk=,tag:cs11EvoIu7ZZBE7Be/60TA==,type:str]
- datasourceUid: ENC[AES256_GCM,data:G7/JyTG9CdU=,iv:u7rWUXaAaRdKck1w1wsSMe2n0ezX/avhDSypPdHUDV4=,tag:MrFBdbS6yBxl6IsmUMNMVA==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:2w==,iv:IuQPuz4Gu4RCccWs4LNvhtoagOr8N+QO1TGr9XJpQ1o=,tag:vOYCk7pymFnK4NKOCVj2Ew==,type:int]
- - ENC[AES256_GCM,data:wA==,iv:Sb0NCB1FyrLL+DCs9NkHPEdUIsPhyNcdxcpCQruhVKk=,tag:LM8dc6EbXDPbiaAQ7Ar16g==,type:int]
- type: ENC[AES256_GCM,data:dlU=,iv:9bibbmn9yvY1uSmX6nj72MroLj5zqliZr+gdE9CRaBU=,tag:FGgxNCQxhwJFT28K5dhNrA==,type:str]
- operator:
- type: ENC[AES256_GCM,data:9gis,iv:dFdOcTb1vvSriaxBv+WKeLKguIMyvW+3oZqVOXQwzSI=,tag:wYXpAa3jVf0LKs2Tn8nIUA==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:1djv,iv:x1Knl4s6FYdoRyOe5gs+PyCzrPI8ozn2uJ4XZSXaa6E=,tag:ze3YC6qCtMgHaDRG8aY94w==,type:str]
- type: ENC[AES256_GCM,data:76lRnoA=,iv:SEYjv6mTD0gIAaFm+lxXG7sgfMFWBNueK2JbPYoLmuc=,tag:IOr8MdNqza/TosGmyiLO3w==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:R+j0IhkhFDexog==,iv:G90fHqrEEqOw0VBDH/bS1rd1IZNZ7WqutY+P/ARdFgM=,tag:nDmcMWmmHujVhZfc9m/hXQ==,type:str]
- type: ENC[AES256_GCM,data:akePqSvpA38=,iv:IFO8DfzC0JTBx9zz8NX73k5sUh1ih0vwQfGZ2tJe17w=,tag:NVT9Jp2yy8fHQ42u4/k0LQ==,type:str]
- uid: ENC[AES256_GCM,data:jkActoaavRs=,iv:HOAUAeiVSxs5ahneJB6HR54eAZxEPqRSk4ZP14jVlkQ=,tag:jekOaCUea82y3zLdpQ1QUA==,type:str]
- expression: ENC[AES256_GCM,data:gw==,iv:EvvYVOwtOjpKPor0p8iz7P7/5hYRpmBuMkGlpY5lWcw=,tag:IMvcfl8WjngGfEA+t/142A==,type:str]
- intervalMs: ENC[AES256_GCM,data:ecJP9A==,iv:+pGhIWqIqLyeuyLELp0wrc/SRpIdji8e10a7xKFz/gg=,tag:Gc23qrWHrctzUGcL5iTm7Q==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:ug2svdc=,iv:7vLctPoadwg6BQOI8hBbh9OxXifCteK2aAC8nbaa83c=,tag:wDIoPYzYZ2oNRKGBh79xaQ==,type:int]
- refId: ENC[AES256_GCM,data:sQ==,iv:bEnqZOJewe0ZeeX62JH4eVfnI7CNi8vFjmi9Hrh7gUU=,tag:ymSznRUIRO9obC86lTfnRg==,type:str]
- type: ENC[AES256_GCM,data:Rp15RgdoEClP,iv:LFroKBGtJQBc8NqXsINsi7dOXmPq6PNLJ+6+yxGetEg=,tag:ywXIgQAuLqymojwY5REfhw==,type:str]
- noDataState: OK
- execErrState: Error
- for: 1m
- annotations: {}
- labels:
- service_id: draupnir4all
- isPaused: false
- - uid: ddz3nijqshjb4e
- title: Draupnir4All Synapse "BackOff"
- condition: B
- data:
- - refId: ENC[AES256_GCM,data:Vg==,iv:dtcOeSBFXNuyXPYyzJM3hhs6KrxEVloR74BdZCtaMG4=,tag:Zw8VkuyWH0msi7R8+jmXDw==,type:str]
- queryType: ENC[AES256_GCM,data:7vUwNHlOcg==,iv:KlaWgP8s3ZVUVVNLr6uzlwkJ+dEzLgpcHXW7jZQcX5c=,tag:g29lNgBIyx83pl4b2ZDoMQ==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:X4Cy,iv:y0kIQcOPR34nXTOncc8iweE5KUxa9H6aXVUaY/vYV7g=,tag:ZMOPDmr2y0amB/Sm/8kWHA==,type:int]
- to: ENC[AES256_GCM,data:8Q==,iv:6MMPMK5rxoAMedyP6YKLKRQj/Kykm0To28amiiy4/QM=,tag:CqjveMDhIElohknvDje9GA==,type:int]
- datasourceUid: ENC[AES256_GCM,data:SnqUjA==,iv:i//JOqxKQE3+xsejsAOQsYDam5bOyQI1pKSzlPk8Oi8=,tag:qpFnUcyS1GtXnLm3oFQa8Q==,type:str]
- model:
- datasource:
- type: ENC[AES256_GCM,data:eDvtaw==,iv:ka0EJWlMBgJBZA2rX/i7f5qunxxETyr41JdFpIaaGT8=,tag:KOniMbdxX6Q1O3swXHQB1w==,type:str]
- uid: ENC[AES256_GCM,data:w3lJDg==,iv:xq2pZ/8AEGUQIw/IIJFg/odn82SQ5TCR9BOEQ6vkFRA=,tag:IN9PSL3uyc27tU8NSSxaEQ==,type:str]
- editorMode: ENC[AES256_GCM,data:4hWS3DXs1A==,iv:oGmr9JmJgs4/7FES7LOUl/KV+ePkgmr905Cjk5WKYLA=,tag:wkn9itrUjNRrnr9oM/N3rQ==,type:str]
- expr: ENC[AES256_GCM,data:lBOL+LWdXgVGQgt7TuwgNkVgroriB3MqcOronk9z33pG7Zu3bNGkCdyeDplly+Jho0CNZxP+THVF//lHT8ouc8ZQdnFB6NUKAaAzmy35vx06HW0J3uP/LiEVSCz3w5eUZhNlfduoPMKouig3sMpY+M0BI6JOply0hF6FyunvkQ5QoRnj3A==,iv:VYgVoiyXeD9iGHSv130eFBTTYv9vuB0ZkOEWXvzmnrY=,tag:a8ubmCufFRyaFU6eQ6NttQ==,type:str]
- intervalMs: ENC[AES256_GCM,data:hfX1Tg==,iv:QKAC3Lj15SLc/A++nTmVwooIXQPooQn4FJ1cdTuGeBE=,tag:khgFb48Ev+YDz2xGFGtm3A==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:GgyLR5k=,iv:cg0SUKizFW8gj3s24HvKqXwrFsPbYClwIipgnyUWQ9s=,tag:ZASd+UPSv4/+tjr9C9DgUg==,type:int]
- queryType: ENC[AES256_GCM,data:sBxK9Nt9nw==,iv:scrDGB8OPB5HiJxr+e4JaiMOtJ+oqdkE7HM0hgjE588=,tag:GiKjgtbubszuv+WEtU6TkQ==,type:str]
- refId: ENC[AES256_GCM,data:pw==,iv:WLVdDBLv4cBEjPGahDTbCJ2GG/ZOrKkcGbg1XRwX018=,tag:5o0qO+yOttusEr+zRRD0ZQ==,type:str]
- - refId: ENC[AES256_GCM,data:cA==,iv:ogPeWXHUPDQRHkaRMlCerQMKR8RSc+gL8zbMX8X5yX8=,tag:sGnrcEh0faY/TFPimc/j6Q==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:ZnNF,iv:caZS0nhBQqasgSAKe2k5jxD5s1Ydb8wq0wZHod5AiiA=,tag:r5uvb3rFb29naOZerYyhxQ==,type:int]
- to: ENC[AES256_GCM,data:qA==,iv:5YTo8F4UTB+bkMqdlCMhXXFeTtERDJiw3Yeyq1YQJC8=,tag:+aj8W8QhhJxweLLRs62Cjw==,type:int]
- datasourceUid: ENC[AES256_GCM,data:c5oTtWuZmD0=,iv:Rkh15LXvMHt7a9lAGJTyGrEdw6KyyrZpzV3EzoONcTY=,tag:lXMG5r/2BDkDB8J7VABlXA==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:cQ==,iv:weX21QCTZohywQmIFTBaSqgIYulHtVrUIySdLXEgO3A=,tag:o5YUpYMR4kTz5+BUpOGRxQ==,type:int]
- - ENC[AES256_GCM,data:AA==,iv:Splpmr7Ph36SNbYNXkBdMk/RfnMoxOAJjZ8eBE60qs4=,tag:ix8wIPVI59+SujEsz+gCCA==,type:int]
- type: ENC[AES256_GCM,data:D/U=,iv:Lrf4vjlGdrdC0KbjV1K8hY0sI882M5VCVuoDBcStBrE=,tag:dnlAHsiOU0TIjKNRuWH2eQ==,type:str]
- operator:
- type: ENC[AES256_GCM,data:vued,iv:ZTwU6S8X/TGjsHe1uwlEkPVPwiqnSLVN0OjKW24MTLQ=,tag:1PISHVO0+gzVdz3CYGTZig==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:Obbx,iv:Xjx89FwwNX1uaHP/Y0LPuZibZ3QVXbC19z0YT47p7O8=,tag:jXUsCS3QJBoKzFlY4g98oA==,type:str]
- type: ENC[AES256_GCM,data:KLj6PtU=,iv:jCoL2RSUUJsSwRSGkUQPYGeY/nvEx3u8cbTGeYMWH0I=,tag:WWI+6SuNYoLdqqji+rCLHw==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:4eWSWXUh5wfBbA==,iv:PT8636Rlv9cB62KuEqSkfXg8QTBXwGNj/VycBgPok00=,tag:fVYFcmL75wgJNCulDVTMNQ==,type:str]
- type: ENC[AES256_GCM,data:i9GADkyjkhQ=,iv:5UbmS/frH1w0MPNhzcIfug3dM+iGu1q0daAOaJ8AJgw=,tag:csYNJlfd1GlqvKbVrX4bzg==,type:str]
- uid: ENC[AES256_GCM,data:MkOdYzwCgPQ=,iv:SVHoVwiVU0HBsV3+oIowsV3jJlbb8XJ3vLB+SUCqCkI=,tag:dz09e8UajUcxZJRQGahg7A==,type:str]
- expression: ENC[AES256_GCM,data:Kw==,iv:R7342RtHM3kvRiMvnhEcBHQ74eypPBztzbsY59Etoc8=,tag:CNOsj6dcM10viOAFtG1jKA==,type:str]
- intervalMs: ENC[AES256_GCM,data:YQKf0g==,iv:Oao3OladPv9DwAA2oV0XBEAFAHi/f097PRyxSU5wcWE=,tag:95ZNB/kH8U3IyVlQ7TLjUw==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:vzWKn/I=,iv:WsBz4VSPHd0zXvSfeR69IMHkqyz9+0FKKa1LKSGGfQQ=,tag:gIVc6fkuaYdGyvn20O/kkw==,type:int]
- reducer: ENC[AES256_GCM,data:8eE8,iv:MxlUxbvM1FBTaC7kkAHcok6lI6seZa2j+GBxmDfbY80=,tag:S/KhcExoNSg+xksexBN1pQ==,type:str]
- refId: ENC[AES256_GCM,data:iQ==,iv:aReAsgqvj/F9nZSFz7OOPaqy2GsbeOsdHTgu0XXP/30=,tag:+3ExHJGvkF1+7uwLDcL3+Q==,type:str]
- type: ENC[AES256_GCM,data:c7UT86jC,iv:a0ogVyHXY4vkSfhlO4oCnyA+ZYoYsBvVz6SR2b8oS+U=,tag:LpFZnySySepgQihRjNHGLg==,type:str]
- - refId: ENC[AES256_GCM,data:yQ==,iv:rCrOITPTDpxB3EtpRhgXdjPfZes/pSNw7tQLG2+cUGU=,tag:C1/6qfUjoY7hko1Xc2pEvQ==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:kF3G,iv:Q5m/oT8afp4hOkgaQU6BBjc8z6uScBinPB58QlOfioE=,tag:X+tY1Azq7W8+dVj7TlNkHA==,type:int]
- to: ENC[AES256_GCM,data:0w==,iv:ltqlKKJ2XRJsUYTNMzlXpdT0371LjZqRte0wRP9DUdE=,tag:3fOqBg14/Bmn/hn/UYQTsw==,type:int]
- datasourceUid: ENC[AES256_GCM,data:FKgjo/+6ZhA=,iv:Qnw1GWI4gfPi3IqYss2nVzrsBSIddSfaXiKJsFLGbfM=,tag:xOtC26i6Ec6v5GrmXikZxA==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:lw==,iv:4Wd4kY6fSeu1Bq0yjRPC2w4+fGmHguoH+mDxZviID1M=,tag:fIO/6uKNu76BBP2HxOOpqQ==,type:int]
- - ENC[AES256_GCM,data:Uw==,iv:T6j0F7Ua7SvsotCGIlB3RIIIRaGO/8dYO6g5FlWshsk=,tag:KZWR+lIs9U+7Qu1g3Gl+bw==,type:int]
- type: ENC[AES256_GCM,data:LRk=,iv:CxnzjybcHNEi7NhnEzoXhhkXDuLO6su29MaLWUrRLf8=,tag:+hPqa3cbxfgdm7mXBKeGdw==,type:str]
- operator:
- type: ENC[AES256_GCM,data:oZc8,iv:ISzOq2crLzxWP12tZp71EDj0upz7qulllDGn3glL6+4=,tag:VmWd4r1bdesGcgDkxmd9lg==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:/XWC,iv:yjRZ+C0eS+idlhrnWM4/FihZxJb+WzpCRgzXGhM0RfU=,tag:DX5BIDuPFujrzEcV3zrhWA==,type:str]
- type: ENC[AES256_GCM,data:3xT2lU8=,iv:pAlMbUZnOjySEqJklckrHnfGFenVEM9Ajjd4mNBa33Y=,tag:l16YLGVQhaH6AU1KyeSgwA==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:QUvdDy3Uj2xC4w==,iv:7ucYbjHrCYCv5HhfZ43p4eb4LEsY8JbUFoCQ5OJOj44=,tag:GWU6/sUPN7U7n2MDOfFBag==,type:str]
- type: ENC[AES256_GCM,data:vc76NoOR3C0=,iv:rXvivVyXKqVRTJ9tI/YmYkBgFeBqmHe6OIrE6RPgBL0=,tag:HfwlueycmnI5dLVqbr6Bww==,type:str]
- uid: ENC[AES256_GCM,data:8cFxvUw97GY=,iv:LGDAeI7EmiEEtMWde9hthaHxXrGy4YqswIQd80qAe/Q=,tag:DMBHeR1viL8iYvlgVkLBQg==,type:str]
- expression: ENC[AES256_GCM,data:TQ==,iv:z0ViUn35xrXz35DayW+Nf8vUyISHps+VDO6FsbrZWPk=,tag:/EMPXCTm05nXbTYYUdlAeg==,type:str]
- intervalMs: ENC[AES256_GCM,data:WQ7+OA==,iv:HMTbvkH89SUIR5akLEaPunYNw6oKlaGUlFoJxfuBRdY=,tag:lptMaD+gtSU3DOy+6JHoNA==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:qoqfFCA=,iv:cZYyHOaUcxE3+zaR5kcZQdWILtXg4IarfrW1zXFGPlI=,tag:YbWulx38f0j28l9DMfxE0Q==,type:int]
- refId: ENC[AES256_GCM,data:1g==,iv:0bScbl9C+ln7v5geW4y0Tn/dRZAsT8idgZPi0xALcj4=,tag:XX59FL9EaZ81OPEZL69ZCQ==,type:str]
- type: ENC[AES256_GCM,data:6DG73nsoQ5Re,iv:77Dga2KTsdQnYV1qgV2zZutvTWztG4D9JAU1QNZGMII=,tag:G3RQ267dpamUDvZxxEO9/g==,type:str]
- noDataState: OK
- execErrState: Error
- for: 1m
- annotations: {}
- labels:
- service_id: draupnir4all
- isPaused: false
- - uid: cdz3njjeosphce
- title: Draupnir4All "BackOff"
- condition: B
- data:
- - refId: ENC[AES256_GCM,data:7w==,iv:ueezk+d6yAQmWAsPVXsnMxsNo7aiCmmseDlkAochZCI=,tag:YarOVoT/EOwel5Leg8x5Sw==,type:str]
- queryType: ENC[AES256_GCM,data:UeMQt3GbNg==,iv:Y8SPSCbFOiaVsplZmve6hKd/wN0zxVlo9aFW8T+G4hU=,tag:eR1yKIKgQgN6BskfyEC3dw==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:rWAd,iv:LhspOfxi9ARbgsMrnTSwVlI3M/EWLYboP/lmvOg4SUg=,tag:yrdeJP+ee2zxRnZXuaWz7Q==,type:int]
- to: ENC[AES256_GCM,data:+A==,iv:6nYRo17JzQf4jGvrzLFiHjg4ujCHmmBL6b5y1sNxtPA=,tag:FFb+clPQkd6rkgsJwPZuMQ==,type:int]
- datasourceUid: ENC[AES256_GCM,data:l0QVsQ==,iv:LYyE3AKqaZwD40fa5gauXKsILFVj0rfzsDXJNHpDqr4=,tag:8knALZ8/hFy2iPEFJBfhDA==,type:str]
- model:
- datasource:
- type: ENC[AES256_GCM,data:mRi5Tw==,iv:WXRXmt5PyfWHsfpkIauOa2OtDWqIZp0R+72w9LSvBMU=,tag:isR9l2jRHSW7jAaSWpbOkQ==,type:str]
- uid: ENC[AES256_GCM,data:N4PYmg==,iv:sMNQtaq3bheTsX6IsSxwIEo93uUUBvEV9eO+6NmsWzE=,tag:DXpkz6++8W9Oirs9ZUrc9Q==,type:str]
- editorMode: ENC[AES256_GCM,data:eWlsbPiHLg==,iv:70CubXvtkM2zxfHdQNW99aHdXfhrlQJMsWwFDHpFiWI=,tag:J6U3DaA7Hf38QlvXlS+nZg==,type:str]
- expr: ENC[AES256_GCM,data:ObWlT0xR/AusxSngPSrO6XzeEU8L/18TtMrHuFIXD0Q1skuM4Hsgsc6v83L4mOpYsrOFFJidZBiqSLX4L/QkUy3MMgMDcrIcmTmyzIqu76XYq9sS6c9/qxyUMbxW259NpXozDgVUz0K8ImvMu0+N8F9JfrzEzistOPRjxeFjJewG,iv:tM2FOdnRzLhUNkqO3WErQLS1ApFSi8UBacFTzm2qWz8=,tag:y4xGtiIiC/Zrvf9edB9Raw==,type:str]
- intervalMs: ENC[AES256_GCM,data:o73mXQ==,iv:MBzH6bqNcx7mKkGd2m/zoZR2pYsQSYg2eUpjo/yLdHY=,tag:jRq7wO6HUcV6xu7vusN2WA==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:kfdwIDo=,iv:mGRG9ZuEGtfIh4DGWf9RYJE94Rp+57XKitijFWTEEQg=,tag:W5cKk3aoc28zIvA3MvfkAA==,type:int]
- queryType: ENC[AES256_GCM,data:AxS60RaS+A==,iv:YaV1y1Lhmxqr2U5cW7CXDe+YKEBASC+n8P97dJoX5+o=,tag:dFTTRnJI72wMXlJJi5jPpQ==,type:str]
- refId: ENC[AES256_GCM,data:Xg==,iv:sx6EAct+ANwBrh080Al8kgiIUhOemWqitDO1thiiIts=,tag:kHne5x1YodsjY8dYjOkX6g==,type:str]
- - refId: ENC[AES256_GCM,data:PA==,iv:Y8k9lRnGKSsD88YQ6zfXXWhV5laIKfVvdWSKtRKvnGo=,tag:ew24iVhFd8+belGnHxUyuQ==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:WabU,iv:QH6R5AwQPjIyxIDR7iNAo2CcbenUWM/qz+8Hz52JwzI=,tag:EPoVK3rBqkcyRK0Ym+T1rQ==,type:int]
- to: ENC[AES256_GCM,data:+Q==,iv:5MyJaUIcPfoYRrLIu6TUwx33I78KxKzj4jo3y1Mlck8=,tag:WUSaDtxj44hqgzKoSnWZzQ==,type:int]
- datasourceUid: ENC[AES256_GCM,data:40AA6jD3ixQ=,iv:EhsLmEhN0mwRfx/85wl9Khqc82IyAC0OEJM/DsNah1E=,tag:2lm8eNRW1syyRYiba5R00w==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:CQ==,iv:dXzoCt7hMX2DuIrQgcZzodk8y5goOFhO2J2T895D0Ws=,tag:wn97USLF3gcl8hcVnMe4sw==,type:int]
- - ENC[AES256_GCM,data:UA==,iv:ncYY9ZlMEAppUUoXW5bBTbsh8cx0fXQvvJoReYmY5Lo=,tag:6bn+nckxKSfFIbWY5mUATg==,type:int]
- type: ENC[AES256_GCM,data:oZY=,iv:HQMyYxR2ohk90IKdx4CpGAIjOdn5EvLqGOULGslg0cg=,tag:f9T6lZ1guAasm2XUAqVpuQ==,type:str]
- operator:
- type: ENC[AES256_GCM,data:DZLj,iv:NvQ3ntVzo93Qrk9/J4sSf4zv9VhhduNxgqKrcl+1TdU=,tag:kAwKBIRg2Nf0ONswcoFp7w==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:VLJd,iv:mumQLrG4rkF5ms+YyZzJki8+1FHSWqEMRrIKmW1IY2o=,tag:qNiHm3VTiYxFJGsgo1zFJg==,type:str]
- type: ENC[AES256_GCM,data:/bHId6c=,iv:pIuGnFaIeBmu9/cGPLHBThbBtz2yArT3l3RM6CcTTb4=,tag:3C65xj/aXw9bPI0hPm4gFA==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:2jSU1+z30hRclw==,iv:98W1DEB7kUQuESl+S5m0PbsefjXb5Za7l9y//PfBgK0=,tag:6n5j5s7ldKOBveyNPndNgw==,type:str]
- type: ENC[AES256_GCM,data:Vy2JJAKeyvg=,iv:yy7DdpJ61cTI7sxK0204MhMRJkIdLooVYiJgfA89Lks=,tag:TzNKYi2XW1FH6yeCUAXaEg==,type:str]
- uid: ENC[AES256_GCM,data:GT1PkcmoIX4=,iv:jI+RK0lHkWY7ea4WW4XzvCWKRGNSh10YmJuyhyOAPtY=,tag:421CGBJhywJ9xAwxiwCdcg==,type:str]
- expression: ENC[AES256_GCM,data:rQ==,iv:OWsDovaw/ZfXDa7xxKGsVRqQg7976HOFn/fJy1Dc8/I=,tag:mpr2uP4jTAN5GdkEEmpZHw==,type:str]
- intervalMs: ENC[AES256_GCM,data:PJisTg==,iv:wENjoDR2VO0DcJtVaupgqVzdNpQO+mNzPV7QvHDAu1w=,tag:AQ86yCqYb31qrWlIg7UECg==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:8UqN9Xc=,iv:bAWybOk4YOlSZCQmexfXzERwi5jaH1EhzuPtjRXF9os=,tag:fAIlPFguMdYKihzEI6W5Hg==,type:int]
- reducer: ENC[AES256_GCM,data:Yx17,iv:rFWuvvuXrcVDDRVJA35My2w/AolMwKfEGRez10JHVTo=,tag:BXRQvpdayRX1+3lkNV7ebA==,type:str]
- refId: ENC[AES256_GCM,data:lA==,iv:IS3q/E3xq+YCOEwZa36FMlpIOlRM5SUFdPctzFm7rCY=,tag:fUJA4wobK49wIx7WsSgI2w==,type:str]
- type: ENC[AES256_GCM,data:02MpAat0,iv:1xFLXGKc4Hn14nrIJtHqLQNrxOwcyDm3cL0jCBXNdVY=,tag:4fJ6H+6lWhbsSY4DmS9n5Q==,type:str]
- - refId: ENC[AES256_GCM,data:PQ==,iv:CwBQ9zVLzWYF4KTNRqPzEWaHGvh2/dUnLAztKTvMylY=,tag:K9P+qNQ7iTw7GWFMH1ZmZQ==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:J9oX,iv:HrG2hC5SHV3IJVjMP2Jeh0wD9goXV3EAeUvd8Cz0n50=,tag:vcdU83g80abopku/MxiT5w==,type:int]
- to: ENC[AES256_GCM,data:7Q==,iv:Ew3Zg6vmQw+Kjg3zlI6+WU3q5n1gtitsPJIB+bvcQcg=,tag:y/KiDQVQuJeYF8pgya53Ew==,type:int]
- datasourceUid: ENC[AES256_GCM,data:inyPNVUP9QI=,iv:GoYfj1zARilV0znGx4mPhBVshZZGTL7wneUZDqJwpBc=,tag:1dh/ZO/OyQAVHKyfSfQB1Q==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:sg==,iv:62GdKYJK4Z0p0tUlH/0heGOZjVXmHkA6DNytr3H/w8w=,tag:tbwtUd/SHChyjDBQ7SC4TQ==,type:int]
- - ENC[AES256_GCM,data:qA==,iv:J0BeynfzuEHCwgRwdHrn+2Lezg17ELoOQ6lhlizz1i4=,tag:yaMACmK/VUPta7WrryDwAg==,type:int]
- type: ENC[AES256_GCM,data:9ho=,iv:1PNEHu5HBt1mxlovQVg5XwOHbn1uvEhOhEUIr5AjVKw=,tag:UyX4mj5kV2M42FwUOySZgg==,type:str]
- operator:
- type: ENC[AES256_GCM,data:UBsm,iv:fXEnx++WPDGXg/3zoFhQzfPw17vuG07IT/4QN3HSN5E=,tag:Y+WyTXyTFrL58vaH6B6uiw==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:5LxV,iv:wt42uIh+92FoGm1DkInXiARf0DkkJ83ZrZeaICV47R8=,tag:lv1lyAljNWx2uWxMJK9IFw==,type:str]
- type: ENC[AES256_GCM,data:5Juy7no=,iv:g7pFmwMXeCyEED34BL49XBX8aj/RwbzqYsVA4T+0/fQ=,tag:x5pqQiF27joKiAO2yF4JiQ==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:0Db4YLhUMJTMkw==,iv:+95X9fZlv7s7ryi7tpqTKCxKVhf/IDN3GyQSEwcKDBs=,tag:BHHIYwQ8nvwt7VOG9NCZrg==,type:str]
- type: ENC[AES256_GCM,data:Yu2ry07nMBE=,iv:2IRwVk9r+Bkg+IkSHAEg2BZL56jaTZVthOR5Ki06EzA=,tag:DMKEvsuR3zjJsMpj8ipBLQ==,type:str]
- uid: ENC[AES256_GCM,data:dFuu5XsMDKs=,iv:JP+RvCh620V12opyMeqlOzdpMm3BEZVtBzPMndMRYdY=,tag:lkjpokjSBBw+IISMMcM1HA==,type:str]
- expression: ENC[AES256_GCM,data:vA==,iv:5ogkjH2apiIrLi5svV47Xh1rP0CFR4A7cEd9FxgVL6U=,tag:hfj60skDICzw2ZTtBA2vHQ==,type:str]
- intervalMs: ENC[AES256_GCM,data:XLxo3A==,iv:FNLEW2FjiFWtaXhnyAWn9WU3dUQBwJummNVaCNCLzhw=,tag:xiUPeXgMpuLZ4Ma9kOKKkw==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:EqWOEBI=,iv:g5e3eMQQcyKT09nY2E+8STb+rz/iyFW4z9/uLlwfpM0=,tag:pPD2nKl9Jt1Ht8UuDaF3wA==,type:int]
- refId: ENC[AES256_GCM,data:hg==,iv:UPGE9ho7PL3bysVBKcM/+TvLOr1mp4wzyPe+7QP9daw=,tag:kvL70t7RapcaYa+/J5Xkmw==,type:str]
- type: ENC[AES256_GCM,data:jbW+VdjYJJ00,iv:EeJplcdWry4QPmhjSVCZyWNfXWvVOy2J5xNnpiGrb0k=,tag:8rao104rqL4McOkv9rl0hg==,type:str]
- noDataState: OK
- execErrState: Error
- for: 1m
- annotations: {}
- labels:
- service_id: draupnir4all
- isPaused: false
- - uid: adz3nkklsv2tcd
- title: Draupnir4All "Failed"
- condition: B
- data:
- - refId: ENC[AES256_GCM,data:5w==,iv:Gue6UfikistbRUBXuNDxTDyfGjQRXM4fLwpRoI+fLFk=,tag:KsVkPZmUqQRYFsrcLY4dgw==,type:str]
- queryType: ENC[AES256_GCM,data:Jaqh6PjxrQ==,iv:5IfNxRsjuVTuq7fEc4RC0v+8tcluzhXJSZ1UFJafpJo=,tag:259lOJ1qH6uosSQPchJpRg==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:mkTK,iv:BHZsMBpVi8w726TwoMbgR1WamabPi3jGMefV33bJfxM=,tag:tW8aHpWyyG24eLsZuZET9A==,type:int]
- to: ENC[AES256_GCM,data:7A==,iv:1rA+MXpTFCP0fG1wKZmI53IiV2vcX+wh3ADmn120WWU=,tag:XbOGKHblWpTRYvlwWB0G5A==,type:int]
- datasourceUid: ENC[AES256_GCM,data:YktlyQ==,iv:KfH8ldMEuCdAWmm6vbaBS011Eg8PhiUVxwgIX1slUDg=,tag:ni23pBlDBhU0QgTZpqevgg==,type:str]
- model:
- datasource:
- type: ENC[AES256_GCM,data:6eaoEA==,iv:BsvqUdhflkpu9sHy8Dyv/YqYezVgrka0695fgF4phFE=,tag:0YrKzuGNfMv5RCyfgqs1Lw==,type:str]
- uid: ENC[AES256_GCM,data:4EinVg==,iv:gKgvkU5BJWduPCQNQf5Xmn7qr1ofm9Ifv9CJ2a8wnN8=,tag:bPVsIBGVuchrjqJ+q8Eq3g==,type:str]
- editorMode: ENC[AES256_GCM,data:wyQlpK7S5A==,iv:as+X8f+ATOKKd9MaQeJyoRF75GG5Mn53U5e+NCwkRPg=,tag:yPdBm8Z1L81cuGLf6LQgPg==,type:str]
- expr: ENC[AES256_GCM,data:OFWkwqFuK+s/TWRd116R3Ttf6l/i1XTg98SwtztB9SSWuRGfPIEreDgVX3oc9ieXFuxV+zC/tEvFVF4teIu+HMRFuLhJp+u+3OcpnaFPHNE8uYNxHArarXm5X1GizIB8j/o8WIc6MG0BLRaaVLuVzn6MDIaOgZNV4cCVIGwsrGs=,iv:0YV0WTKN9SiBdkBq53x7vRiLB0XEq+JPKSCQ9FT1+RE=,tag:xTG1edhOothyCGzQnQ6LRA==,type:str]
- intervalMs: ENC[AES256_GCM,data:F4TkUA==,iv:bvNo2lMu2J75TlblYX6wBmMxVNTaG45YFclGrDwfu74=,tag:c+d2VeppuUmB6ggWz7Nj9g==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:a6FEQ6A=,iv:6Op/orJ+XaeG/UsG9R7BOlqOW4faQrpFKUO6Ol6N0P4=,tag:TW11qS0HVoT3gdrlMoOnAw==,type:int]
- queryType: ENC[AES256_GCM,data:kKBtGom9tQ==,iv:3kIKCwDKHRO0x8zC7KIjUSMyKT64CACCw6yH09abvfs=,tag:3X/GP6kzx204kmC9Jv/fvQ==,type:str]
- refId: ENC[AES256_GCM,data:Mw==,iv:EKgfTgTqG7EFwVIMJtxK5p+a3mucyYpLWRC9tfZUnTE=,tag:9Y4JANB50VoILyj7ZEVOlg==,type:str]
- - refId: ENC[AES256_GCM,data:Ag==,iv:Qa8knzOXSacHxu2FM0qxYmNV+NYne4VnUt+IZIylad8=,tag:YV9ph93yHov8un+k6uTn2g==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:m3c8,iv:DXArnOGAWOv+uPrJHqVCUq8QGaMXDK1/MtR5BOUUwzM=,tag:K/Blbwa8BRHJ7LSESNlxyQ==,type:int]
- to: ENC[AES256_GCM,data:Tg==,iv:nrE5KR5Z4eCxnci8ZUviICpaGcvzfV4uikACTheoII4=,tag:2E8BTRetGS9tDWizsuqReg==,type:int]
- datasourceUid: ENC[AES256_GCM,data:nKQdknxR3FY=,iv:o2b8Bw5K3QK1Gvqyd8ciLXoov/Jz/tjBmV3TWJvyjVI=,tag:nUuCfN4R5yY+j2avSb7evA==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:rw==,iv:IP/ayajEBf7gh9EyBgrou6a9wPGXT6oYQvd0kB/oa6M=,tag:NOD2lyajgJ1TLnxHY49HGg==,type:int]
- - ENC[AES256_GCM,data:ZA==,iv:T9DuYWBNB9mtNsvajlwVoM9g/bWsz9wEw/0g+EjzXAE=,tag:9O+4q/7SigLhbJ9pzi5/9Q==,type:int]
- type: ENC[AES256_GCM,data:xvA=,iv:AYqeXTYVnkv4Mq60ykVAzk0G6LcXS/Y+jw8C163BQgc=,tag:iMik1bM73MZQeCmLDzpl/w==,type:str]
- operator:
- type: ENC[AES256_GCM,data:QP8w,iv:98kA0IzQi1wT9UhvR/DHpqfnNgbfTH1Jmol0hqkViHo=,tag:2eeRIQ64P9yjBU4/ZJuIdQ==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:6bWc,iv:TYoclJElN4DMJ39nSA5tWCzUytywCDRejdUUIU8PQyc=,tag:WMuqFSzBe7kFHIYYaSpVWQ==,type:str]
- type: ENC[AES256_GCM,data:QkW+6CY=,iv:pb19y/8Z3Ro3V+q6pOSiKRcpR5BdRYOYPFjbZ1VKmaA=,tag:QdQ05FBcYFovVCtf6ifhRQ==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:/Jmc1cmCYicMug==,iv:HDV2mMSJ4QbTONrk+kqwXIyn6yl3TKpdKQB2V93cHdE=,tag:NEFwHsByduxNT2fqQHArLg==,type:str]
- type: ENC[AES256_GCM,data:YfxQqmQruB8=,iv:+1UbMxFwbfaSX3pauKx8IL2vcOnjW/zLFNnRsRDY4Hc=,tag:EP1QW9a3XOTxj3LOoFcGyQ==,type:str]
- uid: ENC[AES256_GCM,data:X7EBK+QEKj4=,iv:MINXSPUGGjk/xuakThXA9jQ7fYmCxA74obnFyUYcEUc=,tag:EqpTCpieVOlRKBUI+bLWmg==,type:str]
- expression: ENC[AES256_GCM,data:PQ==,iv:Q+rgba3AarUW9Eo9KoDWbjspBR9T8ZJcomOthXT+l2Y=,tag:wWqiqrwP5CM7w3RMPCtBMg==,type:str]
- intervalMs: ENC[AES256_GCM,data:WXDw7Q==,iv:UkFCNn5MnFZpKFWRScrBA19XF0eHK/S+F8NhJtQL1AY=,tag:FLcNxC1n+GnSiCtYpUCQYA==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:CKnjG4k=,iv:l6crEKusOUIGHk4kDLL6jj//YB+SYA2WcvjeBUCTqqs=,tag:mwm4rVp751flUXLS9oc/3g==,type:int]
- reducer: ENC[AES256_GCM,data:jWxW,iv:ZiK0h+oIpcQW/ACXAE9WN+6w8M6kkESD23DbSSMMUeM=,tag:zZJY/gSDQq3aoX99meeqpw==,type:str]
- refId: ENC[AES256_GCM,data:Ow==,iv:kmzskYw21AlV7RGbI1pbdBHP2Ek8lYKdm9cnUljrnTs=,tag:+Guy90rJRbOT/rKduEyrEA==,type:str]
- type: ENC[AES256_GCM,data:qbtTs3w9,iv:n7oPbJrKrQH1SL35x5HWTb30Il41neKG2FXl0+dUk38=,tag:++1COWAd6uaFXA6uf3Eo+Q==,type:str]
- - refId: ENC[AES256_GCM,data:QQ==,iv:eb5atcWIV38hIliJE4LLbrK6IXXDXGaivlZRkzeQl0E=,tag:RK4Okqn0a8D/7JTpJHWsxQ==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:2Q0n,iv:7hOev0keEzHiKaFB31w16DGCHLknKzU1kuQt9KoGQMI=,tag:gCwlDAubuRnfYW/cIID27Q==,type:int]
- to: ENC[AES256_GCM,data:BQ==,iv:oY3tWjrMHXKVoy0MsJwQLFrjgRTXIpk22CdOcArNCaw=,tag:wT01fOJEqk1jelSzlPNqGA==,type:int]
- datasourceUid: ENC[AES256_GCM,data:Itfguw4yrBA=,iv:iE6QU3H9ttMFeiqOycEH7SUj7xpCbwdtTM9BTF+W+1A=,tag:3nyAn7naiPrzjORy09xa0g==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:7A==,iv:1coEseHn8njOhVmGANZxprRHAz2K3ZIeOTmwsHIrnx0=,tag:iuGDUdtwoXD/zHWcwQEoRQ==,type:int]
- - ENC[AES256_GCM,data:xg==,iv:YLbsH9jUQ/K1u+3+tE3dP+2hubbPDJEmn4lN1IaLFtE=,tag:nnsmSMrtdqDQyFSx6fJQdw==,type:int]
- type: ENC[AES256_GCM,data:RgM=,iv:r6WZ1GfzyVEIdInBEV17uMT8yeLd6YjJFzCmrgTFQ/M=,tag:PskANBeM1AuzwQWj9yRaJw==,type:str]
- operator:
- type: ENC[AES256_GCM,data:6jJ5,iv:TW+F6FRcfx5qW9SXzXmZzhE4kd/1BKw0J/dlbReEUBg=,tag:62R1Czw1u4tOC2jS9C0zTw==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:9ZIB,iv:HYPOn1g84M0MQqAshPkaNJD/RkgD2l+UyztGHjZhdYM=,tag:wPRX6ZjFSY/flqeeZi34Hg==,type:str]
- type: ENC[AES256_GCM,data:xcsapnc=,iv:PhZDhNodF6ORX0wV5bj9PyVMyj/fTeJvetosJQlvi+0=,tag:h9irF522XyvhoYfHyR7WLA==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:WMYnC6chyGdzuQ==,iv:oWGD9sMv1q5eEeQk6GzZQG6PGpnhiiaOqL2jwWifTys=,tag:Nt5kN/DViS8L7/l2YwYLuw==,type:str]
- type: ENC[AES256_GCM,data:8YT9rOmPWPI=,iv:kMpQhSc6Rg3hCeIsDbXQItARrrgwigx6G6xIuIeB8cw=,tag:sYwe1C5XKUv2MzQ+DPhyyQ==,type:str]
- uid: ENC[AES256_GCM,data:r256/JZ5WnI=,iv:4et9h1QlxPfiUC1xZPC0Vo6kjzz/3pqA2wOr1vZPXFA=,tag:JZMlMehTyJ1CXSN9U3B9BA==,type:str]
- expression: ENC[AES256_GCM,data:Tw==,iv:NZw3GgInxDux4tWGSlvYUmUmW50LKFVI5RXuKtikkhc=,tag:FvBNwJMIb1o6rX8WGQtXVg==,type:str]
- intervalMs: ENC[AES256_GCM,data:Gihlgg==,iv:8+JwR88qEyYI11kEj5JOkQHdp+XfHC6tL5bN4Y5Mess=,tag:J8HMsNV4kIV1RbZytk8DrQ==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:ts/dFyo=,iv:j0LRvKiRqBgsjRtRaGCJBwx12MWdRo0ZY67kXzBZUDE=,tag:2wGxDzMmd/sJOIDx38mP7A==,type:int]
- refId: ENC[AES256_GCM,data:6w==,iv:OCXqzCqpbDvPsLfE+AjNG+AhoujnKx9Kl+DOc8wOMQ8=,tag:wb8PtQL3FgIe25JOE13vpQ==,type:str]
- type: ENC[AES256_GCM,data:bsIb+/O+wdSA,iv:BjWgGocaE5QwO5dNrEBeBGRBr1vsPemuYSUikSzQnmk=,tag:zf9yhG43va//ypQB+7v2SA==,type:str]
- noDataState: OK
- execErrState: Error
- for: 1m
- annotations: {}
- labels:
- service_id: draupnir4all
- isPaused: false
- notification_settings:
- receiver: PagerDuty Draupnir4All Warning
+ - uid: adz3nhdki7d34d
+ title: Draupnir4All Synapse "Failed"
+ condition: B
+ data:
+ - refId: ENC[AES256_GCM,data:9Q==,iv:xdJFG2iVuFSsgUC7cnPlOAZndQggOhEI46LkegCWYYc=,tag:Uud6d+Xk+8ZnfY2SiTrMYw==,type:str]
+ queryType: ENC[AES256_GCM,data:e0j1O2Ipwg==,iv:B1ORvnkE6Eokm7vw8cHYOwE4Da+LdVO1tpwQb3e8b5c=,tag:44J1LYDQP5bbcy+a+xcUhA==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:0q6s,iv:6XxPR/sVrPnJpBfnFahWqq7Jg0vnqDT1gtBPWIlzLt4=,tag:guSvkDH8FopYHiMETmOmAw==,type:int]
+ to: ENC[AES256_GCM,data:jA==,iv:f4+nsp7rPa72YMYPWtwyRVoRt0J1iaeeow2Ub0MFz7w=,tag:Rp5cmSfWlV4CWk8EkIS1Ig==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:h6a1zg==,iv:5eZ7wV7t3Lk9AxWb0SSL41duM5cKjQuHKH+DsTRtbsQ=,tag:cBar0O22COv4e5xAtZl5eA==,type:str]
+ model:
+ datasource:
+ type: ENC[AES256_GCM,data:EiP3/g==,iv:zDb85K38fifDIhzQyLWcxBtpFgSiyf1C5018flXmInE=,tag:RGKgtLqgc4lvz3dOqQP+Hw==,type:str]
+ uid: ENC[AES256_GCM,data:jAT/4g==,iv:jtSdwujeAa9H1vBaSkE62VawrEDJfrhx0kNupHWj6VM=,tag:+afVOmfD9yCfW33JFaKlpg==,type:str]
+ editorMode: ENC[AES256_GCM,data:FYfFro9sdw==,iv:osh3hDRuz7u/ye4SCCdOUHVCl7nB7HK5mE+1GSE7tY8=,tag:GeHsAMFz11QktrazAnIWAQ==,type:str]
+ expr: ENC[AES256_GCM,data:blTfuXBtz77BGtSFQWMJl2zKPfLGuNGYmw34TvS6lwJxmZvcSiI3EiwZXEXvBFwtwJnAxeti7l7m5ATLNnpY7cKvOzXeVHJZOg3FuHZB3KoKejKUyUutHBvdldvCnlR5N4P4STJgiZ8Ag9hO8tD00HBDIe2yN0nrLweZsmT2iXjv5dow,iv:okwpC1bd3IM73F+8kZXrSWrzNG6mWllegPASIBCvMUk=,tag:dyyX9fkrppUghgS3cZ9pBQ==,type:str]
+ intervalMs: ENC[AES256_GCM,data:rSK9ig==,iv:iryVu8itZffFMjrP8ZQOnHfPQNZNwWTYLm1P3Mt0T1Y=,tag:4Fz3uIt1M6vWei6ZmVl5AA==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:U/wCy50=,iv:kcllCxGeJwsM6znY0jR8Uo8zg4YSbhvaF3s4RwQQ830=,tag:QVcRAyr86sj1ZE1/zRlu8w==,type:int]
+ queryType: ENC[AES256_GCM,data:I74jRX8edQ==,iv:ChQOlvHyy/r36j3gtocSx6wppRQ2+f4U8ZcErvqertE=,tag:5ofEjGgB2mvirG2OM1kgIA==,type:str]
+ refId: ENC[AES256_GCM,data:gg==,iv:0bCRley0puEk+mK4Ynkpqf+y9s6h7lcrDQ0TD04oGOQ=,tag:DdWKhEdOYmgdQSfRLHYg9Q==,type:str]
+ - refId: ENC[AES256_GCM,data:dA==,iv:Or7NDPZmNyHiejhoNd+4IkBw+wwKPueevgzBarRXc1w=,tag:Qe6N/G0CK5Z7aHuH8ZcpBw==,type:str]
+ datasourceUid: ENC[AES256_GCM,data:sfeWUmqdego=,iv:Efph9pMPEn/p2C/Imsdr4AhtxPlB4b6yoPSQhAdn3/Y=,tag:IOFFhlPqCguGDYNKtneOUQ==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:NA==,iv:sq+zQFGtd0mHbWeYmAUV1FWly+yd183q6tEpoOgz6RY=,tag:Ag7WSEOXnwCzogtwMGoq0A==,type:int]
+ - ENC[AES256_GCM,data:ug==,iv:Y8bZ3vz3YNqN3M58E5pHKSrfmbos9Fw6V2RG5klevXA=,tag:2OCWsDeRQw0t09Xk3PRpig==,type:int]
+ type: ENC[AES256_GCM,data:hgA=,iv:Cc7g8dg3s/KkuL1lTBjSgLOQE2AEqmYoLTHvGNagsJM=,tag:Aq+fNswt80Qs2htdTAD05g==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:D6vG,iv:lzXR0IGn9RAVRpJPhknV5xcf+NK1WQm5bUG5nc4jZkA=,tag:EPI5Gt/3h3q8gEh8pmET9Q==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:pSrU,iv:GuUqZrinLOlutVgeW2SdXN8ojcQmLNfX9t1RWBm1LDE=,tag:dbddvKgvdpGSLBdOITW/ag==,type:str]
+ type: ENC[AES256_GCM,data:1oaHsDI=,iv:Wk8sOg4J0GfSWauTm2BhBF4C5r3taWdMQF1NGa44u+I=,tag:kj2u8n1abA+6ej6D13bmEw==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:Miv1hoV5Kb0vQw==,iv:UYGYmqwM6lujGdr73vrdX0FzlVlpJ19Q11G9mt496s4=,tag:pMv6+G7sPQA/sWPpIN5ADg==,type:str]
+ type: ENC[AES256_GCM,data:lBV/3PdqsRk=,iv:A3prmO+bpb1dRpqznI0tlPkNOsxaijghfJyk9mWEWl8=,tag:nosdTIiw9nfFNKKhZCSfog==,type:str]
+ uid: ENC[AES256_GCM,data:33IuU/CH0RM=,iv:PJ/hT88LuthC9HZ1axspkssv0mKm4+YdhnMTElARJrg=,tag:xS8by6CMnK7DZmy7i5TE/w==,type:str]
+ expression: ENC[AES256_GCM,data:ng==,iv:KoVeGGcbEoMOJl3naQTRWulnpoYUHnipflq6nVEA36I=,tag:KWoeM4MLLiK41fiq3uxAmQ==,type:str]
+ intervalMs: ENC[AES256_GCM,data:0ZNOKA==,iv:OijhhW2Cm1jQpwTsr1kwRN61Pnz0hUQGvEWlRf9h+Vc=,tag:pgtIaUYBVuyLQQ87yMY6oQ==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:zmKcBcA=,iv:15l4UoJmmuhHLNJpcEnC+EqMQxcTupTryqc7EvyGCag=,tag:0xaqBiL0WA5rXg6+uNJT5Q==,type:int]
+ reducer: ENC[AES256_GCM,data:ul66,iv:Cwn8raRtVFrGBhmuDgxreheskTplnt9fyyxjzd0pnXU=,tag:Cx7VaU8vfszGgbIkhj5Buw==,type:str]
+ refId: ENC[AES256_GCM,data:Gw==,iv:Oaffv2YG8ksl5C9NBhHxNcFv4s5raAzeJ+j9BzFuHes=,tag:rCftuRGrcEpcO2cSGsZT7Q==,type:str]
+ type: ENC[AES256_GCM,data:L/8Fwak5,iv:Z3+kPThmsuJihctnAyf4l7OPV8P5A+hMCvHX9OLwI6I=,tag:Ha1z2qJBwpNqkOlpVCoRZg==,type:str]
+ - refId: ENC[AES256_GCM,data:yA==,iv:d1EB15qeJ65J3jGIRnZ8fCGUiVV3s0La5OasdYdYvbk=,tag:cs11EvoIu7ZZBE7Be/60TA==,type:str]
+ datasourceUid: ENC[AES256_GCM,data:G7/JyTG9CdU=,iv:u7rWUXaAaRdKck1w1wsSMe2n0ezX/avhDSypPdHUDV4=,tag:MrFBdbS6yBxl6IsmUMNMVA==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:2w==,iv:IuQPuz4Gu4RCccWs4LNvhtoagOr8N+QO1TGr9XJpQ1o=,tag:vOYCk7pymFnK4NKOCVj2Ew==,type:int]
+ - ENC[AES256_GCM,data:wA==,iv:Sb0NCB1FyrLL+DCs9NkHPEdUIsPhyNcdxcpCQruhVKk=,tag:LM8dc6EbXDPbiaAQ7Ar16g==,type:int]
+ type: ENC[AES256_GCM,data:dlU=,iv:9bibbmn9yvY1uSmX6nj72MroLj5zqliZr+gdE9CRaBU=,tag:FGgxNCQxhwJFT28K5dhNrA==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:9gis,iv:dFdOcTb1vvSriaxBv+WKeLKguIMyvW+3oZqVOXQwzSI=,tag:wYXpAa3jVf0LKs2Tn8nIUA==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:1djv,iv:x1Knl4s6FYdoRyOe5gs+PyCzrPI8ozn2uJ4XZSXaa6E=,tag:ze3YC6qCtMgHaDRG8aY94w==,type:str]
+ type: ENC[AES256_GCM,data:76lRnoA=,iv:SEYjv6mTD0gIAaFm+lxXG7sgfMFWBNueK2JbPYoLmuc=,tag:IOr8MdNqza/TosGmyiLO3w==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:R+j0IhkhFDexog==,iv:G90fHqrEEqOw0VBDH/bS1rd1IZNZ7WqutY+P/ARdFgM=,tag:nDmcMWmmHujVhZfc9m/hXQ==,type:str]
+ type: ENC[AES256_GCM,data:akePqSvpA38=,iv:IFO8DfzC0JTBx9zz8NX73k5sUh1ih0vwQfGZ2tJe17w=,tag:NVT9Jp2yy8fHQ42u4/k0LQ==,type:str]
+ uid: ENC[AES256_GCM,data:jkActoaavRs=,iv:HOAUAeiVSxs5ahneJB6HR54eAZxEPqRSk4ZP14jVlkQ=,tag:jekOaCUea82y3zLdpQ1QUA==,type:str]
+ expression: ENC[AES256_GCM,data:gw==,iv:EvvYVOwtOjpKPor0p8iz7P7/5hYRpmBuMkGlpY5lWcw=,tag:IMvcfl8WjngGfEA+t/142A==,type:str]
+ intervalMs: ENC[AES256_GCM,data:ecJP9A==,iv:+pGhIWqIqLyeuyLELp0wrc/SRpIdji8e10a7xKFz/gg=,tag:Gc23qrWHrctzUGcL5iTm7Q==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:ug2svdc=,iv:7vLctPoadwg6BQOI8hBbh9OxXifCteK2aAC8nbaa83c=,tag:wDIoPYzYZ2oNRKGBh79xaQ==,type:int]
+ refId: ENC[AES256_GCM,data:sQ==,iv:bEnqZOJewe0ZeeX62JH4eVfnI7CNi8vFjmi9Hrh7gUU=,tag:ymSznRUIRO9obC86lTfnRg==,type:str]
+ type: ENC[AES256_GCM,data:Rp15RgdoEClP,iv:LFroKBGtJQBc8NqXsINsi7dOXmPq6PNLJ+6+yxGetEg=,tag:ywXIgQAuLqymojwY5REfhw==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 1m
+ annotations: {}
+ labels:
+ service_id: draupnir4all
+ isPaused: false
+ - uid: ddz3nijqshjb4e
+ title: Draupnir4All Synapse "BackOff"
+ condition: B
+ data:
+ - refId: ENC[AES256_GCM,data:Vg==,iv:dtcOeSBFXNuyXPYyzJM3hhs6KrxEVloR74BdZCtaMG4=,tag:Zw8VkuyWH0msi7R8+jmXDw==,type:str]
+ queryType: ENC[AES256_GCM,data:7vUwNHlOcg==,iv:KlaWgP8s3ZVUVVNLr6uzlwkJ+dEzLgpcHXW7jZQcX5c=,tag:g29lNgBIyx83pl4b2ZDoMQ==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:X4Cy,iv:y0kIQcOPR34nXTOncc8iweE5KUxa9H6aXVUaY/vYV7g=,tag:ZMOPDmr2y0amB/Sm/8kWHA==,type:int]
+ to: ENC[AES256_GCM,data:8Q==,iv:6MMPMK5rxoAMedyP6YKLKRQj/Kykm0To28amiiy4/QM=,tag:CqjveMDhIElohknvDje9GA==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:SnqUjA==,iv:i//JOqxKQE3+xsejsAOQsYDam5bOyQI1pKSzlPk8Oi8=,tag:qpFnUcyS1GtXnLm3oFQa8Q==,type:str]
+ model:
+ datasource:
+ type: ENC[AES256_GCM,data:eDvtaw==,iv:ka0EJWlMBgJBZA2rX/i7f5qunxxETyr41JdFpIaaGT8=,tag:KOniMbdxX6Q1O3swXHQB1w==,type:str]
+ uid: ENC[AES256_GCM,data:w3lJDg==,iv:xq2pZ/8AEGUQIw/IIJFg/odn82SQ5TCR9BOEQ6vkFRA=,tag:IN9PSL3uyc27tU8NSSxaEQ==,type:str]
+ editorMode: ENC[AES256_GCM,data:4hWS3DXs1A==,iv:oGmr9JmJgs4/7FES7LOUl/KV+ePkgmr905Cjk5WKYLA=,tag:wkn9itrUjNRrnr9oM/N3rQ==,type:str]
+ expr: ENC[AES256_GCM,data:lBOL+LWdXgVGQgt7TuwgNkVgroriB3MqcOronk9z33pG7Zu3bNGkCdyeDplly+Jho0CNZxP+THVF//lHT8ouc8ZQdnFB6NUKAaAzmy35vx06HW0J3uP/LiEVSCz3w5eUZhNlfduoPMKouig3sMpY+M0BI6JOply0hF6FyunvkQ5QoRnj3A==,iv:VYgVoiyXeD9iGHSv130eFBTTYv9vuB0ZkOEWXvzmnrY=,tag:a8ubmCufFRyaFU6eQ6NttQ==,type:str]
+ intervalMs: ENC[AES256_GCM,data:hfX1Tg==,iv:QKAC3Lj15SLc/A++nTmVwooIXQPooQn4FJ1cdTuGeBE=,tag:khgFb48Ev+YDz2xGFGtm3A==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:GgyLR5k=,iv:cg0SUKizFW8gj3s24HvKqXwrFsPbYClwIipgnyUWQ9s=,tag:ZASd+UPSv4/+tjr9C9DgUg==,type:int]
+ queryType: ENC[AES256_GCM,data:sBxK9Nt9nw==,iv:scrDGB8OPB5HiJxr+e4JaiMOtJ+oqdkE7HM0hgjE588=,tag:GiKjgtbubszuv+WEtU6TkQ==,type:str]
+ refId: ENC[AES256_GCM,data:pw==,iv:WLVdDBLv4cBEjPGahDTbCJ2GG/ZOrKkcGbg1XRwX018=,tag:5o0qO+yOttusEr+zRRD0ZQ==,type:str]
+ - refId: ENC[AES256_GCM,data:cA==,iv:ogPeWXHUPDQRHkaRMlCerQMKR8RSc+gL8zbMX8X5yX8=,tag:sGnrcEh0faY/TFPimc/j6Q==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:ZnNF,iv:caZS0nhBQqasgSAKe2k5jxD5s1Ydb8wq0wZHod5AiiA=,tag:r5uvb3rFb29naOZerYyhxQ==,type:int]
+ to: ENC[AES256_GCM,data:qA==,iv:5YTo8F4UTB+bkMqdlCMhXXFeTtERDJiw3Yeyq1YQJC8=,tag:+aj8W8QhhJxweLLRs62Cjw==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:c5oTtWuZmD0=,iv:Rkh15LXvMHt7a9lAGJTyGrEdw6KyyrZpzV3EzoONcTY=,tag:lXMG5r/2BDkDB8J7VABlXA==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:cQ==,iv:weX21QCTZohywQmIFTBaSqgIYulHtVrUIySdLXEgO3A=,tag:o5YUpYMR4kTz5+BUpOGRxQ==,type:int]
+ - ENC[AES256_GCM,data:AA==,iv:Splpmr7Ph36SNbYNXkBdMk/RfnMoxOAJjZ8eBE60qs4=,tag:ix8wIPVI59+SujEsz+gCCA==,type:int]
+ type: ENC[AES256_GCM,data:D/U=,iv:Lrf4vjlGdrdC0KbjV1K8hY0sI882M5VCVuoDBcStBrE=,tag:dnlAHsiOU0TIjKNRuWH2eQ==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:vued,iv:ZTwU6S8X/TGjsHe1uwlEkPVPwiqnSLVN0OjKW24MTLQ=,tag:1PISHVO0+gzVdz3CYGTZig==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:Obbx,iv:Xjx89FwwNX1uaHP/Y0LPuZibZ3QVXbC19z0YT47p7O8=,tag:jXUsCS3QJBoKzFlY4g98oA==,type:str]
+ type: ENC[AES256_GCM,data:KLj6PtU=,iv:jCoL2RSUUJsSwRSGkUQPYGeY/nvEx3u8cbTGeYMWH0I=,tag:WWI+6SuNYoLdqqji+rCLHw==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:4eWSWXUh5wfBbA==,iv:PT8636Rlv9cB62KuEqSkfXg8QTBXwGNj/VycBgPok00=,tag:fVYFcmL75wgJNCulDVTMNQ==,type:str]
+ type: ENC[AES256_GCM,data:i9GADkyjkhQ=,iv:5UbmS/frH1w0MPNhzcIfug3dM+iGu1q0daAOaJ8AJgw=,tag:csYNJlfd1GlqvKbVrX4bzg==,type:str]
+ uid: ENC[AES256_GCM,data:MkOdYzwCgPQ=,iv:SVHoVwiVU0HBsV3+oIowsV3jJlbb8XJ3vLB+SUCqCkI=,tag:dz09e8UajUcxZJRQGahg7A==,type:str]
+ expression: ENC[AES256_GCM,data:Kw==,iv:R7342RtHM3kvRiMvnhEcBHQ74eypPBztzbsY59Etoc8=,tag:CNOsj6dcM10viOAFtG1jKA==,type:str]
+ intervalMs: ENC[AES256_GCM,data:YQKf0g==,iv:Oao3OladPv9DwAA2oV0XBEAFAHi/f097PRyxSU5wcWE=,tag:95ZNB/kH8U3IyVlQ7TLjUw==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:vzWKn/I=,iv:WsBz4VSPHd0zXvSfeR69IMHkqyz9+0FKKa1LKSGGfQQ=,tag:gIVc6fkuaYdGyvn20O/kkw==,type:int]
+ reducer: ENC[AES256_GCM,data:8eE8,iv:MxlUxbvM1FBTaC7kkAHcok6lI6seZa2j+GBxmDfbY80=,tag:S/KhcExoNSg+xksexBN1pQ==,type:str]
+ refId: ENC[AES256_GCM,data:iQ==,iv:aReAsgqvj/F9nZSFz7OOPaqy2GsbeOsdHTgu0XXP/30=,tag:+3ExHJGvkF1+7uwLDcL3+Q==,type:str]
+ type: ENC[AES256_GCM,data:c7UT86jC,iv:a0ogVyHXY4vkSfhlO4oCnyA+ZYoYsBvVz6SR2b8oS+U=,tag:LpFZnySySepgQihRjNHGLg==,type:str]
+ - refId: ENC[AES256_GCM,data:yQ==,iv:rCrOITPTDpxB3EtpRhgXdjPfZes/pSNw7tQLG2+cUGU=,tag:C1/6qfUjoY7hko1Xc2pEvQ==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:kF3G,iv:Q5m/oT8afp4hOkgaQU6BBjc8z6uScBinPB58QlOfioE=,tag:X+tY1Azq7W8+dVj7TlNkHA==,type:int]
+ to: ENC[AES256_GCM,data:0w==,iv:ltqlKKJ2XRJsUYTNMzlXpdT0371LjZqRte0wRP9DUdE=,tag:3fOqBg14/Bmn/hn/UYQTsw==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:FKgjo/+6ZhA=,iv:Qnw1GWI4gfPi3IqYss2nVzrsBSIddSfaXiKJsFLGbfM=,tag:xOtC26i6Ec6v5GrmXikZxA==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:lw==,iv:4Wd4kY6fSeu1Bq0yjRPC2w4+fGmHguoH+mDxZviID1M=,tag:fIO/6uKNu76BBP2HxOOpqQ==,type:int]
+ - ENC[AES256_GCM,data:Uw==,iv:T6j0F7Ua7SvsotCGIlB3RIIIRaGO/8dYO6g5FlWshsk=,tag:KZWR+lIs9U+7Qu1g3Gl+bw==,type:int]
+ type: ENC[AES256_GCM,data:LRk=,iv:CxnzjybcHNEi7NhnEzoXhhkXDuLO6su29MaLWUrRLf8=,tag:+hPqa3cbxfgdm7mXBKeGdw==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:oZc8,iv:ISzOq2crLzxWP12tZp71EDj0upz7qulllDGn3glL6+4=,tag:VmWd4r1bdesGcgDkxmd9lg==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:/XWC,iv:yjRZ+C0eS+idlhrnWM4/FihZxJb+WzpCRgzXGhM0RfU=,tag:DX5BIDuPFujrzEcV3zrhWA==,type:str]
+ type: ENC[AES256_GCM,data:3xT2lU8=,iv:pAlMbUZnOjySEqJklckrHnfGFenVEM9Ajjd4mNBa33Y=,tag:l16YLGVQhaH6AU1KyeSgwA==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:QUvdDy3Uj2xC4w==,iv:7ucYbjHrCYCv5HhfZ43p4eb4LEsY8JbUFoCQ5OJOj44=,tag:GWU6/sUPN7U7n2MDOfFBag==,type:str]
+ type: ENC[AES256_GCM,data:vc76NoOR3C0=,iv:rXvivVyXKqVRTJ9tI/YmYkBgFeBqmHe6OIrE6RPgBL0=,tag:HfwlueycmnI5dLVqbr6Bww==,type:str]
+ uid: ENC[AES256_GCM,data:8cFxvUw97GY=,iv:LGDAeI7EmiEEtMWde9hthaHxXrGy4YqswIQd80qAe/Q=,tag:DMBHeR1viL8iYvlgVkLBQg==,type:str]
+ expression: ENC[AES256_GCM,data:TQ==,iv:z0ViUn35xrXz35DayW+Nf8vUyISHps+VDO6FsbrZWPk=,tag:/EMPXCTm05nXbTYYUdlAeg==,type:str]
+ intervalMs: ENC[AES256_GCM,data:WQ7+OA==,iv:HMTbvkH89SUIR5akLEaPunYNw6oKlaGUlFoJxfuBRdY=,tag:lptMaD+gtSU3DOy+6JHoNA==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:qoqfFCA=,iv:cZYyHOaUcxE3+zaR5kcZQdWILtXg4IarfrW1zXFGPlI=,tag:YbWulx38f0j28l9DMfxE0Q==,type:int]
+ refId: ENC[AES256_GCM,data:1g==,iv:0bScbl9C+ln7v5geW4y0Tn/dRZAsT8idgZPi0xALcj4=,tag:XX59FL9EaZ81OPEZL69ZCQ==,type:str]
+ type: ENC[AES256_GCM,data:6DG73nsoQ5Re,iv:77Dga2KTsdQnYV1qgV2zZutvTWztG4D9JAU1QNZGMII=,tag:G3RQ267dpamUDvZxxEO9/g==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 1m
+ annotations: {}
+ labels:
+ service_id: draupnir4all
+ isPaused: false
+ - uid: cdz3njjeosphce
+ title: Draupnir4All "BackOff"
+ condition: B
+ data:
+ - refId: ENC[AES256_GCM,data:7w==,iv:ueezk+d6yAQmWAsPVXsnMxsNo7aiCmmseDlkAochZCI=,tag:YarOVoT/EOwel5Leg8x5Sw==,type:str]
+ queryType: ENC[AES256_GCM,data:UeMQt3GbNg==,iv:Y8SPSCbFOiaVsplZmve6hKd/wN0zxVlo9aFW8T+G4hU=,tag:eR1yKIKgQgN6BskfyEC3dw==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:rWAd,iv:LhspOfxi9ARbgsMrnTSwVlI3M/EWLYboP/lmvOg4SUg=,tag:yrdeJP+ee2zxRnZXuaWz7Q==,type:int]
+ to: ENC[AES256_GCM,data:+A==,iv:6nYRo17JzQf4jGvrzLFiHjg4ujCHmmBL6b5y1sNxtPA=,tag:FFb+clPQkd6rkgsJwPZuMQ==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:l0QVsQ==,iv:LYyE3AKqaZwD40fa5gauXKsILFVj0rfzsDXJNHpDqr4=,tag:8knALZ8/hFy2iPEFJBfhDA==,type:str]
+ model:
+ datasource:
+ type: ENC[AES256_GCM,data:mRi5Tw==,iv:WXRXmt5PyfWHsfpkIauOa2OtDWqIZp0R+72w9LSvBMU=,tag:isR9l2jRHSW7jAaSWpbOkQ==,type:str]
+ uid: ENC[AES256_GCM,data:N4PYmg==,iv:sMNQtaq3bheTsX6IsSxwIEo93uUUBvEV9eO+6NmsWzE=,tag:DXpkz6++8W9Oirs9ZUrc9Q==,type:str]
+ editorMode: ENC[AES256_GCM,data:eWlsbPiHLg==,iv:70CubXvtkM2zxfHdQNW99aHdXfhrlQJMsWwFDHpFiWI=,tag:J6U3DaA7Hf38QlvXlS+nZg==,type:str]
+ expr: ENC[AES256_GCM,data:ObWlT0xR/AusxSngPSrO6XzeEU8L/18TtMrHuFIXD0Q1skuM4Hsgsc6v83L4mOpYsrOFFJidZBiqSLX4L/QkUy3MMgMDcrIcmTmyzIqu76XYq9sS6c9/qxyUMbxW259NpXozDgVUz0K8ImvMu0+N8F9JfrzEzistOPRjxeFjJewG,iv:tM2FOdnRzLhUNkqO3WErQLS1ApFSi8UBacFTzm2qWz8=,tag:y4xGtiIiC/Zrvf9edB9Raw==,type:str]
+ intervalMs: ENC[AES256_GCM,data:o73mXQ==,iv:MBzH6bqNcx7mKkGd2m/zoZR2pYsQSYg2eUpjo/yLdHY=,tag:jRq7wO6HUcV6xu7vusN2WA==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:kfdwIDo=,iv:mGRG9ZuEGtfIh4DGWf9RYJE94Rp+57XKitijFWTEEQg=,tag:W5cKk3aoc28zIvA3MvfkAA==,type:int]
+ queryType: ENC[AES256_GCM,data:AxS60RaS+A==,iv:YaV1y1Lhmxqr2U5cW7CXDe+YKEBASC+n8P97dJoX5+o=,tag:dFTTRnJI72wMXlJJi5jPpQ==,type:str]
+ refId: ENC[AES256_GCM,data:Xg==,iv:sx6EAct+ANwBrh080Al8kgiIUhOemWqitDO1thiiIts=,tag:kHne5x1YodsjY8dYjOkX6g==,type:str]
+ - refId: ENC[AES256_GCM,data:PA==,iv:Y8k9lRnGKSsD88YQ6zfXXWhV5laIKfVvdWSKtRKvnGo=,tag:ew24iVhFd8+belGnHxUyuQ==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:WabU,iv:QH6R5AwQPjIyxIDR7iNAo2CcbenUWM/qz+8Hz52JwzI=,tag:EPoVK3rBqkcyRK0Ym+T1rQ==,type:int]
+ to: ENC[AES256_GCM,data:+Q==,iv:5MyJaUIcPfoYRrLIu6TUwx33I78KxKzj4jo3y1Mlck8=,tag:WUSaDtxj44hqgzKoSnWZzQ==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:40AA6jD3ixQ=,iv:EhsLmEhN0mwRfx/85wl9Khqc82IyAC0OEJM/DsNah1E=,tag:2lm8eNRW1syyRYiba5R00w==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:CQ==,iv:dXzoCt7hMX2DuIrQgcZzodk8y5goOFhO2J2T895D0Ws=,tag:wn97USLF3gcl8hcVnMe4sw==,type:int]
+ - ENC[AES256_GCM,data:UA==,iv:ncYY9ZlMEAppUUoXW5bBTbsh8cx0fXQvvJoReYmY5Lo=,tag:6bn+nckxKSfFIbWY5mUATg==,type:int]
+ type: ENC[AES256_GCM,data:oZY=,iv:HQMyYxR2ohk90IKdx4CpGAIjOdn5EvLqGOULGslg0cg=,tag:f9T6lZ1guAasm2XUAqVpuQ==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:DZLj,iv:NvQ3ntVzo93Qrk9/J4sSf4zv9VhhduNxgqKrcl+1TdU=,tag:kAwKBIRg2Nf0ONswcoFp7w==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:VLJd,iv:mumQLrG4rkF5ms+YyZzJki8+1FHSWqEMRrIKmW1IY2o=,tag:qNiHm3VTiYxFJGsgo1zFJg==,type:str]
+ type: ENC[AES256_GCM,data:/bHId6c=,iv:pIuGnFaIeBmu9/cGPLHBThbBtz2yArT3l3RM6CcTTb4=,tag:3C65xj/aXw9bPI0hPm4gFA==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:2jSU1+z30hRclw==,iv:98W1DEB7kUQuESl+S5m0PbsefjXb5Za7l9y//PfBgK0=,tag:6n5j5s7ldKOBveyNPndNgw==,type:str]
+ type: ENC[AES256_GCM,data:Vy2JJAKeyvg=,iv:yy7DdpJ61cTI7sxK0204MhMRJkIdLooVYiJgfA89Lks=,tag:TzNKYi2XW1FH6yeCUAXaEg==,type:str]
+ uid: ENC[AES256_GCM,data:GT1PkcmoIX4=,iv:jI+RK0lHkWY7ea4WW4XzvCWKRGNSh10YmJuyhyOAPtY=,tag:421CGBJhywJ9xAwxiwCdcg==,type:str]
+ expression: ENC[AES256_GCM,data:rQ==,iv:OWsDovaw/ZfXDa7xxKGsVRqQg7976HOFn/fJy1Dc8/I=,tag:mpr2uP4jTAN5GdkEEmpZHw==,type:str]
+ intervalMs: ENC[AES256_GCM,data:PJisTg==,iv:wENjoDR2VO0DcJtVaupgqVzdNpQO+mNzPV7QvHDAu1w=,tag:AQ86yCqYb31qrWlIg7UECg==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:8UqN9Xc=,iv:bAWybOk4YOlSZCQmexfXzERwi5jaH1EhzuPtjRXF9os=,tag:fAIlPFguMdYKihzEI6W5Hg==,type:int]
+ reducer: ENC[AES256_GCM,data:Yx17,iv:rFWuvvuXrcVDDRVJA35My2w/AolMwKfEGRez10JHVTo=,tag:BXRQvpdayRX1+3lkNV7ebA==,type:str]
+ refId: ENC[AES256_GCM,data:lA==,iv:IS3q/E3xq+YCOEwZa36FMlpIOlRM5SUFdPctzFm7rCY=,tag:fUJA4wobK49wIx7WsSgI2w==,type:str]
+ type: ENC[AES256_GCM,data:02MpAat0,iv:1xFLXGKc4Hn14nrIJtHqLQNrxOwcyDm3cL0jCBXNdVY=,tag:4fJ6H+6lWhbsSY4DmS9n5Q==,type:str]
+ - refId: ENC[AES256_GCM,data:PQ==,iv:CwBQ9zVLzWYF4KTNRqPzEWaHGvh2/dUnLAztKTvMylY=,tag:K9P+qNQ7iTw7GWFMH1ZmZQ==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:J9oX,iv:HrG2hC5SHV3IJVjMP2Jeh0wD9goXV3EAeUvd8Cz0n50=,tag:vcdU83g80abopku/MxiT5w==,type:int]
+ to: ENC[AES256_GCM,data:7Q==,iv:Ew3Zg6vmQw+Kjg3zlI6+WU3q5n1gtitsPJIB+bvcQcg=,tag:y/KiDQVQuJeYF8pgya53Ew==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:inyPNVUP9QI=,iv:GoYfj1zARilV0znGx4mPhBVshZZGTL7wneUZDqJwpBc=,tag:1dh/ZO/OyQAVHKyfSfQB1Q==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:sg==,iv:62GdKYJK4Z0p0tUlH/0heGOZjVXmHkA6DNytr3H/w8w=,tag:tbwtUd/SHChyjDBQ7SC4TQ==,type:int]
+ - ENC[AES256_GCM,data:qA==,iv:J0BeynfzuEHCwgRwdHrn+2Lezg17ELoOQ6lhlizz1i4=,tag:yaMACmK/VUPta7WrryDwAg==,type:int]
+ type: ENC[AES256_GCM,data:9ho=,iv:1PNEHu5HBt1mxlovQVg5XwOHbn1uvEhOhEUIr5AjVKw=,tag:UyX4mj5kV2M42FwUOySZgg==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:UBsm,iv:fXEnx++WPDGXg/3zoFhQzfPw17vuG07IT/4QN3HSN5E=,tag:Y+WyTXyTFrL58vaH6B6uiw==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:5LxV,iv:wt42uIh+92FoGm1DkInXiARf0DkkJ83ZrZeaICV47R8=,tag:lv1lyAljNWx2uWxMJK9IFw==,type:str]
+ type: ENC[AES256_GCM,data:5Juy7no=,iv:g7pFmwMXeCyEED34BL49XBX8aj/RwbzqYsVA4T+0/fQ=,tag:x5pqQiF27joKiAO2yF4JiQ==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:0Db4YLhUMJTMkw==,iv:+95X9fZlv7s7ryi7tpqTKCxKVhf/IDN3GyQSEwcKDBs=,tag:BHHIYwQ8nvwt7VOG9NCZrg==,type:str]
+ type: ENC[AES256_GCM,data:Yu2ry07nMBE=,iv:2IRwVk9r+Bkg+IkSHAEg2BZL56jaTZVthOR5Ki06EzA=,tag:DMKEvsuR3zjJsMpj8ipBLQ==,type:str]
+ uid: ENC[AES256_GCM,data:dFuu5XsMDKs=,iv:JP+RvCh620V12opyMeqlOzdpMm3BEZVtBzPMndMRYdY=,tag:lkjpokjSBBw+IISMMcM1HA==,type:str]
+ expression: ENC[AES256_GCM,data:vA==,iv:5ogkjH2apiIrLi5svV47Xh1rP0CFR4A7cEd9FxgVL6U=,tag:hfj60skDICzw2ZTtBA2vHQ==,type:str]
+ intervalMs: ENC[AES256_GCM,data:XLxo3A==,iv:FNLEW2FjiFWtaXhnyAWn9WU3dUQBwJummNVaCNCLzhw=,tag:xiUPeXgMpuLZ4Ma9kOKKkw==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:EqWOEBI=,iv:g5e3eMQQcyKT09nY2E+8STb+rz/iyFW4z9/uLlwfpM0=,tag:pPD2nKl9Jt1Ht8UuDaF3wA==,type:int]
+ refId: ENC[AES256_GCM,data:hg==,iv:UPGE9ho7PL3bysVBKcM/+TvLOr1mp4wzyPe+7QP9daw=,tag:kvL70t7RapcaYa+/J5Xkmw==,type:str]
+ type: ENC[AES256_GCM,data:jbW+VdjYJJ00,iv:EeJplcdWry4QPmhjSVCZyWNfXWvVOy2J5xNnpiGrb0k=,tag:8rao104rqL4McOkv9rl0hg==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 1m
+ annotations: {}
+ labels:
+ service_id: draupnir4all
+ isPaused: false
+ - uid: adz3nkklsv2tcd
+ title: Draupnir4All "Failed"
+ condition: B
+ data:
+ - refId: ENC[AES256_GCM,data:5w==,iv:Gue6UfikistbRUBXuNDxTDyfGjQRXM4fLwpRoI+fLFk=,tag:KsVkPZmUqQRYFsrcLY4dgw==,type:str]
+ queryType: ENC[AES256_GCM,data:Jaqh6PjxrQ==,iv:5IfNxRsjuVTuq7fEc4RC0v+8tcluzhXJSZ1UFJafpJo=,tag:259lOJ1qH6uosSQPchJpRg==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:mkTK,iv:BHZsMBpVi8w726TwoMbgR1WamabPi3jGMefV33bJfxM=,tag:tW8aHpWyyG24eLsZuZET9A==,type:int]
+ to: ENC[AES256_GCM,data:7A==,iv:1rA+MXpTFCP0fG1wKZmI53IiV2vcX+wh3ADmn120WWU=,tag:XbOGKHblWpTRYvlwWB0G5A==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:YktlyQ==,iv:KfH8ldMEuCdAWmm6vbaBS011Eg8PhiUVxwgIX1slUDg=,tag:ni23pBlDBhU0QgTZpqevgg==,type:str]
+ model:
+ datasource:
+ type: ENC[AES256_GCM,data:6eaoEA==,iv:BsvqUdhflkpu9sHy8Dyv/YqYezVgrka0695fgF4phFE=,tag:0YrKzuGNfMv5RCyfgqs1Lw==,type:str]
+ uid: ENC[AES256_GCM,data:4EinVg==,iv:gKgvkU5BJWduPCQNQf5Xmn7qr1ofm9Ifv9CJ2a8wnN8=,tag:bPVsIBGVuchrjqJ+q8Eq3g==,type:str]
+ editorMode: ENC[AES256_GCM,data:wyQlpK7S5A==,iv:as+X8f+ATOKKd9MaQeJyoRF75GG5Mn53U5e+NCwkRPg=,tag:yPdBm8Z1L81cuGLf6LQgPg==,type:str]
+ expr: ENC[AES256_GCM,data:OFWkwqFuK+s/TWRd116R3Ttf6l/i1XTg98SwtztB9SSWuRGfPIEreDgVX3oc9ieXFuxV+zC/tEvFVF4teIu+HMRFuLhJp+u+3OcpnaFPHNE8uYNxHArarXm5X1GizIB8j/o8WIc6MG0BLRaaVLuVzn6MDIaOgZNV4cCVIGwsrGs=,iv:0YV0WTKN9SiBdkBq53x7vRiLB0XEq+JPKSCQ9FT1+RE=,tag:xTG1edhOothyCGzQnQ6LRA==,type:str]
+ intervalMs: ENC[AES256_GCM,data:F4TkUA==,iv:bvNo2lMu2J75TlblYX6wBmMxVNTaG45YFclGrDwfu74=,tag:c+d2VeppuUmB6ggWz7Nj9g==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:a6FEQ6A=,iv:6Op/orJ+XaeG/UsG9R7BOlqOW4faQrpFKUO6Ol6N0P4=,tag:TW11qS0HVoT3gdrlMoOnAw==,type:int]
+ queryType: ENC[AES256_GCM,data:kKBtGom9tQ==,iv:3kIKCwDKHRO0x8zC7KIjUSMyKT64CACCw6yH09abvfs=,tag:3X/GP6kzx204kmC9Jv/fvQ==,type:str]
+ refId: ENC[AES256_GCM,data:Mw==,iv:EKgfTgTqG7EFwVIMJtxK5p+a3mucyYpLWRC9tfZUnTE=,tag:9Y4JANB50VoILyj7ZEVOlg==,type:str]
+ - refId: ENC[AES256_GCM,data:Ag==,iv:Qa8knzOXSacHxu2FM0qxYmNV+NYne4VnUt+IZIylad8=,tag:YV9ph93yHov8un+k6uTn2g==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:m3c8,iv:DXArnOGAWOv+uPrJHqVCUq8QGaMXDK1/MtR5BOUUwzM=,tag:K/Blbwa8BRHJ7LSESNlxyQ==,type:int]
+ to: ENC[AES256_GCM,data:Tg==,iv:nrE5KR5Z4eCxnci8ZUviICpaGcvzfV4uikACTheoII4=,tag:2E8BTRetGS9tDWizsuqReg==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:nKQdknxR3FY=,iv:o2b8Bw5K3QK1Gvqyd8ciLXoov/Jz/tjBmV3TWJvyjVI=,tag:nUuCfN4R5yY+j2avSb7evA==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:rw==,iv:IP/ayajEBf7gh9EyBgrou6a9wPGXT6oYQvd0kB/oa6M=,tag:NOD2lyajgJ1TLnxHY49HGg==,type:int]
+ - ENC[AES256_GCM,data:ZA==,iv:T9DuYWBNB9mtNsvajlwVoM9g/bWsz9wEw/0g+EjzXAE=,tag:9O+4q/7SigLhbJ9pzi5/9Q==,type:int]
+ type: ENC[AES256_GCM,data:xvA=,iv:AYqeXTYVnkv4Mq60ykVAzk0G6LcXS/Y+jw8C163BQgc=,tag:iMik1bM73MZQeCmLDzpl/w==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:QP8w,iv:98kA0IzQi1wT9UhvR/DHpqfnNgbfTH1Jmol0hqkViHo=,tag:2eeRIQ64P9yjBU4/ZJuIdQ==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:6bWc,iv:TYoclJElN4DMJ39nSA5tWCzUytywCDRejdUUIU8PQyc=,tag:WMuqFSzBe7kFHIYYaSpVWQ==,type:str]
+ type: ENC[AES256_GCM,data:QkW+6CY=,iv:pb19y/8Z3Ro3V+q6pOSiKRcpR5BdRYOYPFjbZ1VKmaA=,tag:QdQ05FBcYFovVCtf6ifhRQ==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:/Jmc1cmCYicMug==,iv:HDV2mMSJ4QbTONrk+kqwXIyn6yl3TKpdKQB2V93cHdE=,tag:NEFwHsByduxNT2fqQHArLg==,type:str]
+ type: ENC[AES256_GCM,data:YfxQqmQruB8=,iv:+1UbMxFwbfaSX3pauKx8IL2vcOnjW/zLFNnRsRDY4Hc=,tag:EP1QW9a3XOTxj3LOoFcGyQ==,type:str]
+ uid: ENC[AES256_GCM,data:X7EBK+QEKj4=,iv:MINXSPUGGjk/xuakThXA9jQ7fYmCxA74obnFyUYcEUc=,tag:EqpTCpieVOlRKBUI+bLWmg==,type:str]
+ expression: ENC[AES256_GCM,data:PQ==,iv:Q+rgba3AarUW9Eo9KoDWbjspBR9T8ZJcomOthXT+l2Y=,tag:wWqiqrwP5CM7w3RMPCtBMg==,type:str]
+ intervalMs: ENC[AES256_GCM,data:WXDw7Q==,iv:UkFCNn5MnFZpKFWRScrBA19XF0eHK/S+F8NhJtQL1AY=,tag:FLcNxC1n+GnSiCtYpUCQYA==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:CKnjG4k=,iv:l6crEKusOUIGHk4kDLL6jj//YB+SYA2WcvjeBUCTqqs=,tag:mwm4rVp751flUXLS9oc/3g==,type:int]
+ reducer: ENC[AES256_GCM,data:jWxW,iv:ZiK0h+oIpcQW/ACXAE9WN+6w8M6kkESD23DbSSMMUeM=,tag:zZJY/gSDQq3aoX99meeqpw==,type:str]
+ refId: ENC[AES256_GCM,data:Ow==,iv:kmzskYw21AlV7RGbI1pbdBHP2Ek8lYKdm9cnUljrnTs=,tag:+Guy90rJRbOT/rKduEyrEA==,type:str]
+ type: ENC[AES256_GCM,data:qbtTs3w9,iv:n7oPbJrKrQH1SL35x5HWTb30Il41neKG2FXl0+dUk38=,tag:++1COWAd6uaFXA6uf3Eo+Q==,type:str]
+ - refId: ENC[AES256_GCM,data:QQ==,iv:eb5atcWIV38hIliJE4LLbrK6IXXDXGaivlZRkzeQl0E=,tag:RK4Okqn0a8D/7JTpJHWsxQ==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:2Q0n,iv:7hOev0keEzHiKaFB31w16DGCHLknKzU1kuQt9KoGQMI=,tag:gCwlDAubuRnfYW/cIID27Q==,type:int]
+ to: ENC[AES256_GCM,data:BQ==,iv:oY3tWjrMHXKVoy0MsJwQLFrjgRTXIpk22CdOcArNCaw=,tag:wT01fOJEqk1jelSzlPNqGA==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:Itfguw4yrBA=,iv:iE6QU3H9ttMFeiqOycEH7SUj7xpCbwdtTM9BTF+W+1A=,tag:3nyAn7naiPrzjORy09xa0g==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:7A==,iv:1coEseHn8njOhVmGANZxprRHAz2K3ZIeOTmwsHIrnx0=,tag:iuGDUdtwoXD/zHWcwQEoRQ==,type:int]
+ - ENC[AES256_GCM,data:xg==,iv:YLbsH9jUQ/K1u+3+tE3dP+2hubbPDJEmn4lN1IaLFtE=,tag:nnsmSMrtdqDQyFSx6fJQdw==,type:int]
+ type: ENC[AES256_GCM,data:RgM=,iv:r6WZ1GfzyVEIdInBEV17uMT8yeLd6YjJFzCmrgTFQ/M=,tag:PskANBeM1AuzwQWj9yRaJw==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:6jJ5,iv:TW+F6FRcfx5qW9SXzXmZzhE4kd/1BKw0J/dlbReEUBg=,tag:62R1Czw1u4tOC2jS9C0zTw==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:9ZIB,iv:HYPOn1g84M0MQqAshPkaNJD/RkgD2l+UyztGHjZhdYM=,tag:wPRX6ZjFSY/flqeeZi34Hg==,type:str]
+ type: ENC[AES256_GCM,data:xcsapnc=,iv:PhZDhNodF6ORX0wV5bj9PyVMyj/fTeJvetosJQlvi+0=,tag:h9irF522XyvhoYfHyR7WLA==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:WMYnC6chyGdzuQ==,iv:oWGD9sMv1q5eEeQk6GzZQG6PGpnhiiaOqL2jwWifTys=,tag:Nt5kN/DViS8L7/l2YwYLuw==,type:str]
+ type: ENC[AES256_GCM,data:8YT9rOmPWPI=,iv:kMpQhSc6Rg3hCeIsDbXQItARrrgwigx6G6xIuIeB8cw=,tag:sYwe1C5XKUv2MzQ+DPhyyQ==,type:str]
+ uid: ENC[AES256_GCM,data:r256/JZ5WnI=,iv:4et9h1QlxPfiUC1xZPC0Vo6kjzz/3pqA2wOr1vZPXFA=,tag:JZMlMehTyJ1CXSN9U3B9BA==,type:str]
+ expression: ENC[AES256_GCM,data:Tw==,iv:NZw3GgInxDux4tWGSlvYUmUmW50LKFVI5RXuKtikkhc=,tag:FvBNwJMIb1o6rX8WGQtXVg==,type:str]
+ intervalMs: ENC[AES256_GCM,data:Gihlgg==,iv:8+JwR88qEyYI11kEj5JOkQHdp+XfHC6tL5bN4Y5Mess=,tag:J8HMsNV4kIV1RbZytk8DrQ==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:ts/dFyo=,iv:j0LRvKiRqBgsjRtRaGCJBwx12MWdRo0ZY67kXzBZUDE=,tag:2wGxDzMmd/sJOIDx38mP7A==,type:int]
+ refId: ENC[AES256_GCM,data:6w==,iv:OCXqzCqpbDvPsLfE+AjNG+AhoujnKx9Kl+DOc8wOMQ8=,tag:wb8PtQL3FgIe25JOE13vpQ==,type:str]
+ type: ENC[AES256_GCM,data:bsIb+/O+wdSA,iv:BjWgGocaE5QwO5dNrEBeBGRBr1vsPemuYSUikSzQnmk=,tag:zf9yhG43va//ypQB+7v2SA==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 1m
+ annotations: {}
+ labels:
+ service_id: draupnir4all
+ isPaused: false
+ notification_settings:
+ receiver: PagerDuty Draupnir4All Warning
- orgId: 1
name: Kube-system
folder: Cluster
interval: 1m
rules:
- - uid: d2b94d3a-2483-4be4-bf2a-2fd9f9fac406
- title: Coredns Failure
- condition: Thresh
- isPaused: false
- data:
- - refId: ENC[AES256_GCM,data:YLIE4WI=,iv:JTn9DdOzgY1Z9yDuC5ioQQd6MIbsZ6WEyO8qOlOUMp0=,tag:qS564VitQLBh6Pg7K/AwMg==,type:str]
- queryType: ENC[AES256_GCM,data:XodqKzc=,iv:OC5qtzhvH2AyusVEpvkzgrtsIPJuEsHo37MV+XKZe7M=,tag:F24fbVaRVkIq7FLY7Ox2Ag==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:jNQJ,iv:96VeonoJJlTrm740h/GgH2z4uHmYokXjMNkAWDCQQUc=,tag:1q2VZ6A+YuN4Azfj0g7rtA==,type:int]
- to: ENC[AES256_GCM,data:XQ==,iv:1DsmkxJBVwqlsRDAFN6vDOkYMYCQKjmJfyTbc4ekxEE=,tag:TyeQyQULtpQ5I2oMEMWDrA==,type:int]
- datasourceUid: ENC[AES256_GCM,data:Y3KJbw==,iv:4nTh5prw9NnG8ZjVsQK3h1NdybBz0ovRm89XHA8+Izw=,tag:5X/z+eGwukZ8e6hyyU7Yrw==,type:str]
- model:
- datasource:
- type: ENC[AES256_GCM,data:CwSkig==,iv:kbqkqrkkJ5uFsrKvecm94xUePAYgxCOanIWi00WZw4E=,tag:jWgSbU6EfJRGfy89oe9Aog==,type:str]
- uid: ENC[AES256_GCM,data:Y1sFFg==,iv:RuKR/AkLN4eSAZOfWZ2MIhvTH/MoqbzzUa2DVd3QYQE=,tag:hY6IJL9TUSTwG/8pNxFzzQ==,type:str]
- editorMode: ENC[AES256_GCM,data:qmKR3UZRzg==,iv:FeIXVXA9uIfAXkGfArKO444FW95ORqbSwV8oSZsqoyM=,tag:euWh7KywzGyCggEf9NQMHA==,type:str]
- expr: ENC[AES256_GCM,data:DgN+rzrxLICd6AxQTNNMi8YcxQaUWg+z/TBYVLjtptxRUC40TtYR/hxIEAebftJDgYmxvf+9BLzLirEUVh2WnyZrdvsi,iv:kiJiuWsj1S7cn6TiCGmYVM1rYssnvyzqN0bwfFnoUkU=,tag:eWv9uBm7r+umxZVceKtW0g==,type:str]
- hide: ENC[AES256_GCM,data:OtALMd0=,iv:N5FuEZ6BwTpPhOXzqDJG4eyUmqgWUaXPvDsY+JLZqqg=,tag:+wrBWZJCIAfo9iCyhANpng==,type:bool]
- intervalMs: ENC[AES256_GCM,data:y30MPA==,iv:13ykjBxU/aKUhl9nXseCuYKHDkPiU4IeTY+XeMpxrbI=,tag:gUUkxaaZpqgAVuSYNxOaPQ==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:7Fc6Zss=,iv:f5i6rf8jd253WlJQARhfty2/g+25G40LNsK2XNwFK6s=,tag:8pucbFXTlihNV7F8LhrsVQ==,type:int]
- queryType: ENC[AES256_GCM,data:ToeyQ8w=,iv:yLlJYSZoZa9sXXN2sZxqY/R6OMxZSHPzo5ok4h/EcYk=,tag:YpM+7WNaPGNVCQF0GL+Y5w==,type:str]
- range: ENC[AES256_GCM,data:pVcctg==,iv:KpCp1r7uQWGBzYZRXlzYFI+Ii7YwDn6CahNDOqneGrE=,tag:OxgkG4PLrtF9V9hhBTd9BA==,type:bool]
- refId: ENC[AES256_GCM,data:GLRSFdg=,iv:lSs4gO2lheXYB59skjA/QH2mtI1cnSlBoLzx3C+fhz8=,tag:FWXo9mqawW1xEJwcMeZvkg==,type:str]
- - refId: ENC[AES256_GCM,data:5dbU28Er,iv:kH69w+BA+HjxA4uqVxo/6E5QiPbIa063QOfsYKNSYq4=,tag:E+uhXvg6gZ/3/1nYLO1EtA==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:gUCn,iv:wlAg2voGUzBCyqY2AK+wgy3HCGsTqp7mtpUDHBidRR8=,tag:4AjU0qB5kkyev+jFpjWMOQ==,type:int]
- to: ENC[AES256_GCM,data:6Q==,iv:XCsX2cU0PTUqo0t9j54hjYCI+5JhYU7racorBeP7t4M=,tag:xSdqFdko5UzZBE/gxD0+Ww==,type:int]
- datasourceUid: ENC[AES256_GCM,data:5rnADR6zcYw=,iv:6bDKvZitt/jrDb0GPO51NpQ2dbLMovHVvcgKxl5Pztc=,tag:6VTAGVyKBtsJk1t6wGYr4g==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:iQ==,iv:lEvofnH2WkPzasp2zyfdflXxPxabXcIq2G9UbQ8b0IQ=,tag:l/4Z4FncLjOGIkvLWVN9yQ==,type:int]
- type: ENC[AES256_GCM,data:Eeg=,iv:wwWkp0g2K5JC3TCbVcT4hQGHcoCDme6X0ONnchvwqnU=,tag:OqHnwbM1SIdWXTu1nOXdWw==,type:str]
- operator:
- type: ENC[AES256_GCM,data:1A6k,iv:R0Ux9JyEU2RAUc0pCY4gSggE2EYgjx05k+3r4x+4zSE=,tag:UpOD0JG6YulqwXUfAvE0eA==,type:str]
- query:
- params:
- - ENC[AES256_GCM,data:tA==,iv:/2JgJopAyWqYZIFuGfxG4UN1ApdFDHRlTDhW4llCP70=,tag:six5kzaceqYC0cYudjwoBQ==,type:str]
- reducer:
- params: []
- type: ENC[AES256_GCM,data:nWyXng==,iv:lssJzj+d2cBCeZnvfeK0w9DgQu6UHLFZi+t8fU5CBMk=,tag:o+bJHlyMNQaCJltfpEg17A==,type:str]
- type: ENC[AES256_GCM,data:uLHqb0M=,iv:rS9cT1KyFJ2b2KpqbTpAU9r4XyUPmO4NHFXLZ6otJ/4=,tag:SAmHl7BtFPvEmjn7j/nP9w==,type:str]
- datasource:
- type: ENC[AES256_GCM,data:UHQP+Rlo6s8=,iv:tcW5AvR0zJLuG6C0LX2kAIQj6S2KWHFJXRUAt0bNW8E=,tag:rBtTpAP3u70gPT7KxT8ZBg==,type:str]
- uid: ENC[AES256_GCM,data:Qfbm2uMGQ5Y=,iv:bqISUvqLKIIbbYKnuPXQWuf7KE4+4jMYq6A5+XUKN5w=,tag:8SpvUupvrl8HcUKfKjDpHw==,type:str]
- expression: ENC[AES256_GCM,data:Rw==,iv:LY0ryT2H5oQIu3NBE3zY+I/i3QJbdSCUIMJ2B2chiY0=,tag:/ULFX3K1peCY8/14x4YExg==,type:str]
- hide: ENC[AES256_GCM,data:bF7AGPA=,iv:VnxDNgLX+20nNAzmIxp6bN0FyjWrNrRyS3d/6vDMdVE=,tag:zk5tI3vEHTXOo6jeb6Sd8g==,type:bool]
- intervalMs: ENC[AES256_GCM,data:rooYpw==,iv:B6kmVMOelOYaY+CEbG1RmMPS0cXWKnJvNy57KHC1MFc=,tag:mCvRLLFpTHMshuOomNrkNg==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:HiF5KlI=,iv:TKlI00RaG81Snuq6g3h4qBAjT0tdZHnqKBbiRPOwjNE=,tag:0muVeeP5RjkiGZV1U1zPFA==,type:int]
- refId: ENC[AES256_GCM,data:xMqgNXRq,iv:bWTmXyzByNC8linZKDhIpvkuEH2JkitTf2Ywt77rBt4=,tag:gQ/11jq314Tcg8X+kNtr3w==,type:str]
- type: ENC[AES256_GCM,data:MPaB5roFwPrw,iv:8Hyxft7aFeHmWW4G3qRR1Xf4uxTtSgPzNWJJeJrSg6Y=,tag:fHmS9PknNtooW6YLYC9WLw==,type:str]
- - refId: ENC[AES256_GCM,data:tQ==,iv:+5KMrEeC9mWObjxfOC+324rLHmES8TZTcHIppksyFV4=,tag:RxQ/dWe41nr6/y7j0PIR4Q==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:yWRq,iv:XaXxOBaQu7TGvfzkh1Pa3ywLVrSz2fLpDJg0MPD150I=,tag:eyigdJHcd8BuxEUbiVCJHQ==,type:int]
- to: ENC[AES256_GCM,data:HA==,iv:N+xvFSUZm0V/9EQo/etQEQmQB2NziQCRpp9k2xmnhEA=,tag:bZp3O6OW2m/RyHVigQn8tg==,type:int]
- datasourceUid: ENC[AES256_GCM,data:bE/VjQDc0us=,iv:6wFfKVR5Lr/HmPJesvtyLtvqz6hH9AdogjN5BCAMZZ4=,tag:GY7r5BR2quH2DDz8vP8KyQ==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:pA==,iv:Cpkc8JMVpZlkt3wiLWiMdxS2Y+wINsV8jVcaYTkl7Rg=,tag:QfZ4d3fEbqh7GXxDtbYa6g==,type:int]
- - ENC[AES256_GCM,data:Ng==,iv:PUQXdjeRGdIuUUyyq/yiW8T10zZqIAqKhJGHXMkaPGw=,tag:c27CvCve0skRMr0rUhnkmw==,type:int]
- type: ENC[AES256_GCM,data:jK0=,iv:Rif3sQBsQVxQcOkvzWia7pGK+KEiSh8bfPNCn4p53/0=,tag:lyddFibnFvmxDjw1w6VJtw==,type:str]
- operator:
- type: ENC[AES256_GCM,data:D51C,iv:07bcBppLjE+Jv0Nqy1HEHE57MrP8/LcOtNMNF5CVUEI=,tag:MQGM4gieNZNIbrsy87n49w==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:CSlB,iv:yBktox2Y3vB08zPfpvhVsoRy2jlV2Nu6Lzo+90W09n8=,tag:jSdf1Z/VyyI+CbAcPfpQ/w==,type:str]
- type: ENC[AES256_GCM,data:I2TOCrs=,iv:XOmFDmtCuw342d99lkcTo7y1pOAwwolArXaerrIekiM=,tag:Mi/TEzS1DfWfgg/Dq9NvTA==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:Dmm757OdhhOeQA==,iv:bGvBdq/vtJYXmsNkCM2iIfsQR11bjI+lp3xOJwUk7Dw=,tag:88l+FpOWOUG5y3pXmCoCIQ==,type:str]
- type: ENC[AES256_GCM,data:ytQmZx9tZz0=,iv:3qYhUUYB0WUli+iJGrc1D7vS3J6ErS4bdawNt2/9ErI=,tag:ftkNjNT1/xjTGteiipCU1Q==,type:str]
- uid: ENC[AES256_GCM,data:klON1y+48o4=,iv:kBmc1+4VSsGrQoiR7hrEozMC4c3D611TWCSVmjrC8Ms=,tag:A5TL+sgYIn5iLZfdCXzYgw==,type:str]
- expression: ENC[AES256_GCM,data:Ft5vi1c=,iv:MA1fyJ2UfrXy9gCupRhUrTDy2Y5fD4C69jIhnypJkN8=,tag:LE8D6XhHLexhGeo4ylJJQQ==,type:str]
- intervalMs: ENC[AES256_GCM,data:3PyeLA==,iv:1Fbirt73nNvLQ/xQqOWKIaqvr4csXduQmQovx0nOWRg=,tag:uQxo14IkqCOfwbaYYYpfVA==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:SI4YRIM=,iv:nxbY1UMmVvSylwC6SG3nmIRc6pbA0d3yNcBnrRuPl8s=,tag:CGBhI5615VCjafIsV2E03w==,type:int]
- reducer: ENC[AES256_GCM,data:IHS+jQ==,iv:c8XDy9Na60cXFlLUGEZWYlkIaBMz40FXyL59IC2GPhc=,tag:L0JfoXKFkopyf+wHewKPRQ==,type:str]
- refId: ENC[AES256_GCM,data:dg==,iv:kAF4kV45Hs/6QnC/OEfUGAhHMtg2E0OJXKVmaYtXCMQ=,tag:BZk6t22jqWxkojWEjh08DQ==,type:str]
- settings:
- mode: ENC[AES256_GCM,data:eglE3SYor9Yd,iv:pzwXNk2SudMZbdinaArwAa3gkJIR0u+V60/3D3jyVys=,tag:FgNiP/RpSRw6HZHAiNOsqw==,type:str]
- replaceWithValue: ENC[AES256_GCM,data:6A==,iv:/oFX3uJcvpysTOqwFYBIt/qgrYOH9iif7ASECCDM5mQ=,tag:5bihhiW55vBgNPPAoGg+6g==,type:int]
- type: ENC[AES256_GCM,data:e6cspiJ7,iv:QlnmScvzce4xjNmdO5W1KmYGX2V6lCAm7Uyqcrc4tio=,tag:JNpsQ3u1i39qKy2v8IXz4Q==,type:str]
- noDataState: OK
- execErrState: Error
- for: 2m
- annotations:
- summary: High amount of i/o timeouts in coredns
+ - uid: d2b94d3a-2483-4be4-bf2a-2fd9f9fac406
+ title: Coredns Failure
+ condition: Thresh
+ isPaused: false
+ data:
+ - refId: ENC[AES256_GCM,data:YLIE4WI=,iv:JTn9DdOzgY1Z9yDuC5ioQQd6MIbsZ6WEyO8qOlOUMp0=,tag:qS564VitQLBh6Pg7K/AwMg==,type:str]
+ queryType: ENC[AES256_GCM,data:XodqKzc=,iv:OC5qtzhvH2AyusVEpvkzgrtsIPJuEsHo37MV+XKZe7M=,tag:F24fbVaRVkIq7FLY7Ox2Ag==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:jNQJ,iv:96VeonoJJlTrm740h/GgH2z4uHmYokXjMNkAWDCQQUc=,tag:1q2VZ6A+YuN4Azfj0g7rtA==,type:int]
+ to: ENC[AES256_GCM,data:XQ==,iv:1DsmkxJBVwqlsRDAFN6vDOkYMYCQKjmJfyTbc4ekxEE=,tag:TyeQyQULtpQ5I2oMEMWDrA==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:Y3KJbw==,iv:4nTh5prw9NnG8ZjVsQK3h1NdybBz0ovRm89XHA8+Izw=,tag:5X/z+eGwukZ8e6hyyU7Yrw==,type:str]
+ model:
+ datasource:
+ type: ENC[AES256_GCM,data:CwSkig==,iv:kbqkqrkkJ5uFsrKvecm94xUePAYgxCOanIWi00WZw4E=,tag:jWgSbU6EfJRGfy89oe9Aog==,type:str]
+ uid: ENC[AES256_GCM,data:Y1sFFg==,iv:RuKR/AkLN4eSAZOfWZ2MIhvTH/MoqbzzUa2DVd3QYQE=,tag:hY6IJL9TUSTwG/8pNxFzzQ==,type:str]
+ editorMode: ENC[AES256_GCM,data:qmKR3UZRzg==,iv:FeIXVXA9uIfAXkGfArKO444FW95ORqbSwV8oSZsqoyM=,tag:euWh7KywzGyCggEf9NQMHA==,type:str]
+ expr: ENC[AES256_GCM,data:DgN+rzrxLICd6AxQTNNMi8YcxQaUWg+z/TBYVLjtptxRUC40TtYR/hxIEAebftJDgYmxvf+9BLzLirEUVh2WnyZrdvsi,iv:kiJiuWsj1S7cn6TiCGmYVM1rYssnvyzqN0bwfFnoUkU=,tag:eWv9uBm7r+umxZVceKtW0g==,type:str]
+ hide: ENC[AES256_GCM,data:OtALMd0=,iv:N5FuEZ6BwTpPhOXzqDJG4eyUmqgWUaXPvDsY+JLZqqg=,tag:+wrBWZJCIAfo9iCyhANpng==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:y30MPA==,iv:13ykjBxU/aKUhl9nXseCuYKHDkPiU4IeTY+XeMpxrbI=,tag:gUUkxaaZpqgAVuSYNxOaPQ==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:7Fc6Zss=,iv:f5i6rf8jd253WlJQARhfty2/g+25G40LNsK2XNwFK6s=,tag:8pucbFXTlihNV7F8LhrsVQ==,type:int]
+ queryType: ENC[AES256_GCM,data:ToeyQ8w=,iv:yLlJYSZoZa9sXXN2sZxqY/R6OMxZSHPzo5ok4h/EcYk=,tag:YpM+7WNaPGNVCQF0GL+Y5w==,type:str]
+ range: ENC[AES256_GCM,data:pVcctg==,iv:KpCp1r7uQWGBzYZRXlzYFI+Ii7YwDn6CahNDOqneGrE=,tag:OxgkG4PLrtF9V9hhBTd9BA==,type:bool]
+ refId: ENC[AES256_GCM,data:GLRSFdg=,iv:lSs4gO2lheXYB59skjA/QH2mtI1cnSlBoLzx3C+fhz8=,tag:FWXo9mqawW1xEJwcMeZvkg==,type:str]
+ - refId: ENC[AES256_GCM,data:5dbU28Er,iv:kH69w+BA+HjxA4uqVxo/6E5QiPbIa063QOfsYKNSYq4=,tag:E+uhXvg6gZ/3/1nYLO1EtA==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:gUCn,iv:wlAg2voGUzBCyqY2AK+wgy3HCGsTqp7mtpUDHBidRR8=,tag:4AjU0qB5kkyev+jFpjWMOQ==,type:int]
+ to: ENC[AES256_GCM,data:6Q==,iv:XCsX2cU0PTUqo0t9j54hjYCI+5JhYU7racorBeP7t4M=,tag:xSdqFdko5UzZBE/gxD0+Ww==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:5rnADR6zcYw=,iv:6bDKvZitt/jrDb0GPO51NpQ2dbLMovHVvcgKxl5Pztc=,tag:6VTAGVyKBtsJk1t6wGYr4g==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:iQ==,iv:lEvofnH2WkPzasp2zyfdflXxPxabXcIq2G9UbQ8b0IQ=,tag:l/4Z4FncLjOGIkvLWVN9yQ==,type:int]
+ type: ENC[AES256_GCM,data:Eeg=,iv:wwWkp0g2K5JC3TCbVcT4hQGHcoCDme6X0ONnchvwqnU=,tag:OqHnwbM1SIdWXTu1nOXdWw==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:1A6k,iv:R0Ux9JyEU2RAUc0pCY4gSggE2EYgjx05k+3r4x+4zSE=,tag:UpOD0JG6YulqwXUfAvE0eA==,type:str]
+ query:
+ params:
+ - ENC[AES256_GCM,data:tA==,iv:/2JgJopAyWqYZIFuGfxG4UN1ApdFDHRlTDhW4llCP70=,tag:six5kzaceqYC0cYudjwoBQ==,type:str]
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:nWyXng==,iv:lssJzj+d2cBCeZnvfeK0w9DgQu6UHLFZi+t8fU5CBMk=,tag:o+bJHlyMNQaCJltfpEg17A==,type:str]
+ type: ENC[AES256_GCM,data:uLHqb0M=,iv:rS9cT1KyFJ2b2KpqbTpAU9r4XyUPmO4NHFXLZ6otJ/4=,tag:SAmHl7BtFPvEmjn7j/nP9w==,type:str]
+ datasource:
+ type: ENC[AES256_GCM,data:UHQP+Rlo6s8=,iv:tcW5AvR0zJLuG6C0LX2kAIQj6S2KWHFJXRUAt0bNW8E=,tag:rBtTpAP3u70gPT7KxT8ZBg==,type:str]
+ uid: ENC[AES256_GCM,data:Qfbm2uMGQ5Y=,iv:bqISUvqLKIIbbYKnuPXQWuf7KE4+4jMYq6A5+XUKN5w=,tag:8SpvUupvrl8HcUKfKjDpHw==,type:str]
+ expression: ENC[AES256_GCM,data:Rw==,iv:LY0ryT2H5oQIu3NBE3zY+I/i3QJbdSCUIMJ2B2chiY0=,tag:/ULFX3K1peCY8/14x4YExg==,type:str]
+ hide: ENC[AES256_GCM,data:bF7AGPA=,iv:VnxDNgLX+20nNAzmIxp6bN0FyjWrNrRyS3d/6vDMdVE=,tag:zk5tI3vEHTXOo6jeb6Sd8g==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:rooYpw==,iv:B6kmVMOelOYaY+CEbG1RmMPS0cXWKnJvNy57KHC1MFc=,tag:mCvRLLFpTHMshuOomNrkNg==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:HiF5KlI=,iv:TKlI00RaG81Snuq6g3h4qBAjT0tdZHnqKBbiRPOwjNE=,tag:0muVeeP5RjkiGZV1U1zPFA==,type:int]
+ refId: ENC[AES256_GCM,data:xMqgNXRq,iv:bWTmXyzByNC8linZKDhIpvkuEH2JkitTf2Ywt77rBt4=,tag:gQ/11jq314Tcg8X+kNtr3w==,type:str]
+ type: ENC[AES256_GCM,data:MPaB5roFwPrw,iv:8Hyxft7aFeHmWW4G3qRR1Xf4uxTtSgPzNWJJeJrSg6Y=,tag:fHmS9PknNtooW6YLYC9WLw==,type:str]
+ - refId: ENC[AES256_GCM,data:tQ==,iv:+5KMrEeC9mWObjxfOC+324rLHmES8TZTcHIppksyFV4=,tag:RxQ/dWe41nr6/y7j0PIR4Q==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:yWRq,iv:XaXxOBaQu7TGvfzkh1Pa3ywLVrSz2fLpDJg0MPD150I=,tag:eyigdJHcd8BuxEUbiVCJHQ==,type:int]
+ to: ENC[AES256_GCM,data:HA==,iv:N+xvFSUZm0V/9EQo/etQEQmQB2NziQCRpp9k2xmnhEA=,tag:bZp3O6OW2m/RyHVigQn8tg==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:bE/VjQDc0us=,iv:6wFfKVR5Lr/HmPJesvtyLtvqz6hH9AdogjN5BCAMZZ4=,tag:GY7r5BR2quH2DDz8vP8KyQ==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:pA==,iv:Cpkc8JMVpZlkt3wiLWiMdxS2Y+wINsV8jVcaYTkl7Rg=,tag:QfZ4d3fEbqh7GXxDtbYa6g==,type:int]
+ - ENC[AES256_GCM,data:Ng==,iv:PUQXdjeRGdIuUUyyq/yiW8T10zZqIAqKhJGHXMkaPGw=,tag:c27CvCve0skRMr0rUhnkmw==,type:int]
+ type: ENC[AES256_GCM,data:jK0=,iv:Rif3sQBsQVxQcOkvzWia7pGK+KEiSh8bfPNCn4p53/0=,tag:lyddFibnFvmxDjw1w6VJtw==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:D51C,iv:07bcBppLjE+Jv0Nqy1HEHE57MrP8/LcOtNMNF5CVUEI=,tag:MQGM4gieNZNIbrsy87n49w==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:CSlB,iv:yBktox2Y3vB08zPfpvhVsoRy2jlV2Nu6Lzo+90W09n8=,tag:jSdf1Z/VyyI+CbAcPfpQ/w==,type:str]
+ type: ENC[AES256_GCM,data:I2TOCrs=,iv:XOmFDmtCuw342d99lkcTo7y1pOAwwolArXaerrIekiM=,tag:Mi/TEzS1DfWfgg/Dq9NvTA==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:Dmm757OdhhOeQA==,iv:bGvBdq/vtJYXmsNkCM2iIfsQR11bjI+lp3xOJwUk7Dw=,tag:88l+FpOWOUG5y3pXmCoCIQ==,type:str]
+ type: ENC[AES256_GCM,data:ytQmZx9tZz0=,iv:3qYhUUYB0WUli+iJGrc1D7vS3J6ErS4bdawNt2/9ErI=,tag:ftkNjNT1/xjTGteiipCU1Q==,type:str]
+ uid: ENC[AES256_GCM,data:klON1y+48o4=,iv:kBmc1+4VSsGrQoiR7hrEozMC4c3D611TWCSVmjrC8Ms=,tag:A5TL+sgYIn5iLZfdCXzYgw==,type:str]
+ expression: ENC[AES256_GCM,data:Ft5vi1c=,iv:MA1fyJ2UfrXy9gCupRhUrTDy2Y5fD4C69jIhnypJkN8=,tag:LE8D6XhHLexhGeo4ylJJQQ==,type:str]
+ intervalMs: ENC[AES256_GCM,data:3PyeLA==,iv:1Fbirt73nNvLQ/xQqOWKIaqvr4csXduQmQovx0nOWRg=,tag:uQxo14IkqCOfwbaYYYpfVA==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:SI4YRIM=,iv:nxbY1UMmVvSylwC6SG3nmIRc6pbA0d3yNcBnrRuPl8s=,tag:CGBhI5615VCjafIsV2E03w==,type:int]
+ reducer: ENC[AES256_GCM,data:IHS+jQ==,iv:c8XDy9Na60cXFlLUGEZWYlkIaBMz40FXyL59IC2GPhc=,tag:L0JfoXKFkopyf+wHewKPRQ==,type:str]
+ refId: ENC[AES256_GCM,data:dg==,iv:kAF4kV45Hs/6QnC/OEfUGAhHMtg2E0OJXKVmaYtXCMQ=,tag:BZk6t22jqWxkojWEjh08DQ==,type:str]
+ settings:
+ mode: ENC[AES256_GCM,data:eglE3SYor9Yd,iv:pzwXNk2SudMZbdinaArwAa3gkJIR0u+V60/3D3jyVys=,tag:FgNiP/RpSRw6HZHAiNOsqw==,type:str]
+ replaceWithValue: ENC[AES256_GCM,data:6A==,iv:/oFX3uJcvpysTOqwFYBIt/qgrYOH9iif7ASECCDM5mQ=,tag:5bihhiW55vBgNPPAoGg+6g==,type:int]
+ type: ENC[AES256_GCM,data:e6cspiJ7,iv:QlnmScvzce4xjNmdO5W1KmYGX2V6lCAm7Uyqcrc4tio=,tag:JNpsQ3u1i39qKy2v8IXz4Q==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 2m
+ annotations:
+ summary: High amount of i/o timeouts in coredns
- orgId: 1
name: Matrix Synapse
folder: Matrix
interval: 1m
rules:
- - uid: b6db960c-c0bd-4d4f-b0f1-e1f216ec81cc
- title: DNSQueryRefusedError
- condition: Thresh
- isPaused: false
- data:
- - refId: ENC[AES256_GCM,data:n2qY1Wg=,iv:sYOwKOsXOxiQlS8cC02y9gpDaw88wMmUbYvSvAS7y8w=,tag:w3sn1o28GnwHFBSyr3ePlA==,type:str]
- queryType: ENC[AES256_GCM,data:AIBCGio=,iv:h+kgDoN+PCkYVrtSn3b45ZsCiUPFYqY0B2aIGz29aVY=,tag:E4uIVb1HVwbe4U5ArJD7sQ==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:VkTF,iv:oJnmXIUP4Ywd8lINMBtbk1Z7EV5/8mYbYBlTMgV6Ibo=,tag:/8etZfoVyJT5kOVwMfthvg==,type:int]
- to: ENC[AES256_GCM,data:kQ==,iv:aElTcFOi+rM8j8PFd20I2/7/dI74oExeM+XZF+Eaf9A=,tag:uIUb1FjCOPHxzI+YhwtpbQ==,type:int]
- datasourceUid: ENC[AES256_GCM,data:XedhsQ==,iv:xeEuQyRofWE6seYCqLikV95v/UyxDUZPPBa2t18+szQ=,tag:oGEGYgydyrwNrWdXepS3IA==,type:str]
- model:
- datasource:
- type: ENC[AES256_GCM,data:iaHiZA==,iv:8EqVnhGbnYyz9svjtGkzgq/NJZRK27l36NYZnsJzQuk=,tag:ed8rKdISPG+EsaKeA7k+sA==,type:str]
- uid: ENC[AES256_GCM,data:cQ8HRg==,iv:MN6EsNQbRxlw/2vhes+y4SjhQoCRo2SIJ4Fdf6iONNI=,tag:odRL9ryJNk6Csshy8Mu1cw==,type:str]
- editorMode: ENC[AES256_GCM,data:BYYaW2BpZQ==,iv:hiFudjt+Y8QhNXFuRJ5g3ZUbPmw8alsCGiKjISbYEcE=,tag:kee8f2SqwIlQL+mZv6pGOw==,type:str]
- expr: ENC[AES256_GCM,data:1iniNKWpeeVJu7UVXvegqrDd85YLuN9NppMqiZ4g93eP6snzfUuxa6brDDlb8hLpfnIwAQQnY9y4gpn1zNh9LsnthoEpX0sZyHlvYrZ6r7g=,iv:gPgXnGv+5nn5zYs0CYAzk5kO5eHCzyzJ5sZJenu0lKs=,tag:AI84+A6Qe5++3m3fkgzesw==,type:str]
- hide: ENC[AES256_GCM,data:dsrwQKI=,iv:pCc61Hx1AgLa2wNQl6YT8SspjlV50rUJVkhfAPZm34g=,tag:4v5r5nnviLdx49aTE9/d5g==,type:bool]
- intervalMs: ENC[AES256_GCM,data:SMhyCQ==,iv:qJUb8uvOR5Wtt8p8lnhtjX6nrpRcXHztEhDsDafiwd8=,tag:C9OYwIGNF5OJbMhY+IvcAw==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:gt8WzaU=,iv:m2EmNTMbKOTVA16yq4fbGoVIG+dS8ZqtrCZ28Iefny0=,tag:j2gNp/ltjByWOAxBXi/IWw==,type:int]
- queryType: ENC[AES256_GCM,data:SINdqqE=,iv:Ksx+ki1x9+EtBqEmEoQ3ZqryjYcoVhzJtMC6I56ClQI=,tag:exdV0JRXJiKyBuhi90BNFw==,type:str]
- range: ENC[AES256_GCM,data:wpqEZA==,iv:86p3KPSzNFieGbV3p8xPrXvK2p6x6bWmynHWPg4k+cI=,tag:DWtVgycsDThR9rR49DqRXA==,type:bool]
- refId: ENC[AES256_GCM,data:G2iXR4Y=,iv:JpZ3Zzw5k5SqcH7IiFkGheWo07tEQ1yuELd5uVcmgb0=,tag:vuHvFNwe8TXJmB1geLrkLA==,type:str]
- - refId: ENC[AES256_GCM,data:prlO0rGf,iv:XFd0axSyGQ30Nyl54kvU2QOBZIe6ia1wy0hct5y5jac=,tag:soX9eRM3ayy4L/y4tF/www==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:baNG,iv:7hke+4A31L6q4FwnWLojEvHLQbCASIp0ilnCo9wp+BA=,tag:vbjkpphxie++De2FkKwepw==,type:int]
- to: ENC[AES256_GCM,data:XA==,iv:ms18Skjz+fFfo5x1AtsOhLj6C5zLDj6exAdIYL++1/I=,tag:W0uGdPJJncaK1IxHmP5I4g==,type:int]
- datasourceUid: ENC[AES256_GCM,data:AVKndMI/hVw=,iv:rJNMC8eykAu6LyFE5kXwRxNSstPTjMkTrjxdyXRyvRk=,tag:X1m+vWlqkJRUdAE0ieQ5mQ==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:2g==,iv:iEq+CjjrIH2FqbYoXJf+bLUf/1WrvloUVhbseEkI5o4=,tag:TyVWrrePOro9b2UxaviJJA==,type:int]
- type: ENC[AES256_GCM,data:YJs=,iv:IiS2rO1FOaL75JO2Ryi8yfC8jH/AHl+dCpW1QG8GlQE=,tag:STe5F4gT3fKoWhe+SeITOw==,type:str]
- operator:
- type: ENC[AES256_GCM,data:kvpK,iv:5ENT9LxMg/qlQMz40Rwcpg+fAt/iMdF867Yqf6Ll2xo=,tag:JYW4+1vZE6WfrIxcEv/vew==,type:str]
- query:
- params:
- - ENC[AES256_GCM,data:xg==,iv:vWQT+OTTlCpWAqEmse/MpjEqN29wEsxEzfYBHcxrpPY=,tag:Swomqyhlh0u9kljrUJa11Q==,type:str]
- reducer:
- params: []
- type: ENC[AES256_GCM,data:3LGqQw==,iv:+CaRA5tchRbGvzKxujPq3v9WnoqRHrceod0P2kjpg8I=,tag:1ZA6gvpnaBcyPHJxZcZ1VA==,type:str]
- type: ENC[AES256_GCM,data:if/qmso=,iv:+KzRSND5MGS7gRgM4XFo9+kvtnGjosFxAkMNhs/XgyM=,tag:NTNYJy9Tvpzw2aKKohAUTQ==,type:str]
- datasource:
- type: ENC[AES256_GCM,data:6I5nCrcJZds=,iv:D5J8QYOocNtV5gomwXZj4tDmRV74NhblIDZbqpy6Q74=,tag:kS6BNeimn3JUaY8LnB9+Fg==,type:str]
- uid: ENC[AES256_GCM,data:89V12WI5Z0o=,iv:RarrRZW1qqWJ8XC1wG0Tlo0Kpgbf2xVBgxAsuL0eM+Q=,tag:T68zMMRcPmNw82oSQPzyQQ==,type:str]
- expression: ENC[AES256_GCM,data:vA==,iv:Z0RYQC25dPeBZRp6f4Q1dJki4nCh5nMk8pbaSJA1Hjg=,tag:mWVbVem6Cc5r8AYaHyflGA==,type:str]
- hide: ENC[AES256_GCM,data:AgqJaEA=,iv:xYTzA7NVwA6XCKb8TObjAwcDlvpdaE7U+JW5ODd3mWM=,tag:fXMfH/5jRntSQEAj9BBwxA==,type:bool]
- intervalMs: ENC[AES256_GCM,data:Kf792A==,iv:Nhgq1ewa6O0XV/oEGol3m1GQJ0TSIzv4sgm5lS8eioc=,tag:v/Ck+Al5/7dcKbRJnZHrUg==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:+F2Aou0=,iv:YGMSqKmHdfp3JE350esXcW5WZ72roSjdQJ+wNumOA8M=,tag:pW+up4mx0UNPIU/3UaYplw==,type:int]
- refId: ENC[AES256_GCM,data:50jXpZrw,iv:JoIw/rhWSGM6U6nkz4pTXPiDjGigH62Pr58f3kFsEgw=,tag:Ur4zm3Ma9Gl8syo2KhyHJw==,type:str]
- type: ENC[AES256_GCM,data:7mLuZN+29yK6,iv:qjf1cRBffTm37UwP46z28SysYplrStfX4lD3EBg3Oas=,tag:F4e7PhcEh5GzRfwHi7N0hA==,type:str]
- - refId: ENC[AES256_GCM,data:TQ==,iv:ltQGL9uedpc/hozdWfYUuGdq34yD7NAnt62SxYy3Txw=,tag:SOQZa5wDw3tYhVyVr5uMBg==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:OuFZ,iv:BnsJztLHvRqpu3EPWbjYiXy5cXTYUmz+oY3r3MxKVns=,tag:RUDedI87n/juMCqGU6UMPA==,type:int]
- to: ENC[AES256_GCM,data:dA==,iv:LewqOAETjwqABCoOaERBSNfuqcmnk7lcN0ov6BexBhc=,tag:uLm2LEHfHV7G4VeRBPhf+w==,type:int]
- datasourceUid: ENC[AES256_GCM,data:Vz+ive5+eQs=,iv:HNib4DYbNcEdjPvn8S6ZVSonBI876iUO2GEyVJYXe3Y=,tag:jdA3phmSmlhfEzweQw6Zxg==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:gg==,iv:j95rImkfb8zh//TSigqmXIegLU/34aMdnbfL438LN90=,tag:ZwjvLMArZyZHeAvbMoGOfg==,type:int]
- - ENC[AES256_GCM,data:4w==,iv:i55T3PwXztFe5QxO1czxG6Rn+arkJ1okAViRft76KHE=,tag:DHvcgCknVFXkQbFrA4ei8A==,type:int]
- type: ENC[AES256_GCM,data:BOQ=,iv:wzeWZlIQuNwIhlziRtF1T06SwH2ZgdnlHPAdZhON+jk=,tag:7mg4R+9bjsq2xDdqiR182A==,type:str]
- operator:
- type: ENC[AES256_GCM,data:Wo60,iv:111kAr7iw2uCnlODQQ2DS+4KL7syI+hIRXfwfQNsVFg=,tag:WuQL440AvDCPZ0py19YahA==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:Gref,iv:ay7tw3puWIvCNa6B7bC+gaeqQHosSLSyzcNx1CAkOLA=,tag:N8fAiNtITsM5bQG+ESaLfA==,type:str]
- type: ENC[AES256_GCM,data:577PYQ8=,iv:RPeFEzTd2EVk6BYwdvbo43aWOt+l9WN6cUEJQI2PEz0=,tag:Q5qQ9VjEd1gBrIJagSRqEQ==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:zaZ4BQufalGrZg==,iv:isO11RNzkaUTPTyl05Rs/Yy4V4S+sGoZMwbxPD0bYSo=,tag:ZlVbrihEV+VF+Oympa51Ug==,type:str]
- type: ENC[AES256_GCM,data:PQpX5A7PcIU=,iv:Pe1udsQsozVtTI7VfBWlhGC+w6lJejnp1tSGxAYUbfc=,tag:jt/0H8nbz6lU7vdavMpcxg==,type:str]
- uid: ENC[AES256_GCM,data:ycvelPkoJB8=,iv:DY8peFmgozoqPlz+5VyJeHm4qO+UvvpB78lbcbGK+GI=,tag:z23V9AxT9SpS78yLC7AsBA==,type:str]
- expression: ENC[AES256_GCM,data:NpPWdc0=,iv:Gz8WfZhHrBGUo9ysTRH+QJLyjXpZk0VCDr4lucaY66A=,tag:ipgZk9QZnf0TLscBinDZHw==,type:str]
- intervalMs: ENC[AES256_GCM,data:zSTxYQ==,iv:EnVbBg/cfjwQIs6RiKx0h4o7/mR4GcdCYAnEvlkk2n4=,tag:oUO5QXXZFAQcVPV1F4APdw==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:4YifGXM=,iv:wniMAlxBRvAIZo9elpEIjrmIOqT1qukPRBlOOnufapw=,tag:o0F0xLM4yrp5u/ZBWTmMQQ==,type:int]
- reducer: ENC[AES256_GCM,data:NM4iwA==,iv:WAWdz0IrDwzv5GyYCzRAk0qhtiO8ljMH5VUglcEnFbQ=,tag:jzovO81rUOijzoWizR65WA==,type:str]
- refId: ENC[AES256_GCM,data:EQ==,iv:0/WF1sC2U+nuFzPEVTFWHEuj8bezsnkzniIHk52uCZI=,tag:st2lXOcD4s/ySfhxs1DGog==,type:str]
- settings:
- mode: ENC[AES256_GCM,data:bzY18OLDkcji,iv:ggrP8HMLyWWPuok7sem3xGlUfkrbITxfompa27lGUQ8=,tag:K3iFdhKHEB8c+MV55eZLQw==,type:str]
- replaceWithValue: ENC[AES256_GCM,data:ng==,iv:1/cJgN7rN/AtxeHdaQYyLL9roLbCkE5jPbb4VzG505Q=,tag:fi24/eFpKZ6hLWHPIUgGrg==,type:int]
- type: ENC[AES256_GCM,data:ghubSnCw,iv:0i0VDzKv75uYCQYBtITAgUkNWl3dKd+7lzloRacM/f8=,tag:eqPIH7L6IVRzLV5Fsad0kQ==,type:str]
- noDataState: OK
- execErrState: Error
- for: 2m
- annotations:
- summary: Matrix Synapse DNS Errors are higher than usual
- notification_settings:
- receiver: PagerDuty Warning
- - uid: de30a1da-84da-45f4-a4a5-d641dbd92e6c
- title: psycopg2.errors.ReadOnlySqlTransaction
- condition: Thresh
- isPaused: false
- data:
- - refId: ENC[AES256_GCM,data:4FSfD5U=,iv:TautCbBChguKdymhWNZYdqeXi+8duZUst0Qw8WUJQaM=,tag:3ud5sXVi88fZtqBbHKaTNA==,type:str]
- queryType: ENC[AES256_GCM,data:KyYwgWA=,iv:DGJBE2+Zv2i/3yN6n25E5fIClpNzUk0vufv0/0GBfHY=,tag:nTs9G+cU0Nay0EjC3jgYTg==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:aHG6,iv:vLd/1wiN6zAiOfHzbB6qoIwHPiLqnHMNvf66jMuqzs0=,tag:/Xp+jDJA/7vrKqJt7c85Fg==,type:int]
- to: ENC[AES256_GCM,data:6g==,iv:RTQky52PK49d/ZGe6T8D2Cd4JuAMeRcc+HurjpXclPo=,tag:TPKXBwfCQPgs3vOblSZUmw==,type:int]
- datasourceUid: ENC[AES256_GCM,data:HbMyIQ==,iv:cLwtDeaIaTNOs9/7wFwARLYp8PlY/vawr9chkxAjn+U=,tag:Hy1/aAe6cmd/Nm36E9CXyQ==,type:str]
- model:
- datasource:
- type: ENC[AES256_GCM,data:tFq3sg==,iv:8DMqD9nbilRGLcSnHnHUb2zgTv277CuxyEh/3pGQvK4=,tag:hHqBj71I1wai3aFqYjHYAg==,type:str]
- uid: ENC[AES256_GCM,data:ANATww==,iv:Fx2CnLqCt18Vv9hWUu+msqknTCQv0ePRuhLYnPBL66k=,tag:zQd+AtVRZK6cmzPmjiAMuw==,type:str]
- editorMode: ENC[AES256_GCM,data:iFy9Pvxkxw==,iv:PnBHzC58MO81F6oWyNHHZbxiC0yTPSaZSjgRuKtNE2U=,tag:3u6eXWKxwFAFKLGZImVPBA==,type:str]
- expr: ENC[AES256_GCM,data:GKI16rW+IyyVlnOy68g83Q+zumxlJg/YBMHuBWmoYcH0tO8ErjSuB67YxtYRhqzm9/2QeaS605Svydc3CCiMB+YpUGH374WnlXTZ69B2R1YRX9mifBAP1XJ9PyGRdSu/6M0=,iv:+VMbTi5trvHtdJ8YiJXF6FjYD6cEN+WnplmXf/4hZAk=,tag:D3nedJTHZi5lB3mmC6Qbcg==,type:str]
- hide: ENC[AES256_GCM,data:97FTMoI=,iv:na1iIDStxGlGPHYXUeE/5+q0qcg7ixtCGJdsio9hlEw=,tag:n0iHDpwISoSxhLsfb/C2HQ==,type:bool]
- intervalMs: ENC[AES256_GCM,data:xO4rUw==,iv:hubLsNnHpTyxtgdxq7QxT6Yi+I2RUPXaJhIPr2M/vdE=,tag:9kFUxZaV4+F5GmG5Qns7xw==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:ybwEjcA=,iv:j1XydTxbjCUEyWIAVXA8p+VQ1QRlDPJue8xdvsH2LN8=,tag:ejo3LhbjAYf19D1VMAoqdw==,type:int]
- queryType: ENC[AES256_GCM,data:ZXW3kk8=,iv:AZboPCZpFSPA/OKks20hryW8dcz9ReAGBYnxfJoTfQo=,tag:ng14XH5Ffy58/ADePPtrpQ==,type:str]
- range: ENC[AES256_GCM,data:VegTkw==,iv:JFc8Z8y7y/3kd831TQGqrDjAS14KOmuRpZGLy8j0w00=,tag:sPsohCGKbqzxkLrBS8psdA==,type:bool]
- refId: ENC[AES256_GCM,data:/zhq6T0=,iv:gJy8Pn6C/hoj6Omase/Wsz/fpnr4i2A5UBLUdnAy0hw=,tag:IjsnEz6ZaNH5CjhqAH+CyQ==,type:str]
- - refId: ENC[AES256_GCM,data:HrVyezUv,iv:fd/VVYR9rxWj3vxhJq9vcM507HqmawzkUXOE37m/4Q0=,tag:Me69F6RFE45qSaXZ2/T2CA==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:lMx+,iv:AhOYBrR0DIfaPOQtWN57uzy6FyIBpTqZqiY2ZzFsvYc=,tag:L7iIQZgGyELpexvRt5U1dw==,type:int]
- to: ENC[AES256_GCM,data:hw==,iv:dJMnsAXqMTtbfdNH1X6rjgYPrUhs7IPJEkGyWo677ck=,tag:h87xGPHgmMgV+U2VV9Q5Ow==,type:int]
- datasourceUid: ENC[AES256_GCM,data:V5lGTa9mkqM=,iv:4t/FDUw7e6pTsKjYzAny+u26Ad6PVByPArE6NE0GL6o=,tag:TIMEZpkC3qKiiYGtztH5Vw==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:Waro,iv:LZOx3ypE1D+PABuOFnlmGSfZfqQKLLJQbKk01VEAmNA=,tag:+1eV5fErxkCK5g5SAHoAyQ==,type:float]
- type: ENC[AES256_GCM,data:KKg=,iv:rMh9jiyB+ufx7LAP69M/TXMx20vZxtsP3Zw365qHvX0=,tag:ksfE/fp8tmaK1qu4IoGYaA==,type:str]
- operator:
- type: ENC[AES256_GCM,data:SLHL,iv:B/l+zaIqyco+zFbAh4ak335LM2cDNasK+qh+FO96XZI=,tag:wYBbzvZZdBDxrphC8LzexA==,type:str]
- query:
- params:
- - ENC[AES256_GCM,data:Rg==,iv:xx7hkEN6kfTHnG8Z0Tkab7Q133JX1sq8R9TI5Os2KnA=,tag:08liAjId4ihIjWMpz8ZoUg==,type:str]
- reducer:
- params: []
- type: ENC[AES256_GCM,data:xQza6A==,iv:M2yYpJyPf/3/DlCgjgpYjl9ULJLbJK+c1C2yi96GuZM=,tag:6DTKw4lWZq81Nn1GpM6M2A==,type:str]
- type: ENC[AES256_GCM,data:ITLK+DI=,iv:+LvXcHHb+PtOrdVEoOyhBtoVJbXb3Fzo4Rp2WY2GzBs=,tag:3DyuzRCG7BX/D2Ghb5tSsw==,type:str]
- datasource:
- type: ENC[AES256_GCM,data:deiC5blqCR4=,iv:0JBiF6c5RSP1Mw+CbawhjI+/hFb/zOOjepvyL+1J9+0=,tag:4wOpRNiaK2k4V/hjMh4khQ==,type:str]
- uid: ENC[AES256_GCM,data:bkVg07YkgNQ=,iv:ucZ4QAb9GkbvbXpVsVORFLhIZ3LfciXhukpTP9s/li8=,tag:NmZOCY6WjCDt3J1Vg5i44w==,type:str]
- expression: ENC[AES256_GCM,data:zw==,iv:d+2Y+ZpvJ5sjrnkZPAnO9rgIuB/krtpSp1uOemvjfVg=,tag:UqxbFxry6LsKDS4NUVErVw==,type:str]
- hide: ENC[AES256_GCM,data:gId2YjE=,iv:JBJUJXI4raA36lskJl/l/5KoItV9gmxXP4sKW+EBoPw=,tag:k6Ou//hspQa8yriZ1hEoMA==,type:bool]
- intervalMs: ENC[AES256_GCM,data:bhJC7A==,iv:2y+iwbjOyIxQPBXX97igQ/x2iY8F8zODmy9FuTBN0Gc=,tag:PVNsVcu/naABRIrNNJsvQA==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:AO5rVt0=,iv:GTAlhdENQfJy60kPPLdL46RKKV7xs+Vjg1iYArNjsfI=,tag:qVp2y+t+cxW7vPJ8furC+g==,type:int]
- refId: ENC[AES256_GCM,data:TgK0xbph,iv:4W6EFZGuOlOaQp89wXZ0+OaYQTVzGEP7NICEUa33f98=,tag:lqor32BDaugBfXBQKpVVFw==,type:str]
- type: ENC[AES256_GCM,data:rn+I0LF7wC1S,iv:ZMo/Xczw+tpwcoAEzPoETDxQyO6/RrVwr4Pg9g5+CTA=,tag:oLM4cEo4RS3oOXle0Ewstg==,type:str]
- - refId: ENC[AES256_GCM,data:VA==,iv:xZSjIe1OvrcKqqubS9+fF6OvFl95rEoZDcBHi1UBkgI=,tag:hI/3a/1GSSj1DyQQ0A/WDg==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:7Mhz,iv:VxPmmVe158s8Mi09LApt6ad20oFsmGY/kfsdGZx6rPU=,tag:WEIqyqZXeOQvM2Gge6ln1g==,type:int]
- to: ENC[AES256_GCM,data:nw==,iv:tgd/hiJ9C84riYmZixPhxUyelKnPbOmHUM4H2wCCOY4=,tag:NeckTn7kT04mU4m35H9Peg==,type:int]
- datasourceUid: ENC[AES256_GCM,data:ESDqKsw1raY=,iv:g9AWTkGyeMreQiikZOBwS1F2prvFQ4E4tApZrdwXFuU=,tag:hnWIRv6o7Ila+8W/zLtT7A==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:QA==,iv:dTiD/POS6LgkzCC2q6/vFQOvXLB7oBiKsCjWGSTPIVM=,tag:fDROejJgOCc3329dBVck7Q==,type:int]
- - ENC[AES256_GCM,data:Xg==,iv:3gfou7YbHRwyGkI6uZ8HwwZG72vLt2vDZkq6H2htka4=,tag:zJ0ZBhiDe5bBM6WlLDIAnA==,type:int]
- type: ENC[AES256_GCM,data:70M=,iv:b9IbvGdkJP9GolbPI+0Bbf37ifV/zcBa3ojE+GzvaDk=,tag:VJGdjOCn3L+qI+b2MjEmeg==,type:str]
- operator:
- type: ENC[AES256_GCM,data:o5qn,iv:H7A6fX0EPx4tdenfbBkOj0EMiDt7iGWu0mYGl75iy4U=,tag:IG3195gXRuvvVVQqDp2PVA==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:f64Q,iv:u83doNd/fdHQjHmW/P+nf72YLfOJ227J6NaX8+/25bw=,tag:QijR1oD1k3QJlog7vsxO2A==,type:str]
- type: ENC[AES256_GCM,data:tf5+xrE=,iv:B94zzqxDXF0vpbjqOTKdEo62y1s8mvc+rwIplY0grJU=,tag:g4YzwmGRY/9ZHNDa6Z7XCA==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:sJR6kjHwfsGwMw==,iv:f6WGU0d7Xf+txQ/TR5lc7cpbMqljk5jLf4jTuVpMMFc=,tag:DvsN71oNQ2dSEpK3rgYrZQ==,type:str]
- type: ENC[AES256_GCM,data:SXariJ1oMEI=,iv:TOQLj98cDdOowRXxs8pfIQpAr6aIFjDpkufvTQiAfL8=,tag:ihmGUw76PWSS3kMXOEtxHw==,type:str]
- uid: ENC[AES256_GCM,data:XL4Bp6tMfZs=,iv:vizPS4C98/sfOW7YlQJ5RSbUkDMtJb2xVZit8tZBilA=,tag:Oeags11yDXEY8YEYBFkcIA==,type:str]
- expression: ENC[AES256_GCM,data:u1tawGI=,iv:gh1ceLTbwJyd7/dXIPXGd0u3sFtrxzE+aRW4TgzbL7g=,tag:wke1ojaslI+Cq/4gg1JDRw==,type:str]
- intervalMs: ENC[AES256_GCM,data:c5x6jw==,iv:G3nRX4f2csdxDv9Ki1c5NIpHAmuY0WWbZTcPK1M6eoM=,tag:Ujv9r55Sr0wY/26MaUNj2Q==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:cTDVStw=,iv:5oNAUZExpJEI+pxTSjqaWroE5KDOQhID6hwMvujR5qM=,tag:YcoBVkdtMg4Do8lEBkGjmQ==,type:int]
- reducer: ENC[AES256_GCM,data:PKascA==,iv:TZHRE1Ap/klHjEcsmw712ZUdYkawAuNZBuE5DjtOVTA=,tag:/KP7n3SioyDPmobFU2xRlw==,type:str]
- refId: ENC[AES256_GCM,data:Ig==,iv:zREl4ZsbP5onisLiCEHNXyiEq7C2rXsHkSPJFv8DJjg=,tag:B3N5wz5MnY53pjvtL7dTPQ==,type:str]
- settings:
- mode: ENC[AES256_GCM,data:jF6u3xb19dsj,iv:092bMWTbCODLwds7toybiXdYwjvJlj6fUBnyEgo0Rmc=,tag:fMnrDbMe0vd6i/w4Lp0EOQ==,type:str]
- replaceWithValue: ENC[AES256_GCM,data:6A==,iv:kIVrHgsZChlZ1cZoATQjfTqEzI3Mvz2QG+SOINlFlGg=,tag:9WbVim9dwT6tLaNgwfuFoQ==,type:int]
- type: ENC[AES256_GCM,data:fguf340B,iv:bYCKR9vuevo9LOh9aOWiUuL8WB7T8xZuRlzO4v2OV2k=,tag:vmC4GmckHYI9jxjKygqOtw==,type:str]
- noDataState: OK
- execErrState: Error
- for: 2m
- annotations:
- summary: Matrix Database rolled over but synapse crashed
- - uid: d16dc343-31d9-466a-bdc1-c2466b0bc18f
- title: Unusual synapse logging
- condition: Thresh
- isPaused: false
- data:
- - refId: ENC[AES256_GCM,data:TnX9nLA=,iv:rDWVYrCGM3y4gBhaB6t2eOYz2hDP8IEkFosNdDXq4Z4=,tag:9o3aIciPsF0y4OEpttNE5g==,type:str]
- queryType: ENC[AES256_GCM,data:AVT7wDI=,iv:8MqhFTshNnJXNpU8xAiczGp9gRa2M7/dZNcxNBQSY6Y=,tag:EQpGgbA2tz5GSkitTQcXyg==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:Cpi9,iv:Pyl/i1MIi0Ie5namY03zzMKgKZ1rNBunbOLmRrGO4n8=,tag:kCwYPTBxjAK6g99197FU/Q==,type:int]
- to: ENC[AES256_GCM,data:sw==,iv:TJ3RdwgLewt/45jXNLsvlgs+0lVGufRpXMdkgANagPw=,tag:DZMovwgwD7hULRWsE0u73A==,type:int]
- datasourceUid: ENC[AES256_GCM,data:0404mA==,iv:zdYFYmGBNnKpJX1KApQRVKDF1GmcVKuOYB/2f9XDpY4=,tag:is+xgoBRlWLfY1M2eA1W+Q==,type:str]
- model:
- datasource:
- type: ENC[AES256_GCM,data:Dz9KoQ==,iv:A0+EmUS45SRTRr5RkDNkWqxPtIygNYV8NlOTLlCSbTw=,tag:x82CdlF2jUyqNUj+1+kz2A==,type:str]
- uid: ENC[AES256_GCM,data:50n3hg==,iv:+hK4R0hv0YSIBFFz3kotSAcIZoq8A6LKPg1lDjbqfvI=,tag:VfXQxq2Tmp5U1iCyF75goQ==,type:str]
- editorMode: ENC[AES256_GCM,data:5aO9S/l0cQ==,iv:lfI4s7OltPUK9bwOaOCcrt7NlnEFnS0oXnjaQYe1mYI=,tag:FOgc+WpH9nOuVCZoyIGr4Q==,type:str]
- expr: ENC[AES256_GCM,data:i908l0/zXzHvm6JZ6/I9nQgenaguBu5YUT4gq7NI9v8W9Mv5C/Y1eNJ5SpkY6lBdLfoUPg3lNMzJ/Sy8UPm8yA9ctnp0Z+O4GsUuwmjf0sv0akdYPtdrSK+EfFSjdFFuQ9Ay4xLTqVkmcAyWvsyQqnONU39PhDHpwpVHoUlTsLnUW9h+KuCXqckuvhpK1hT9ib0xfoDclUsmimJ5be50753Ff1O1OeLWL12DNS8=,iv:XsxWlmT36GL/oJVBff36fNAiDWN5FnTzXCJrsTFhQUU=,tag:vmOrMJqSePPRe0dSc1BAcw==,type:str]
- hide: ENC[AES256_GCM,data:zTHiDA4=,iv:C7EocZ9q76gM3KpHEROTncMIgy16HUNsPiN3ZXwdeg4=,tag:r8NBAyoabi88kTR1UvTl7g==,type:bool]
- intervalMs: ENC[AES256_GCM,data:LsAFTQ==,iv:jrBBeNlTZBk4ITFu/42Lsv9FusB4fckxPmF/zJ17RCM=,tag:lJo2+f+A9cUj31dPCElVfQ==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:EFbczy0=,iv:aF9lfosfPKCntRMncgwmQ8oIp4cO3mr9eUVymKQgbTA=,tag:ZbAY6W6LqWhmiSCyzWyZDg==,type:int]
- queryType: ENC[AES256_GCM,data:6kPSsLI=,iv:NyM1A1kQwwaCWZRTkOtJLXDcxMMOSbfyT5P1WDDxWa8=,tag:RqIsjZSx8D52b5DfyIfsqg==,type:str]
- range: ENC[AES256_GCM,data:As/A2g==,iv:dx3Uyc4I+9J5tN+aqdReehmsfPGDCqXnTr53H6RLi48=,tag:7vBJJQkRk6XqBeG2Cpl0Dg==,type:bool]
- refId: ENC[AES256_GCM,data:BUswBRU=,iv:TbpHFnaxPG1mvdgCnuAtfFt9h1IAC/AACrUyyJ05xgA=,tag:TD3e5cL5qmgmFAKLnzjHfQ==,type:str]
- - refId: ENC[AES256_GCM,data:766+wLn3,iv:ei4EownA+BcmLQWsCyJJCo4AHWo0+2vjHkywSw1IxN0=,tag:hhXJ/4d88XKcsEz/nCzwaQ==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:yGuA,iv:o1+zpeAKTVyQo1uMKthzFAzC4KBfrAYQVwoRN1I8Hs4=,tag:yHAd8cBRH4287mV6L3EJHQ==,type:int]
- to: ENC[AES256_GCM,data:Kw==,iv:q2LtCK+gucZXFTdly0y1CS2X/lWzWBEmQdXY4DXKMqg=,tag:U1yO6QUr3XhgE3WtczHyUg==,type:int]
- datasourceUid: ENC[AES256_GCM,data:WEtIjChqtNQ=,iv:04cYpu1T+//HvgwLM1FssnrIYOX74NIotu6aIXT3fn0=,tag:1nzhyR8b2omM4pJ2rfnGIQ==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:QmA=,iv:2/klon7WP8V8NOVDH8TMri7x6uhJggFfGkclbJbRP6Q=,tag:WUVFcTirdWUhDP0cBsqkCw==,type:int]
- type: ENC[AES256_GCM,data:mCA=,iv:e1h98JC6sGuaA01inx9S5nI2qNoASsiDNYg4Tne8zrg=,tag:lUYPAYfZbftFEY1CgWejCg==,type:str]
- operator:
- type: ENC[AES256_GCM,data:rJH5,iv:nG3WC/RD9xxKlEfqab91ZBwA11PkfMiGkbzyfoYgB4I=,tag:qUC/0JydhpLVJqO2RLVYnQ==,type:str]
- query:
- params:
- - ENC[AES256_GCM,data:og==,iv:YlxVdb+MB4BxeM59fqPsVxcwWcxAEV8UfktJ390qCFE=,tag:TL50YFnbe29NlGEVmhWvMA==,type:str]
- reducer:
- params: []
- type: ENC[AES256_GCM,data:zECUnA==,iv:rvs9XCMfYuH6lPW/c6IlxamEFrmNlvc7IfJULjufi1w=,tag:UEJGX/ismCz40f+nkmKn5Q==,type:str]
- type: ENC[AES256_GCM,data:/xv/nJI=,iv:M4XI6qVWbRbu4LLK3oOrZXeWRSA3FcNhfEEKD785LYo=,tag:uK26RCPc6FAijn0tQ6/TnA==,type:str]
- datasource:
- type: ENC[AES256_GCM,data:pPEXmApjOeE=,iv:xiCL28oNcKlyKoPFF3P9WSdU0hbge547IyGgEkL+CtM=,tag:rDWDl9tf0I0+xOSIyOx8hA==,type:str]
- uid: ENC[AES256_GCM,data:J44zXyfNpdg=,iv:jYYh79MosB1WEE5bzWvHVQ8frfA6XjGl6vuNDiqqwV8=,tag:aW/J0r5j+fRM+X0gscnyZw==,type:str]
- expression: ENC[AES256_GCM,data:6Q==,iv:yQMoOZhBTixfQUVxzODtiZjCvVz068r2C3s05ZJvPUI=,tag:NseUTUCUkeBFQX2uBg0oyA==,type:str]
- hide: ENC[AES256_GCM,data:dO8ifNc=,iv:TgerRP0ldT4xvXZ9xBvAbsvVOlg4ImObXIkerDwo2yc=,tag:kuOdriIhtbMmQGSDkYIMHg==,type:bool]
- intervalMs: ENC[AES256_GCM,data:WDFsVg==,iv:onG5EiHABrBBb4nJI/7mjWl0nDnGzwCqNj3XbCsOyb8=,tag:/uutXtY4SzDOrxtPO+mGDA==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:oglo+Cc=,iv:Kif1qVped53PPUNkoPYQ/zN3GvZPcCZmwBZUM8zyVa0=,tag:k+/zXjCcbsbsyXmAmJ4uLg==,type:int]
- refId: ENC[AES256_GCM,data:gWnybj72,iv:oXLEgobjEPMbblhc3eYhpQlfvjzqQXCSa5GcqPkMd8s=,tag:l2Bj0J9SuMfOuu3ZJdU2CA==,type:str]
- type: ENC[AES256_GCM,data:2OqcMAeNwmss,iv:+RmH0NnWncQr1UBOFkN5spmrPLaGvn3/EaUoVI5FQOo=,tag:PPM8dovJ/IfCV8dH/6do+Q==,type:str]
- - refId: ENC[AES256_GCM,data:nQ==,iv:WozM36dtysBvKLCsn8nhIxmsicncv6jHWeeMZPo8dck=,tag:cz1M2RbiD6dQKDsnfpzkMg==,type:str]
- relativeTimeRange:
- from: ENC[AES256_GCM,data:BBzu,iv:f19NHOXFZ+VIyfFTBMWFbb4pDWfHj0oBoB1ZHqQdrVU=,tag:CCv4zXZtULK1SakYPi/n9A==,type:int]
- to: ENC[AES256_GCM,data:XQ==,iv:pN8+zcCqpSt0U9El5O0kNA2vN+GDmIN2rieZTkJHiQA=,tag:LuTfnRjNb3gxoEDlY6zYEg==,type:int]
- datasourceUid: ENC[AES256_GCM,data:DBSQWku5l3Y=,iv:6VMaRTBGBrkcvUktcVwDZ5gG7LJCbjW0QQAbRrlv+so=,tag:ZxGKBUZcF4Sd854H1Vgo3g==,type:str]
- model:
- conditions:
- - evaluator:
- params:
- - ENC[AES256_GCM,data:AA==,iv:p+7QX117VfyLt+J+g3+GylWjYvKWqjGkFU8oJ0TF0MU=,tag:gzPARIf6e4C3coUOZX2MaA==,type:int]
- - ENC[AES256_GCM,data:Sw==,iv:9PAMr7LLoNGXe2I7gXszMmTPYSrseJdm9Y5GPBSCGe4=,tag:/A6s8Zw1CfsCPhIBG0WPbQ==,type:int]
- type: ENC[AES256_GCM,data:2Vw=,iv:YYnzsGXDoR0waB8LTqaIctUR5KSPx8zPzYIYRixZmBs=,tag:BeyEzNQ9K8VEU1drMo8mbA==,type:str]
- operator:
- type: ENC[AES256_GCM,data:9fie,iv:WSUvc4fAWaYiZn+h36CxXUAMCn2qS9yUOLLdSFeuR9g=,tag:JJFQCiXoWTYTuFZeAX6Srg==,type:str]
- query:
- params: []
- reducer:
- params: []
- type: ENC[AES256_GCM,data:tQrd,iv:Q76gNv1pUKaeRLDIMlV9j5GvmeDYAfpzI4nfFU4aYtQ=,tag:+tsF2mIpJDyknPFz/xcugw==,type:str]
- type: ENC[AES256_GCM,data:xI3MiOo=,iv:J1Sxam/mk1jVQ25xAEqPDLvq+NNy0rxeCWZMmNqq4LQ=,tag:fhn6W1lBd8yh/IaQceiG+w==,type:str]
- datasource:
- name: ENC[AES256_GCM,data:6YlNRXco9kLelQ==,iv:eKEb9unr7+IU2yhcNAXrfpy6t8j0OOIEbbpZ2OQCc8E=,tag:jhsHiyk2sVkHEoeMvU/BqQ==,type:str]
- type: ENC[AES256_GCM,data:cMaOJ6CjRLc=,iv:m46oNWzSOiHoqzP5Rka12SAYjKTpokOCxCl9ZHS8IMU=,tag:yVFxhw0R3EL6Tw1A9uLNGw==,type:str]
- uid: ENC[AES256_GCM,data:xJqkFCUFwIw=,iv:yCMlfENxH97X9bVOwQ9kta82weFrRG5TzjwoDEBL7is=,tag:FyYtJ8Zt9muymvd0Yr/SVw==,type:str]
- expression: ENC[AES256_GCM,data:32VTC3o=,iv:FXa9RDx4svHa0mfdVQTvQky7bSRrzLxPO9KAqET/loY=,tag:+w0JjrTxbXuaZEaHvcHbCg==,type:str]
- intervalMs: ENC[AES256_GCM,data:Y6Vjtg==,iv:TEnrdUKUXqRhgEZodznpBhfbC/IKMgtJmXZbXznRT2g=,tag:0QuXVcVFN8lZquPsgJ5XEw==,type:int]
- maxDataPoints: ENC[AES256_GCM,data:V1LTLJ0=,iv:eLbR307AGDFrT2O4aYYbm6We/M4Au1tckX4q/dxf4zk=,tag:P0CSyFi/65XkOn6zW3eB8g==,type:int]
- reducer: ENC[AES256_GCM,data:JYuPcA==,iv:R3gQSGYY7sK28zB5U2gAsIbwSHdjYuOFmjhXqz67Qgg=,tag:Wz5oLEc5/g8Pu+Pg1RPJgA==,type:str]
- refId: ENC[AES256_GCM,data:CQ==,iv:JyM9TiUlWYEN3aG6OpWsTMTorZ0T+6bSmjNjiaQmpYg=,tag:I+7C4Hsi6d6SPhCURenhgg==,type:str]
- settings:
- mode: ENC[AES256_GCM,data:CxIqZYOu0tO4,iv:kqoBjqNpfAjRb+AeUcgnLjAKN0dP4+7rj8R2CAI2Frg=,tag:97N18DjMckJIIYTIxvMrUw==,type:str]
- replaceWithValue: ENC[AES256_GCM,data:aQ==,iv:iLTOkrnxcFDjV1n6Dc8rppzVBh1cGfGIObKJ2drdrTk=,tag:EgHIwW27W8Fpg2uD8YV/Pg==,type:int]
- type: ENC[AES256_GCM,data:XNHi4I7+,iv:comPfsH7uH7+4J2UOX2QYu0bQbrEwzTo/3oRqZtXfkg=,tag:T5oleHOARzxdf8LBWg611g==,type:str]
- noDataState: OK
- execErrState: Error
- for: 2m
- annotations:
- description: High amount of notifications which aren't INFO level
- summary: Unusual logging for synapse.
- notification_settings:
- receiver: PagerDuty Warning
+ - uid: b6db960c-c0bd-4d4f-b0f1-e1f216ec81cc
+ title: DNSQueryRefusedError
+ condition: Thresh
+ isPaused: false
+ data:
+ - refId: ENC[AES256_GCM,data:n2qY1Wg=,iv:sYOwKOsXOxiQlS8cC02y9gpDaw88wMmUbYvSvAS7y8w=,tag:w3sn1o28GnwHFBSyr3ePlA==,type:str]
+ queryType: ENC[AES256_GCM,data:AIBCGio=,iv:h+kgDoN+PCkYVrtSn3b45ZsCiUPFYqY0B2aIGz29aVY=,tag:E4uIVb1HVwbe4U5ArJD7sQ==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:VkTF,iv:oJnmXIUP4Ywd8lINMBtbk1Z7EV5/8mYbYBlTMgV6Ibo=,tag:/8etZfoVyJT5kOVwMfthvg==,type:int]
+ to: ENC[AES256_GCM,data:kQ==,iv:aElTcFOi+rM8j8PFd20I2/7/dI74oExeM+XZF+Eaf9A=,tag:uIUb1FjCOPHxzI+YhwtpbQ==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:XedhsQ==,iv:xeEuQyRofWE6seYCqLikV95v/UyxDUZPPBa2t18+szQ=,tag:oGEGYgydyrwNrWdXepS3IA==,type:str]
+ model:
+ datasource:
+ type: ENC[AES256_GCM,data:iaHiZA==,iv:8EqVnhGbnYyz9svjtGkzgq/NJZRK27l36NYZnsJzQuk=,tag:ed8rKdISPG+EsaKeA7k+sA==,type:str]
+ uid: ENC[AES256_GCM,data:cQ8HRg==,iv:MN6EsNQbRxlw/2vhes+y4SjhQoCRo2SIJ4Fdf6iONNI=,tag:odRL9ryJNk6Csshy8Mu1cw==,type:str]
+ editorMode: ENC[AES256_GCM,data:BYYaW2BpZQ==,iv:hiFudjt+Y8QhNXFuRJ5g3ZUbPmw8alsCGiKjISbYEcE=,tag:kee8f2SqwIlQL+mZv6pGOw==,type:str]
+ expr: ENC[AES256_GCM,data:1iniNKWpeeVJu7UVXvegqrDd85YLuN9NppMqiZ4g93eP6snzfUuxa6brDDlb8hLpfnIwAQQnY9y4gpn1zNh9LsnthoEpX0sZyHlvYrZ6r7g=,iv:gPgXnGv+5nn5zYs0CYAzk5kO5eHCzyzJ5sZJenu0lKs=,tag:AI84+A6Qe5++3m3fkgzesw==,type:str]
+ hide: ENC[AES256_GCM,data:dsrwQKI=,iv:pCc61Hx1AgLa2wNQl6YT8SspjlV50rUJVkhfAPZm34g=,tag:4v5r5nnviLdx49aTE9/d5g==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:SMhyCQ==,iv:qJUb8uvOR5Wtt8p8lnhtjX6nrpRcXHztEhDsDafiwd8=,tag:C9OYwIGNF5OJbMhY+IvcAw==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:gt8WzaU=,iv:m2EmNTMbKOTVA16yq4fbGoVIG+dS8ZqtrCZ28Iefny0=,tag:j2gNp/ltjByWOAxBXi/IWw==,type:int]
+ queryType: ENC[AES256_GCM,data:SINdqqE=,iv:Ksx+ki1x9+EtBqEmEoQ3ZqryjYcoVhzJtMC6I56ClQI=,tag:exdV0JRXJiKyBuhi90BNFw==,type:str]
+ range: ENC[AES256_GCM,data:wpqEZA==,iv:86p3KPSzNFieGbV3p8xPrXvK2p6x6bWmynHWPg4k+cI=,tag:DWtVgycsDThR9rR49DqRXA==,type:bool]
+ refId: ENC[AES256_GCM,data:G2iXR4Y=,iv:JpZ3Zzw5k5SqcH7IiFkGheWo07tEQ1yuELd5uVcmgb0=,tag:vuHvFNwe8TXJmB1geLrkLA==,type:str]
+ - refId: ENC[AES256_GCM,data:prlO0rGf,iv:XFd0axSyGQ30Nyl54kvU2QOBZIe6ia1wy0hct5y5jac=,tag:soX9eRM3ayy4L/y4tF/www==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:baNG,iv:7hke+4A31L6q4FwnWLojEvHLQbCASIp0ilnCo9wp+BA=,tag:vbjkpphxie++De2FkKwepw==,type:int]
+ to: ENC[AES256_GCM,data:XA==,iv:ms18Skjz+fFfo5x1AtsOhLj6C5zLDj6exAdIYL++1/I=,tag:W0uGdPJJncaK1IxHmP5I4g==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:AVKndMI/hVw=,iv:rJNMC8eykAu6LyFE5kXwRxNSstPTjMkTrjxdyXRyvRk=,tag:X1m+vWlqkJRUdAE0ieQ5mQ==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:2g==,iv:iEq+CjjrIH2FqbYoXJf+bLUf/1WrvloUVhbseEkI5o4=,tag:TyVWrrePOro9b2UxaviJJA==,type:int]
+ type: ENC[AES256_GCM,data:YJs=,iv:IiS2rO1FOaL75JO2Ryi8yfC8jH/AHl+dCpW1QG8GlQE=,tag:STe5F4gT3fKoWhe+SeITOw==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:kvpK,iv:5ENT9LxMg/qlQMz40Rwcpg+fAt/iMdF867Yqf6Ll2xo=,tag:JYW4+1vZE6WfrIxcEv/vew==,type:str]
+ query:
+ params:
+ - ENC[AES256_GCM,data:xg==,iv:vWQT+OTTlCpWAqEmse/MpjEqN29wEsxEzfYBHcxrpPY=,tag:Swomqyhlh0u9kljrUJa11Q==,type:str]
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:3LGqQw==,iv:+CaRA5tchRbGvzKxujPq3v9WnoqRHrceod0P2kjpg8I=,tag:1ZA6gvpnaBcyPHJxZcZ1VA==,type:str]
+ type: ENC[AES256_GCM,data:if/qmso=,iv:+KzRSND5MGS7gRgM4XFo9+kvtnGjosFxAkMNhs/XgyM=,tag:NTNYJy9Tvpzw2aKKohAUTQ==,type:str]
+ datasource:
+ type: ENC[AES256_GCM,data:6I5nCrcJZds=,iv:D5J8QYOocNtV5gomwXZj4tDmRV74NhblIDZbqpy6Q74=,tag:kS6BNeimn3JUaY8LnB9+Fg==,type:str]
+ uid: ENC[AES256_GCM,data:89V12WI5Z0o=,iv:RarrRZW1qqWJ8XC1wG0Tlo0Kpgbf2xVBgxAsuL0eM+Q=,tag:T68zMMRcPmNw82oSQPzyQQ==,type:str]
+ expression: ENC[AES256_GCM,data:vA==,iv:Z0RYQC25dPeBZRp6f4Q1dJki4nCh5nMk8pbaSJA1Hjg=,tag:mWVbVem6Cc5r8AYaHyflGA==,type:str]
+ hide: ENC[AES256_GCM,data:AgqJaEA=,iv:xYTzA7NVwA6XCKb8TObjAwcDlvpdaE7U+JW5ODd3mWM=,tag:fXMfH/5jRntSQEAj9BBwxA==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:Kf792A==,iv:Nhgq1ewa6O0XV/oEGol3m1GQJ0TSIzv4sgm5lS8eioc=,tag:v/Ck+Al5/7dcKbRJnZHrUg==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:+F2Aou0=,iv:YGMSqKmHdfp3JE350esXcW5WZ72roSjdQJ+wNumOA8M=,tag:pW+up4mx0UNPIU/3UaYplw==,type:int]
+ refId: ENC[AES256_GCM,data:50jXpZrw,iv:JoIw/rhWSGM6U6nkz4pTXPiDjGigH62Pr58f3kFsEgw=,tag:Ur4zm3Ma9Gl8syo2KhyHJw==,type:str]
+ type: ENC[AES256_GCM,data:7mLuZN+29yK6,iv:qjf1cRBffTm37UwP46z28SysYplrStfX4lD3EBg3Oas=,tag:F4e7PhcEh5GzRfwHi7N0hA==,type:str]
+ - refId: ENC[AES256_GCM,data:TQ==,iv:ltQGL9uedpc/hozdWfYUuGdq34yD7NAnt62SxYy3Txw=,tag:SOQZa5wDw3tYhVyVr5uMBg==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:OuFZ,iv:BnsJztLHvRqpu3EPWbjYiXy5cXTYUmz+oY3r3MxKVns=,tag:RUDedI87n/juMCqGU6UMPA==,type:int]
+ to: ENC[AES256_GCM,data:dA==,iv:LewqOAETjwqABCoOaERBSNfuqcmnk7lcN0ov6BexBhc=,tag:uLm2LEHfHV7G4VeRBPhf+w==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:Vz+ive5+eQs=,iv:HNib4DYbNcEdjPvn8S6ZVSonBI876iUO2GEyVJYXe3Y=,tag:jdA3phmSmlhfEzweQw6Zxg==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:gg==,iv:j95rImkfb8zh//TSigqmXIegLU/34aMdnbfL438LN90=,tag:ZwjvLMArZyZHeAvbMoGOfg==,type:int]
+ - ENC[AES256_GCM,data:4w==,iv:i55T3PwXztFe5QxO1czxG6Rn+arkJ1okAViRft76KHE=,tag:DHvcgCknVFXkQbFrA4ei8A==,type:int]
+ type: ENC[AES256_GCM,data:BOQ=,iv:wzeWZlIQuNwIhlziRtF1T06SwH2ZgdnlHPAdZhON+jk=,tag:7mg4R+9bjsq2xDdqiR182A==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:Wo60,iv:111kAr7iw2uCnlODQQ2DS+4KL7syI+hIRXfwfQNsVFg=,tag:WuQL440AvDCPZ0py19YahA==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:Gref,iv:ay7tw3puWIvCNa6B7bC+gaeqQHosSLSyzcNx1CAkOLA=,tag:N8fAiNtITsM5bQG+ESaLfA==,type:str]
+ type: ENC[AES256_GCM,data:577PYQ8=,iv:RPeFEzTd2EVk6BYwdvbo43aWOt+l9WN6cUEJQI2PEz0=,tag:Q5qQ9VjEd1gBrIJagSRqEQ==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:zaZ4BQufalGrZg==,iv:isO11RNzkaUTPTyl05Rs/Yy4V4S+sGoZMwbxPD0bYSo=,tag:ZlVbrihEV+VF+Oympa51Ug==,type:str]
+ type: ENC[AES256_GCM,data:PQpX5A7PcIU=,iv:Pe1udsQsozVtTI7VfBWlhGC+w6lJejnp1tSGxAYUbfc=,tag:jt/0H8nbz6lU7vdavMpcxg==,type:str]
+ uid: ENC[AES256_GCM,data:ycvelPkoJB8=,iv:DY8peFmgozoqPlz+5VyJeHm4qO+UvvpB78lbcbGK+GI=,tag:z23V9AxT9SpS78yLC7AsBA==,type:str]
+ expression: ENC[AES256_GCM,data:NpPWdc0=,iv:Gz8WfZhHrBGUo9ysTRH+QJLyjXpZk0VCDr4lucaY66A=,tag:ipgZk9QZnf0TLscBinDZHw==,type:str]
+ intervalMs: ENC[AES256_GCM,data:zSTxYQ==,iv:EnVbBg/cfjwQIs6RiKx0h4o7/mR4GcdCYAnEvlkk2n4=,tag:oUO5QXXZFAQcVPV1F4APdw==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:4YifGXM=,iv:wniMAlxBRvAIZo9elpEIjrmIOqT1qukPRBlOOnufapw=,tag:o0F0xLM4yrp5u/ZBWTmMQQ==,type:int]
+ reducer: ENC[AES256_GCM,data:NM4iwA==,iv:WAWdz0IrDwzv5GyYCzRAk0qhtiO8ljMH5VUglcEnFbQ=,tag:jzovO81rUOijzoWizR65WA==,type:str]
+ refId: ENC[AES256_GCM,data:EQ==,iv:0/WF1sC2U+nuFzPEVTFWHEuj8bezsnkzniIHk52uCZI=,tag:st2lXOcD4s/ySfhxs1DGog==,type:str]
+ settings:
+ mode: ENC[AES256_GCM,data:bzY18OLDkcji,iv:ggrP8HMLyWWPuok7sem3xGlUfkrbITxfompa27lGUQ8=,tag:K3iFdhKHEB8c+MV55eZLQw==,type:str]
+ replaceWithValue: ENC[AES256_GCM,data:ng==,iv:1/cJgN7rN/AtxeHdaQYyLL9roLbCkE5jPbb4VzG505Q=,tag:fi24/eFpKZ6hLWHPIUgGrg==,type:int]
+ type: ENC[AES256_GCM,data:ghubSnCw,iv:0i0VDzKv75uYCQYBtITAgUkNWl3dKd+7lzloRacM/f8=,tag:eqPIH7L6IVRzLV5Fsad0kQ==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 2m
+ annotations:
+ summary: Matrix Synapse DNS Errors are higher than usual
+ notification_settings:
+ receiver: PagerDuty Warning
+ - uid: de30a1da-84da-45f4-a4a5-d641dbd92e6c
+ title: psycopg2.errors.ReadOnlySqlTransaction
+ condition: Thresh
+ isPaused: false
+ data:
+ - refId: ENC[AES256_GCM,data:4FSfD5U=,iv:TautCbBChguKdymhWNZYdqeXi+8duZUst0Qw8WUJQaM=,tag:3ud5sXVi88fZtqBbHKaTNA==,type:str]
+ queryType: ENC[AES256_GCM,data:KyYwgWA=,iv:DGJBE2+Zv2i/3yN6n25E5fIClpNzUk0vufv0/0GBfHY=,tag:nTs9G+cU0Nay0EjC3jgYTg==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:aHG6,iv:vLd/1wiN6zAiOfHzbB6qoIwHPiLqnHMNvf66jMuqzs0=,tag:/Xp+jDJA/7vrKqJt7c85Fg==,type:int]
+ to: ENC[AES256_GCM,data:6g==,iv:RTQky52PK49d/ZGe6T8D2Cd4JuAMeRcc+HurjpXclPo=,tag:TPKXBwfCQPgs3vOblSZUmw==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:HbMyIQ==,iv:cLwtDeaIaTNOs9/7wFwARLYp8PlY/vawr9chkxAjn+U=,tag:Hy1/aAe6cmd/Nm36E9CXyQ==,type:str]
+ model:
+ datasource:
+ type: ENC[AES256_GCM,data:tFq3sg==,iv:8DMqD9nbilRGLcSnHnHUb2zgTv277CuxyEh/3pGQvK4=,tag:hHqBj71I1wai3aFqYjHYAg==,type:str]
+ uid: ENC[AES256_GCM,data:ANATww==,iv:Fx2CnLqCt18Vv9hWUu+msqknTCQv0ePRuhLYnPBL66k=,tag:zQd+AtVRZK6cmzPmjiAMuw==,type:str]
+ editorMode: ENC[AES256_GCM,data:iFy9Pvxkxw==,iv:PnBHzC58MO81F6oWyNHHZbxiC0yTPSaZSjgRuKtNE2U=,tag:3u6eXWKxwFAFKLGZImVPBA==,type:str]
+ expr: ENC[AES256_GCM,data:GKI16rW+IyyVlnOy68g83Q+zumxlJg/YBMHuBWmoYcH0tO8ErjSuB67YxtYRhqzm9/2QeaS605Svydc3CCiMB+YpUGH374WnlXTZ69B2R1YRX9mifBAP1XJ9PyGRdSu/6M0=,iv:+VMbTi5trvHtdJ8YiJXF6FjYD6cEN+WnplmXf/4hZAk=,tag:D3nedJTHZi5lB3mmC6Qbcg==,type:str]
+ hide: ENC[AES256_GCM,data:97FTMoI=,iv:na1iIDStxGlGPHYXUeE/5+q0qcg7ixtCGJdsio9hlEw=,tag:n0iHDpwISoSxhLsfb/C2HQ==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:xO4rUw==,iv:hubLsNnHpTyxtgdxq7QxT6Yi+I2RUPXaJhIPr2M/vdE=,tag:9kFUxZaV4+F5GmG5Qns7xw==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:ybwEjcA=,iv:j1XydTxbjCUEyWIAVXA8p+VQ1QRlDPJue8xdvsH2LN8=,tag:ejo3LhbjAYf19D1VMAoqdw==,type:int]
+ queryType: ENC[AES256_GCM,data:ZXW3kk8=,iv:AZboPCZpFSPA/OKks20hryW8dcz9ReAGBYnxfJoTfQo=,tag:ng14XH5Ffy58/ADePPtrpQ==,type:str]
+ range: ENC[AES256_GCM,data:VegTkw==,iv:JFc8Z8y7y/3kd831TQGqrDjAS14KOmuRpZGLy8j0w00=,tag:sPsohCGKbqzxkLrBS8psdA==,type:bool]
+ refId: ENC[AES256_GCM,data:/zhq6T0=,iv:gJy8Pn6C/hoj6Omase/Wsz/fpnr4i2A5UBLUdnAy0hw=,tag:IjsnEz6ZaNH5CjhqAH+CyQ==,type:str]
+ - refId: ENC[AES256_GCM,data:HrVyezUv,iv:fd/VVYR9rxWj3vxhJq9vcM507HqmawzkUXOE37m/4Q0=,tag:Me69F6RFE45qSaXZ2/T2CA==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:lMx+,iv:AhOYBrR0DIfaPOQtWN57uzy6FyIBpTqZqiY2ZzFsvYc=,tag:L7iIQZgGyELpexvRt5U1dw==,type:int]
+ to: ENC[AES256_GCM,data:hw==,iv:dJMnsAXqMTtbfdNH1X6rjgYPrUhs7IPJEkGyWo677ck=,tag:h87xGPHgmMgV+U2VV9Q5Ow==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:V5lGTa9mkqM=,iv:4t/FDUw7e6pTsKjYzAny+u26Ad6PVByPArE6NE0GL6o=,tag:TIMEZpkC3qKiiYGtztH5Vw==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:Waro,iv:LZOx3ypE1D+PABuOFnlmGSfZfqQKLLJQbKk01VEAmNA=,tag:+1eV5fErxkCK5g5SAHoAyQ==,type:float]
+ type: ENC[AES256_GCM,data:KKg=,iv:rMh9jiyB+ufx7LAP69M/TXMx20vZxtsP3Zw365qHvX0=,tag:ksfE/fp8tmaK1qu4IoGYaA==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:SLHL,iv:B/l+zaIqyco+zFbAh4ak335LM2cDNasK+qh+FO96XZI=,tag:wYBbzvZZdBDxrphC8LzexA==,type:str]
+ query:
+ params:
+ - ENC[AES256_GCM,data:Rg==,iv:xx7hkEN6kfTHnG8Z0Tkab7Q133JX1sq8R9TI5Os2KnA=,tag:08liAjId4ihIjWMpz8ZoUg==,type:str]
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:xQza6A==,iv:M2yYpJyPf/3/DlCgjgpYjl9ULJLbJK+c1C2yi96GuZM=,tag:6DTKw4lWZq81Nn1GpM6M2A==,type:str]
+ type: ENC[AES256_GCM,data:ITLK+DI=,iv:+LvXcHHb+PtOrdVEoOyhBtoVJbXb3Fzo4Rp2WY2GzBs=,tag:3DyuzRCG7BX/D2Ghb5tSsw==,type:str]
+ datasource:
+ type: ENC[AES256_GCM,data:deiC5blqCR4=,iv:0JBiF6c5RSP1Mw+CbawhjI+/hFb/zOOjepvyL+1J9+0=,tag:4wOpRNiaK2k4V/hjMh4khQ==,type:str]
+ uid: ENC[AES256_GCM,data:bkVg07YkgNQ=,iv:ucZ4QAb9GkbvbXpVsVORFLhIZ3LfciXhukpTP9s/li8=,tag:NmZOCY6WjCDt3J1Vg5i44w==,type:str]
+ expression: ENC[AES256_GCM,data:zw==,iv:d+2Y+ZpvJ5sjrnkZPAnO9rgIuB/krtpSp1uOemvjfVg=,tag:UqxbFxry6LsKDS4NUVErVw==,type:str]
+ hide: ENC[AES256_GCM,data:gId2YjE=,iv:JBJUJXI4raA36lskJl/l/5KoItV9gmxXP4sKW+EBoPw=,tag:k6Ou//hspQa8yriZ1hEoMA==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:bhJC7A==,iv:2y+iwbjOyIxQPBXX97igQ/x2iY8F8zODmy9FuTBN0Gc=,tag:PVNsVcu/naABRIrNNJsvQA==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:AO5rVt0=,iv:GTAlhdENQfJy60kPPLdL46RKKV7xs+Vjg1iYArNjsfI=,tag:qVp2y+t+cxW7vPJ8furC+g==,type:int]
+ refId: ENC[AES256_GCM,data:TgK0xbph,iv:4W6EFZGuOlOaQp89wXZ0+OaYQTVzGEP7NICEUa33f98=,tag:lqor32BDaugBfXBQKpVVFw==,type:str]
+ type: ENC[AES256_GCM,data:rn+I0LF7wC1S,iv:ZMo/Xczw+tpwcoAEzPoETDxQyO6/RrVwr4Pg9g5+CTA=,tag:oLM4cEo4RS3oOXle0Ewstg==,type:str]
+ - refId: ENC[AES256_GCM,data:VA==,iv:xZSjIe1OvrcKqqubS9+fF6OvFl95rEoZDcBHi1UBkgI=,tag:hI/3a/1GSSj1DyQQ0A/WDg==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:7Mhz,iv:VxPmmVe158s8Mi09LApt6ad20oFsmGY/kfsdGZx6rPU=,tag:WEIqyqZXeOQvM2Gge6ln1g==,type:int]
+ to: ENC[AES256_GCM,data:nw==,iv:tgd/hiJ9C84riYmZixPhxUyelKnPbOmHUM4H2wCCOY4=,tag:NeckTn7kT04mU4m35H9Peg==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:ESDqKsw1raY=,iv:g9AWTkGyeMreQiikZOBwS1F2prvFQ4E4tApZrdwXFuU=,tag:hnWIRv6o7Ila+8W/zLtT7A==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:QA==,iv:dTiD/POS6LgkzCC2q6/vFQOvXLB7oBiKsCjWGSTPIVM=,tag:fDROejJgOCc3329dBVck7Q==,type:int]
+ - ENC[AES256_GCM,data:Xg==,iv:3gfou7YbHRwyGkI6uZ8HwwZG72vLt2vDZkq6H2htka4=,tag:zJ0ZBhiDe5bBM6WlLDIAnA==,type:int]
+ type: ENC[AES256_GCM,data:70M=,iv:b9IbvGdkJP9GolbPI+0Bbf37ifV/zcBa3ojE+GzvaDk=,tag:VJGdjOCn3L+qI+b2MjEmeg==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:o5qn,iv:H7A6fX0EPx4tdenfbBkOj0EMiDt7iGWu0mYGl75iy4U=,tag:IG3195gXRuvvVVQqDp2PVA==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:f64Q,iv:u83doNd/fdHQjHmW/P+nf72YLfOJ227J6NaX8+/25bw=,tag:QijR1oD1k3QJlog7vsxO2A==,type:str]
+ type: ENC[AES256_GCM,data:tf5+xrE=,iv:B94zzqxDXF0vpbjqOTKdEo62y1s8mvc+rwIplY0grJU=,tag:g4YzwmGRY/9ZHNDa6Z7XCA==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:sJR6kjHwfsGwMw==,iv:f6WGU0d7Xf+txQ/TR5lc7cpbMqljk5jLf4jTuVpMMFc=,tag:DvsN71oNQ2dSEpK3rgYrZQ==,type:str]
+ type: ENC[AES256_GCM,data:SXariJ1oMEI=,iv:TOQLj98cDdOowRXxs8pfIQpAr6aIFjDpkufvTQiAfL8=,tag:ihmGUw76PWSS3kMXOEtxHw==,type:str]
+ uid: ENC[AES256_GCM,data:XL4Bp6tMfZs=,iv:vizPS4C98/sfOW7YlQJ5RSbUkDMtJb2xVZit8tZBilA=,tag:Oeags11yDXEY8YEYBFkcIA==,type:str]
+ expression: ENC[AES256_GCM,data:u1tawGI=,iv:gh1ceLTbwJyd7/dXIPXGd0u3sFtrxzE+aRW4TgzbL7g=,tag:wke1ojaslI+Cq/4gg1JDRw==,type:str]
+ intervalMs: ENC[AES256_GCM,data:c5x6jw==,iv:G3nRX4f2csdxDv9Ki1c5NIpHAmuY0WWbZTcPK1M6eoM=,tag:Ujv9r55Sr0wY/26MaUNj2Q==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:cTDVStw=,iv:5oNAUZExpJEI+pxTSjqaWroE5KDOQhID6hwMvujR5qM=,tag:YcoBVkdtMg4Do8lEBkGjmQ==,type:int]
+ reducer: ENC[AES256_GCM,data:PKascA==,iv:TZHRE1Ap/klHjEcsmw712ZUdYkawAuNZBuE5DjtOVTA=,tag:/KP7n3SioyDPmobFU2xRlw==,type:str]
+ refId: ENC[AES256_GCM,data:Ig==,iv:zREl4ZsbP5onisLiCEHNXyiEq7C2rXsHkSPJFv8DJjg=,tag:B3N5wz5MnY53pjvtL7dTPQ==,type:str]
+ settings:
+ mode: ENC[AES256_GCM,data:jF6u3xb19dsj,iv:092bMWTbCODLwds7toybiXdYwjvJlj6fUBnyEgo0Rmc=,tag:fMnrDbMe0vd6i/w4Lp0EOQ==,type:str]
+ replaceWithValue: ENC[AES256_GCM,data:6A==,iv:kIVrHgsZChlZ1cZoATQjfTqEzI3Mvz2QG+SOINlFlGg=,tag:9WbVim9dwT6tLaNgwfuFoQ==,type:int]
+ type: ENC[AES256_GCM,data:fguf340B,iv:bYCKR9vuevo9LOh9aOWiUuL8WB7T8xZuRlzO4v2OV2k=,tag:vmC4GmckHYI9jxjKygqOtw==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 2m
+ annotations:
+ summary: Matrix Database rolled over but synapse crashed
+ - uid: d16dc343-31d9-466a-bdc1-c2466b0bc18f
+ title: Unusual synapse logging
+ condition: Thresh
+ isPaused: false
+ data:
+ - refId: ENC[AES256_GCM,data:TnX9nLA=,iv:rDWVYrCGM3y4gBhaB6t2eOYz2hDP8IEkFosNdDXq4Z4=,tag:9o3aIciPsF0y4OEpttNE5g==,type:str]
+ queryType: ENC[AES256_GCM,data:AVT7wDI=,iv:8MqhFTshNnJXNpU8xAiczGp9gRa2M7/dZNcxNBQSY6Y=,tag:EQpGgbA2tz5GSkitTQcXyg==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:Cpi9,iv:Pyl/i1MIi0Ie5namY03zzMKgKZ1rNBunbOLmRrGO4n8=,tag:kCwYPTBxjAK6g99197FU/Q==,type:int]
+ to: ENC[AES256_GCM,data:sw==,iv:TJ3RdwgLewt/45jXNLsvlgs+0lVGufRpXMdkgANagPw=,tag:DZMovwgwD7hULRWsE0u73A==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:0404mA==,iv:zdYFYmGBNnKpJX1KApQRVKDF1GmcVKuOYB/2f9XDpY4=,tag:is+xgoBRlWLfY1M2eA1W+Q==,type:str]
+ model:
+ datasource:
+ type: ENC[AES256_GCM,data:Dz9KoQ==,iv:A0+EmUS45SRTRr5RkDNkWqxPtIygNYV8NlOTLlCSbTw=,tag:x82CdlF2jUyqNUj+1+kz2A==,type:str]
+ uid: ENC[AES256_GCM,data:50n3hg==,iv:+hK4R0hv0YSIBFFz3kotSAcIZoq8A6LKPg1lDjbqfvI=,tag:VfXQxq2Tmp5U1iCyF75goQ==,type:str]
+ editorMode: ENC[AES256_GCM,data:5aO9S/l0cQ==,iv:lfI4s7OltPUK9bwOaOCcrt7NlnEFnS0oXnjaQYe1mYI=,tag:FOgc+WpH9nOuVCZoyIGr4Q==,type:str]
+ expr: ENC[AES256_GCM,data:i908l0/zXzHvm6JZ6/I9nQgenaguBu5YUT4gq7NI9v8W9Mv5C/Y1eNJ5SpkY6lBdLfoUPg3lNMzJ/Sy8UPm8yA9ctnp0Z+O4GsUuwmjf0sv0akdYPtdrSK+EfFSjdFFuQ9Ay4xLTqVkmcAyWvsyQqnONU39PhDHpwpVHoUlTsLnUW9h+KuCXqckuvhpK1hT9ib0xfoDclUsmimJ5be50753Ff1O1OeLWL12DNS8=,iv:XsxWlmT36GL/oJVBff36fNAiDWN5FnTzXCJrsTFhQUU=,tag:vmOrMJqSePPRe0dSc1BAcw==,type:str]
+ hide: ENC[AES256_GCM,data:zTHiDA4=,iv:C7EocZ9q76gM3KpHEROTncMIgy16HUNsPiN3ZXwdeg4=,tag:r8NBAyoabi88kTR1UvTl7g==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:LsAFTQ==,iv:jrBBeNlTZBk4ITFu/42Lsv9FusB4fckxPmF/zJ17RCM=,tag:lJo2+f+A9cUj31dPCElVfQ==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:EFbczy0=,iv:aF9lfosfPKCntRMncgwmQ8oIp4cO3mr9eUVymKQgbTA=,tag:ZbAY6W6LqWhmiSCyzWyZDg==,type:int]
+ queryType: ENC[AES256_GCM,data:6kPSsLI=,iv:NyM1A1kQwwaCWZRTkOtJLXDcxMMOSbfyT5P1WDDxWa8=,tag:RqIsjZSx8D52b5DfyIfsqg==,type:str]
+ range: ENC[AES256_GCM,data:As/A2g==,iv:dx3Uyc4I+9J5tN+aqdReehmsfPGDCqXnTr53H6RLi48=,tag:7vBJJQkRk6XqBeG2Cpl0Dg==,type:bool]
+ refId: ENC[AES256_GCM,data:BUswBRU=,iv:TbpHFnaxPG1mvdgCnuAtfFt9h1IAC/AACrUyyJ05xgA=,tag:TD3e5cL5qmgmFAKLnzjHfQ==,type:str]
+ - refId: ENC[AES256_GCM,data:766+wLn3,iv:ei4EownA+BcmLQWsCyJJCo4AHWo0+2vjHkywSw1IxN0=,tag:hhXJ/4d88XKcsEz/nCzwaQ==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:yGuA,iv:o1+zpeAKTVyQo1uMKthzFAzC4KBfrAYQVwoRN1I8Hs4=,tag:yHAd8cBRH4287mV6L3EJHQ==,type:int]
+ to: ENC[AES256_GCM,data:Kw==,iv:q2LtCK+gucZXFTdly0y1CS2X/lWzWBEmQdXY4DXKMqg=,tag:U1yO6QUr3XhgE3WtczHyUg==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:WEtIjChqtNQ=,iv:04cYpu1T+//HvgwLM1FssnrIYOX74NIotu6aIXT3fn0=,tag:1nzhyR8b2omM4pJ2rfnGIQ==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:QmA=,iv:2/klon7WP8V8NOVDH8TMri7x6uhJggFfGkclbJbRP6Q=,tag:WUVFcTirdWUhDP0cBsqkCw==,type:int]
+ type: ENC[AES256_GCM,data:mCA=,iv:e1h98JC6sGuaA01inx9S5nI2qNoASsiDNYg4Tne8zrg=,tag:lUYPAYfZbftFEY1CgWejCg==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:rJH5,iv:nG3WC/RD9xxKlEfqab91ZBwA11PkfMiGkbzyfoYgB4I=,tag:qUC/0JydhpLVJqO2RLVYnQ==,type:str]
+ query:
+ params:
+ - ENC[AES256_GCM,data:og==,iv:YlxVdb+MB4BxeM59fqPsVxcwWcxAEV8UfktJ390qCFE=,tag:TL50YFnbe29NlGEVmhWvMA==,type:str]
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:zECUnA==,iv:rvs9XCMfYuH6lPW/c6IlxamEFrmNlvc7IfJULjufi1w=,tag:UEJGX/ismCz40f+nkmKn5Q==,type:str]
+ type: ENC[AES256_GCM,data:/xv/nJI=,iv:M4XI6qVWbRbu4LLK3oOrZXeWRSA3FcNhfEEKD785LYo=,tag:uK26RCPc6FAijn0tQ6/TnA==,type:str]
+ datasource:
+ type: ENC[AES256_GCM,data:pPEXmApjOeE=,iv:xiCL28oNcKlyKoPFF3P9WSdU0hbge547IyGgEkL+CtM=,tag:rDWDl9tf0I0+xOSIyOx8hA==,type:str]
+ uid: ENC[AES256_GCM,data:J44zXyfNpdg=,iv:jYYh79MosB1WEE5bzWvHVQ8frfA6XjGl6vuNDiqqwV8=,tag:aW/J0r5j+fRM+X0gscnyZw==,type:str]
+ expression: ENC[AES256_GCM,data:6Q==,iv:yQMoOZhBTixfQUVxzODtiZjCvVz068r2C3s05ZJvPUI=,tag:NseUTUCUkeBFQX2uBg0oyA==,type:str]
+ hide: ENC[AES256_GCM,data:dO8ifNc=,iv:TgerRP0ldT4xvXZ9xBvAbsvVOlg4ImObXIkerDwo2yc=,tag:kuOdriIhtbMmQGSDkYIMHg==,type:bool]
+ intervalMs: ENC[AES256_GCM,data:WDFsVg==,iv:onG5EiHABrBBb4nJI/7mjWl0nDnGzwCqNj3XbCsOyb8=,tag:/uutXtY4SzDOrxtPO+mGDA==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:oglo+Cc=,iv:Kif1qVped53PPUNkoPYQ/zN3GvZPcCZmwBZUM8zyVa0=,tag:k+/zXjCcbsbsyXmAmJ4uLg==,type:int]
+ refId: ENC[AES256_GCM,data:gWnybj72,iv:oXLEgobjEPMbblhc3eYhpQlfvjzqQXCSa5GcqPkMd8s=,tag:l2Bj0J9SuMfOuu3ZJdU2CA==,type:str]
+ type: ENC[AES256_GCM,data:2OqcMAeNwmss,iv:+RmH0NnWncQr1UBOFkN5spmrPLaGvn3/EaUoVI5FQOo=,tag:PPM8dovJ/IfCV8dH/6do+Q==,type:str]
+ - refId: ENC[AES256_GCM,data:nQ==,iv:WozM36dtysBvKLCsn8nhIxmsicncv6jHWeeMZPo8dck=,tag:cz1M2RbiD6dQKDsnfpzkMg==,type:str]
+ relativeTimeRange:
+ from: ENC[AES256_GCM,data:BBzu,iv:f19NHOXFZ+VIyfFTBMWFbb4pDWfHj0oBoB1ZHqQdrVU=,tag:CCv4zXZtULK1SakYPi/n9A==,type:int]
+ to: ENC[AES256_GCM,data:XQ==,iv:pN8+zcCqpSt0U9El5O0kNA2vN+GDmIN2rieZTkJHiQA=,tag:LuTfnRjNb3gxoEDlY6zYEg==,type:int]
+ datasourceUid: ENC[AES256_GCM,data:DBSQWku5l3Y=,iv:6VMaRTBGBrkcvUktcVwDZ5gG7LJCbjW0QQAbRrlv+so=,tag:ZxGKBUZcF4Sd854H1Vgo3g==,type:str]
+ model:
+ conditions:
+ - evaluator:
+ params:
+ - ENC[AES256_GCM,data:AA==,iv:p+7QX117VfyLt+J+g3+GylWjYvKWqjGkFU8oJ0TF0MU=,tag:gzPARIf6e4C3coUOZX2MaA==,type:int]
+ - ENC[AES256_GCM,data:Sw==,iv:9PAMr7LLoNGXe2I7gXszMmTPYSrseJdm9Y5GPBSCGe4=,tag:/A6s8Zw1CfsCPhIBG0WPbQ==,type:int]
+ type: ENC[AES256_GCM,data:2Vw=,iv:YYnzsGXDoR0waB8LTqaIctUR5KSPx8zPzYIYRixZmBs=,tag:BeyEzNQ9K8VEU1drMo8mbA==,type:str]
+ operator:
+ type: ENC[AES256_GCM,data:9fie,iv:WSUvc4fAWaYiZn+h36CxXUAMCn2qS9yUOLLdSFeuR9g=,tag:JJFQCiXoWTYTuFZeAX6Srg==,type:str]
+ query:
+ params: []
+ reducer:
+ params: []
+ type: ENC[AES256_GCM,data:tQrd,iv:Q76gNv1pUKaeRLDIMlV9j5GvmeDYAfpzI4nfFU4aYtQ=,tag:+tsF2mIpJDyknPFz/xcugw==,type:str]
+ type: ENC[AES256_GCM,data:xI3MiOo=,iv:J1Sxam/mk1jVQ25xAEqPDLvq+NNy0rxeCWZMmNqq4LQ=,tag:fhn6W1lBd8yh/IaQceiG+w==,type:str]
+ datasource:
+ name: ENC[AES256_GCM,data:6YlNRXco9kLelQ==,iv:eKEb9unr7+IU2yhcNAXrfpy6t8j0OOIEbbpZ2OQCc8E=,tag:jhsHiyk2sVkHEoeMvU/BqQ==,type:str]
+ type: ENC[AES256_GCM,data:cMaOJ6CjRLc=,iv:m46oNWzSOiHoqzP5Rka12SAYjKTpokOCxCl9ZHS8IMU=,tag:yVFxhw0R3EL6Tw1A9uLNGw==,type:str]
+ uid: ENC[AES256_GCM,data:xJqkFCUFwIw=,iv:yCMlfENxH97X9bVOwQ9kta82weFrRG5TzjwoDEBL7is=,tag:FyYtJ8Zt9muymvd0Yr/SVw==,type:str]
+ expression: ENC[AES256_GCM,data:32VTC3o=,iv:FXa9RDx4svHa0mfdVQTvQky7bSRrzLxPO9KAqET/loY=,tag:+w0JjrTxbXuaZEaHvcHbCg==,type:str]
+ intervalMs: ENC[AES256_GCM,data:Y6Vjtg==,iv:TEnrdUKUXqRhgEZodznpBhfbC/IKMgtJmXZbXznRT2g=,tag:0QuXVcVFN8lZquPsgJ5XEw==,type:int]
+ maxDataPoints: ENC[AES256_GCM,data:V1LTLJ0=,iv:eLbR307AGDFrT2O4aYYbm6We/M4Au1tckX4q/dxf4zk=,tag:P0CSyFi/65XkOn6zW3eB8g==,type:int]
+ reducer: ENC[AES256_GCM,data:JYuPcA==,iv:R3gQSGYY7sK28zB5U2gAsIbwSHdjYuOFmjhXqz67Qgg=,tag:Wz5oLEc5/g8Pu+Pg1RPJgA==,type:str]
+ refId: ENC[AES256_GCM,data:CQ==,iv:JyM9TiUlWYEN3aG6OpWsTMTorZ0T+6bSmjNjiaQmpYg=,tag:I+7C4Hsi6d6SPhCURenhgg==,type:str]
+ settings:
+ mode: ENC[AES256_GCM,data:CxIqZYOu0tO4,iv:kqoBjqNpfAjRb+AeUcgnLjAKN0dP4+7rj8R2CAI2Frg=,tag:97N18DjMckJIIYTIxvMrUw==,type:str]
+ replaceWithValue: ENC[AES256_GCM,data:aQ==,iv:iLTOkrnxcFDjV1n6Dc8rppzVBh1cGfGIObKJ2drdrTk=,tag:EgHIwW27W8Fpg2uD8YV/Pg==,type:int]
+ type: ENC[AES256_GCM,data:XNHi4I7+,iv:comPfsH7uH7+4J2UOX2QYu0bQbrEwzTo/3oRqZtXfkg=,tag:T5oleHOARzxdf8LBWg611g==,type:str]
+ noDataState: OK
+ execErrState: Error
+ for: 2m
+ annotations:
+ description: High amount of notifications which aren't INFO level
+ summary: Unusual logging for synapse.
+ notification_settings:
+ receiver: PagerDuty Warning
contactpoints.yaml:
apiVersion: 1
contactPoints:
- orgId: 1
name: MidnightThoughts Ops
receivers:
- - uid: "1"
- name: MidnightThoughts Ops
- type: email
- sendReminder: true
- frequency: 1m
- settings:
- addresses: ops@nordgedanken.dev;mtrnord@nordgedanken.dev;mtrnord1@gmail.com
+ - uid: "1"
+ name: MidnightThoughts Ops
+ type: email
+ sendReminder: true
+ frequency: 1m
+ settings:
+ addresses: ops@nordgedanken.dev;mtrnord@nordgedanken.dev;mtrnord1@gmail.com
- orgId: 1
name: PagerDuty
receivers:
- - uid: adz30e8814wshd
- type: pagerduty
- settings:
- integrationKey: ENC[AES256_GCM,data:jQUWsSqKF5i3qL/W/PnZv8dZgWQpait/kp80hos/tHY=,iv:9yHYOdQEN8UpTeG+0s9dolx92c6rgcBfsZcFiCPayXw=,tag:p1n3qVd+dPiyEvEQU+49sA==,type:str]
- url: https://events.eu.pagerduty.com/v2/enqueue
- disableResolveMessage: false
+ - uid: adz30e8814wshd
+ type: pagerduty
+ settings:
+ integrationKey: ENC[AES256_GCM,data:jQUWsSqKF5i3qL/W/PnZv8dZgWQpait/kp80hos/tHY=,iv:9yHYOdQEN8UpTeG+0s9dolx92c6rgcBfsZcFiCPayXw=,tag:p1n3qVd+dPiyEvEQU+49sA==,type:str]
+ url: https://events.eu.pagerduty.com/v2/enqueue
+ disableResolveMessage: false
- orgId: 1
name: PagerDuty Warning
receivers:
- - uid: pg_warning
- type: pagerduty
- settings:
- integrationKey: ENC[AES256_GCM,data:vuQB+vgdMp9ZAirUjE6t5u0Mpl2NYANdJ4bLeE3d8yU=,iv:2Ypzf8W1QbIWuQArSEYf2nMUXcWpV/3JzFo0vXKvbws=,tag:IduiwiMl5YnoyKbyfuakdA==,type:str]
- severity: warning
- url: https://events.eu.pagerduty.com/v2/enqueue
- disableResolveMessage: false
+ - uid: pg_warning
+ type: pagerduty
+ settings:
+ integrationKey: ENC[AES256_GCM,data:vuQB+vgdMp9ZAirUjE6t5u0Mpl2NYANdJ4bLeE3d8yU=,iv:2Ypzf8W1QbIWuQArSEYf2nMUXcWpV/3JzFo0vXKvbws=,tag:IduiwiMl5YnoyKbyfuakdA==,type:str]
+ severity: warning
+ url: https://events.eu.pagerduty.com/v2/enqueue
+ disableResolveMessage: false
- orgId: 1
name: PagerDuty Draupnir4All
receivers:
- - uid: 3
- type: pagerduty
- settings:
- integrationKey: ENC[AES256_GCM,data:fPwsETQ46jjn3h/oVOv/X9tQSGLgO/v5weYQu1ocC5s=,iv:0zQItMs/yynhrcnSHTIOQQPZva/BTZi4ZT5jBMZtQ4E=,tag:JvYnpTu6rX1JIhiKpXhxYg==,type:str]
- url: https://events.eu.pagerduty.com/v2/enqueue
- disableResolveMessage: false
+ - uid: 3
+ type: pagerduty
+ settings:
+ integrationKey: ENC[AES256_GCM,data:fPwsETQ46jjn3h/oVOv/X9tQSGLgO/v5weYQu1ocC5s=,iv:0zQItMs/yynhrcnSHTIOQQPZva/BTZi4ZT5jBMZtQ4E=,tag:JvYnpTu6rX1JIhiKpXhxYg==,type:str]
+ url: https://events.eu.pagerduty.com/v2/enqueue
+ disableResolveMessage: false
- orgId: 1
name: PagerDuty Draupnir4All Warning
receivers:
- - uid: de02at943dxxcf
- type: pagerduty
- settings:
- integrationKey: ENC[AES256_GCM,data:MQsCflId5h+siqIXf/DQU7uYucCqVUG8eusNnLVYIUc=,iv:v9SZuTwy+Qma8GAOdw3h+g7PlGQry7eMmDF7AabPU5o=,tag:4YDWweVyou0w/K/YgzERAQ==,type:str]
- severity: warning
- url: https://events.eu.pagerduty.com/v2/enqueue
- disableResolveMessage: false
+ - uid: de02at943dxxcf
+ type: pagerduty
+ settings:
+ integrationKey: ENC[AES256_GCM,data:MQsCflId5h+siqIXf/DQU7uYucCqVUG8eusNnLVYIUc=,iv:v9SZuTwy+Qma8GAOdw3h+g7PlGQry7eMmDF7AabPU5o=,tag:4YDWweVyou0w/K/YgzERAQ==,type:str]
+ severity: warning
+ url: https://events.eu.pagerduty.com/v2/enqueue
+ disableResolveMessage: false
policies.yaml:
apiVersion: 1
policies:
- orgId: 1
receiver: PagerDuty
group_by:
- - grafana_folder
- - alertname
+ - grafana_folder
+ - alertname
routes:
- - receiver: PagerDuty
- object_matchers:
- - - service_id
- - "!="
- - draupnir4all
- - receiver: PagerDuty Draupnir4All
- object_matchers:
- - - service_id
- - =
- - draupnir4all
- - receiver: MidnightThoughts Ops
+ - receiver: PagerDuty
+ object_matchers:
+ - - service_id
+ - "!="
+ - draupnir4all
+ - receiver: PagerDuty Draupnir4All
+ object_matchers:
+ - - service_id
+ - =
+ - draupnir4all
+ - receiver: MidnightThoughts Ops
plugins:
- grafana-piechart-panel
- grafana-worldmap-panel
@@ -1213,7 +1214,7 @@ spec:
disableDeletion: false
editable: true
options:
- path: /var/lib/grafana/dashboards/default
+ path: /var/lib/grafana/dashboards/default
- name: monitoring
orgId: 1
folder: Monitoring
@@ -1223,7 +1224,7 @@ spec:
updateIntervalSeconds: 10
allowUiUpdates: false
options:
- path: /var/lib/grafana/dashboards/monitoring
+ path: /var/lib/grafana/dashboards/monitoring
- name: cluster
orgId: 1
folder: Cluster
@@ -1233,7 +1234,7 @@ spec:
updateIntervalSeconds: 10
allowUiUpdates: false
options:
- path: /var/lib/grafana/dashboards/cluster
+ path: /var/lib/grafana/dashboards/cluster
- name: matrix
orgId: 1
folder: Matrix
@@ -1243,7 +1244,7 @@ spec:
updateIntervalSeconds: 10
allowUiUpdates: false
options:
- path: /var/lib/grafana/dashboards/matrix
+ path: /var/lib/grafana/dashboards/matrix
datasources:
datasources.yaml:
apiVersion: 1
@@ -1286,10 +1287,10 @@ spec:
url: http://hubble-relay.kube-system.svc.cluster.local:80
isDefault: false
jsonData:
- serviceMap:
- datasourceUid: prometheus
- tempoDatasourceUid: tempo
- prometheusDatasourceUid: prometheus
+ serviceMap:
+ datasourceUid: prometheus
+ tempoDatasourceUid: tempo
+ prometheusDatasourceUid: prometheus
dashboards:
default:
nodeexporter:
@@ -1364,7 +1365,7 @@ spec:
tls:
- secretName: grafana.midnightthoughts.space
hosts:
- - grafana.midnightthoughts.space
+ - grafana.midnightthoughts.space
grafana.ini:
server:
root_url: https://grafana.midnightthoughts.space
@@ -1395,29 +1396,30 @@ spec:
auth_url: https://auth.midnightthoughts.space/application/o/authorize/
token_url: https://auth.midnightthoughts.space/application/o/token/
api_url: https://auth.midnightthoughts.space/application/o/userinfo/
- role_attribute_path: contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors') && 'Editor' || 'Viewer'
+ role_attribute_path: contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors')
+ && 'Editor' || 'Viewer'
imageRenderer:
enabled: true
kubeProxy:
enabled: false
postRenderers:
- kustomize:
- patches:
- - target:
- # Ignore these objects from Flux diff as they are mutated from chart hooks
- kind: (ValidatingWebhookConfiguration|MutatingWebhookConfiguration)
- name: kube-prometheus-stack-admission
- patch: |
- - op: add
- path: /metadata/annotations/helm.toolkit.fluxcd.io~1driftDetection
- value: disabled
- - target:
- # Ignore these objects from Flux diff as they are mutated at apply time but not at dry-run time
- kind: PrometheusRule
- patch: |
- - op: add
- path: /metadata/annotations/helm.toolkit.fluxcd.io~1driftDetection
- value: disabled
+ patches:
+ - target:
+ # Ignore these objects from Flux diff as they are mutated from chart hooks
+ kind: (ValidatingWebhookConfiguration|MutatingWebhookConfiguration)
+ name: kube-prometheus-stack-admission
+ patch: |
+ - op: add
+ path: /metadata/annotations/helm.toolkit.fluxcd.io~1driftDetection
+ value: disabled
+ - target:
+ # Ignore these objects from Flux diff as they are mutated at apply time but not at dry-run time
+ kind: PrometheusRule
+ patch: |
+ - op: add
+ path: /metadata/annotations/helm.toolkit.fluxcd.io~1driftDetection
+ value: disabled
sops:
kms: []
gcp_kms: []
@@ -1426,13 +1428,13 @@ sops:
age:
- recipient: age1esjyg2qfy49awv0ptkzvpk425adczjr38m37w2mmcahzc4p8n54sll2nzh
enc: |
- -----BEGIN AGE ENCRYPTED FILE-----
- YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQUnJLTmZHRDYwK1V3bnZL
- MWNvRkh6VW53VmU3VC8zN0xQZFlabW0zY3lVCk5OYzJpRkcvZjFQN3IyOGhQa3JH
- WDFJMDlOT0sxa3JnL3VnOStFRVd0dVUKLS0tIGtGaFpTRkFyOWhXYTlnSlFpVGts
- SVlmaWUxeWQvRHBaNzdwem9IMDBtV2cKG2JS1Y6aq8X8Fi7ytPpBKesrCfN1qY0s
- ZnDQaH2z9ceySHAHhvh8ETpzStHuGfmSCLGmdMpL4VqmILEQ08j1EQ==
- -----END AGE ENCRYPTED FILE-----
+ -----BEGIN AGE ENCRYPTED FILE-----
+ YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQUnJLTmZHRDYwK1V3bnZL
+ MWNvRkh6VW53VmU3VC8zN0xQZFlabW0zY3lVCk5OYzJpRkcvZjFQN3IyOGhQa3JH
+ WDFJMDlOT0sxa3JnL3VnOStFRVd0dVUKLS0tIGtGaFpTRkFyOWhXYTlnSlFpVGts
+ SVlmaWUxeWQvRHBaNzdwem9IMDBtV2cKG2JS1Y6aq8X8Fi7ytPpBKesrCfN1qY0s
+ ZnDQaH2z9ceySHAHhvh8ETpzStHuGfmSCLGmdMpL4VqmILEQ08j1EQ==
+ -----END AGE ENCRYPTED FILE-----
lastmodified: "2024-11-06T18:28:27Z"
mac: ENC[AES256_GCM,data:ethuFGzDhBpl0f8RXhD6D7ny37P3ASqrxmlSuhKbRSJycYJAOVCDVjkqcNJQN1v5W39owhqCnPkZp9v89M956KSgryc3Q0JtV8vHL7Iy3kKRZcdCurOft+gmzGoAWGWmh3CSPPYjho3cEY3PFFotkYGikYqA6lVmp0AX0hNkhxE=,iv:Qo21khqIlW+mHUv0IOEvNl8QTsI+JFj5uC0hZ8ff6qw=,tag:W2A8ZcoBCqFKBaKBnVVNVw==,type:str]
pgp: []
