synapse.json (530519B)
1 { 2 "__elements": {}, 3 "__requires": [ 4 { 5 "type": "grafana", 6 "id": "grafana", 7 "name": "Grafana", 8 "version": "9.2.2" 9 }, 10 { 11 "type": "panel", 12 "id": "graph", 13 "name": "Graph (old)", 14 "version": "" 15 }, 16 { 17 "type": "panel", 18 "id": "heatmap", 19 "name": "Heatmap", 20 "version": "" 21 }, 22 { 23 "type": "datasource", 24 "id": "prometheus", 25 "name": "Prometheus", 26 "version": "1.0.0" 27 }, 28 { 29 "type": "panel", 30 "id": "timeseries", 31 "name": "Time series", 32 "version": "" 33 } 34 ], 35 "annotations": { 36 "list": [ 37 { 38 "builtIn": 1, 39 "datasource": { 40 "uid": "${DS_PROMETHEUS}" 41 }, 42 "enable": false, 43 "hide": true, 44 "iconColor": "rgba(0, 211, 255, 1)", 45 "limit": 100, 46 "name": "Annotations & Alerts", 47 "showIn": 0, 48 "type": "dashboard" 49 }, 50 { 51 "datasource": { 52 "type": "prometheus", 53 "uid": "${DS_PROMETHEUS}" 54 }, 55 "enable": true, 56 "expr": "changes(process_start_time_seconds{instance=\"$instance\",job=~\"synapse\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"$instance\",job=\"synapse\"}", 57 "iconColor": "purple", 58 "name": "deploys", 59 "titleFormat": "Deployed {{version}}" 60 } 61 ] 62 }, 63 "editable": true, 64 "fiscalYearStartMonth": 0, 65 "graphTooltip": 0, 66 "id": null, 67 "links": [ 68 { 69 "asDropdown": false, 70 "icon": "external link", 71 "includeVars": true, 72 "keepTime": true, 73 "tags": [ 74 "matrix" 75 ], 76 "title": "Dashboards", 77 "type": "dashboards" 78 } 79 ], 80 "liveNow": false, 81 "panels": [ 82 { 83 "collapsed": false, 84 "datasource": { 85 "type": "prometheus", 86 "uid": "${DS_PROMETHEUS}" 87 }, 88 "gridPos": { 89 "h": 1, 90 "w": 24, 91 "x": 0, 92 "y": 0 93 }, 94 "id": 73, 95 "panels": [], 96 "targets": [ 97 { 98 "datasource": { 99 "type": "prometheus", 100 "uid": "${DS_PROMETHEUS}" 101 }, 102 "refId": "A" 103 } 104 ], 105 "title": "Overview", 106 "type": "row" 107 }, 108 { 109 "cards": { 110 "cardPadding": -1, 111 "cardRound": 0 112 }, 113 "color": { 114 "cardColor": "#b4ff00", 115 "colorScale": "sqrt", 116 "colorScheme": "interpolateInferno", 117 "exponent": 0.5, 118 "mode": "spectrum" 119 }, 120 "dataFormat": "tsbuckets", 121 "datasource": { 122 "uid": "${DS_PROMETHEUS}" 123 }, 124 "fieldConfig": { 125 "defaults": { 126 "custom": { 127 "hideFrom": { 128 "legend": false, 129 "tooltip": false, 130 "viz": false 131 }, 132 "scaleDistribution": { 133 "type": "linear" 134 } 135 } 136 }, 137 "overrides": [] 138 }, 139 "gridPos": { 140 "h": 9, 141 "w": 12, 142 "x": 0, 143 "y": 1 144 }, 145 "heatmap": {}, 146 "hideZeroBuckets": false, 147 "highlightCards": true, 148 "id": 189, 149 "legend": { 150 "show": false 151 }, 152 "links": [], 153 "options": { 154 "calculate": false, 155 "calculation": {}, 156 "cellGap": -1, 157 "cellRadius": 0, 158 "cellValues": {}, 159 "color": { 160 "exponent": 0.5, 161 "fill": "#b4ff00", 162 "mode": "scheme", 163 "reverse": false, 164 "scale": "exponential", 165 "scheme": "Inferno", 166 "steps": 128 167 }, 168 "exemplars": { 169 "color": "rgba(255,0,255,0.7)" 170 }, 171 "filterValues": { 172 "le": 1e-9 173 }, 174 "legend": { 175 "show": false 176 }, 177 "rowsFrame": { 178 "layout": "auto" 179 }, 180 "showValue": "never", 181 "tooltip": { 182 "show": true, 183 "yHistogram": true 184 }, 185 "yAxis": { 186 "axisPlacement": "left", 187 "reverse": false, 188 "unit": "s" 189 } 190 }, 191 "pluginVersion": "9.2.2", 192 "reverseYBuckets": false, 193 "targets": [ 194 { 195 "datasource": { 196 "uid": "${DS_PROMETHEUS}" 197 }, 198 "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le)", 199 "format": "heatmap", 200 "interval": "", 201 "intervalFactor": 1, 202 "legendFormat": "{{le}}", 203 "refId": "A" 204 } 205 ], 206 "title": "Event Send Time (excluding errors, all workers)", 207 "tooltip": { 208 "show": true, 209 "showHistogram": true 210 }, 211 "type": "heatmap", 212 "xAxis": { 213 "show": true 214 }, 215 "yAxis": { 216 "format": "s", 217 "logBase": 2, 218 "show": true 219 }, 220 "yBucketBound": "auto" 221 }, 222 { 223 "aliasColors": {}, 224 "bars": false, 225 "dashLength": 10, 226 "dashes": false, 227 "datasource": { 228 "uid": "${DS_PROMETHEUS}" 229 }, 230 "description": "", 231 "fieldConfig": { 232 "defaults": { 233 "links": [] 234 }, 235 "overrides": [] 236 }, 237 "fill": 0, 238 "fillGradient": 0, 239 "gridPos": { 240 "h": 9, 241 "w": 12, 242 "x": 12, 243 "y": 1 244 }, 245 "hiddenSeries": false, 246 "id": 152, 247 "legend": { 248 "avg": false, 249 "current": false, 250 "max": false, 251 "min": false, 252 "rightSide": false, 253 "show": true, 254 "total": false, 255 "values": false 256 }, 257 "lines": true, 258 "linewidth": 0, 259 "links": [], 260 "nullPointMode": "connected", 261 "options": { 262 "alertThreshold": true 263 }, 264 "paceLength": 10, 265 "percentage": false, 266 "pluginVersion": "9.2.2", 267 "pointradius": 5, 268 "points": false, 269 "renderer": "flot", 270 "seriesOverrides": [ 271 { 272 "alias": "Avg", 273 "fill": 0, 274 "linewidth": 3 275 }, 276 { 277 "alias": "99%", 278 "color": "#C4162A", 279 "fillBelowTo": "90%" 280 }, 281 { 282 "alias": "90%", 283 "color": "#FF7383", 284 "fillBelowTo": "75%" 285 }, 286 { 287 "alias": "75%", 288 "color": "#FFEE52", 289 "fillBelowTo": "50%" 290 }, 291 { 292 "alias": "50%", 293 "color": "#73BF69", 294 "fillBelowTo": "25%" 295 }, 296 { 297 "alias": "25%", 298 "color": "#1F60C4", 299 "fillBelowTo": "5%" 300 }, 301 { 302 "alias": "5%", 303 "lines": false 304 }, 305 { 306 "alias": "Average", 307 "color": "rgb(255, 255, 255)", 308 "lines": true, 309 "linewidth": 3 310 }, 311 { 312 "alias": "Events", 313 "color": "#B877D9", 314 "hideTooltip": true, 315 "points": true, 316 "yaxis": 2, 317 "zindex": -3 318 } 319 ], 320 "spaceLength": 10, 321 "stack": false, 322 "steppedLine": false, 323 "targets": [ 324 { 325 "datasource": { 326 "uid": "${DS_PROMETHEUS}" 327 }, 328 "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", 329 "format": "time_series", 330 "intervalFactor": 1, 331 "legendFormat": "99%", 332 "refId": "D" 333 }, 334 { 335 "datasource": { 336 "uid": "${DS_PROMETHEUS}" 337 }, 338 "expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", 339 "format": "time_series", 340 "interval": "", 341 "intervalFactor": 1, 342 "legendFormat": "90%", 343 "refId": "A" 344 }, 345 { 346 "datasource": { 347 "uid": "${DS_PROMETHEUS}" 348 }, 349 "expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", 350 "format": "time_series", 351 "intervalFactor": 1, 352 "legendFormat": "75%", 353 "refId": "C" 354 }, 355 { 356 "datasource": { 357 "uid": "${DS_PROMETHEUS}" 358 }, 359 "expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", 360 "format": "time_series", 361 "intervalFactor": 1, 362 "legendFormat": "50%", 363 "refId": "B" 364 }, 365 { 366 "datasource": { 367 "uid": "${DS_PROMETHEUS}" 368 }, 369 "expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", 370 "legendFormat": "25%", 371 "refId": "F" 372 }, 373 { 374 "datasource": { 375 "uid": "${DS_PROMETHEUS}" 376 }, 377 "expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", 378 "legendFormat": "5%", 379 "refId": "G" 380 }, 381 { 382 "datasource": { 383 "uid": "${DS_PROMETHEUS}" 384 }, 385 "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))", 386 "legendFormat": "Average", 387 "refId": "H" 388 }, 389 { 390 "datasource": { 391 "uid": "${DS_PROMETHEUS}" 392 }, 393 "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size]))", 394 "hide": false, 395 "instant": false, 396 "legendFormat": "Events", 397 "refId": "E" 398 } 399 ], 400 "thresholds": [ 401 { 402 "$$hashKey": "object:283", 403 "colorMode": "warning", 404 "fill": false, 405 "line": true, 406 "op": "gt", 407 "value": 1, 408 "yaxis": "left" 409 }, 410 { 411 "$$hashKey": "object:284", 412 "colorMode": "critical", 413 "fill": false, 414 "line": true, 415 "op": "gt", 416 "value": 2, 417 "yaxis": "left" 418 } 419 ], 420 "timeRegions": [], 421 "title": "Event Send Time Quantiles (excluding errors, all workers)", 422 "tooltip": { 423 "shared": true, 424 "sort": 2, 425 "value_type": "individual" 426 }, 427 "type": "graph", 428 "xaxis": { 429 "mode": "time", 430 "show": true, 431 "values": [] 432 }, 433 "yaxes": [ 434 { 435 "$$hashKey": "object:255", 436 "format": "s", 437 "label": "", 438 "logBase": 1, 439 "min": "0", 440 "show": true 441 }, 442 { 443 "$$hashKey": "object:256", 444 "format": "hertz", 445 "label": "", 446 "logBase": 1, 447 "min": "0", 448 "show": true 449 } 450 ], 451 "yaxis": { 452 "align": false 453 } 454 }, 455 { 456 "aliasColors": {}, 457 "bars": false, 458 "dashLength": 10, 459 "dashes": false, 460 "datasource": { 461 "uid": "${DS_PROMETHEUS}" 462 }, 463 "fieldConfig": { 464 "defaults": { 465 "links": [] 466 }, 467 "overrides": [] 468 }, 469 "fill": 1, 470 "fillGradient": 0, 471 "gridPos": { 472 "h": 9, 473 "w": 12, 474 "x": 0, 475 "y": 10 476 }, 477 "hiddenSeries": false, 478 "id": 75, 479 "legend": { 480 "avg": false, 481 "current": false, 482 "max": false, 483 "min": false, 484 "show": true, 485 "total": false, 486 "values": false 487 }, 488 "lines": true, 489 "linewidth": 3, 490 "links": [], 491 "nullPointMode": "null", 492 "options": { 493 "alertThreshold": true 494 }, 495 "paceLength": 10, 496 "percentage": false, 497 "pluginVersion": "9.2.2", 498 "pointradius": 5, 499 "points": false, 500 "renderer": "flot", 501 "seriesOverrides": [], 502 "spaceLength": 10, 503 "stack": false, 504 "steppedLine": false, 505 "targets": [ 506 { 507 "datasource": { 508 "uid": "${DS_PROMETHEUS}" 509 }, 510 "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 511 "format": "time_series", 512 "interval": "", 513 "intervalFactor": 1, 514 "legendFormat": "{{job}}-{{index}} ", 515 "refId": "A" 516 } 517 ], 518 "thresholds": [ 519 { 520 "$$hashKey": "object:566", 521 "colorMode": "critical", 522 "fill": true, 523 "line": true, 524 "op": "gt", 525 "value": 1, 526 "yaxis": "left" 527 } 528 ], 529 "timeRegions": [], 530 "title": "CPU usage", 531 "tooltip": { 532 "shared": false, 533 "sort": 0, 534 "value_type": "individual" 535 }, 536 "type": "graph", 537 "xaxis": { 538 "mode": "time", 539 "show": true, 540 "values": [] 541 }, 542 "yaxes": [ 543 { 544 "$$hashKey": "object:538", 545 "format": "percentunit", 546 "logBase": 1, 547 "max": "1.5", 548 "min": "0", 549 "show": true 550 }, 551 { 552 "$$hashKey": "object:539", 553 "format": "short", 554 "logBase": 1, 555 "show": true 556 } 557 ], 558 "yaxis": { 559 "align": false 560 } 561 }, 562 { 563 "aliasColors": {}, 564 "bars": false, 565 "dashLength": 10, 566 "dashes": false, 567 "datasource": { 568 "uid": "${DS_PROMETHEUS}" 569 }, 570 "editable": true, 571 "error": false, 572 "fieldConfig": { 573 "defaults": { 574 "links": [] 575 }, 576 "overrides": [] 577 }, 578 "fill": 1, 579 "fillGradient": 0, 580 "grid": {}, 581 "gridPos": { 582 "h": 9, 583 "w": 12, 584 "x": 12, 585 "y": 10 586 }, 587 "hiddenSeries": false, 588 "id": 198, 589 "legend": { 590 "avg": false, 591 "current": false, 592 "max": false, 593 "min": false, 594 "show": true, 595 "total": false, 596 "values": false 597 }, 598 "lines": true, 599 "linewidth": 3, 600 "links": [], 601 "nullPointMode": "null", 602 "options": { 603 "alertThreshold": true 604 }, 605 "paceLength": 10, 606 "percentage": false, 607 "pluginVersion": "9.2.2", 608 "pointradius": 5, 609 "points": false, 610 "renderer": "flot", 611 "seriesOverrides": [], 612 "spaceLength": 10, 613 "stack": false, 614 "steppedLine": false, 615 "targets": [ 616 { 617 "datasource": { 618 "uid": "${DS_PROMETHEUS}" 619 }, 620 "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 621 "format": "time_series", 622 "interval": "", 623 "intervalFactor": 2, 624 "legendFormat": "{{job}} {{index}}", 625 "refId": "A", 626 "step": 20, 627 "target": "" 628 }, 629 { 630 "datasource": { 631 "uid": "${DS_PROMETHEUS}" 632 }, 633 "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})", 634 "hide": true, 635 "interval": "", 636 "legendFormat": "total", 637 "refId": "B" 638 } 639 ], 640 "thresholds": [], 641 "timeRegions": [], 642 "title": "Memory", 643 "tooltip": { 644 "shared": false, 645 "sort": 0, 646 "value_type": "cumulative" 647 }, 648 "transformations": [], 649 "type": "graph", 650 "xaxis": { 651 "mode": "time", 652 "show": true, 653 "values": [] 654 }, 655 "yaxes": [ 656 { 657 "$$hashKey": "object:1560", 658 "format": "bytes", 659 "logBase": 1, 660 "min": "0", 661 "show": true 662 }, 663 { 664 "$$hashKey": "object:1561", 665 "format": "short", 666 "logBase": 1, 667 "show": true 668 } 669 ], 670 "yaxis": { 671 "align": false 672 } 673 }, 674 { 675 "datasource": { 676 "type": "prometheus", 677 "uid": "${DS_PROMETHEUS}" 678 }, 679 "fieldConfig": { 680 "defaults": { 681 "color": { 682 "mode": "palette-classic" 683 }, 684 "custom": { 685 "axisCenteredZero": false, 686 "axisColorMode": "text", 687 "axisLabel": "", 688 "axisPlacement": "auto", 689 "axisSoftMax": 1, 690 "barAlignment": 0, 691 "drawStyle": "line", 692 "fillOpacity": 0, 693 "gradientMode": "none", 694 "hideFrom": { 695 "legend": false, 696 "tooltip": false, 697 "viz": false 698 }, 699 "lineInterpolation": "linear", 700 "lineWidth": 10, 701 "pointSize": 5, 702 "scaleDistribution": { 703 "type": "linear" 704 }, 705 "showPoints": "never", 706 "spanNulls": false, 707 "stacking": { 708 "group": "A", 709 "mode": "none" 710 }, 711 "thresholdsStyle": { 712 "mode": "off" 713 } 714 }, 715 "mappings": [], 716 "thresholds": { 717 "mode": "absolute", 718 "steps": [ 719 { 720 "color": "green", 721 "value": null 722 } 723 ] 724 } 725 }, 726 "overrides": [] 727 }, 728 "gridPos": { 729 "h": 7, 730 "w": 12, 731 "x": 0, 732 "y": 19 733 }, 734 "id": 245, 735 "options": { 736 "legend": { 737 "calcs": [], 738 "displayMode": "list", 739 "placement": "bottom", 740 "showLegend": true 741 }, 742 "tooltip": { 743 "mode": "single", 744 "sort": "none" 745 } 746 }, 747 "targets": [ 748 { 749 "datasource": { 750 "type": "prometheus", 751 "uid": "${DS_PROMETHEUS}" 752 }, 753 "editorMode": "code", 754 "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1", 755 "legendFormat": "version {{version}}", 756 "range": true, 757 "refId": "deployed_synapse_versions" 758 } 759 ], 760 "title": "Deployed Synapse versions over time", 761 "type": "timeseries" 762 }, 763 { 764 "aliasColors": {}, 765 "bars": false, 766 "dashLength": 10, 767 "dashes": false, 768 "datasource": { 769 "uid": "${DS_PROMETHEUS}" 770 }, 771 "fieldConfig": { 772 "defaults": { 773 "links": [] 774 }, 775 "overrides": [] 776 }, 777 "fill": 1, 778 "fillGradient": 0, 779 "gridPos": { 780 "h": 7, 781 "w": 12, 782 "x": 12, 783 "y": 19 784 }, 785 "hiddenSeries": false, 786 "id": 37, 787 "legend": { 788 "avg": false, 789 "current": false, 790 "max": false, 791 "min": false, 792 "show": true, 793 "total": false, 794 "values": false 795 }, 796 "lines": true, 797 "linewidth": 1, 798 "links": [], 799 "nullPointMode": "null", 800 "options": { 801 "alertThreshold": true 802 }, 803 "paceLength": 10, 804 "percentage": false, 805 "pluginVersion": "9.2.2", 806 "pointradius": 5, 807 "points": false, 808 "renderer": "flot", 809 "seriesOverrides": [ 810 { 811 "$$hashKey": "object:639", 812 "alias": "/max$/", 813 "color": "#890F02", 814 "fill": 0, 815 "legend": false 816 } 817 ], 818 "spaceLength": 10, 819 "stack": false, 820 "steppedLine": false, 821 "targets": [ 822 { 823 "datasource": { 824 "uid": "${DS_PROMETHEUS}" 825 }, 826 "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 827 "format": "time_series", 828 "hide": false, 829 "interval": "", 830 "intervalFactor": 2, 831 "legendFormat": "{{job}}-{{index}}", 832 "refId": "A", 833 "step": 20 834 }, 835 { 836 "datasource": { 837 "uid": "${DS_PROMETHEUS}" 838 }, 839 "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 840 "format": "time_series", 841 "hide": true, 842 "interval": "", 843 "intervalFactor": 2, 844 "legendFormat": "{{job}}-{{index}} max", 845 "refId": "B", 846 "step": 20 847 } 848 ], 849 "thresholds": [], 850 "timeRegions": [], 851 "title": "Open FDs", 852 "tooltip": { 853 "shared": false, 854 "sort": 0, 855 "value_type": "individual" 856 }, 857 "type": "graph", 858 "xaxis": { 859 "mode": "time", 860 "show": true, 861 "values": [] 862 }, 863 "yaxes": [ 864 { 865 "$$hashKey": "object:650", 866 "format": "none", 867 "label": "", 868 "logBase": 1, 869 "show": true 870 }, 871 { 872 "$$hashKey": "object:651", 873 "format": "short", 874 "logBase": 1, 875 "show": true 876 } 877 ], 878 "yaxis": { 879 "align": false 880 } 881 }, 882 { 883 "collapsed": true, 884 "datasource": { 885 "type": "prometheus", 886 "uid": "${DS_PROMETHEUS}" 887 }, 888 "gridPos": { 889 "h": 1, 890 "w": 24, 891 "x": 0, 892 "y": 26 893 }, 894 "id": 54, 895 "panels": [ 896 { 897 "aliasColors": {}, 898 "bars": false, 899 "dashLength": 10, 900 "dashes": false, 901 "datasource": { 902 "type": "prometheus", 903 "uid": "${DS_PROMETHEUS}" 904 }, 905 "editable": true, 906 "error": false, 907 "fieldConfig": { 908 "defaults": { 909 "links": [] 910 }, 911 "overrides": [] 912 }, 913 "fill": 1, 914 "fillGradient": 0, 915 "grid": {}, 916 "gridPos": { 917 "h": 7, 918 "w": 12, 919 "x": 0, 920 "y": 27 921 }, 922 "hiddenSeries": false, 923 "id": 5, 924 "legend": { 925 "alignAsTable": false, 926 "avg": false, 927 "current": false, 928 "hideEmpty": false, 929 "hideZero": false, 930 "max": false, 931 "min": false, 932 "rightSide": false, 933 "show": true, 934 "total": false, 935 "values": false 936 }, 937 "lines": true, 938 "linewidth": 3, 939 "links": [], 940 "nullPointMode": "null", 941 "options": { 942 "alertThreshold": true 943 }, 944 "paceLength": 10, 945 "percentage": false, 946 "pluginVersion": "9.2.2", 947 "pointradius": 5, 948 "points": false, 949 "renderer": "flot", 950 "seriesOverrides": [ 951 { 952 "$$hashKey": "object:1240", 953 "alias": "/user/" 954 }, 955 { 956 "$$hashKey": "object:1241", 957 "alias": "/system/" 958 } 959 ], 960 "spaceLength": 10, 961 "stack": false, 962 "steppedLine": false, 963 "targets": [ 964 { 965 "datasource": { 966 "uid": "${DS_PROMETHEUS}" 967 }, 968 "editorMode": "code", 969 "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 970 "format": "time_series", 971 "intervalFactor": 1, 972 "legendFormat": "{{job}}-{{index}} system ", 973 "metric": "", 974 "range": true, 975 "refId": "B", 976 "step": 20 977 }, 978 { 979 "datasource": { 980 "uid": "${DS_PROMETHEUS}" 981 }, 982 "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 983 "format": "time_series", 984 "hide": false, 985 "interval": "", 986 "intervalFactor": 1, 987 "legendFormat": "{{job}}-{{index}} user", 988 "refId": "A", 989 "step": 20 990 } 991 ], 992 "thresholds": [ 993 { 994 "$$hashKey": "object:1278", 995 "colorMode": "custom", 996 "fillColor": "rgba(255, 255, 255, 1)", 997 "line": true, 998 "lineColor": "rgba(216, 200, 27, 0.27)", 999 "op": "gt", 1000 "value": 0.5, 1001 "yaxis": "left" 1002 }, 1003 { 1004 "$$hashKey": "object:1279", 1005 "colorMode": "custom", 1006 "fillColor": "rgba(255, 255, 255, 1)", 1007 "line": true, 1008 "lineColor": "rgb(87, 6, 16)", 1009 "op": "gt", 1010 "value": 0.8, 1011 "yaxis": "left" 1012 }, 1013 { 1014 "$$hashKey": "object:1498", 1015 "colorMode": "critical", 1016 "fill": true, 1017 "line": true, 1018 "op": "gt", 1019 "value": 1, 1020 "yaxis": "left" 1021 } 1022 ], 1023 "timeRegions": [], 1024 "title": "CPU", 1025 "tooltip": { 1026 "shared": false, 1027 "sort": 0, 1028 "value_type": "individual" 1029 }, 1030 "type": "graph", 1031 "xaxis": { 1032 "mode": "time", 1033 "show": true, 1034 "values": [] 1035 }, 1036 "yaxes": [ 1037 { 1038 "$$hashKey": "object:1250", 1039 "format": "percentunit", 1040 "label": "", 1041 "logBase": 1, 1042 "max": "1.2", 1043 "min": 0, 1044 "show": true 1045 }, 1046 { 1047 "$$hashKey": "object:1251", 1048 "format": "short", 1049 "logBase": 1, 1050 "show": true 1051 } 1052 ], 1053 "yaxis": { 1054 "align": false 1055 } 1056 }, 1057 { 1058 "datasource": { 1059 "type": "prometheus", 1060 "uid": "${DS_PROMETHEUS}" 1061 }, 1062 "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan", 1063 "fieldConfig": { 1064 "defaults": { 1065 "color": { 1066 "mode": "palette-classic" 1067 }, 1068 "custom": { 1069 "axisCenteredZero": false, 1070 "axisColorMode": "text", 1071 "axisLabel": "", 1072 "axisPlacement": "auto", 1073 "barAlignment": 0, 1074 "drawStyle": "line", 1075 "fillOpacity": 10, 1076 "gradientMode": "none", 1077 "hideFrom": { 1078 "legend": false, 1079 "tooltip": false, 1080 "viz": false 1081 }, 1082 "lineInterpolation": "linear", 1083 "lineWidth": 1, 1084 "pointSize": 5, 1085 "scaleDistribution": { 1086 "type": "linear" 1087 }, 1088 "showPoints": "never", 1089 "spanNulls": true, 1090 "stacking": { 1091 "group": "A", 1092 "mode": "none" 1093 }, 1094 "thresholdsStyle": { 1095 "mode": "off" 1096 } 1097 }, 1098 "links": [], 1099 "mappings": [], 1100 "thresholds": { 1101 "mode": "absolute", 1102 "steps": [ 1103 { 1104 "color": "green" 1105 }, 1106 { 1107 "color": "red", 1108 "value": 80 1109 } 1110 ] 1111 }, 1112 "unit": "s" 1113 }, 1114 "overrides": [] 1115 }, 1116 "gridPos": { 1117 "h": 7, 1118 "w": 12, 1119 "x": 12, 1120 "y": 27 1121 }, 1122 "id": 105, 1123 "interval": "", 1124 "links": [], 1125 "options": { 1126 "legend": { 1127 "calcs": [], 1128 "displayMode": "list", 1129 "placement": "bottom", 1130 "showLegend": true 1131 }, 1132 "tooltip": { 1133 "mode": "single", 1134 "sort": "none" 1135 } 1136 }, 1137 "pluginVersion": "8.3.2", 1138 "targets": [ 1139 { 1140 "datasource": { 1141 "type": "prometheus", 1142 "uid": "${DS_PROMETHEUS}" 1143 }, 1144 "exemplar": true, 1145 "expr": "histogram_quantile(0.999, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))", 1146 "hide": false, 1147 "interval": "", 1148 "legendFormat": "{{job}}-{{index}} 99.9%", 1149 "refId": "E" 1150 }, 1151 { 1152 "datasource": { 1153 "type": "prometheus", 1154 "uid": "${DS_PROMETHEUS}" 1155 }, 1156 "exemplar": true, 1157 "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))", 1158 "format": "time_series", 1159 "interval": "", 1160 "intervalFactor": 2, 1161 "legendFormat": "{{job}}-{{index}} 99%", 1162 "refId": "A", 1163 "step": 20 1164 }, 1165 { 1166 "datasource": { 1167 "type": "prometheus", 1168 "uid": "${DS_PROMETHEUS}" 1169 }, 1170 "exemplar": true, 1171 "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))", 1172 "format": "time_series", 1173 "interval": "", 1174 "intervalFactor": 1, 1175 "legendFormat": "{{job}}-{{index}} 95%", 1176 "refId": "B" 1177 }, 1178 { 1179 "datasource": { 1180 "type": "prometheus", 1181 "uid": "${DS_PROMETHEUS}" 1182 }, 1183 "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))", 1184 "format": "time_series", 1185 "intervalFactor": 1, 1186 "legendFormat": "{{job}}-{{index}} 90%", 1187 "refId": "C" 1188 }, 1189 { 1190 "datasource": { 1191 "type": "prometheus", 1192 "uid": "${DS_PROMETHEUS}" 1193 }, 1194 "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])", 1195 "format": "time_series", 1196 "intervalFactor": 1, 1197 "legendFormat": "{{job}}-{{index}} mean", 1198 "refId": "D" 1199 } 1200 ], 1201 "title": "Reactor tick quantiles", 1202 "type": "timeseries" 1203 }, 1204 { 1205 "aliasColors": {}, 1206 "bars": false, 1207 "dashLength": 10, 1208 "dashes": false, 1209 "datasource": { 1210 "uid": "${DS_PROMETHEUS}" 1211 }, 1212 "editable": true, 1213 "error": false, 1214 "fieldConfig": { 1215 "defaults": { 1216 "links": [] 1217 }, 1218 "overrides": [] 1219 }, 1220 "fill": 0, 1221 "fillGradient": 0, 1222 "grid": {}, 1223 "gridPos": { 1224 "h": 7, 1225 "w": 12, 1226 "x": 0, 1227 "y": 34 1228 }, 1229 "hiddenSeries": false, 1230 "id": 34, 1231 "legend": { 1232 "avg": false, 1233 "current": false, 1234 "max": false, 1235 "min": false, 1236 "show": true, 1237 "total": false, 1238 "values": false 1239 }, 1240 "lines": true, 1241 "linewidth": 1, 1242 "links": [], 1243 "nullPointMode": "null", 1244 "options": { 1245 "alertThreshold": true 1246 }, 1247 "paceLength": 10, 1248 "percentage": false, 1249 "pluginVersion": "9.2.2", 1250 "pointradius": 5, 1251 "points": false, 1252 "renderer": "flot", 1253 "seriesOverrides": [], 1254 "spaceLength": 10, 1255 "stack": false, 1256 "steppedLine": false, 1257 "targets": [ 1258 { 1259 "datasource": { 1260 "uid": "${DS_PROMETHEUS}" 1261 }, 1262 "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 1263 "format": "time_series", 1264 "interval": "", 1265 "intervalFactor": 2, 1266 "legendFormat": "{{job}} {{index}}", 1267 "refId": "A", 1268 "step": 20, 1269 "target": "" 1270 }, 1271 { 1272 "datasource": { 1273 "uid": "${DS_PROMETHEUS}" 1274 }, 1275 "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})", 1276 "interval": "", 1277 "legendFormat": "total", 1278 "refId": "B" 1279 } 1280 ], 1281 "thresholds": [], 1282 "timeRegions": [], 1283 "title": "Memory", 1284 "tooltip": { 1285 "shared": false, 1286 "sort": 0, 1287 "value_type": "cumulative" 1288 }, 1289 "transformations": [], 1290 "type": "graph", 1291 "xaxis": { 1292 "mode": "time", 1293 "show": true, 1294 "values": [] 1295 }, 1296 "yaxes": [ 1297 { 1298 "format": "bytes", 1299 "logBase": 1, 1300 "min": "0", 1301 "show": true 1302 }, 1303 { 1304 "format": "short", 1305 "logBase": 1, 1306 "show": true 1307 } 1308 ], 1309 "yaxis": { 1310 "align": false 1311 } 1312 }, 1313 { 1314 "aliasColors": {}, 1315 "bars": false, 1316 "dashLength": 10, 1317 "dashes": false, 1318 "datasource": { 1319 "uid": "${DS_PROMETHEUS}" 1320 }, 1321 "fieldConfig": { 1322 "defaults": { 1323 "links": [] 1324 }, 1325 "overrides": [] 1326 }, 1327 "fill": 1, 1328 "fillGradient": 0, 1329 "gridPos": { 1330 "h": 7, 1331 "w": 12, 1332 "x": 12, 1333 "y": 34 1334 }, 1335 "hiddenSeries": false, 1336 "id": 49, 1337 "legend": { 1338 "avg": false, 1339 "current": false, 1340 "max": false, 1341 "min": false, 1342 "show": true, 1343 "total": false, 1344 "values": false 1345 }, 1346 "lines": true, 1347 "linewidth": 1, 1348 "links": [], 1349 "nullPointMode": "null", 1350 "options": { 1351 "alertThreshold": true 1352 }, 1353 "paceLength": 10, 1354 "percentage": false, 1355 "pluginVersion": "9.2.2", 1356 "pointradius": 5, 1357 "points": false, 1358 "renderer": "flot", 1359 "seriesOverrides": [ 1360 { 1361 "alias": "/^up/", 1362 "legend": false, 1363 "yaxis": 2 1364 } 1365 ], 1366 "spaceLength": 10, 1367 "stack": false, 1368 "steppedLine": false, 1369 "targets": [ 1370 { 1371 "datasource": { 1372 "uid": "${DS_PROMETHEUS}" 1373 }, 1374 "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 1375 "format": "time_series", 1376 "interval": "", 1377 "intervalFactor": 2, 1378 "legendFormat": "{{job}}-{{index}}", 1379 "refId": "A", 1380 "step": 20 1381 } 1382 ], 1383 "thresholds": [], 1384 "timeRegions": [], 1385 "title": "Prometheus scrape time", 1386 "tooltip": { 1387 "shared": false, 1388 "sort": 0, 1389 "value_type": "individual" 1390 }, 1391 "type": "graph", 1392 "xaxis": { 1393 "mode": "time", 1394 "show": true, 1395 "values": [] 1396 }, 1397 "yaxes": [ 1398 { 1399 "format": "s", 1400 "logBase": 1, 1401 "min": "0", 1402 "show": true 1403 }, 1404 { 1405 "decimals": 0, 1406 "format": "none", 1407 "label": "", 1408 "logBase": 1, 1409 "max": "0", 1410 "min": "-1", 1411 "show": false 1412 } 1413 ], 1414 "yaxis": { 1415 "align": false 1416 } 1417 }, 1418 { 1419 "aliasColors": {}, 1420 "bars": false, 1421 "dashLength": 10, 1422 "dashes": false, 1423 "datasource": { 1424 "type": "prometheus", 1425 "uid": "${DS_PROMETHEUS}" 1426 }, 1427 "fieldConfig": { 1428 "defaults": { 1429 "links": [] 1430 }, 1431 "overrides": [] 1432 }, 1433 "fill": 0, 1434 "fillGradient": 0, 1435 "gridPos": { 1436 "h": 7, 1437 "w": 12, 1438 "x": 0, 1439 "y": 41 1440 }, 1441 "hiddenSeries": false, 1442 "id": 53, 1443 "legend": { 1444 "avg": false, 1445 "current": false, 1446 "max": false, 1447 "min": false, 1448 "show": true, 1449 "total": false, 1450 "values": false 1451 }, 1452 "lines": true, 1453 "linewidth": 1, 1454 "links": [], 1455 "nullPointMode": "null", 1456 "options": { 1457 "alertThreshold": true 1458 }, 1459 "paceLength": 10, 1460 "percentage": false, 1461 "pluginVersion": "9.2.2", 1462 "pointradius": 5, 1463 "points": false, 1464 "renderer": "flot", 1465 "seriesOverrides": [ 1466 { 1467 "$$hashKey": "object:116", 1468 "alias": "/^version .*/", 1469 "lines": true, 1470 "linewidth": 6, 1471 "points": false 1472 } 1473 ], 1474 "spaceLength": 10, 1475 "stack": false, 1476 "steppedLine": false, 1477 "targets": [ 1478 { 1479 "datasource": { 1480 "uid": "${DS_PROMETHEUS}" 1481 }, 1482 "editorMode": "code", 1483 "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 1484 "format": "time_series", 1485 "intervalFactor": 2, 1486 "legendFormat": "{{job}}-{{index}}", 1487 "range": true, 1488 "refId": "A" 1489 }, 1490 { 1491 "datasource": { 1492 "type": "prometheus", 1493 "uid": "${DS_PROMETHEUS}" 1494 }, 1495 "editorMode": "code", 1496 "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1", 1497 "hide": false, 1498 "legendFormat": "version {{version}}", 1499 "range": true, 1500 "refId": "deployed_synapse_versions" 1501 } 1502 ], 1503 "thresholds": [], 1504 "timeRegions": [], 1505 "title": "Up", 1506 "tooltip": { 1507 "shared": false, 1508 "sort": 0, 1509 "value_type": "individual" 1510 }, 1511 "type": "graph", 1512 "xaxis": { 1513 "mode": "time", 1514 "show": true, 1515 "values": [] 1516 }, 1517 "yaxes": [ 1518 { 1519 "format": "short", 1520 "logBase": 1, 1521 "show": true 1522 }, 1523 { 1524 "format": "short", 1525 "logBase": 1, 1526 "show": true 1527 } 1528 ], 1529 "yaxis": { 1530 "align": false 1531 } 1532 }, 1533 { 1534 "aliasColors": {}, 1535 "bars": false, 1536 "dashLength": 10, 1537 "dashes": false, 1538 "datasource": { 1539 "uid": "${DS_PROMETHEUS}" 1540 }, 1541 "fieldConfig": { 1542 "defaults": { 1543 "links": [] 1544 }, 1545 "overrides": [] 1546 }, 1547 "fill": 1, 1548 "fillGradient": 0, 1549 "gridPos": { 1550 "h": 7, 1551 "w": 12, 1552 "x": 12, 1553 "y": 41 1554 }, 1555 "hiddenSeries": false, 1556 "id": 120, 1557 "legend": { 1558 "avg": false, 1559 "current": false, 1560 "max": false, 1561 "min": false, 1562 "show": true, 1563 "total": false, 1564 "values": false 1565 }, 1566 "lines": true, 1567 "linewidth": 1, 1568 "links": [], 1569 "nullPointMode": "null as zero", 1570 "options": { 1571 "alertThreshold": true 1572 }, 1573 "percentage": false, 1574 "pluginVersion": "9.2.2", 1575 "pointradius": 2, 1576 "points": false, 1577 "renderer": "flot", 1578 "seriesOverrides": [], 1579 "spaceLength": 10, 1580 "stack": true, 1581 "steppedLine": false, 1582 "targets": [ 1583 { 1584 "datasource": { 1585 "uid": "${DS_PROMETHEUS}" 1586 }, 1587 "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 1588 "format": "time_series", 1589 "hide": false, 1590 "instant": false, 1591 "intervalFactor": 1, 1592 "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}", 1593 "refId": "A" 1594 }, 1595 { 1596 "datasource": { 1597 "uid": "${DS_PROMETHEUS}" 1598 }, 1599 "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 1600 "format": "time_series", 1601 "hide": false, 1602 "instant": false, 1603 "interval": "", 1604 "intervalFactor": 1, 1605 "legendFormat": "{{job}}-{{index}} {{name}}", 1606 "refId": "B" 1607 } 1608 ], 1609 "thresholds": [ 1610 { 1611 "colorMode": "critical", 1612 "fill": true, 1613 "line": true, 1614 "op": "gt", 1615 "value": 1, 1616 "yaxis": "left" 1617 } 1618 ], 1619 "timeRegions": [], 1620 "title": "Stacked CPU usage", 1621 "tooltip": { 1622 "shared": false, 1623 "sort": 0, 1624 "value_type": "individual" 1625 }, 1626 "type": "graph", 1627 "xaxis": { 1628 "mode": "time", 1629 "show": true, 1630 "values": [] 1631 }, 1632 "yaxes": [ 1633 { 1634 "$$hashKey": "object:572", 1635 "format": "percentunit", 1636 "logBase": 1, 1637 "show": true 1638 }, 1639 { 1640 "$$hashKey": "object:573", 1641 "format": "short", 1642 "logBase": 1, 1643 "show": true 1644 } 1645 ], 1646 "yaxis": { 1647 "align": false 1648 } 1649 }, 1650 { 1651 "aliasColors": {}, 1652 "bars": false, 1653 "dashLength": 10, 1654 "dashes": false, 1655 "datasource": { 1656 "type": "prometheus", 1657 "uid": "${DS_PROMETHEUS}" 1658 }, 1659 "fieldConfig": { 1660 "defaults": { 1661 "links": [] 1662 }, 1663 "overrides": [] 1664 }, 1665 "fill": 1, 1666 "fillGradient": 0, 1667 "gridPos": { 1668 "h": 7, 1669 "w": 12, 1670 "x": 0, 1671 "y": 48 1672 }, 1673 "hiddenSeries": false, 1674 "id": 136, 1675 "interval": "", 1676 "legend": { 1677 "avg": false, 1678 "current": false, 1679 "max": false, 1680 "min": false, 1681 "show": true, 1682 "total": false, 1683 "values": false 1684 }, 1685 "lines": true, 1686 "linewidth": 1, 1687 "nullPointMode": "null", 1688 "options": { 1689 "alertThreshold": true 1690 }, 1691 "percentage": false, 1692 "pluginVersion": "9.2.2", 1693 "pointradius": 2, 1694 "points": false, 1695 "renderer": "flot", 1696 "seriesOverrides": [], 1697 "spaceLength": 10, 1698 "stack": false, 1699 "steppedLine": false, 1700 "targets": [ 1701 { 1702 "datasource": { 1703 "uid": "${DS_PROMETHEUS}" 1704 }, 1705 "editorMode": "code", 1706 "expr": "rate(synapse_http_client_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 1707 "legendFormat": "{{job}}-{{index}} {{method}}", 1708 "range": true, 1709 "refId": "A" 1710 }, 1711 { 1712 "datasource": { 1713 "uid": "${DS_PROMETHEUS}" 1714 }, 1715 "editorMode": "code", 1716 "expr": "rate(synapse_http_matrixfederationclient_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 1717 "legendFormat": "{{job}}-{{index}} {{method}} (federation)", 1718 "range": true, 1719 "refId": "B" 1720 } 1721 ], 1722 "thresholds": [], 1723 "timeRegions": [], 1724 "title": "Outgoing HTTP request rate", 1725 "tooltip": { 1726 "shared": false, 1727 "sort": 0, 1728 "value_type": "individual" 1729 }, 1730 "type": "graph", 1731 "xaxis": { 1732 "mode": "time", 1733 "show": true, 1734 "values": [] 1735 }, 1736 "yaxes": [ 1737 { 1738 "$$hashKey": "object:123", 1739 "format": "reqps", 1740 "logBase": 1, 1741 "show": true 1742 }, 1743 { 1744 "$$hashKey": "object:124", 1745 "format": "short", 1746 "logBase": 1, 1747 "show": true 1748 } 1749 ], 1750 "yaxis": { 1751 "align": false 1752 } 1753 }, 1754 { 1755 "datasource": { 1756 "type": "prometheus", 1757 "uid": "${DS_PROMETHEUS}" 1758 }, 1759 "fieldConfig": { 1760 "defaults": { 1761 "color": { 1762 "mode": "palette-classic" 1763 }, 1764 "custom": { 1765 "axisCenteredZero": false, 1766 "axisColorMode": "text", 1767 "axisLabel": "active threads", 1768 "axisPlacement": "auto", 1769 "barAlignment": 0, 1770 "drawStyle": "line", 1771 "fillOpacity": 0, 1772 "gradientMode": "none", 1773 "hideFrom": { 1774 "legend": false, 1775 "tooltip": false, 1776 "viz": false 1777 }, 1778 "lineInterpolation": "linear", 1779 "lineWidth": 1, 1780 "pointSize": 5, 1781 "scaleDistribution": { 1782 "type": "linear" 1783 }, 1784 "showPoints": "auto", 1785 "spanNulls": false, 1786 "stacking": { 1787 "group": "A", 1788 "mode": "none" 1789 }, 1790 "thresholdsStyle": { 1791 "mode": "off" 1792 } 1793 }, 1794 "mappings": [], 1795 "thresholds": { 1796 "mode": "absolute", 1797 "steps": [ 1798 { 1799 "color": "green" 1800 }, 1801 { 1802 "color": "red", 1803 "value": 80 1804 } 1805 ] 1806 } 1807 }, 1808 "overrides": [] 1809 }, 1810 "gridPos": { 1811 "h": 7, 1812 "w": 12, 1813 "x": 12, 1814 "y": 48 1815 }, 1816 "id": 207, 1817 "options": { 1818 "legend": { 1819 "calcs": [], 1820 "displayMode": "list", 1821 "placement": "bottom", 1822 "showLegend": true 1823 }, 1824 "tooltip": { 1825 "mode": "single", 1826 "sort": "none" 1827 } 1828 }, 1829 "targets": [ 1830 { 1831 "datasource": { 1832 "type": "prometheus", 1833 "uid": "${DS_PROMETHEUS}" 1834 }, 1835 "exemplar": true, 1836 "expr": "synapse_threadpool_working_threads{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 1837 "interval": "", 1838 "legendFormat": "{{job}}-{{index}} {{name}}", 1839 "refId": "A" 1840 } 1841 ], 1842 "title": "Threadpool activity", 1843 "type": "timeseries" 1844 } 1845 ], 1846 "targets": [ 1847 { 1848 "datasource": { 1849 "type": "prometheus", 1850 "uid": "${DS_PROMETHEUS}" 1851 }, 1852 "refId": "A" 1853 } 1854 ], 1855 "title": "Process info", 1856 "type": "row" 1857 }, 1858 { 1859 "collapsed": true, 1860 "datasource": { 1861 "type": "prometheus", 1862 "uid": "${DS_PROMETHEUS}" 1863 }, 1864 "gridPos": { 1865 "h": 1, 1866 "w": 24, 1867 "x": 0, 1868 "y": 27 1869 }, 1870 "id": 56, 1871 "panels": [ 1872 { 1873 "cards": { 1874 "cardPadding": -1, 1875 "cardRound": 0 1876 }, 1877 "color": { 1878 "cardColor": "#b4ff00", 1879 "colorScale": "sqrt", 1880 "colorScheme": "interpolateInferno", 1881 "exponent": 0.5, 1882 "mode": "spectrum" 1883 }, 1884 "dataFormat": "tsbuckets", 1885 "datasource": { 1886 "uid": "${DS_PROMETHEUS}" 1887 }, 1888 "fieldConfig": { 1889 "defaults": { 1890 "custom": { 1891 "hideFrom": { 1892 "legend": false, 1893 "tooltip": false, 1894 "viz": false 1895 }, 1896 "scaleDistribution": { 1897 "type": "linear" 1898 } 1899 } 1900 }, 1901 "overrides": [] 1902 }, 1903 "gridPos": { 1904 "h": 9, 1905 "w": 12, 1906 "x": 0, 1907 "y": 28 1908 }, 1909 "heatmap": {}, 1910 "hideZeroBuckets": false, 1911 "highlightCards": true, 1912 "id": 85, 1913 "legend": { 1914 "show": false 1915 }, 1916 "links": [], 1917 "options": { 1918 "calculate": false, 1919 "calculation": {}, 1920 "cellGap": -1, 1921 "cellRadius": 0, 1922 "cellValues": {}, 1923 "color": { 1924 "exponent": 0.5, 1925 "fill": "#b4ff00", 1926 "mode": "scheme", 1927 "reverse": false, 1928 "scale": "exponential", 1929 "scheme": "Inferno", 1930 "steps": 128 1931 }, 1932 "exemplars": { 1933 "color": "rgba(255,0,255,0.7)" 1934 }, 1935 "filterValues": { 1936 "le": 1e-9 1937 }, 1938 "legend": { 1939 "show": false 1940 }, 1941 "rowsFrame": { 1942 "layout": "auto" 1943 }, 1944 "showValue": "never", 1945 "tooltip": { 1946 "show": true, 1947 "yHistogram": true 1948 }, 1949 "yAxis": { 1950 "axisPlacement": "left", 1951 "reverse": false, 1952 "unit": "s" 1953 } 1954 }, 1955 "pluginVersion": "9.2.2", 1956 "reverseYBuckets": false, 1957 "targets": [ 1958 { 1959 "datasource": { 1960 "uid": "${DS_PROMETHEUS}" 1961 }, 1962 "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)", 1963 "format": "heatmap", 1964 "intervalFactor": 1, 1965 "legendFormat": "{{le}}", 1966 "refId": "A" 1967 } 1968 ], 1969 "title": "Event Send Time (Including errors, across all workers)", 1970 "tooltip": { 1971 "show": true, 1972 "showHistogram": true 1973 }, 1974 "type": "heatmap", 1975 "xAxis": { 1976 "show": true 1977 }, 1978 "yAxis": { 1979 "format": "s", 1980 "logBase": 2, 1981 "show": true 1982 }, 1983 "yBucketBound": "auto" 1984 }, 1985 { 1986 "aliasColors": {}, 1987 "bars": false, 1988 "dashLength": 10, 1989 "dashes": false, 1990 "datasource": { 1991 "uid": "${DS_PROMETHEUS}" 1992 }, 1993 "description": "", 1994 "editable": true, 1995 "error": false, 1996 "fieldConfig": { 1997 "defaults": { 1998 "links": [] 1999 }, 2000 "overrides": [] 2001 }, 2002 "fill": 1, 2003 "fillGradient": 0, 2004 "grid": {}, 2005 "gridPos": { 2006 "h": 9, 2007 "w": 12, 2008 "x": 12, 2009 "y": 28 2010 }, 2011 "hiddenSeries": false, 2012 "id": 33, 2013 "legend": { 2014 "avg": false, 2015 "current": false, 2016 "max": false, 2017 "min": false, 2018 "show": false, 2019 "total": false, 2020 "values": false 2021 }, 2022 "lines": true, 2023 "linewidth": 2, 2024 "links": [], 2025 "nullPointMode": "null", 2026 "options": { 2027 "alertThreshold": true 2028 }, 2029 "paceLength": 10, 2030 "percentage": false, 2031 "pluginVersion": "9.2.2", 2032 "pointradius": 5, 2033 "points": false, 2034 "renderer": "flot", 2035 "seriesOverrides": [], 2036 "spaceLength": 10, 2037 "stack": false, 2038 "steppedLine": false, 2039 "targets": [ 2040 { 2041 "datasource": { 2042 "uid": "${DS_PROMETHEUS}" 2043 }, 2044 "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size])) without (job,index)", 2045 "format": "time_series", 2046 "interval": "", 2047 "intervalFactor": 2, 2048 "legendFormat": "", 2049 "refId": "A", 2050 "step": 20, 2051 "target": "" 2052 } 2053 ], 2054 "thresholds": [], 2055 "timeRegions": [], 2056 "title": "Events Persisted (all workers)", 2057 "tooltip": { 2058 "shared": true, 2059 "sort": 0, 2060 "value_type": "cumulative" 2061 }, 2062 "type": "graph", 2063 "xaxis": { 2064 "mode": "time", 2065 "show": true, 2066 "values": [] 2067 }, 2068 "yaxes": [ 2069 { 2070 "$$hashKey": "object:102", 2071 "format": "hertz", 2072 "logBase": 1, 2073 "show": true 2074 }, 2075 { 2076 "$$hashKey": "object:103", 2077 "format": "short", 2078 "logBase": 1, 2079 "show": true 2080 } 2081 ], 2082 "yaxis": { 2083 "align": false 2084 } 2085 }, 2086 { 2087 "aliasColors": {}, 2088 "bars": false, 2089 "dashLength": 10, 2090 "dashes": false, 2091 "datasource": { 2092 "uid": "${DS_PROMETHEUS}" 2093 }, 2094 "decimals": 1, 2095 "fill": 1, 2096 "fillGradient": 0, 2097 "gridPos": { 2098 "h": 7, 2099 "w": 12, 2100 "x": 0, 2101 "y": 37 2102 }, 2103 "hiddenSeries": false, 2104 "id": 40, 2105 "legend": { 2106 "avg": false, 2107 "current": false, 2108 "max": false, 2109 "min": false, 2110 "show": true, 2111 "total": false, 2112 "values": false 2113 }, 2114 "lines": true, 2115 "linewidth": 1, 2116 "links": [], 2117 "nullPointMode": "null", 2118 "options": { 2119 "alertThreshold": true 2120 }, 2121 "percentage": false, 2122 "pluginVersion": "9.2.2", 2123 "pointradius": 5, 2124 "points": false, 2125 "renderer": "flot", 2126 "seriesOverrides": [], 2127 "spaceLength": 10, 2128 "stack": false, 2129 "steppedLine": false, 2130 "targets": [ 2131 { 2132 "datasource": { 2133 "uid": "${DS_PROMETHEUS}" 2134 }, 2135 "expr": "rate(synapse_storage_events_persisted_by_source_type{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 2136 "format": "time_series", 2137 "intervalFactor": 2, 2138 "legendFormat": "{{type}}", 2139 "refId": "D" 2140 } 2141 ], 2142 "thresholds": [], 2143 "timeRegions": [], 2144 "title": "Events/s Local vs Remote", 2145 "tooltip": { 2146 "shared": true, 2147 "sort": 2, 2148 "value_type": "individual" 2149 }, 2150 "type": "graph", 2151 "xaxis": { 2152 "mode": "time", 2153 "show": true, 2154 "values": [] 2155 }, 2156 "yaxes": [ 2157 { 2158 "format": "hertz", 2159 "label": "", 2160 "logBase": 1, 2161 "min": "0", 2162 "show": true 2163 }, 2164 { 2165 "format": "short", 2166 "logBase": 1, 2167 "show": true 2168 } 2169 ], 2170 "yaxis": { 2171 "align": false 2172 } 2173 }, 2174 { 2175 "aliasColors": {}, 2176 "bars": false, 2177 "dashLength": 10, 2178 "dashes": false, 2179 "datasource": { 2180 "uid": "${DS_PROMETHEUS}" 2181 }, 2182 "decimals": 1, 2183 "fill": 1, 2184 "fillGradient": 0, 2185 "gridPos": { 2186 "h": 7, 2187 "w": 12, 2188 "x": 12, 2189 "y": 37 2190 }, 2191 "hiddenSeries": false, 2192 "id": 46, 2193 "legend": { 2194 "avg": false, 2195 "current": false, 2196 "max": false, 2197 "min": false, 2198 "show": true, 2199 "total": false, 2200 "values": false 2201 }, 2202 "lines": true, 2203 "linewidth": 1, 2204 "links": [], 2205 "nullPointMode": "null", 2206 "options": { 2207 "alertThreshold": true 2208 }, 2209 "percentage": false, 2210 "pluginVersion": "9.2.2", 2211 "pointradius": 5, 2212 "points": false, 2213 "renderer": "flot", 2214 "seriesOverrides": [], 2215 "spaceLength": 10, 2216 "stack": false, 2217 "steppedLine": false, 2218 "targets": [ 2219 { 2220 "datasource": { 2221 "uid": "${DS_PROMETHEUS}" 2222 }, 2223 "expr": "rate(synapse_storage_events_persisted_by_event_type{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 2224 "format": "time_series", 2225 "instant": false, 2226 "intervalFactor": 2, 2227 "legendFormat": "{{type}}", 2228 "refId": "A", 2229 "step": 20 2230 } 2231 ], 2232 "thresholds": [], 2233 "timeRegions": [], 2234 "title": "Events/s by Type", 2235 "tooltip": { 2236 "shared": false, 2237 "sort": 2, 2238 "value_type": "individual" 2239 }, 2240 "type": "graph", 2241 "xaxis": { 2242 "mode": "time", 2243 "show": true, 2244 "values": [] 2245 }, 2246 "yaxes": [ 2247 { 2248 "format": "hertz", 2249 "logBase": 1, 2250 "min": "0", 2251 "show": true 2252 }, 2253 { 2254 "format": "short", 2255 "logBase": 1, 2256 "show": true 2257 } 2258 ], 2259 "yaxis": { 2260 "align": false 2261 } 2262 }, 2263 { 2264 "aliasColors": { 2265 "irc-freenode (local)": "#EAB839" 2266 }, 2267 "bars": false, 2268 "dashLength": 10, 2269 "dashes": false, 2270 "datasource": { 2271 "uid": "${DS_PROMETHEUS}" 2272 }, 2273 "decimals": 1, 2274 "fill": 1, 2275 "fillGradient": 0, 2276 "gridPos": { 2277 "h": 7, 2278 "w": 12, 2279 "x": 0, 2280 "y": 44 2281 }, 2282 "hiddenSeries": false, 2283 "id": 44, 2284 "legend": { 2285 "alignAsTable": true, 2286 "avg": false, 2287 "current": false, 2288 "hideEmpty": true, 2289 "hideZero": true, 2290 "max": false, 2291 "min": false, 2292 "show": true, 2293 "total": false, 2294 "values": false 2295 }, 2296 "lines": true, 2297 "linewidth": 1, 2298 "links": [], 2299 "nullPointMode": "null", 2300 "options": { 2301 "alertThreshold": true 2302 }, 2303 "percentage": false, 2304 "pluginVersion": "9.2.2", 2305 "pointradius": 5, 2306 "points": false, 2307 "renderer": "flot", 2308 "seriesOverrides": [], 2309 "spaceLength": 10, 2310 "stack": false, 2311 "steppedLine": false, 2312 "targets": [ 2313 { 2314 "datasource": { 2315 "uid": "${DS_PROMETHEUS}" 2316 }, 2317 "expr": "rate(synapse_storage_events_persisted_by_origin{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 2318 "format": "time_series", 2319 "intervalFactor": 2, 2320 "legendFormat": "{{origin_entity}} ({{origin_type}})", 2321 "refId": "A", 2322 "step": 20 2323 } 2324 ], 2325 "thresholds": [], 2326 "timeRegions": [], 2327 "title": "Events/s by Origin", 2328 "tooltip": { 2329 "shared": false, 2330 "sort": 2, 2331 "value_type": "individual" 2332 }, 2333 "type": "graph", 2334 "xaxis": { 2335 "mode": "time", 2336 "show": true, 2337 "values": [] 2338 }, 2339 "yaxes": [ 2340 { 2341 "format": "hertz", 2342 "logBase": 1, 2343 "min": "0", 2344 "show": true 2345 }, 2346 { 2347 "format": "short", 2348 "logBase": 1, 2349 "show": true 2350 } 2351 ], 2352 "yaxis": { 2353 "align": false 2354 } 2355 }, 2356 { 2357 "aliasColors": {}, 2358 "bars": false, 2359 "dashLength": 10, 2360 "dashes": false, 2361 "datasource": { 2362 "type": "prometheus", 2363 "uid": "${DS_PROMETHEUS}" 2364 }, 2365 "decimals": 1, 2366 "fill": 1, 2367 "fillGradient": 0, 2368 "gridPos": { 2369 "h": 7, 2370 "w": 12, 2371 "x": 12, 2372 "y": 44 2373 }, 2374 "hiddenSeries": false, 2375 "id": 45, 2376 "legend": { 2377 "alignAsTable": true, 2378 "avg": false, 2379 "current": false, 2380 "hideEmpty": true, 2381 "hideZero": true, 2382 "max": false, 2383 "min": false, 2384 "show": true, 2385 "total": false, 2386 "values": false 2387 }, 2388 "lines": true, 2389 "linewidth": 1, 2390 "links": [], 2391 "nullPointMode": "null", 2392 "options": { 2393 "alertThreshold": true 2394 }, 2395 "percentage": false, 2396 "pluginVersion": "9.2.2", 2397 "pointradius": 5, 2398 "points": false, 2399 "renderer": "flot", 2400 "seriesOverrides": [], 2401 "spaceLength": 10, 2402 "stack": false, 2403 "steppedLine": false, 2404 "targets": [ 2405 { 2406 "datasource": { 2407 "uid": "${DS_PROMETHEUS}" 2408 }, 2409 "editorMode": "code", 2410 "expr": "sum(rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)", 2411 "format": "time_series", 2412 "intervalFactor": 2, 2413 "legendFormat": "{{origin_entity}} ({{origin_type}})", 2414 "range": true, 2415 "refId": "A", 2416 "step": 20 2417 } 2418 ], 2419 "thresholds": [], 2420 "timeRegions": [], 2421 "title": "Memberships/s by Origin", 2422 "tooltip": { 2423 "shared": true, 2424 "sort": 2, 2425 "value_type": "individual" 2426 }, 2427 "type": "graph", 2428 "xaxis": { 2429 "mode": "time", 2430 "show": true, 2431 "values": [] 2432 }, 2433 "yaxes": [ 2434 { 2435 "$$hashKey": "object:232", 2436 "format": "hertz", 2437 "logBase": 1, 2438 "min": "0", 2439 "show": true 2440 }, 2441 { 2442 "$$hashKey": "object:233", 2443 "format": "short", 2444 "logBase": 1, 2445 "show": true 2446 } 2447 ], 2448 "yaxis": { 2449 "align": false 2450 } 2451 }, 2452 { 2453 "aliasColors": {}, 2454 "bars": false, 2455 "dashLength": 10, 2456 "dashes": false, 2457 "datasource": { 2458 "type": "prometheus", 2459 "uid": "${DS_PROMETHEUS}" 2460 }, 2461 "fieldConfig": { 2462 "defaults": { 2463 "links": [] 2464 }, 2465 "overrides": [] 2466 }, 2467 "fill": 0, 2468 "fillGradient": 0, 2469 "gridPos": { 2470 "h": 9, 2471 "w": 12, 2472 "x": 0, 2473 "y": 51 2474 }, 2475 "hiddenSeries": false, 2476 "id": 118, 2477 "legend": { 2478 "avg": false, 2479 "current": false, 2480 "max": false, 2481 "min": false, 2482 "show": true, 2483 "total": false, 2484 "values": false 2485 }, 2486 "lines": true, 2487 "linewidth": 1, 2488 "links": [], 2489 "nullPointMode": "null", 2490 "options": { 2491 "alertThreshold": true 2492 }, 2493 "paceLength": 10, 2494 "percentage": false, 2495 "pluginVersion": "9.2.2", 2496 "pointradius": 5, 2497 "points": false, 2498 "renderer": "flot", 2499 "repeatDirection": "h", 2500 "seriesOverrides": [ 2501 { 2502 "$$hashKey": "object:316", 2503 "alias": "mean", 2504 "linewidth": 2 2505 } 2506 ], 2507 "spaceLength": 10, 2508 "stack": false, 2509 "steppedLine": false, 2510 "targets": [ 2511 { 2512 "datasource": { 2513 "type": "prometheus", 2514 "uid": "${DS_PROMETHEUS}" 2515 }, 2516 "exemplar": true, 2517 "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", 2518 "format": "time_series", 2519 "interval": "", 2520 "intervalFactor": 1, 2521 "legendFormat": "{{job}}-{{index}} 99%", 2522 "refId": "A" 2523 }, 2524 { 2525 "datasource": { 2526 "type": "prometheus", 2527 "uid": "${DS_PROMETHEUS}" 2528 }, 2529 "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", 2530 "format": "time_series", 2531 "interval": "", 2532 "intervalFactor": 1, 2533 "legendFormat": "{{job}}-{{index}} 95%", 2534 "refId": "B" 2535 }, 2536 { 2537 "datasource": { 2538 "type": "prometheus", 2539 "uid": "${DS_PROMETHEUS}" 2540 }, 2541 "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", 2542 "format": "time_series", 2543 "intervalFactor": 1, 2544 "legendFormat": "{{job}}-{{index}} 90%", 2545 "refId": "C" 2546 }, 2547 { 2548 "datasource": { 2549 "type": "prometheus", 2550 "uid": "${DS_PROMETHEUS}" 2551 }, 2552 "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", 2553 "format": "time_series", 2554 "intervalFactor": 1, 2555 "legendFormat": "{{job}}-{{index}} 50%", 2556 "refId": "D" 2557 }, 2558 { 2559 "datasource": { 2560 "type": "prometheus", 2561 "uid": "${DS_PROMETHEUS}" 2562 }, 2563 "editorMode": "code", 2564 "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)", 2565 "format": "time_series", 2566 "intervalFactor": 1, 2567 "legendFormat": "{{job}}-{{index}} mean", 2568 "range": true, 2569 "refId": "E" 2570 } 2571 ], 2572 "thresholds": [], 2573 "timeRegions": [], 2574 "title": "Event send time quantiles by worker", 2575 "tooltip": { 2576 "shared": true, 2577 "sort": 0, 2578 "value_type": "individual" 2579 }, 2580 "type": "graph", 2581 "xaxis": { 2582 "mode": "time", 2583 "show": true, 2584 "values": [] 2585 }, 2586 "yaxes": [ 2587 { 2588 "$$hashKey": "object:263", 2589 "format": "s", 2590 "logBase": 1, 2591 "show": true 2592 }, 2593 { 2594 "$$hashKey": "object:264", 2595 "format": "short", 2596 "logBase": 1, 2597 "show": true 2598 } 2599 ], 2600 "yaxis": { 2601 "align": false 2602 } 2603 }, 2604 { 2605 "datasource": { 2606 "type": "prometheus", 2607 "uid": "${DS_PROMETHEUS}" 2608 }, 2609 "description": "CPU and DB time spent on most expensive state resolution in a room, summed over all workers. This is a very rough proxy for \"how fast is state res\", but it doesn't accurately represent the system load (e.g. it completely ignores cheap state resolutions).\n", 2610 "fieldConfig": { 2611 "defaults": { 2612 "color": { 2613 "mode": "palette-classic" 2614 }, 2615 "custom": { 2616 "axisCenteredZero": false, 2617 "axisColorMode": "text", 2618 "axisLabel": "", 2619 "axisPlacement": "auto", 2620 "barAlignment": 0, 2621 "drawStyle": "line", 2622 "fillOpacity": 30, 2623 "gradientMode": "none", 2624 "hideFrom": { 2625 "legend": false, 2626 "tooltip": false, 2627 "viz": false 2628 }, 2629 "lineInterpolation": "linear", 2630 "lineWidth": 1, 2631 "pointSize": 5, 2632 "scaleDistribution": { 2633 "type": "linear" 2634 }, 2635 "showPoints": "auto", 2636 "spanNulls": false, 2637 "stacking": { 2638 "group": "A", 2639 "mode": "normal" 2640 }, 2641 "thresholdsStyle": { 2642 "mode": "off" 2643 } 2644 }, 2645 "mappings": [], 2646 "thresholds": { 2647 "mode": "absolute", 2648 "steps": [ 2649 { 2650 "color": "green" 2651 }, 2652 { 2653 "color": "red", 2654 "value": 80 2655 } 2656 ] 2657 }, 2658 "unit": "s/s" 2659 }, 2660 "overrides": [] 2661 }, 2662 "gridPos": { 2663 "h": 9, 2664 "w": 12, 2665 "x": 12, 2666 "y": 51 2667 }, 2668 "id": 222, 2669 "options": { 2670 "legend": { 2671 "calcs": [], 2672 "displayMode": "list", 2673 "placement": "bottom", 2674 "showLegend": false 2675 }, 2676 "tooltip": { 2677 "mode": "multi", 2678 "sort": "none" 2679 } 2680 }, 2681 "targets": [ 2682 { 2683 "datasource": { 2684 "type": "prometheus", 2685 "uid": "${DS_PROMETHEUS}" 2686 }, 2687 "exemplar": false, 2688 "expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))", 2689 "format": "time_series", 2690 "hide": false, 2691 "instant": false, 2692 "interval": "", 2693 "legendFormat": "DB time", 2694 "refId": "B" 2695 }, 2696 { 2697 "datasource": { 2698 "type": "prometheus", 2699 "uid": "${DS_PROMETHEUS}" 2700 }, 2701 "exemplar": false, 2702 "expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))", 2703 "format": "time_series", 2704 "hide": false, 2705 "instant": false, 2706 "interval": "", 2707 "legendFormat": "CPU time", 2708 "refId": "C" 2709 } 2710 ], 2711 "title": "Stateres worst-case", 2712 "type": "timeseries" 2713 } 2714 ], 2715 "targets": [ 2716 { 2717 "datasource": { 2718 "type": "prometheus", 2719 "uid": "${DS_PROMETHEUS}" 2720 }, 2721 "refId": "A" 2722 } 2723 ], 2724 "title": "Event persistence", 2725 "type": "row" 2726 }, 2727 { 2728 "collapsed": true, 2729 "datasource": { 2730 "type": "prometheus", 2731 "uid": "${DS_PROMETHEUS}" 2732 }, 2733 "gridPos": { 2734 "h": 1, 2735 "w": 24, 2736 "x": 0, 2737 "y": 28 2738 }, 2739 "id": 57, 2740 "panels": [ 2741 { 2742 "aliasColors": {}, 2743 "bars": false, 2744 "dashLength": 10, 2745 "dashes": false, 2746 "datasource": { 2747 "type": "prometheus", 2748 "uid": "${DS_PROMETHEUS}" 2749 }, 2750 "editable": true, 2751 "error": false, 2752 "fieldConfig": { 2753 "defaults": { 2754 "links": [] 2755 }, 2756 "overrides": [] 2757 }, 2758 "fill": 2, 2759 "fillGradient": 0, 2760 "grid": {}, 2761 "gridPos": { 2762 "h": 8, 2763 "w": 12, 2764 "x": 0, 2765 "y": 29 2766 }, 2767 "hiddenSeries": false, 2768 "id": 4, 2769 "legend": { 2770 "alignAsTable": true, 2771 "avg": false, 2772 "current": false, 2773 "hideEmpty": false, 2774 "hideZero": true, 2775 "max": false, 2776 "min": false, 2777 "rightSide": false, 2778 "show": true, 2779 "total": false, 2780 "values": false 2781 }, 2782 "lines": true, 2783 "linewidth": 1, 2784 "links": [], 2785 "nullPointMode": "null", 2786 "options": { 2787 "alertThreshold": true 2788 }, 2789 "percentage": false, 2790 "pluginVersion": "9.2.2", 2791 "pointradius": 5, 2792 "points": false, 2793 "renderer": "flot", 2794 "seriesOverrides": [], 2795 "spaceLength": 10, 2796 "stack": false, 2797 "steppedLine": false, 2798 "targets": [ 2799 { 2800 "datasource": { 2801 "uid": "${DS_PROMETHEUS}" 2802 }, 2803 "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 2804 "format": "time_series", 2805 "interval": "", 2806 "intervalFactor": 2, 2807 "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}", 2808 "refId": "A", 2809 "step": 20 2810 } 2811 ], 2812 "thresholds": [ 2813 { 2814 "$$hashKey": "object:234", 2815 "colorMode": "custom", 2816 "fill": true, 2817 "fillColor": "rgba(216, 200, 27, 0.27)", 2818 "op": "gt", 2819 "value": 100, 2820 "yaxis": "left" 2821 }, 2822 { 2823 "$$hashKey": "object:235", 2824 "colorMode": "custom", 2825 "fill": true, 2826 "fillColor": "rgba(234, 112, 112, 0.22)", 2827 "op": "gt", 2828 "value": 250, 2829 "yaxis": "left" 2830 } 2831 ], 2832 "timeRegions": [], 2833 "title": "Request Count by arrival time", 2834 "tooltip": { 2835 "shared": false, 2836 "sort": 2, 2837 "value_type": "individual" 2838 }, 2839 "type": "graph", 2840 "xaxis": { 2841 "mode": "time", 2842 "show": true, 2843 "values": [] 2844 }, 2845 "yaxes": [ 2846 { 2847 "$$hashKey": "object:206", 2848 "format": "hertz", 2849 "logBase": 1, 2850 "show": true 2851 }, 2852 { 2853 "$$hashKey": "object:207", 2854 "format": "short", 2855 "logBase": 1, 2856 "show": true 2857 } 2858 ], 2859 "yaxis": { 2860 "align": false 2861 } 2862 }, 2863 { 2864 "aliasColors": {}, 2865 "bars": false, 2866 "dashLength": 10, 2867 "dashes": false, 2868 "datasource": { 2869 "type": "prometheus", 2870 "uid": "${DS_PROMETHEUS}" 2871 }, 2872 "editable": true, 2873 "error": false, 2874 "fieldConfig": { 2875 "defaults": { 2876 "links": [] 2877 }, 2878 "overrides": [] 2879 }, 2880 "fill": 1, 2881 "fillGradient": 0, 2882 "grid": {}, 2883 "gridPos": { 2884 "h": 8, 2885 "w": 12, 2886 "x": 12, 2887 "y": 29 2888 }, 2889 "hiddenSeries": false, 2890 "id": 32, 2891 "legend": { 2892 "avg": false, 2893 "current": false, 2894 "max": false, 2895 "min": false, 2896 "show": true, 2897 "total": false, 2898 "values": false 2899 }, 2900 "lines": true, 2901 "linewidth": 2, 2902 "links": [], 2903 "nullPointMode": "null", 2904 "options": { 2905 "alertThreshold": true 2906 }, 2907 "percentage": false, 2908 "pluginVersion": "9.2.2", 2909 "pointradius": 5, 2910 "points": false, 2911 "renderer": "flot", 2912 "seriesOverrides": [], 2913 "spaceLength": 10, 2914 "stack": false, 2915 "steppedLine": false, 2916 "targets": [ 2917 { 2918 "datasource": { 2919 "uid": "${DS_PROMETHEUS}" 2920 }, 2921 "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})", 2922 "format": "time_series", 2923 "intervalFactor": 2, 2924 "legendFormat": "{{method}} {{servlet}} {{job}}-{{index}}", 2925 "refId": "A", 2926 "step": 20, 2927 "target": "" 2928 } 2929 ], 2930 "thresholds": [], 2931 "timeRegions": [], 2932 "title": "Top 10 Request Counts", 2933 "tooltip": { 2934 "shared": false, 2935 "sort": 2, 2936 "value_type": "cumulative" 2937 }, 2938 "type": "graph", 2939 "xaxis": { 2940 "mode": "time", 2941 "show": true, 2942 "values": [] 2943 }, 2944 "yaxes": [ 2945 { 2946 "$$hashKey": "object:305", 2947 "format": "hertz", 2948 "logBase": 1, 2949 "show": true 2950 }, 2951 { 2952 "$$hashKey": "object:306", 2953 "format": "short", 2954 "logBase": 1, 2955 "show": true 2956 } 2957 ], 2958 "yaxis": { 2959 "align": false 2960 } 2961 }, 2962 { 2963 "aliasColors": {}, 2964 "bars": false, 2965 "dashLength": 10, 2966 "dashes": false, 2967 "datasource": { 2968 "type": "prometheus", 2969 "uid": "${DS_PROMETHEUS}" 2970 }, 2971 "editable": true, 2972 "error": false, 2973 "fieldConfig": { 2974 "defaults": { 2975 "links": [] 2976 }, 2977 "overrides": [] 2978 }, 2979 "fill": 2, 2980 "fillGradient": 0, 2981 "grid": {}, 2982 "gridPos": { 2983 "h": 8, 2984 "w": 12, 2985 "x": 0, 2986 "y": 37 2987 }, 2988 "hiddenSeries": false, 2989 "id": 139, 2990 "legend": { 2991 "alignAsTable": true, 2992 "avg": false, 2993 "current": false, 2994 "hideEmpty": false, 2995 "hideZero": true, 2996 "max": false, 2997 "min": false, 2998 "rightSide": false, 2999 "show": true, 3000 "total": false, 3001 "values": false 3002 }, 3003 "lines": true, 3004 "linewidth": 1, 3005 "links": [], 3006 "nullPointMode": "null", 3007 "options": { 3008 "alertThreshold": true 3009 }, 3010 "percentage": false, 3011 "pluginVersion": "9.2.2", 3012 "pointradius": 5, 3013 "points": false, 3014 "renderer": "flot", 3015 "seriesOverrides": [], 3016 "spaceLength": 10, 3017 "stack": false, 3018 "steppedLine": false, 3019 "targets": [ 3020 { 3021 "datasource": { 3022 "uid": "${DS_PROMETHEUS}" 3023 }, 3024 "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 3025 "format": "time_series", 3026 "interval": "", 3027 "intervalFactor": 1, 3028 "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}", 3029 "refId": "A", 3030 "step": 20 3031 } 3032 ], 3033 "thresholds": [ 3034 { 3035 "$$hashKey": "object:135", 3036 "colorMode": "custom", 3037 "fill": true, 3038 "fillColor": "rgba(216, 200, 27, 0.27)", 3039 "op": "gt", 3040 "value": 100, 3041 "yaxis": "left" 3042 }, 3043 { 3044 "$$hashKey": "object:136", 3045 "colorMode": "custom", 3046 "fill": true, 3047 "fillColor": "rgba(234, 112, 112, 0.22)", 3048 "op": "gt", 3049 "value": 250, 3050 "yaxis": "left" 3051 } 3052 ], 3053 "timeRegions": [], 3054 "title": "Total CPU Usage by Endpoint", 3055 "tooltip": { 3056 "shared": false, 3057 "sort": 2, 3058 "value_type": "individual" 3059 }, 3060 "type": "graph", 3061 "xaxis": { 3062 "mode": "time", 3063 "show": true, 3064 "values": [] 3065 }, 3066 "yaxes": [ 3067 { 3068 "$$hashKey": "object:107", 3069 "format": "percentunit", 3070 "logBase": 1, 3071 "show": true 3072 }, 3073 { 3074 "$$hashKey": "object:108", 3075 "format": "short", 3076 "logBase": 1, 3077 "show": true 3078 } 3079 ], 3080 "yaxis": { 3081 "align": false 3082 } 3083 }, 3084 { 3085 "aliasColors": {}, 3086 "bars": false, 3087 "dashLength": 10, 3088 "dashes": false, 3089 "datasource": { 3090 "type": "prometheus", 3091 "uid": "${DS_PROMETHEUS}" 3092 }, 3093 "editable": true, 3094 "error": false, 3095 "fieldConfig": { 3096 "defaults": { 3097 "links": [] 3098 }, 3099 "overrides": [] 3100 }, 3101 "fill": 0, 3102 "fillGradient": 0, 3103 "grid": {}, 3104 "gridPos": { 3105 "h": 8, 3106 "w": 12, 3107 "x": 12, 3108 "y": 37 3109 }, 3110 "hiddenSeries": false, 3111 "id": 52, 3112 "legend": { 3113 "alignAsTable": true, 3114 "avg": false, 3115 "current": false, 3116 "hideEmpty": false, 3117 "hideZero": true, 3118 "max": false, 3119 "min": false, 3120 "rightSide": false, 3121 "show": true, 3122 "total": false, 3123 "values": false 3124 }, 3125 "lines": true, 3126 "linewidth": 1, 3127 "links": [], 3128 "nullPointMode": "null", 3129 "options": { 3130 "alertThreshold": true 3131 }, 3132 "percentage": false, 3133 "pluginVersion": "9.2.2", 3134 "pointradius": 5, 3135 "points": false, 3136 "renderer": "flot", 3137 "seriesOverrides": [], 3138 "spaceLength": 10, 3139 "stack": false, 3140 "steppedLine": false, 3141 "targets": [ 3142 { 3143 "datasource": { 3144 "uid": "${DS_PROMETHEUS}" 3145 }, 3146 "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 3147 "format": "time_series", 3148 "interval": "", 3149 "intervalFactor": 1, 3150 "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}", 3151 "refId": "A", 3152 "step": 20 3153 } 3154 ], 3155 "thresholds": [ 3156 { 3157 "$$hashKey": "object:417", 3158 "colorMode": "custom", 3159 "fill": true, 3160 "fillColor": "rgba(216, 200, 27, 0.27)", 3161 "op": "gt", 3162 "value": 100, 3163 "yaxis": "left" 3164 }, 3165 { 3166 "$$hashKey": "object:418", 3167 "colorMode": "custom", 3168 "fill": true, 3169 "fillColor": "rgba(234, 112, 112, 0.22)", 3170 "op": "gt", 3171 "value": 250, 3172 "yaxis": "left" 3173 } 3174 ], 3175 "timeRegions": [], 3176 "title": "Average CPU Usage by Endpoint", 3177 "tooltip": { 3178 "shared": false, 3179 "sort": 2, 3180 "value_type": "individual" 3181 }, 3182 "type": "graph", 3183 "xaxis": { 3184 "mode": "time", 3185 "show": true, 3186 "values": [] 3187 }, 3188 "yaxes": [ 3189 { 3190 "$$hashKey": "object:389", 3191 "format": "s", 3192 "logBase": 1, 3193 "show": true 3194 }, 3195 { 3196 "$$hashKey": "object:390", 3197 "format": "short", 3198 "logBase": 1, 3199 "show": true 3200 } 3201 ], 3202 "yaxis": { 3203 "align": false 3204 } 3205 }, 3206 { 3207 "aliasColors": {}, 3208 "bars": false, 3209 "dashLength": 10, 3210 "dashes": false, 3211 "datasource": { 3212 "type": "prometheus", 3213 "uid": "${DS_PROMETHEUS}" 3214 }, 3215 "editable": true, 3216 "error": false, 3217 "fieldConfig": { 3218 "defaults": { 3219 "links": [] 3220 }, 3221 "overrides": [] 3222 }, 3223 "fill": 1, 3224 "fillGradient": 0, 3225 "grid": {}, 3226 "gridPos": { 3227 "h": 8, 3228 "w": 12, 3229 "x": 0, 3230 "y": 45 3231 }, 3232 "hiddenSeries": false, 3233 "id": 7, 3234 "legend": { 3235 "alignAsTable": true, 3236 "avg": false, 3237 "current": false, 3238 "hideEmpty": true, 3239 "hideZero": true, 3240 "max": false, 3241 "min": false, 3242 "show": true, 3243 "total": false, 3244 "values": false 3245 }, 3246 "lines": true, 3247 "linewidth": 1, 3248 "links": [], 3249 "nullPointMode": "null", 3250 "options": { 3251 "alertThreshold": true 3252 }, 3253 "percentage": false, 3254 "pluginVersion": "9.2.2", 3255 "pointradius": 5, 3256 "points": false, 3257 "renderer": "flot", 3258 "seriesOverrides": [], 3259 "spaceLength": 10, 3260 "stack": false, 3261 "steppedLine": false, 3262 "targets": [ 3263 { 3264 "datasource": { 3265 "uid": "${DS_PROMETHEUS}" 3266 }, 3267 "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 3268 "format": "time_series", 3269 "interval": "", 3270 "intervalFactor": 1, 3271 "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}", 3272 "refId": "A", 3273 "step": 20 3274 } 3275 ], 3276 "thresholds": [], 3277 "timeRegions": [], 3278 "title": "DB Usage by endpoint", 3279 "tooltip": { 3280 "shared": false, 3281 "sort": 2, 3282 "value_type": "cumulative" 3283 }, 3284 "type": "graph", 3285 "xaxis": { 3286 "mode": "time", 3287 "show": true, 3288 "values": [] 3289 }, 3290 "yaxes": [ 3291 { 3292 "$$hashKey": "object:488", 3293 "format": "percentunit", 3294 "logBase": 1, 3295 "show": true 3296 }, 3297 { 3298 "$$hashKey": "object:489", 3299 "format": "short", 3300 "logBase": 1, 3301 "show": true 3302 } 3303 ], 3304 "yaxis": { 3305 "align": false 3306 } 3307 }, 3308 { 3309 "aliasColors": {}, 3310 "bars": false, 3311 "dashLength": 10, 3312 "dashes": false, 3313 "datasource": { 3314 "uid": "${DS_PROMETHEUS}" 3315 }, 3316 "editable": true, 3317 "error": false, 3318 "fieldConfig": { 3319 "defaults": { 3320 "links": [] 3321 }, 3322 "overrides": [] 3323 }, 3324 "fill": 2, 3325 "fillGradient": 0, 3326 "grid": {}, 3327 "gridPos": { 3328 "h": 8, 3329 "w": 12, 3330 "x": 12, 3331 "y": 45 3332 }, 3333 "hiddenSeries": false, 3334 "id": 47, 3335 "legend": { 3336 "alignAsTable": true, 3337 "avg": true, 3338 "current": false, 3339 "hideEmpty": false, 3340 "hideZero": true, 3341 "max": true, 3342 "min": false, 3343 "rightSide": false, 3344 "show": true, 3345 "total": false, 3346 "values": true 3347 }, 3348 "lines": true, 3349 "linewidth": 1, 3350 "links": [], 3351 "nullPointMode": "null", 3352 "options": { 3353 "alertThreshold": true 3354 }, 3355 "percentage": false, 3356 "pluginVersion": "9.2.2", 3357 "pointradius": 5, 3358 "points": false, 3359 "renderer": "flot", 3360 "seriesOverrides": [], 3361 "spaceLength": 10, 3362 "stack": false, 3363 "steppedLine": false, 3364 "targets": [ 3365 { 3366 "datasource": { 3367 "uid": "${DS_PROMETHEUS}" 3368 }, 3369 "expr": "(sum(rate(synapse_http_server_response_time_seconds_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))/(sum(rate(synapse_http_server_response_time_seconds_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))", 3370 "format": "time_series", 3371 "hide": false, 3372 "interval": "", 3373 "intervalFactor": 1, 3374 "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}", 3375 "refId": "A", 3376 "step": 20 3377 } 3378 ], 3379 "thresholds": [], 3380 "timeRegions": [], 3381 "title": "Non-sync avg response time", 3382 "tooltip": { 3383 "shared": false, 3384 "sort": 2, 3385 "value_type": "individual" 3386 }, 3387 "type": "graph", 3388 "xaxis": { 3389 "mode": "time", 3390 "show": true, 3391 "values": [] 3392 }, 3393 "yaxes": [ 3394 { 3395 "format": "s", 3396 "logBase": 1, 3397 "show": true 3398 }, 3399 { 3400 "format": "short", 3401 "logBase": 1, 3402 "show": false 3403 } 3404 ], 3405 "yaxis": { 3406 "align": false 3407 } 3408 }, 3409 { 3410 "aliasColors": {}, 3411 "bars": false, 3412 "dashLength": 10, 3413 "dashes": false, 3414 "datasource": { 3415 "uid": "${DS_PROMETHEUS}" 3416 }, 3417 "fieldConfig": { 3418 "defaults": { 3419 "links": [] 3420 }, 3421 "overrides": [] 3422 }, 3423 "fill": 1, 3424 "fillGradient": 0, 3425 "gridPos": { 3426 "h": 9, 3427 "w": 12, 3428 "x": 0, 3429 "y": 53 3430 }, 3431 "hiddenSeries": false, 3432 "id": 103, 3433 "legend": { 3434 "avg": false, 3435 "current": false, 3436 "max": false, 3437 "min": false, 3438 "show": true, 3439 "total": false, 3440 "values": false 3441 }, 3442 "lines": true, 3443 "linewidth": 1, 3444 "links": [], 3445 "nullPointMode": "null", 3446 "options": { 3447 "alertThreshold": true 3448 }, 3449 "percentage": false, 3450 "pluginVersion": "9.2.2", 3451 "pointradius": 5, 3452 "points": false, 3453 "renderer": "flot", 3454 "seriesOverrides": [ 3455 { 3456 "alias": "Total", 3457 "color": "rgb(255, 255, 255)", 3458 "fill": 0, 3459 "linewidth": 3 3460 } 3461 ], 3462 "spaceLength": 10, 3463 "stack": false, 3464 "steppedLine": false, 3465 "targets": [ 3466 { 3467 "datasource": { 3468 "uid": "${DS_PROMETHEUS}" 3469 }, 3470 "expr": "topk(10,synapse_http_server_in_flight_requests_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})", 3471 "format": "time_series", 3472 "interval": "", 3473 "intervalFactor": 1, 3474 "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}", 3475 "refId": "A" 3476 }, 3477 { 3478 "datasource": { 3479 "uid": "${DS_PROMETHEUS}" 3480 }, 3481 "expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", 3482 "interval": "", 3483 "legendFormat": "Total", 3484 "refId": "B" 3485 } 3486 ], 3487 "thresholds": [], 3488 "timeRegions": [], 3489 "title": "Requests in flight", 3490 "tooltip": { 3491 "shared": false, 3492 "sort": 2, 3493 "value_type": "individual" 3494 }, 3495 "type": "graph", 3496 "xaxis": { 3497 "mode": "time", 3498 "show": true, 3499 "values": [] 3500 }, 3501 "yaxes": [ 3502 { 3503 "format": "short", 3504 "logBase": 1, 3505 "show": true 3506 }, 3507 { 3508 "format": "short", 3509 "logBase": 1, 3510 "show": true 3511 } 3512 ], 3513 "yaxis": { 3514 "align": false 3515 } 3516 } 3517 ], 3518 "targets": [ 3519 { 3520 "datasource": { 3521 "type": "prometheus", 3522 "uid": "${DS_PROMETHEUS}" 3523 }, 3524 "refId": "A" 3525 } 3526 ], 3527 "title": "Requests", 3528 "type": "row" 3529 }, 3530 { 3531 "collapsed": true, 3532 "datasource": { 3533 "type": "prometheus", 3534 "uid": "${DS_PROMETHEUS}" 3535 }, 3536 "gridPos": { 3537 "h": 1, 3538 "w": 24, 3539 "x": 0, 3540 "y": 29 3541 }, 3542 "id": 97, 3543 "panels": [ 3544 { 3545 "aliasColors": {}, 3546 "bars": false, 3547 "dashLength": 10, 3548 "dashes": false, 3549 "datasource": { 3550 "uid": "${DS_PROMETHEUS}" 3551 }, 3552 "fieldConfig": { 3553 "defaults": { 3554 "links": [] 3555 }, 3556 "overrides": [] 3557 }, 3558 "fill": 1, 3559 "fillGradient": 0, 3560 "gridPos": { 3561 "h": 9, 3562 "w": 12, 3563 "x": 0, 3564 "y": 30 3565 }, 3566 "hiddenSeries": false, 3567 "id": 99, 3568 "legend": { 3569 "avg": false, 3570 "current": false, 3571 "max": false, 3572 "min": false, 3573 "show": true, 3574 "total": false, 3575 "values": false 3576 }, 3577 "lines": true, 3578 "linewidth": 1, 3579 "links": [], 3580 "nullPointMode": "null", 3581 "options": { 3582 "alertThreshold": true 3583 }, 3584 "paceLength": 10, 3585 "percentage": false, 3586 "pluginVersion": "9.2.2", 3587 "pointradius": 5, 3588 "points": false, 3589 "renderer": "flot", 3590 "seriesOverrides": [], 3591 "spaceLength": 10, 3592 "stack": false, 3593 "steppedLine": false, 3594 "targets": [ 3595 { 3596 "datasource": { 3597 "uid": "${DS_PROMETHEUS}" 3598 }, 3599 "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 3600 "format": "time_series", 3601 "interval": "", 3602 "intervalFactor": 1, 3603 "legendFormat": "{{job}}-{{index}} {{name}}", 3604 "refId": "A" 3605 } 3606 ], 3607 "thresholds": [], 3608 "timeRegions": [], 3609 "title": "CPU usage by background jobs", 3610 "tooltip": { 3611 "shared": false, 3612 "sort": 0, 3613 "value_type": "individual" 3614 }, 3615 "type": "graph", 3616 "xaxis": { 3617 "mode": "time", 3618 "show": true, 3619 "values": [] 3620 }, 3621 "yaxes": [ 3622 { 3623 "format": "percentunit", 3624 "logBase": 1, 3625 "show": true 3626 }, 3627 { 3628 "format": "short", 3629 "logBase": 1, 3630 "show": true 3631 } 3632 ], 3633 "yaxis": { 3634 "align": false 3635 } 3636 }, 3637 { 3638 "aliasColors": {}, 3639 "bars": false, 3640 "dashLength": 10, 3641 "dashes": false, 3642 "datasource": { 3643 "uid": "${DS_PROMETHEUS}" 3644 }, 3645 "fieldConfig": { 3646 "defaults": { 3647 "links": [] 3648 }, 3649 "overrides": [] 3650 }, 3651 "fill": 1, 3652 "fillGradient": 0, 3653 "gridPos": { 3654 "h": 9, 3655 "w": 12, 3656 "x": 12, 3657 "y": 30 3658 }, 3659 "hiddenSeries": false, 3660 "id": 101, 3661 "legend": { 3662 "avg": false, 3663 "current": false, 3664 "max": false, 3665 "min": false, 3666 "show": true, 3667 "total": false, 3668 "values": false 3669 }, 3670 "lines": true, 3671 "linewidth": 1, 3672 "links": [], 3673 "nullPointMode": "null", 3674 "options": { 3675 "alertThreshold": true 3676 }, 3677 "paceLength": 10, 3678 "percentage": false, 3679 "pluginVersion": "9.2.2", 3680 "pointradius": 5, 3681 "points": false, 3682 "renderer": "flot", 3683 "seriesOverrides": [], 3684 "spaceLength": 10, 3685 "stack": false, 3686 "steppedLine": false, 3687 "targets": [ 3688 { 3689 "datasource": { 3690 "uid": "${DS_PROMETHEUS}" 3691 }, 3692 "expr": "rate(synapse_background_process_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 3693 "format": "time_series", 3694 "hide": false, 3695 "intervalFactor": 1, 3696 "legendFormat": "{{job}}-{{index}} {{name}}", 3697 "refId": "A" 3698 } 3699 ], 3700 "thresholds": [], 3701 "timeRegions": [], 3702 "title": "DB usage by background jobs (including scheduling time)", 3703 "tooltip": { 3704 "shared": false, 3705 "sort": 0, 3706 "value_type": "individual" 3707 }, 3708 "type": "graph", 3709 "xaxis": { 3710 "mode": "time", 3711 "show": true, 3712 "values": [] 3713 }, 3714 "yaxes": [ 3715 { 3716 "format": "percentunit", 3717 "logBase": 1, 3718 "show": true 3719 }, 3720 { 3721 "format": "short", 3722 "logBase": 1, 3723 "show": true 3724 } 3725 ], 3726 "yaxis": { 3727 "align": false 3728 } 3729 }, 3730 { 3731 "aliasColors": {}, 3732 "bars": false, 3733 "dashLength": 10, 3734 "dashes": false, 3735 "datasource": { 3736 "uid": "${DS_PROMETHEUS}" 3737 }, 3738 "fieldConfig": { 3739 "defaults": { 3740 "links": [] 3741 }, 3742 "overrides": [] 3743 }, 3744 "fill": 1, 3745 "fillGradient": 0, 3746 "gridPos": { 3747 "h": 8, 3748 "w": 12, 3749 "x": 0, 3750 "y": 39 3751 }, 3752 "hiddenSeries": false, 3753 "id": 138, 3754 "legend": { 3755 "avg": false, 3756 "current": false, 3757 "max": false, 3758 "min": false, 3759 "show": true, 3760 "total": false, 3761 "values": false 3762 }, 3763 "lines": true, 3764 "linewidth": 1, 3765 "nullPointMode": "null", 3766 "options": { 3767 "alertThreshold": true 3768 }, 3769 "percentage": false, 3770 "pluginVersion": "9.2.2", 3771 "pointradius": 2, 3772 "points": false, 3773 "renderer": "flot", 3774 "seriesOverrides": [], 3775 "spaceLength": 10, 3776 "stack": false, 3777 "steppedLine": false, 3778 "targets": [ 3779 { 3780 "datasource": { 3781 "uid": "${DS_PROMETHEUS}" 3782 }, 3783 "expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", 3784 "legendFormat": "{{job}}-{{index}} {{name}}", 3785 "refId": "A" 3786 } 3787 ], 3788 "thresholds": [], 3789 "timeRegions": [], 3790 "title": "Background jobs in flight", 3791 "tooltip": { 3792 "shared": false, 3793 "sort": 0, 3794 "value_type": "individual" 3795 }, 3796 "type": "graph", 3797 "xaxis": { 3798 "mode": "time", 3799 "show": true, 3800 "values": [] 3801 }, 3802 "yaxes": [ 3803 { 3804 "format": "short", 3805 "logBase": 1, 3806 "show": true 3807 }, 3808 { 3809 "format": "short", 3810 "logBase": 1, 3811 "show": true 3812 } 3813 ], 3814 "yaxis": { 3815 "align": false 3816 } 3817 } 3818 ], 3819 "targets": [ 3820 { 3821 "datasource": { 3822 "type": "prometheus", 3823 "uid": "${DS_PROMETHEUS}" 3824 }, 3825 "refId": "A" 3826 } 3827 ], 3828 "title": "Background jobs", 3829 "type": "row" 3830 }, 3831 { 3832 "collapsed": true, 3833 "datasource": { 3834 "type": "prometheus", 3835 "uid": "${DS_PROMETHEUS}" 3836 }, 3837 "gridPos": { 3838 "h": 1, 3839 "w": 24, 3840 "x": 0, 3841 "y": 30 3842 }, 3843 "id": 81, 3844 "panels": [ 3845 { 3846 "aliasColors": {}, 3847 "bars": false, 3848 "dashLength": 10, 3849 "dashes": false, 3850 "datasource": { 3851 "uid": "${DS_PROMETHEUS}" 3852 }, 3853 "fieldConfig": { 3854 "defaults": { 3855 "links": [] 3856 }, 3857 "overrides": [] 3858 }, 3859 "fill": 1, 3860 "fillGradient": 0, 3861 "gridPos": { 3862 "h": 9, 3863 "w": 12, 3864 "x": 0, 3865 "y": 31 3866 }, 3867 "hiddenSeries": false, 3868 "id": 79, 3869 "legend": { 3870 "avg": false, 3871 "current": false, 3872 "max": false, 3873 "min": false, 3874 "show": true, 3875 "total": false, 3876 "values": false 3877 }, 3878 "lines": true, 3879 "linewidth": 1, 3880 "links": [], 3881 "nullPointMode": "null", 3882 "options": { 3883 "alertThreshold": true 3884 }, 3885 "paceLength": 10, 3886 "percentage": false, 3887 "pluginVersion": "9.2.2", 3888 "pointradius": 5, 3889 "points": false, 3890 "renderer": "flot", 3891 "seriesOverrides": [], 3892 "spaceLength": 10, 3893 "stack": false, 3894 "steppedLine": false, 3895 "targets": [ 3896 { 3897 "datasource": { 3898 "uid": "${DS_PROMETHEUS}" 3899 }, 3900 "expr": "sum(rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))", 3901 "format": "time_series", 3902 "intervalFactor": 1, 3903 "legendFormat": "successful txn rate", 3904 "refId": "A" 3905 }, 3906 { 3907 "datasource": { 3908 "uid": "${DS_PROMETHEUS}" 3909 }, 3910 "expr": "sum(rate(synapse_util_metrics_block_count_total{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))", 3911 "legendFormat": "failed txn rate", 3912 "refId": "B" 3913 } 3914 ], 3915 "thresholds": [], 3916 "timeRegions": [], 3917 "title": "Outgoing federation transaction rate", 3918 "tooltip": { 3919 "shared": true, 3920 "sort": 0, 3921 "value_type": "individual" 3922 }, 3923 "type": "graph", 3924 "xaxis": { 3925 "mode": "time", 3926 "show": true, 3927 "values": [] 3928 }, 3929 "yaxes": [ 3930 { 3931 "format": "hertz", 3932 "logBase": 1, 3933 "show": true 3934 }, 3935 { 3936 "format": "short", 3937 "logBase": 1, 3938 "show": true 3939 } 3940 ], 3941 "yaxis": { 3942 "align": false 3943 } 3944 }, 3945 { 3946 "aliasColors": {}, 3947 "bars": false, 3948 "dashLength": 10, 3949 "dashes": false, 3950 "datasource": { 3951 "uid": "${DS_PROMETHEUS}" 3952 }, 3953 "fieldConfig": { 3954 "defaults": { 3955 "links": [] 3956 }, 3957 "overrides": [] 3958 }, 3959 "fill": 1, 3960 "fillGradient": 0, 3961 "gridPos": { 3962 "h": 9, 3963 "w": 12, 3964 "x": 12, 3965 "y": 31 3966 }, 3967 "hiddenSeries": false, 3968 "id": 83, 3969 "legend": { 3970 "avg": false, 3971 "current": false, 3972 "max": false, 3973 "min": false, 3974 "show": true, 3975 "total": false, 3976 "values": false 3977 }, 3978 "lines": true, 3979 "linewidth": 1, 3980 "links": [], 3981 "nullPointMode": "null", 3982 "options": { 3983 "alertThreshold": true 3984 }, 3985 "paceLength": 10, 3986 "percentage": false, 3987 "pluginVersion": "9.2.2", 3988 "pointradius": 5, 3989 "points": false, 3990 "renderer": "flot", 3991 "seriesOverrides": [], 3992 "spaceLength": 10, 3993 "stack": false, 3994 "steppedLine": false, 3995 "targets": [ 3996 { 3997 "datasource": { 3998 "uid": "${DS_PROMETHEUS}" 3999 }, 4000 "expr": "sum(rate(synapse_federation_server_received_pdus_total{instance=~\"$instance\"}[$bucket_size]))", 4001 "format": "time_series", 4002 "intervalFactor": 1, 4003 "legendFormat": "pdus", 4004 "refId": "A" 4005 }, 4006 { 4007 "datasource": { 4008 "uid": "${DS_PROMETHEUS}" 4009 }, 4010 "expr": "sum(rate(synapse_federation_server_received_edus_total{instance=~\"$instance\"}[$bucket_size]))", 4011 "format": "time_series", 4012 "intervalFactor": 1, 4013 "legendFormat": "edus", 4014 "refId": "B" 4015 } 4016 ], 4017 "thresholds": [], 4018 "timeRegions": [], 4019 "title": "Incoming PDU/EDU rate", 4020 "tooltip": { 4021 "shared": true, 4022 "sort": 0, 4023 "value_type": "individual" 4024 }, 4025 "type": "graph", 4026 "xaxis": { 4027 "mode": "time", 4028 "show": true, 4029 "values": [] 4030 }, 4031 "yaxes": [ 4032 { 4033 "format": "hertz", 4034 "logBase": 1, 4035 "show": true 4036 }, 4037 { 4038 "format": "short", 4039 "logBase": 1, 4040 "show": true 4041 } 4042 ], 4043 "yaxis": { 4044 "align": false 4045 } 4046 }, 4047 { 4048 "aliasColors": {}, 4049 "bars": false, 4050 "dashLength": 10, 4051 "dashes": false, 4052 "datasource": { 4053 "type": "prometheus", 4054 "uid": "${DS_PROMETHEUS}" 4055 }, 4056 "fieldConfig": { 4057 "defaults": { 4058 "links": [] 4059 }, 4060 "overrides": [] 4061 }, 4062 "fill": 1, 4063 "fillGradient": 0, 4064 "gridPos": { 4065 "h": 9, 4066 "w": 12, 4067 "x": 0, 4068 "y": 40 4069 }, 4070 "hiddenSeries": false, 4071 "id": 109, 4072 "legend": { 4073 "avg": false, 4074 "current": false, 4075 "max": false, 4076 "min": false, 4077 "show": true, 4078 "total": false, 4079 "values": false 4080 }, 4081 "lines": true, 4082 "linewidth": 1, 4083 "links": [], 4084 "nullPointMode": "null", 4085 "options": { 4086 "alertThreshold": true 4087 }, 4088 "paceLength": 10, 4089 "percentage": false, 4090 "pluginVersion": "9.2.2", 4091 "pointradius": 5, 4092 "points": false, 4093 "renderer": "flot", 4094 "seriesOverrides": [], 4095 "spaceLength": 10, 4096 "stack": false, 4097 "steppedLine": false, 4098 "targets": [ 4099 { 4100 "datasource": { 4101 "uid": "${DS_PROMETHEUS}" 4102 }, 4103 "editorMode": "code", 4104 "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations_count_total{instance=\"$instance\"}[$bucket_size]))", 4105 "format": "time_series", 4106 "interval": "", 4107 "intervalFactor": 1, 4108 "legendFormat": "pdus", 4109 "range": true, 4110 "refId": "A" 4111 }, 4112 { 4113 "datasource": { 4114 "uid": "${DS_PROMETHEUS}" 4115 }, 4116 "expr": "sum(rate(synapse_federation_client_sent_edus_total{instance=\"$instance\"}[$bucket_size]))", 4117 "format": "time_series", 4118 "intervalFactor": 1, 4119 "legendFormat": "edus", 4120 "refId": "B" 4121 } 4122 ], 4123 "thresholds": [], 4124 "timeRegions": [], 4125 "title": "Outgoing PDU/EDU rate", 4126 "tooltip": { 4127 "shared": true, 4128 "sort": 0, 4129 "value_type": "individual" 4130 }, 4131 "type": "graph", 4132 "xaxis": { 4133 "mode": "time", 4134 "show": true, 4135 "values": [] 4136 }, 4137 "yaxes": [ 4138 { 4139 "format": "hertz", 4140 "logBase": 1, 4141 "show": true 4142 }, 4143 { 4144 "format": "short", 4145 "logBase": 1, 4146 "show": true 4147 } 4148 ], 4149 "yaxis": { 4150 "align": false 4151 } 4152 }, 4153 { 4154 "aliasColors": {}, 4155 "bars": false, 4156 "dashLength": 10, 4157 "dashes": false, 4158 "datasource": { 4159 "type": "prometheus", 4160 "uid": "${DS_PROMETHEUS}" 4161 }, 4162 "fieldConfig": { 4163 "defaults": { 4164 "links": [] 4165 }, 4166 "overrides": [] 4167 }, 4168 "fill": 1, 4169 "fillGradient": 0, 4170 "gridPos": { 4171 "h": 9, 4172 "w": 12, 4173 "x": 12, 4174 "y": 40 4175 }, 4176 "hiddenSeries": false, 4177 "id": 111, 4178 "legend": { 4179 "avg": false, 4180 "current": false, 4181 "max": false, 4182 "min": false, 4183 "show": true, 4184 "total": false, 4185 "values": false 4186 }, 4187 "lines": true, 4188 "linewidth": 1, 4189 "links": [], 4190 "nullPointMode": "null", 4191 "options": { 4192 "alertThreshold": true 4193 }, 4194 "paceLength": 10, 4195 "percentage": false, 4196 "pluginVersion": "9.2.2", 4197 "pointradius": 5, 4198 "points": false, 4199 "renderer": "flot", 4200 "seriesOverrides": [], 4201 "spaceLength": 10, 4202 "stack": false, 4203 "steppedLine": false, 4204 "targets": [ 4205 { 4206 "datasource": { 4207 "uid": "${DS_PROMETHEUS}" 4208 }, 4209 "editorMode": "code", 4210 "expr": "rate(synapse_federation_client_sent_edus_by_type_total{instance=\"$instance\"}[$bucket_size])", 4211 "format": "time_series", 4212 "interval": "", 4213 "intervalFactor": 1, 4214 "legendFormat": "{{type}}", 4215 "range": true, 4216 "refId": "A" 4217 } 4218 ], 4219 "thresholds": [], 4220 "timeRegions": [], 4221 "title": "Outgoing EDUs by type", 4222 "tooltip": { 4223 "shared": true, 4224 "sort": 0, 4225 "value_type": "individual" 4226 }, 4227 "type": "graph", 4228 "xaxis": { 4229 "mode": "time", 4230 "show": true, 4231 "values": [] 4232 }, 4233 "yaxes": [ 4234 { 4235 "$$hashKey": "object:462", 4236 "format": "hertz", 4237 "logBase": 1, 4238 "show": true 4239 }, 4240 { 4241 "$$hashKey": "object:463", 4242 "format": "short", 4243 "logBase": 1, 4244 "show": true 4245 } 4246 ], 4247 "yaxis": { 4248 "align": false 4249 } 4250 }, 4251 { 4252 "datasource": { 4253 "type": "prometheus", 4254 "uid": "${DS_PROMETHEUS}" 4255 }, 4256 "description": "Triangular growth may indicate a problem with federation sending from the remote host --- but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains", 4257 "fieldConfig": { 4258 "defaults": { 4259 "color": { 4260 "mode": "palette-classic" 4261 }, 4262 "custom": { 4263 "axisCenteredZero": false, 4264 "axisColorMode": "text", 4265 "axisLabel": "", 4266 "axisPlacement": "auto", 4267 "axisSoftMax": 60, 4268 "axisSoftMin": 0, 4269 "barAlignment": 0, 4270 "drawStyle": "line", 4271 "fillOpacity": 0, 4272 "gradientMode": "none", 4273 "hideFrom": { 4274 "legend": false, 4275 "tooltip": false, 4276 "viz": false 4277 }, 4278 "lineInterpolation": "linear", 4279 "lineWidth": 1, 4280 "pointSize": 5, 4281 "scaleDistribution": { 4282 "type": "linear" 4283 }, 4284 "showPoints": "auto", 4285 "spanNulls": false, 4286 "stacking": { 4287 "group": "A", 4288 "mode": "none" 4289 }, 4290 "thresholdsStyle": { 4291 "mode": "line" 4292 } 4293 }, 4294 "mappings": [], 4295 "thresholds": { 4296 "mode": "absolute", 4297 "steps": [ 4298 { 4299 "color": "green" 4300 }, 4301 { 4302 "color": "red", 4303 "value": 60 4304 } 4305 ] 4306 }, 4307 "unit": "m" 4308 }, 4309 "overrides": [ 4310 { 4311 "__systemRef": "hideSeriesFrom", 4312 "matcher": { 4313 "id": "byNames", 4314 "options": { 4315 "mode": "exclude", 4316 "names": [ 4317 "libera.chat " 4318 ], 4319 "prefix": "All except:", 4320 "readOnly": true 4321 } 4322 }, 4323 "properties": [ 4324 { 4325 "id": "custom.hideFrom", 4326 "value": { 4327 "legend": false, 4328 "tooltip": false, 4329 "viz": true 4330 } 4331 } 4332 ] 4333 } 4334 ] 4335 }, 4336 "gridPos": { 4337 "h": 8, 4338 "w": 12, 4339 "x": 0, 4340 "y": 49 4341 }, 4342 "id": 243, 4343 "options": { 4344 "legend": { 4345 "calcs": [], 4346 "displayMode": "list", 4347 "placement": "bottom", 4348 "showLegend": true 4349 }, 4350 "tooltip": { 4351 "mode": "single", 4352 "sort": "none" 4353 } 4354 }, 4355 "targets": [ 4356 { 4357 "datasource": { 4358 "type": "prometheus", 4359 "uid": "${DS_PROMETHEUS}" 4360 }, 4361 "editorMode": "code", 4362 "exemplar": false, 4363 "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_received_pdu_time[10m]))) / 60", 4364 "instant": false, 4365 "legendFormat": "{{server_name}} ", 4366 "range": true, 4367 "refId": "A" 4368 } 4369 ], 4370 "title": "Age of last PDU received from nominated hosts", 4371 "type": "timeseries" 4372 }, 4373 { 4374 "datasource": { 4375 "type": "prometheus", 4376 "uid": "${DS_PROMETHEUS}" 4377 }, 4378 "description": "Triangular growth may indicate a problem with federation senders on the monitored instance---but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains", 4379 "fieldConfig": { 4380 "defaults": { 4381 "color": { 4382 "mode": "palette-classic" 4383 }, 4384 "custom": { 4385 "axisCenteredZero": false, 4386 "axisColorMode": "text", 4387 "axisLabel": "", 4388 "axisPlacement": "auto", 4389 "axisSoftMax": 60, 4390 "axisSoftMin": 0, 4391 "barAlignment": 0, 4392 "drawStyle": "line", 4393 "fillOpacity": 0, 4394 "gradientMode": "none", 4395 "hideFrom": { 4396 "legend": false, 4397 "tooltip": false, 4398 "viz": false 4399 }, 4400 "lineInterpolation": "linear", 4401 "lineWidth": 1, 4402 "pointSize": 5, 4403 "scaleDistribution": { 4404 "type": "linear" 4405 }, 4406 "showPoints": "auto", 4407 "spanNulls": false, 4408 "stacking": { 4409 "group": "A", 4410 "mode": "none" 4411 }, 4412 "thresholdsStyle": { 4413 "mode": "line" 4414 } 4415 }, 4416 "mappings": [], 4417 "thresholds": { 4418 "mode": "absolute", 4419 "steps": [ 4420 { 4421 "color": "green" 4422 }, 4423 { 4424 "color": "red", 4425 "value": 60 4426 } 4427 ] 4428 }, 4429 "unit": "m" 4430 }, 4431 "overrides": [ 4432 { 4433 "__systemRef": "hideSeriesFrom", 4434 "matcher": { 4435 "id": "byNames", 4436 "options": { 4437 "mode": "exclude", 4438 "names": [ 4439 "libera.chat" 4440 ], 4441 "prefix": "All except:", 4442 "readOnly": true 4443 } 4444 }, 4445 "properties": [ 4446 { 4447 "id": "custom.hideFrom", 4448 "value": { 4449 "legend": false, 4450 "tooltip": false, 4451 "viz": true 4452 } 4453 } 4454 ] 4455 } 4456 ] 4457 }, 4458 "gridPos": { 4459 "h": 8, 4460 "w": 12, 4461 "x": 12, 4462 "y": 49 4463 }, 4464 "id": 241, 4465 "options": { 4466 "legend": { 4467 "calcs": [], 4468 "displayMode": "list", 4469 "placement": "bottom", 4470 "showLegend": true 4471 }, 4472 "tooltip": { 4473 "mode": "single", 4474 "sort": "none" 4475 } 4476 }, 4477 "targets": [ 4478 { 4479 "datasource": { 4480 "type": "prometheus", 4481 "uid": "${DS_PROMETHEUS}" 4482 }, 4483 "editorMode": "code", 4484 "exemplar": false, 4485 "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_sent_pdu_time[10m]))) / 60", 4486 "instant": false, 4487 "legendFormat": "{{server_name}}", 4488 "range": true, 4489 "refId": "A" 4490 } 4491 ], 4492 "title": "Age of last PDU sent to nominated hosts", 4493 "type": "timeseries" 4494 }, 4495 { 4496 "aliasColors": {}, 4497 "bars": false, 4498 "dashLength": 10, 4499 "dashes": false, 4500 "datasource": { 4501 "type": "prometheus", 4502 "uid": "${DS_PROMETHEUS}" 4503 }, 4504 "description": "The number of events in the in-memory queues ", 4505 "fieldConfig": { 4506 "defaults": { 4507 "links": [] 4508 }, 4509 "overrides": [] 4510 }, 4511 "fill": 1, 4512 "fillGradient": 0, 4513 "gridPos": { 4514 "h": 9, 4515 "w": 12, 4516 "x": 0, 4517 "y": 57 4518 }, 4519 "hiddenSeries": false, 4520 "id": 142, 4521 "legend": { 4522 "avg": false, 4523 "current": false, 4524 "max": false, 4525 "min": false, 4526 "show": true, 4527 "total": false, 4528 "values": false 4529 }, 4530 "lines": true, 4531 "linewidth": 1, 4532 "nullPointMode": "null", 4533 "options": { 4534 "alertThreshold": true 4535 }, 4536 "percentage": false, 4537 "pluginVersion": "9.2.2", 4538 "pointradius": 2, 4539 "points": false, 4540 "renderer": "flot", 4541 "seriesOverrides": [], 4542 "spaceLength": 10, 4543 "stack": false, 4544 "steppedLine": false, 4545 "targets": [ 4546 { 4547 "datasource": { 4548 "type": "prometheus", 4549 "uid": "${DS_PROMETHEUS}" 4550 }, 4551 "editorMode": "code", 4552 "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 4553 "interval": "", 4554 "legendFormat": "pending PDUs {{job}}-{{index}}", 4555 "range": true, 4556 "refId": "A" 4557 }, 4558 { 4559 "datasource": { 4560 "type": "prometheus", 4561 "uid": "${DS_PROMETHEUS}" 4562 }, 4563 "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 4564 "interval": "", 4565 "legendFormat": "pending EDUs {{job}}-{{index}}", 4566 "refId": "B" 4567 } 4568 ], 4569 "thresholds": [], 4570 "timeRegions": [], 4571 "title": "In-memory federation transmission queues", 4572 "tooltip": { 4573 "shared": true, 4574 "sort": 0, 4575 "value_type": "individual" 4576 }, 4577 "type": "graph", 4578 "xaxis": { 4579 "mode": "time", 4580 "show": true, 4581 "values": [] 4582 }, 4583 "yaxes": [ 4584 { 4585 "$$hashKey": "object:547", 4586 "format": "short", 4587 "label": "events", 4588 "logBase": 1, 4589 "min": "0", 4590 "show": true 4591 }, 4592 { 4593 "$$hashKey": "object:548", 4594 "format": "short", 4595 "label": "", 4596 "logBase": 1, 4597 "show": true 4598 } 4599 ], 4600 "yaxis": { 4601 "align": false 4602 } 4603 }, 4604 { 4605 "aliasColors": {}, 4606 "bars": false, 4607 "dashLength": 10, 4608 "dashes": false, 4609 "datasource": { 4610 "uid": "${DS_PROMETHEUS}" 4611 }, 4612 "description": "Number of events queued up on the master process for processing by the federation sender", 4613 "fieldConfig": { 4614 "defaults": { 4615 "links": [] 4616 }, 4617 "overrides": [] 4618 }, 4619 "fill": 1, 4620 "fillGradient": 0, 4621 "gridPos": { 4622 "h": 9, 4623 "w": 12, 4624 "x": 12, 4625 "y": 57 4626 }, 4627 "hiddenSeries": false, 4628 "id": 140, 4629 "legend": { 4630 "avg": false, 4631 "current": false, 4632 "max": false, 4633 "min": false, 4634 "show": true, 4635 "total": false, 4636 "values": false 4637 }, 4638 "lines": true, 4639 "linewidth": 1, 4640 "links": [], 4641 "nullPointMode": "null", 4642 "options": { 4643 "alertThreshold": true 4644 }, 4645 "paceLength": 10, 4646 "percentage": false, 4647 "pluginVersion": "9.2.2", 4648 "pointradius": 5, 4649 "points": false, 4650 "renderer": "flot", 4651 "seriesOverrides": [], 4652 "spaceLength": 10, 4653 "stack": false, 4654 "steppedLine": false, 4655 "targets": [ 4656 { 4657 "datasource": { 4658 "uid": "${DS_PROMETHEUS}" 4659 }, 4660 "expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 4661 "format": "time_series", 4662 "interval": "", 4663 "intervalFactor": 1, 4664 "legendFormat": "presence changed", 4665 "refId": "A" 4666 }, 4667 { 4668 "datasource": { 4669 "uid": "${DS_PROMETHEUS}" 4670 }, 4671 "expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 4672 "format": "time_series", 4673 "hide": false, 4674 "interval": "", 4675 "intervalFactor": 1, 4676 "legendFormat": "presence map", 4677 "refId": "B" 4678 }, 4679 { 4680 "datasource": { 4681 "uid": "${DS_PROMETHEUS}" 4682 }, 4683 "expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 4684 "format": "time_series", 4685 "hide": false, 4686 "interval": "", 4687 "intervalFactor": 1, 4688 "legendFormat": "presence destinations", 4689 "refId": "E" 4690 }, 4691 { 4692 "datasource": { 4693 "uid": "${DS_PROMETHEUS}" 4694 }, 4695 "expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 4696 "format": "time_series", 4697 "hide": false, 4698 "interval": "", 4699 "intervalFactor": 1, 4700 "legendFormat": "keyed edus", 4701 "refId": "C" 4702 }, 4703 { 4704 "datasource": { 4705 "uid": "${DS_PROMETHEUS}" 4706 }, 4707 "expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 4708 "format": "time_series", 4709 "hide": false, 4710 "interval": "", 4711 "intervalFactor": 1, 4712 "legendFormat": "other edus", 4713 "refId": "D" 4714 }, 4715 { 4716 "datasource": { 4717 "uid": "${DS_PROMETHEUS}" 4718 }, 4719 "expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 4720 "format": "time_series", 4721 "hide": false, 4722 "interval": "", 4723 "intervalFactor": 1, 4724 "legendFormat": "stream positions", 4725 "refId": "F" 4726 } 4727 ], 4728 "thresholds": [], 4729 "timeRegions": [], 4730 "title": "Outgoing EDU queues on master", 4731 "tooltip": { 4732 "shared": true, 4733 "sort": 0, 4734 "value_type": "individual" 4735 }, 4736 "type": "graph", 4737 "xaxis": { 4738 "mode": "time", 4739 "show": true, 4740 "values": [] 4741 }, 4742 "yaxes": [ 4743 { 4744 "format": "none", 4745 "logBase": 1, 4746 "min": "0", 4747 "show": true 4748 }, 4749 { 4750 "format": "short", 4751 "logBase": 1, 4752 "show": true 4753 } 4754 ], 4755 "yaxis": { 4756 "align": false 4757 } 4758 }, 4759 { 4760 "cards": { 4761 "cardPadding": -1 4762 }, 4763 "color": { 4764 "cardColor": "#b4ff00", 4765 "colorScale": "sqrt", 4766 "colorScheme": "interpolateInferno", 4767 "exponent": 0.5, 4768 "min": 0, 4769 "mode": "spectrum" 4770 }, 4771 "dataFormat": "tsbuckets", 4772 "datasource": { 4773 "uid": "${DS_PROMETHEUS}" 4774 }, 4775 "fieldConfig": { 4776 "defaults": { 4777 "custom": { 4778 "hideFrom": { 4779 "legend": false, 4780 "tooltip": false, 4781 "viz": false 4782 }, 4783 "scaleDistribution": { 4784 "type": "linear" 4785 } 4786 } 4787 }, 4788 "overrides": [] 4789 }, 4790 "gridPos": { 4791 "h": 9, 4792 "w": 12, 4793 "x": 0, 4794 "y": 66 4795 }, 4796 "heatmap": {}, 4797 "hideZeroBuckets": false, 4798 "highlightCards": true, 4799 "id": 166, 4800 "legend": { 4801 "show": false 4802 }, 4803 "links": [], 4804 "options": { 4805 "calculate": false, 4806 "calculation": {}, 4807 "cellGap": -1, 4808 "cellValues": { 4809 "decimals": 2 4810 }, 4811 "color": { 4812 "exponent": 0.5, 4813 "fill": "#b4ff00", 4814 "min": 0, 4815 "mode": "scheme", 4816 "reverse": false, 4817 "scale": "exponential", 4818 "scheme": "Inferno", 4819 "steps": 128 4820 }, 4821 "exemplars": { 4822 "color": "rgba(255,0,255,0.7)" 4823 }, 4824 "filterValues": { 4825 "le": 1e-9 4826 }, 4827 "legend": { 4828 "show": false 4829 }, 4830 "rowsFrame": { 4831 "layout": "auto" 4832 }, 4833 "showValue": "never", 4834 "tooltip": { 4835 "show": true, 4836 "yHistogram": true 4837 }, 4838 "yAxis": { 4839 "axisPlacement": "left", 4840 "decimals": 0, 4841 "reverse": false, 4842 "unit": "s" 4843 } 4844 }, 4845 "pluginVersion": "9.2.2", 4846 "reverseYBuckets": false, 4847 "targets": [ 4848 { 4849 "datasource": { 4850 "uid": "${DS_PROMETHEUS}" 4851 }, 4852 "expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{instance=\"$instance\",name=\"federation_sender\"}[$bucket_size])) by (le)", 4853 "format": "heatmap", 4854 "instant": false, 4855 "interval": "", 4856 "intervalFactor": 1, 4857 "legendFormat": "{{ le }}", 4858 "refId": "A" 4859 } 4860 ], 4861 "title": "Federation send PDU lag", 4862 "tooltip": { 4863 "show": true, 4864 "showHistogram": true 4865 }, 4866 "tooltipDecimals": 2, 4867 "type": "heatmap", 4868 "xAxis": { 4869 "show": true 4870 }, 4871 "yAxis": { 4872 "decimals": 0, 4873 "format": "s", 4874 "logBase": 1, 4875 "show": true 4876 }, 4877 "yBucketBound": "auto" 4878 }, 4879 { 4880 "aliasColors": {}, 4881 "bars": false, 4882 "dashLength": 10, 4883 "dashes": false, 4884 "datasource": { 4885 "uid": "${DS_PROMETHEUS}" 4886 }, 4887 "fieldConfig": { 4888 "defaults": { 4889 "links": [] 4890 }, 4891 "overrides": [] 4892 }, 4893 "fill": 0, 4894 "fillGradient": 0, 4895 "gridPos": { 4896 "h": 9, 4897 "w": 12, 4898 "x": 12, 4899 "y": 66 4900 }, 4901 "hiddenSeries": false, 4902 "id": 162, 4903 "legend": { 4904 "avg": false, 4905 "current": false, 4906 "max": false, 4907 "min": false, 4908 "rightSide": false, 4909 "show": true, 4910 "total": false, 4911 "values": false 4912 }, 4913 "lines": true, 4914 "linewidth": 0, 4915 "links": [], 4916 "nullPointMode": "connected", 4917 "options": { 4918 "alertThreshold": true 4919 }, 4920 "paceLength": 10, 4921 "percentage": false, 4922 "pluginVersion": "9.2.2", 4923 "pointradius": 5, 4924 "points": false, 4925 "renderer": "flot", 4926 "seriesOverrides": [ 4927 { 4928 "alias": "Avg", 4929 "fill": 0, 4930 "linewidth": 3 4931 }, 4932 { 4933 "alias": "99%", 4934 "color": "#C4162A", 4935 "fillBelowTo": "90%" 4936 }, 4937 { 4938 "alias": "90%", 4939 "color": "#FF7383", 4940 "fillBelowTo": "75%" 4941 }, 4942 { 4943 "alias": "75%", 4944 "color": "#FFEE52", 4945 "fillBelowTo": "50%" 4946 }, 4947 { 4948 "alias": "50%", 4949 "color": "#73BF69", 4950 "fillBelowTo": "25%" 4951 }, 4952 { 4953 "alias": "25%", 4954 "color": "#1F60C4", 4955 "fillBelowTo": "5%" 4956 }, 4957 { 4958 "alias": "5%", 4959 "lines": false 4960 }, 4961 { 4962 "alias": "Average", 4963 "color": "rgb(255, 255, 255)", 4964 "lines": true, 4965 "linewidth": 3 4966 } 4967 ], 4968 "spaceLength": 10, 4969 "stack": false, 4970 "steppedLine": false, 4971 "targets": [ 4972 { 4973 "datasource": { 4974 "uid": "${DS_PROMETHEUS}" 4975 }, 4976 "expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 4977 "format": "time_series", 4978 "interval": "", 4979 "intervalFactor": 1, 4980 "legendFormat": "99%", 4981 "refId": "D" 4982 }, 4983 { 4984 "datasource": { 4985 "uid": "${DS_PROMETHEUS}" 4986 }, 4987 "expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 4988 "format": "time_series", 4989 "interval": "", 4990 "intervalFactor": 1, 4991 "legendFormat": "90%", 4992 "refId": "A" 4993 }, 4994 { 4995 "datasource": { 4996 "uid": "${DS_PROMETHEUS}" 4997 }, 4998 "expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 4999 "format": "time_series", 5000 "interval": "", 5001 "intervalFactor": 1, 5002 "legendFormat": "75%", 5003 "refId": "C" 5004 }, 5005 { 5006 "datasource": { 5007 "uid": "${DS_PROMETHEUS}" 5008 }, 5009 "expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 5010 "format": "time_series", 5011 "interval": "", 5012 "intervalFactor": 1, 5013 "legendFormat": "50%", 5014 "refId": "B" 5015 }, 5016 { 5017 "datasource": { 5018 "uid": "${DS_PROMETHEUS}" 5019 }, 5020 "expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 5021 "interval": "", 5022 "legendFormat": "25%", 5023 "refId": "F" 5024 }, 5025 { 5026 "datasource": { 5027 "uid": "${DS_PROMETHEUS}" 5028 }, 5029 "expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 5030 "interval": "", 5031 "legendFormat": "5%", 5032 "refId": "G" 5033 }, 5034 { 5035 "datasource": { 5036 "uid": "${DS_PROMETHEUS}" 5037 }, 5038 "expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", 5039 "interval": "", 5040 "legendFormat": "Average", 5041 "refId": "H" 5042 } 5043 ], 5044 "thresholds": [ 5045 { 5046 "colorMode": "warning", 5047 "fill": false, 5048 "line": true, 5049 "op": "gt", 5050 "value": 0.25, 5051 "yaxis": "left" 5052 }, 5053 { 5054 "colorMode": "critical", 5055 "fill": false, 5056 "line": true, 5057 "op": "gt", 5058 "value": 1, 5059 "yaxis": "left" 5060 } 5061 ], 5062 "timeRegions": [], 5063 "title": "Federation send PDU lag quantiles", 5064 "tooltip": { 5065 "shared": true, 5066 "sort": 2, 5067 "value_type": "individual" 5068 }, 5069 "type": "graph", 5070 "xaxis": { 5071 "mode": "time", 5072 "show": true, 5073 "values": [] 5074 }, 5075 "yaxes": [ 5076 { 5077 "format": "s", 5078 "label": "", 5079 "logBase": 1, 5080 "min": "0", 5081 "show": true 5082 }, 5083 { 5084 "format": "hertz", 5085 "label": "", 5086 "logBase": 1, 5087 "min": "0", 5088 "show": true 5089 } 5090 ], 5091 "yaxis": { 5092 "align": false 5093 } 5094 }, 5095 { 5096 "cards": { 5097 "cardPadding": -1 5098 }, 5099 "color": { 5100 "cardColor": "#b4ff00", 5101 "colorScale": "sqrt", 5102 "colorScheme": "interpolateInferno", 5103 "exponent": 0.5, 5104 "min": 0, 5105 "mode": "spectrum" 5106 }, 5107 "dataFormat": "tsbuckets", 5108 "datasource": { 5109 "uid": "${DS_PROMETHEUS}" 5110 }, 5111 "fieldConfig": { 5112 "defaults": { 5113 "custom": { 5114 "hideFrom": { 5115 "legend": false, 5116 "tooltip": false, 5117 "viz": false 5118 }, 5119 "scaleDistribution": { 5120 "type": "linear" 5121 } 5122 } 5123 }, 5124 "overrides": [] 5125 }, 5126 "gridPos": { 5127 "h": 9, 5128 "w": 12, 5129 "x": 0, 5130 "y": 75 5131 }, 5132 "heatmap": {}, 5133 "hideZeroBuckets": false, 5134 "highlightCards": true, 5135 "id": 164, 5136 "legend": { 5137 "show": false 5138 }, 5139 "links": [], 5140 "options": { 5141 "calculate": false, 5142 "calculation": {}, 5143 "cellGap": -1, 5144 "cellValues": { 5145 "decimals": 2 5146 }, 5147 "color": { 5148 "exponent": 0.5, 5149 "fill": "#b4ff00", 5150 "min": 0, 5151 "mode": "scheme", 5152 "reverse": false, 5153 "scale": "exponential", 5154 "scheme": "Inferno", 5155 "steps": 128 5156 }, 5157 "exemplars": { 5158 "color": "rgba(255,0,255,0.7)" 5159 }, 5160 "filterValues": { 5161 "le": 1e-9 5162 }, 5163 "legend": { 5164 "show": false 5165 }, 5166 "rowsFrame": { 5167 "layout": "auto" 5168 }, 5169 "showValue": "never", 5170 "tooltip": { 5171 "show": true, 5172 "yHistogram": true 5173 }, 5174 "yAxis": { 5175 "axisPlacement": "left", 5176 "decimals": 0, 5177 "reverse": false, 5178 "unit": "s" 5179 } 5180 }, 5181 "pluginVersion": "9.2.2", 5182 "reverseYBuckets": false, 5183 "targets": [ 5184 { 5185 "datasource": { 5186 "uid": "${DS_PROMETHEUS}" 5187 }, 5188 "expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{instance=\"$instance\"}[$bucket_size])) by (le)", 5189 "format": "heatmap", 5190 "instant": false, 5191 "interval": "", 5192 "intervalFactor": 1, 5193 "legendFormat": "{{ le }}", 5194 "refId": "A" 5195 } 5196 ], 5197 "title": "Handle inbound PDU time", 5198 "tooltip": { 5199 "show": true, 5200 "showHistogram": true 5201 }, 5202 "tooltipDecimals": 2, 5203 "type": "heatmap", 5204 "xAxis": { 5205 "show": true 5206 }, 5207 "yAxis": { 5208 "decimals": 0, 5209 "format": "s", 5210 "logBase": 1, 5211 "show": true 5212 }, 5213 "yBucketBound": "auto" 5214 }, 5215 { 5216 "aliasColors": {}, 5217 "bars": false, 5218 "dashLength": 10, 5219 "dashes": false, 5220 "datasource": { 5221 "type": "prometheus", 5222 "uid": "${DS_PROMETHEUS}" 5223 }, 5224 "editable": true, 5225 "error": false, 5226 "fieldConfig": { 5227 "defaults": { 5228 "links": [] 5229 }, 5230 "overrides": [] 5231 }, 5232 "fill": 1, 5233 "fillGradient": 0, 5234 "grid": {}, 5235 "gridPos": { 5236 "h": 9, 5237 "w": 12, 5238 "x": 12, 5239 "y": 75 5240 }, 5241 "hiddenSeries": false, 5242 "id": 203, 5243 "legend": { 5244 "avg": false, 5245 "current": false, 5246 "max": false, 5247 "min": false, 5248 "show": true, 5249 "total": false, 5250 "values": false 5251 }, 5252 "lines": true, 5253 "linewidth": 2, 5254 "links": [], 5255 "nullPointMode": "null", 5256 "options": { 5257 "alertThreshold": true 5258 }, 5259 "paceLength": 10, 5260 "percentage": false, 5261 "pluginVersion": "9.2.2", 5262 "pointradius": 5, 5263 "points": false, 5264 "renderer": "flot", 5265 "seriesOverrides": [], 5266 "spaceLength": 10, 5267 "stack": false, 5268 "steppedLine": false, 5269 "targets": [ 5270 { 5271 "datasource": { 5272 "uid": "${DS_PROMETHEUS}" 5273 }, 5274 "editorMode": "code", 5275 "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", 5276 "format": "time_series", 5277 "interval": "", 5278 "intervalFactor": 1, 5279 "legendFormat": "{{job}}-{{index}}", 5280 "range": true, 5281 "refId": "A", 5282 "step": 4 5283 } 5284 ], 5285 "thresholds": [], 5286 "timeRegions": [], 5287 "title": "Age of oldest event in staging area", 5288 "tooltip": { 5289 "msResolution": false, 5290 "shared": true, 5291 "sort": 0, 5292 "value_type": "cumulative" 5293 }, 5294 "type": "graph", 5295 "xaxis": { 5296 "mode": "time", 5297 "show": true, 5298 "values": [] 5299 }, 5300 "yaxes": [ 5301 { 5302 "$$hashKey": "object:92", 5303 "format": "ms", 5304 "logBase": 1, 5305 "min": 0, 5306 "show": true 5307 }, 5308 { 5309 "$$hashKey": "object:93", 5310 "format": "short", 5311 "logBase": 1, 5312 "show": true 5313 } 5314 ], 5315 "yaxis": { 5316 "align": false 5317 } 5318 }, 5319 { 5320 "aliasColors": {}, 5321 "bars": false, 5322 "dashLength": 10, 5323 "dashes": false, 5324 "datasource": { 5325 "type": "prometheus", 5326 "uid": "${DS_PROMETHEUS}" 5327 }, 5328 "editable": true, 5329 "error": false, 5330 "fieldConfig": { 5331 "defaults": { 5332 "links": [] 5333 }, 5334 "overrides": [] 5335 }, 5336 "fill": 1, 5337 "fillGradient": 0, 5338 "grid": {}, 5339 "gridPos": { 5340 "h": 9, 5341 "w": 12, 5342 "x": 0, 5343 "y": 84 5344 }, 5345 "hiddenSeries": false, 5346 "id": 202, 5347 "legend": { 5348 "avg": false, 5349 "current": false, 5350 "max": false, 5351 "min": false, 5352 "show": true, 5353 "total": false, 5354 "values": false 5355 }, 5356 "lines": true, 5357 "linewidth": 2, 5358 "links": [], 5359 "nullPointMode": "null", 5360 "options": { 5361 "alertThreshold": true 5362 }, 5363 "paceLength": 10, 5364 "percentage": false, 5365 "pluginVersion": "9.2.2", 5366 "pointradius": 5, 5367 "points": false, 5368 "renderer": "flot", 5369 "seriesOverrides": [], 5370 "spaceLength": 10, 5371 "stack": false, 5372 "steppedLine": false, 5373 "targets": [ 5374 { 5375 "datasource": { 5376 "uid": "${DS_PROMETHEUS}" 5377 }, 5378 "editorMode": "code", 5379 "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", 5380 "format": "time_series", 5381 "interval": "", 5382 "intervalFactor": 1, 5383 "legendFormat": "{{job}}-{{index}}", 5384 "range": true, 5385 "refId": "A", 5386 "step": 4 5387 } 5388 ], 5389 "thresholds": [], 5390 "timeRegions": [], 5391 "title": "Number of events in federation staging area", 5392 "tooltip": { 5393 "msResolution": false, 5394 "shared": true, 5395 "sort": 0, 5396 "value_type": "cumulative" 5397 }, 5398 "type": "graph", 5399 "xaxis": { 5400 "mode": "time", 5401 "show": true, 5402 "values": [] 5403 }, 5404 "yaxes": [ 5405 { 5406 "$$hashKey": "object:92", 5407 "format": "none", 5408 "logBase": 1, 5409 "min": 0, 5410 "show": true 5411 }, 5412 { 5413 "$$hashKey": "object:93", 5414 "format": "short", 5415 "logBase": 1, 5416 "show": true 5417 } 5418 ], 5419 "yaxis": { 5420 "align": false 5421 } 5422 }, 5423 { 5424 "aliasColors": {}, 5425 "bars": false, 5426 "dashLength": 10, 5427 "dashes": false, 5428 "datasource": { 5429 "type": "prometheus", 5430 "uid": "${DS_PROMETHEUS}" 5431 }, 5432 "fill": 1, 5433 "fillGradient": 0, 5434 "gridPos": { 5435 "h": 8, 5436 "w": 12, 5437 "x": 12, 5438 "y": 84 5439 }, 5440 "hiddenSeries": false, 5441 "id": 205, 5442 "legend": { 5443 "avg": false, 5444 "current": false, 5445 "max": false, 5446 "min": false, 5447 "show": true, 5448 "total": false, 5449 "values": false 5450 }, 5451 "lines": true, 5452 "linewidth": 1, 5453 "nullPointMode": "null", 5454 "options": { 5455 "alertThreshold": true 5456 }, 5457 "percentage": false, 5458 "pluginVersion": "9.2.2", 5459 "pointradius": 2, 5460 "points": false, 5461 "renderer": "flot", 5462 "seriesOverrides": [], 5463 "spaceLength": 10, 5464 "stack": false, 5465 "steppedLine": false, 5466 "targets": [ 5467 { 5468 "datasource": { 5469 "type": "prometheus", 5470 "uid": "${DS_PROMETHEUS}" 5471 }, 5472 "expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))", 5473 "interval": "", 5474 "legendFormat": "soft-failed events", 5475 "refId": "A" 5476 } 5477 ], 5478 "thresholds": [], 5479 "timeRegions": [], 5480 "title": "Soft-failed event rate", 5481 "tooltip": { 5482 "shared": true, 5483 "sort": 0, 5484 "value_type": "individual" 5485 }, 5486 "type": "graph", 5487 "xaxis": { 5488 "mode": "time", 5489 "show": true, 5490 "values": [] 5491 }, 5492 "yaxes": [ 5493 { 5494 "$$hashKey": "object:131", 5495 "format": "hertz", 5496 "logBase": 1, 5497 "show": true 5498 }, 5499 { 5500 "$$hashKey": "object:132", 5501 "format": "short", 5502 "logBase": 1, 5503 "show": false 5504 } 5505 ], 5506 "yaxis": { 5507 "align": false 5508 } 5509 } 5510 ], 5511 "targets": [ 5512 { 5513 "datasource": { 5514 "type": "prometheus", 5515 "uid": "${DS_PROMETHEUS}" 5516 }, 5517 "refId": "A" 5518 } 5519 ], 5520 "title": "Federation", 5521 "type": "row" 5522 }, 5523 { 5524 "collapsed": true, 5525 "gridPos": { 5526 "h": 1, 5527 "w": 24, 5528 "x": 0, 5529 "y": 31 5530 }, 5531 "id": 227, 5532 "panels": [ 5533 { 5534 "datasource": { 5535 "type": "prometheus", 5536 "uid": "${DS_PROMETHEUS}" 5537 }, 5538 "fieldConfig": { 5539 "defaults": { 5540 "color": { 5541 "mode": "palette-classic" 5542 }, 5543 "custom": { 5544 "axisCenteredZero": false, 5545 "axisColorMode": "text", 5546 "axisLabel": "", 5547 "axisPlacement": "auto", 5548 "barAlignment": 0, 5549 "drawStyle": "line", 5550 "fillOpacity": 0, 5551 "gradientMode": "none", 5552 "hideFrom": { 5553 "legend": false, 5554 "tooltip": false, 5555 "viz": false 5556 }, 5557 "lineInterpolation": "linear", 5558 "lineWidth": 1, 5559 "pointSize": 5, 5560 "scaleDistribution": { 5561 "type": "linear" 5562 }, 5563 "showPoints": "auto", 5564 "spanNulls": false, 5565 "stacking": { 5566 "group": "A", 5567 "mode": "none" 5568 }, 5569 "thresholdsStyle": { 5570 "mode": "off" 5571 } 5572 }, 5573 "mappings": [], 5574 "thresholds": { 5575 "mode": "absolute", 5576 "steps": [ 5577 { 5578 "color": "green" 5579 }, 5580 { 5581 "color": "red", 5582 "value": 80 5583 } 5584 ] 5585 } 5586 }, 5587 "overrides": [] 5588 }, 5589 "gridPos": { 5590 "h": 8, 5591 "w": 12, 5592 "x": 0, 5593 "y": 154 5594 }, 5595 "id": 239, 5596 "options": { 5597 "legend": { 5598 "calcs": [], 5599 "displayMode": "list", 5600 "placement": "bottom", 5601 "showLegend": true 5602 }, 5603 "tooltip": { 5604 "mode": "single", 5605 "sort": "none" 5606 } 5607 }, 5608 "targets": [ 5609 { 5610 "datasource": { 5611 "type": "prometheus", 5612 "uid": "${DS_PROMETHEUS}" 5613 }, 5614 "expr": "sum(increase(synapse_rate_limit_reject_total{instance=\"$instance\"}[$bucket_size]))", 5615 "refId": "A" 5616 } 5617 ], 5618 "title": "Number of rate limit rejected requests", 5619 "type": "timeseries" 5620 }, 5621 { 5622 "datasource": { 5623 "type": "prometheus", 5624 "uid": "${DS_PROMETHEUS}" 5625 }, 5626 "fieldConfig": { 5627 "defaults": { 5628 "color": { 5629 "mode": "palette-classic" 5630 }, 5631 "custom": { 5632 "axisCenteredZero": false, 5633 "axisColorMode": "text", 5634 "axisLabel": "", 5635 "axisPlacement": "auto", 5636 "barAlignment": 0, 5637 "drawStyle": "line", 5638 "fillOpacity": 0, 5639 "gradientMode": "none", 5640 "hideFrom": { 5641 "legend": false, 5642 "tooltip": false, 5643 "viz": false 5644 }, 5645 "lineInterpolation": "linear", 5646 "lineWidth": 1, 5647 "pointSize": 5, 5648 "scaleDistribution": { 5649 "type": "linear" 5650 }, 5651 "showPoints": "auto", 5652 "spanNulls": false, 5653 "stacking": { 5654 "group": "A", 5655 "mode": "none" 5656 }, 5657 "thresholdsStyle": { 5658 "mode": "off" 5659 } 5660 }, 5661 "mappings": [], 5662 "thresholds": { 5663 "mode": "absolute", 5664 "steps": [ 5665 { 5666 "color": "green" 5667 }, 5668 { 5669 "color": "red", 5670 "value": 80 5671 } 5672 ] 5673 } 5674 }, 5675 "overrides": [] 5676 }, 5677 "gridPos": { 5678 "h": 8, 5679 "w": 12, 5680 "x": 12, 5681 "y": 154 5682 }, 5683 "id": 235, 5684 "options": { 5685 "legend": { 5686 "calcs": [], 5687 "displayMode": "list", 5688 "placement": "bottom", 5689 "showLegend": true 5690 }, 5691 "tooltip": { 5692 "mode": "single", 5693 "sort": "none" 5694 } 5695 }, 5696 "targets": [ 5697 { 5698 "datasource": { 5699 "type": "prometheus", 5700 "uid": "${DS_PROMETHEUS}" 5701 }, 5702 "expr": "sum(increase(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))", 5703 "refId": "A" 5704 } 5705 ], 5706 "title": "Number of requests being slept by the rate limiter", 5707 "type": "timeseries" 5708 }, 5709 { 5710 "datasource": { 5711 "type": "prometheus", 5712 "uid": "${DS_PROMETHEUS}" 5713 }, 5714 "description": "Why is the data zero (0)? https://github.com/matrix-org/synapse/pull/13541#discussion_r951926322", 5715 "fieldConfig": { 5716 "defaults": { 5717 "color": { 5718 "mode": "palette-classic" 5719 }, 5720 "custom": { 5721 "axisCenteredZero": false, 5722 "axisColorMode": "text", 5723 "axisLabel": "", 5724 "axisPlacement": "auto", 5725 "barAlignment": 0, 5726 "drawStyle": "line", 5727 "fillOpacity": 0, 5728 "gradientMode": "none", 5729 "hideFrom": { 5730 "legend": false, 5731 "tooltip": false, 5732 "viz": false 5733 }, 5734 "lineInterpolation": "linear", 5735 "lineWidth": 1, 5736 "pointSize": 5, 5737 "scaleDistribution": { 5738 "type": "linear" 5739 }, 5740 "showPoints": "auto", 5741 "spanNulls": false, 5742 "stacking": { 5743 "group": "A", 5744 "mode": "none" 5745 }, 5746 "thresholdsStyle": { 5747 "mode": "off" 5748 } 5749 }, 5750 "mappings": [], 5751 "thresholds": { 5752 "mode": "absolute", 5753 "steps": [ 5754 { 5755 "color": "green" 5756 }, 5757 { 5758 "color": "red", 5759 "value": 80 5760 } 5761 ] 5762 } 5763 }, 5764 "overrides": [] 5765 }, 5766 "gridPos": { 5767 "h": 8, 5768 "w": 12, 5769 "x": 0, 5770 "y": 162 5771 }, 5772 "id": 237, 5773 "options": { 5774 "legend": { 5775 "calcs": [], 5776 "displayMode": "list", 5777 "placement": "bottom", 5778 "showLegend": true 5779 }, 5780 "tooltip": { 5781 "mode": "single", 5782 "sort": "none" 5783 } 5784 }, 5785 "pluginVersion": "9.0.4", 5786 "targets": [ 5787 { 5788 "datasource": { 5789 "type": "prometheus", 5790 "uid": "${DS_PROMETHEUS}" 5791 }, 5792 "expr": "sum(increase(synapse_rate_limit_reject_affected_hosts{instance=\"$instance\"}[$bucket_size]))", 5793 "refId": "A" 5794 } 5795 ], 5796 "title": "Number of hosts being rejected by the rate limiter", 5797 "type": "timeseries" 5798 }, 5799 { 5800 "datasource": { 5801 "type": "prometheus", 5802 "uid": "${DS_PROMETHEUS}" 5803 }, 5804 "description": "https://github.com/matrix-org/synapse/pull/13541", 5805 "fieldConfig": { 5806 "defaults": { 5807 "color": { 5808 "mode": "palette-classic" 5809 }, 5810 "custom": { 5811 "axisCenteredZero": false, 5812 "axisColorMode": "text", 5813 "axisLabel": "", 5814 "axisPlacement": "auto", 5815 "barAlignment": 0, 5816 "drawStyle": "line", 5817 "fillOpacity": 0, 5818 "gradientMode": "none", 5819 "hideFrom": { 5820 "legend": false, 5821 "tooltip": false, 5822 "viz": false 5823 }, 5824 "lineInterpolation": "linear", 5825 "lineWidth": 1, 5826 "pointSize": 5, 5827 "scaleDistribution": { 5828 "type": "linear" 5829 }, 5830 "showPoints": "auto", 5831 "spanNulls": false, 5832 "stacking": { 5833 "group": "A", 5834 "mode": "none" 5835 }, 5836 "thresholdsStyle": { 5837 "mode": "off" 5838 } 5839 }, 5840 "mappings": [], 5841 "thresholds": { 5842 "mode": "absolute", 5843 "steps": [ 5844 { 5845 "color": "green" 5846 }, 5847 { 5848 "color": "red", 5849 "value": 80 5850 } 5851 ] 5852 } 5853 }, 5854 "overrides": [] 5855 }, 5856 "gridPos": { 5857 "h": 8, 5858 "w": 12, 5859 "x": 12, 5860 "y": 162 5861 }, 5862 "id": 233, 5863 "options": { 5864 "legend": { 5865 "calcs": [], 5866 "displayMode": "list", 5867 "placement": "bottom", 5868 "showLegend": true 5869 }, 5870 "tooltip": { 5871 "mode": "single", 5872 "sort": "none" 5873 } 5874 }, 5875 "pluginVersion": "9.0.4", 5876 "targets": [ 5877 { 5878 "datasource": { 5879 "type": "prometheus", 5880 "uid": "${DS_PROMETHEUS}" 5881 }, 5882 "expr": "sum(increase(synapse_rate_limit_sleep_affected_hosts{instance=\"$instance\"}[$bucket_size]))", 5883 "refId": "A" 5884 } 5885 ], 5886 "title": "Number of hosts being slept by the rate limiter", 5887 "type": "timeseries" 5888 }, 5889 { 5890 "aliasColors": {}, 5891 "bars": false, 5892 "dashLength": 10, 5893 "dashes": false, 5894 "datasource": { 5895 "type": "prometheus", 5896 "uid": "${DS_PROMETHEUS}" 5897 }, 5898 "description": "", 5899 "fieldConfig": { 5900 "defaults": { 5901 "links": [] 5902 }, 5903 "overrides": [] 5904 }, 5905 "fill": 0, 5906 "fillGradient": 0, 5907 "gridPos": { 5908 "h": 8, 5909 "w": 12, 5910 "x": 0, 5911 "y": 170 5912 }, 5913 "hiddenSeries": false, 5914 "id": 229, 5915 "legend": { 5916 "avg": false, 5917 "current": false, 5918 "max": false, 5919 "min": false, 5920 "rightSide": false, 5921 "show": true, 5922 "total": false, 5923 "values": false 5924 }, 5925 "lines": true, 5926 "linewidth": 0, 5927 "links": [], 5928 "nullPointMode": "connected", 5929 "options": { 5930 "alertThreshold": true 5931 }, 5932 "paceLength": 10, 5933 "percentage": false, 5934 "pluginVersion": "9.2.2", 5935 "pointradius": 5, 5936 "points": false, 5937 "renderer": "flot", 5938 "seriesOverrides": [ 5939 { 5940 "$$hashKey": "object:276", 5941 "alias": "Avg", 5942 "fill": 0, 5943 "linewidth": 3 5944 }, 5945 { 5946 "$$hashKey": "object:277", 5947 "alias": "99%", 5948 "color": "#C4162A", 5949 "fillBelowTo": "90%" 5950 }, 5951 { 5952 "$$hashKey": "object:278", 5953 "alias": "90%", 5954 "color": "#FF7383", 5955 "fillBelowTo": "75%" 5956 }, 5957 { 5958 "$$hashKey": "object:279", 5959 "alias": "75%", 5960 "color": "#FFEE52", 5961 "fillBelowTo": "50%" 5962 }, 5963 { 5964 "$$hashKey": "object:280", 5965 "alias": "50%", 5966 "color": "#73BF69", 5967 "fillBelowTo": "25%" 5968 }, 5969 { 5970 "$$hashKey": "object:281", 5971 "alias": "25%", 5972 "color": "#1F60C4", 5973 "fillBelowTo": "5%" 5974 }, 5975 { 5976 "$$hashKey": "object:282", 5977 "alias": "5%", 5978 "lines": false 5979 }, 5980 { 5981 "$$hashKey": "object:283", 5982 "alias": "Average", 5983 "color": "rgb(255, 255, 255)", 5984 "lines": true, 5985 "linewidth": 3 5986 }, 5987 { 5988 "$$hashKey": "object:284", 5989 "alias": ">99%", 5990 "color": "#B877D9", 5991 "fill": 3, 5992 "lines": true 5993 } 5994 ], 5995 "spaceLength": 10, 5996 "stack": false, 5997 "steppedLine": false, 5998 "targets": [ 5999 { 6000 "datasource": { 6001 "uid": "${DS_PROMETHEUS}" 6002 }, 6003 "editorMode": "code", 6004 "expr": "histogram_quantile(0.9995, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 6005 "format": "time_series", 6006 "hide": false, 6007 "intervalFactor": 1, 6008 "legendFormat": ">99%", 6009 "range": true, 6010 "refId": "E" 6011 }, 6012 { 6013 "datasource": { 6014 "uid": "${DS_PROMETHEUS}" 6015 }, 6016 "editorMode": "code", 6017 "expr": "histogram_quantile(0.99, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 6018 "format": "time_series", 6019 "intervalFactor": 1, 6020 "legendFormat": "99%", 6021 "range": true, 6022 "refId": "D" 6023 }, 6024 { 6025 "datasource": { 6026 "uid": "${DS_PROMETHEUS}" 6027 }, 6028 "expr": "histogram_quantile(0.9, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 6029 "format": "time_series", 6030 "interval": "", 6031 "intervalFactor": 1, 6032 "legendFormat": "90%", 6033 "refId": "A" 6034 }, 6035 { 6036 "datasource": { 6037 "uid": "${DS_PROMETHEUS}" 6038 }, 6039 "expr": "histogram_quantile(0.75, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 6040 "format": "time_series", 6041 "intervalFactor": 1, 6042 "legendFormat": "75%", 6043 "refId": "C" 6044 }, 6045 { 6046 "datasource": { 6047 "uid": "${DS_PROMETHEUS}" 6048 }, 6049 "expr": "histogram_quantile(0.5, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 6050 "format": "time_series", 6051 "intervalFactor": 1, 6052 "legendFormat": "50%", 6053 "refId": "B" 6054 }, 6055 { 6056 "datasource": { 6057 "uid": "${DS_PROMETHEUS}" 6058 }, 6059 "expr": "histogram_quantile(0.25, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 6060 "legendFormat": "25%", 6061 "refId": "F" 6062 }, 6063 { 6064 "datasource": { 6065 "uid": "${DS_PROMETHEUS}" 6066 }, 6067 "expr": "histogram_quantile(0.05, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", 6068 "legendFormat": "5%", 6069 "refId": "G" 6070 }, 6071 { 6072 "datasource": { 6073 "uid": "${DS_PROMETHEUS}" 6074 }, 6075 "expr": "sum(rate(synapse_rate_limit_queue_wait_time_seconds_sum{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_rate_limit_queue_wait_time_seconds_count{index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", 6076 "legendFormat": "Average", 6077 "refId": "H" 6078 } 6079 ], 6080 "thresholds": [ 6081 { 6082 "$$hashKey": "object:283", 6083 "colorMode": "warning", 6084 "fill": false, 6085 "line": true, 6086 "op": "gt", 6087 "value": 1, 6088 "yaxis": "left" 6089 }, 6090 { 6091 "$$hashKey": "object:284", 6092 "colorMode": "critical", 6093 "fill": false, 6094 "line": true, 6095 "op": "gt", 6096 "value": 2, 6097 "yaxis": "left" 6098 } 6099 ], 6100 "timeRegions": [], 6101 "title": "Rate limit queue wait time Quantiles (all workers)", 6102 "tooltip": { 6103 "shared": true, 6104 "sort": 2, 6105 "value_type": "individual" 6106 }, 6107 "type": "graph", 6108 "xaxis": { 6109 "mode": "time", 6110 "show": true, 6111 "values": [] 6112 }, 6113 "yaxes": [ 6114 { 6115 "$$hashKey": "object:255", 6116 "format": "s", 6117 "label": "", 6118 "logBase": 1, 6119 "min": "0", 6120 "show": true 6121 }, 6122 { 6123 "$$hashKey": "object:256", 6124 "format": "hertz", 6125 "label": "", 6126 "logBase": 1, 6127 "min": "0", 6128 "show": true 6129 } 6130 ], 6131 "yaxis": { 6132 "align": false 6133 } 6134 }, 6135 { 6136 "datasource": { 6137 "type": "prometheus", 6138 "uid": "${DS_PROMETHEUS}" 6139 }, 6140 "fieldConfig": { 6141 "defaults": { 6142 "color": { 6143 "mode": "palette-classic" 6144 }, 6145 "custom": { 6146 "axisCenteredZero": false, 6147 "axisColorMode": "text", 6148 "axisLabel": "", 6149 "axisPlacement": "auto", 6150 "barAlignment": 0, 6151 "drawStyle": "line", 6152 "fillOpacity": 0, 6153 "gradientMode": "none", 6154 "hideFrom": { 6155 "legend": false, 6156 "tooltip": false, 6157 "viz": false 6158 }, 6159 "lineInterpolation": "linear", 6160 "lineWidth": 1, 6161 "pointSize": 5, 6162 "scaleDistribution": { 6163 "type": "linear" 6164 }, 6165 "showPoints": "auto", 6166 "spanNulls": false, 6167 "stacking": { 6168 "group": "A", 6169 "mode": "none" 6170 }, 6171 "thresholdsStyle": { 6172 "mode": "line" 6173 } 6174 }, 6175 "mappings": [], 6176 "thresholds": { 6177 "mode": "absolute", 6178 "steps": [ 6179 { 6180 "color": "green" 6181 } 6182 ] 6183 }, 6184 "unit": "hertz" 6185 }, 6186 "overrides": [ 6187 { 6188 "matcher": { 6189 "id": "byName", 6190 "options": "Default reject threshold (50 requests within a second)" 6191 }, 6192 "properties": [ 6193 { 6194 "id": "color", 6195 "value": { 6196 "fixedColor": "red", 6197 "mode": "fixed" 6198 } 6199 }, 6200 { 6201 "id": "custom.lineWidth", 6202 "value": 2 6203 } 6204 ] 6205 } 6206 ] 6207 }, 6208 "gridPos": { 6209 "h": 8, 6210 "w": 12, 6211 "x": 12, 6212 "y": 170 6213 }, 6214 "id": 231, 6215 "options": { 6216 "legend": { 6217 "calcs": [], 6218 "displayMode": "list", 6219 "placement": "bottom", 6220 "showLegend": true 6221 }, 6222 "tooltip": { 6223 "mode": "single", 6224 "sort": "none" 6225 } 6226 }, 6227 "targets": [ 6228 { 6229 "datasource": { 6230 "type": "prometheus", 6231 "uid": "${DS_PROMETHEUS}" 6232 }, 6233 "expr": "sum(rate(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))", 6234 "refId": "A" 6235 }, 6236 { 6237 "datasource": { 6238 "type": "prometheus", 6239 "uid": "${DS_PROMETHEUS}" 6240 }, 6241 "editorMode": "code", 6242 "expr": "50", 6243 "hide": false, 6244 "legendFormat": "Default reject threshold (50 requests within a second)", 6245 "range": true, 6246 "refId": "B" 6247 } 6248 ], 6249 "title": "Rate of requests being slept by the rate limiter", 6250 "type": "timeseries" 6251 } 6252 ], 6253 "title": "Federation rate limiter", 6254 "type": "row" 6255 }, 6256 { 6257 "collapsed": true, 6258 "datasource": { 6259 "type": "prometheus", 6260 "uid": "${DS_PROMETHEUS}" 6261 }, 6262 "gridPos": { 6263 "h": 1, 6264 "w": 24, 6265 "x": 0, 6266 "y": 32 6267 }, 6268 "id": 60, 6269 "panels": [ 6270 { 6271 "datasource": { 6272 "type": "prometheus", 6273 "uid": "${DS_PROMETHEUS}" 6274 }, 6275 "fieldConfig": { 6276 "defaults": { 6277 "color": { 6278 "mode": "palette-classic" 6279 }, 6280 "custom": { 6281 "axisCenteredZero": false, 6282 "axisColorMode": "text", 6283 "axisLabel": "", 6284 "axisPlacement": "auto", 6285 "barAlignment": 0, 6286 "drawStyle": "line", 6287 "fillOpacity": 10, 6288 "gradientMode": "none", 6289 "hideFrom": { 6290 "legend": false, 6291 "tooltip": false, 6292 "viz": false 6293 }, 6294 "lineInterpolation": "linear", 6295 "lineWidth": 1, 6296 "pointSize": 5, 6297 "scaleDistribution": { 6298 "type": "linear" 6299 }, 6300 "showPoints": "never", 6301 "spanNulls": false, 6302 "stacking": { 6303 "group": "A", 6304 "mode": "none" 6305 }, 6306 "thresholdsStyle": { 6307 "mode": "off" 6308 } 6309 }, 6310 "links": [], 6311 "mappings": [], 6312 "thresholds": { 6313 "mode": "absolute", 6314 "steps": [ 6315 { 6316 "color": "green" 6317 }, 6318 { 6319 "color": "red", 6320 "value": 80 6321 } 6322 ] 6323 }, 6324 "unit": "hertz" 6325 }, 6326 "overrides": [] 6327 }, 6328 "gridPos": { 6329 "h": 8, 6330 "w": 12, 6331 "x": 0, 6332 "y": 155 6333 }, 6334 "id": 51, 6335 "links": [], 6336 "options": { 6337 "legend": { 6338 "calcs": [], 6339 "displayMode": "list", 6340 "placement": "bottom", 6341 "showLegend": true 6342 }, 6343 "tooltip": { 6344 "mode": "multi", 6345 "sort": "none" 6346 } 6347 }, 6348 "pluginVersion": "9.2.2", 6349 "targets": [ 6350 { 6351 "datasource": { 6352 "uid": "${DS_PROMETHEUS}" 6353 }, 6354 "editorMode": "code", 6355 "expr": "rate(synapse_http_httppusher_http_pushes_processed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0", 6356 "format": "time_series", 6357 "interval": "", 6358 "intervalFactor": 2, 6359 "legendFormat": "processed {{job}}-{{index}}", 6360 "range": true, 6361 "refId": "A", 6362 "step": 20 6363 }, 6364 { 6365 "datasource": { 6366 "uid": "${DS_PROMETHEUS}" 6367 }, 6368 "editorMode": "code", 6369 "expr": "rate(synapse_http_httppusher_http_pushes_failed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0", 6370 "format": "time_series", 6371 "intervalFactor": 2, 6372 "legendFormat": "failed {{job}}-{{index}}", 6373 "range": true, 6374 "refId": "B", 6375 "step": 20 6376 } 6377 ], 6378 "title": "HTTP Push rate", 6379 "type": "timeseries" 6380 }, 6381 { 6382 "aliasColors": {}, 6383 "bars": false, 6384 "dashLength": 10, 6385 "dashes": false, 6386 "datasource": { 6387 "uid": "${DS_PROMETHEUS}" 6388 }, 6389 "description": "", 6390 "fieldConfig": { 6391 "defaults": { 6392 "links": [] 6393 }, 6394 "overrides": [] 6395 }, 6396 "fill": 1, 6397 "fillGradient": 0, 6398 "gridPos": { 6399 "h": 8, 6400 "w": 12, 6401 "x": 12, 6402 "y": 155 6403 }, 6404 "hiddenSeries": false, 6405 "id": 134, 6406 "legend": { 6407 "avg": false, 6408 "current": false, 6409 "hideZero": false, 6410 "max": false, 6411 "min": false, 6412 "show": true, 6413 "total": false, 6414 "values": false 6415 }, 6416 "lines": true, 6417 "linewidth": 1, 6418 "nullPointMode": "null", 6419 "options": { 6420 "alertThreshold": true 6421 }, 6422 "percentage": false, 6423 "pluginVersion": "9.2.2", 6424 "pointradius": 2, 6425 "points": false, 6426 "renderer": "flot", 6427 "seriesOverrides": [], 6428 "spaceLength": 10, 6429 "stack": false, 6430 "steppedLine": false, 6431 "targets": [ 6432 { 6433 "datasource": { 6434 "uid": "${DS_PROMETHEUS}" 6435 }, 6436 "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})", 6437 "legendFormat": "{{kind}} {{app_id}}", 6438 "refId": "A" 6439 } 6440 ], 6441 "thresholds": [], 6442 "timeRegions": [], 6443 "title": "Active pusher instances by app", 6444 "tooltip": { 6445 "shared": false, 6446 "sort": 2, 6447 "value_type": "individual" 6448 }, 6449 "type": "graph", 6450 "xaxis": { 6451 "mode": "time", 6452 "show": true, 6453 "values": [] 6454 }, 6455 "yaxes": [ 6456 { 6457 "format": "short", 6458 "logBase": 1, 6459 "show": true 6460 }, 6461 { 6462 "format": "short", 6463 "logBase": 1, 6464 "show": true 6465 } 6466 ], 6467 "yaxis": { 6468 "align": false 6469 } 6470 } 6471 ], 6472 "targets": [ 6473 { 6474 "datasource": { 6475 "type": "prometheus", 6476 "uid": "${DS_PROMETHEUS}" 6477 }, 6478 "refId": "A" 6479 } 6480 ], 6481 "title": "Pushes", 6482 "type": "row" 6483 }, 6484 { 6485 "collapsed": true, 6486 "datasource": { 6487 "type": "prometheus", 6488 "uid": "${DS_PROMETHEUS}" 6489 }, 6490 "gridPos": { 6491 "h": 1, 6492 "w": 24, 6493 "x": 0, 6494 "y": 33 6495 }, 6496 "id": 219, 6497 "panels": [ 6498 { 6499 "aliasColors": {}, 6500 "bars": false, 6501 "dashLength": 10, 6502 "dashes": false, 6503 "datasource": { 6504 "type": "prometheus", 6505 "uid": "${DS_PROMETHEUS}" 6506 }, 6507 "description": "How many entries in current state that we are iterating over while calculating push rules.", 6508 "fieldConfig": { 6509 "defaults": { 6510 "links": [] 6511 }, 6512 "overrides": [] 6513 }, 6514 "fill": 1, 6515 "fillGradient": 0, 6516 "gridPos": { 6517 "h": 7, 6518 "w": 12, 6519 "x": 0, 6520 "y": 33 6521 }, 6522 "hiddenSeries": false, 6523 "id": 209, 6524 "legend": { 6525 "avg": false, 6526 "current": false, 6527 "max": false, 6528 "min": false, 6529 "show": false, 6530 "total": false, 6531 "values": false 6532 }, 6533 "lines": true, 6534 "linewidth": 1, 6535 "links": [], 6536 "nullPointMode": "null", 6537 "options": { 6538 "alertThreshold": true 6539 }, 6540 "paceLength": 10, 6541 "percentage": false, 6542 "pluginVersion": "8.4.3", 6543 "pointradius": 5, 6544 "points": false, 6545 "renderer": "flot", 6546 "seriesOverrides": [], 6547 "spaceLength": 10, 6548 "stack": false, 6549 "steppedLine": false, 6550 "targets": [ 6551 { 6552 "datasource": { 6553 "type": "prometheus", 6554 "uid": "${DS_PROMETHEUS}" 6555 }, 6556 "exemplar": true, 6557 "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", 6558 "format": "time_series", 6559 "interval": "", 6560 "intervalFactor": 2, 6561 "legendFormat": "{{index}}", 6562 "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total", 6563 "refId": "A", 6564 "step": 2 6565 } 6566 ], 6567 "thresholds": [], 6568 "timeRegions": [], 6569 "title": "Iterations over State", 6570 "tooltip": { 6571 "shared": true, 6572 "sort": 0, 6573 "value_type": "individual" 6574 }, 6575 "type": "graph", 6576 "xaxis": { 6577 "mode": "time", 6578 "show": true, 6579 "values": [] 6580 }, 6581 "yaxes": [ 6582 { 6583 "format": "hertz", 6584 "label": "", 6585 "logBase": 1, 6586 "min": "0", 6587 "show": true 6588 }, 6589 { 6590 "format": "short", 6591 "logBase": 1, 6592 "show": true 6593 } 6594 ], 6595 "yaxis": { 6596 "align": false 6597 } 6598 }, 6599 { 6600 "aliasColors": {}, 6601 "bars": false, 6602 "dashLength": 10, 6603 "dashes": false, 6604 "datasource": { 6605 "type": "prometheus", 6606 "uid": "${DS_PROMETHEUS}" 6607 }, 6608 "description": "Rate that the cached push rules for a room get invalidated due to underlying push rules being changed. ", 6609 "fieldConfig": { 6610 "defaults": { 6611 "links": [] 6612 }, 6613 "overrides": [] 6614 }, 6615 "fill": 1, 6616 "fillGradient": 0, 6617 "gridPos": { 6618 "h": 7, 6619 "w": 12, 6620 "x": 12, 6621 "y": 33 6622 }, 6623 "hiddenSeries": false, 6624 "id": 211, 6625 "legend": { 6626 "avg": false, 6627 "current": false, 6628 "max": false, 6629 "min": false, 6630 "show": false, 6631 "total": false, 6632 "values": false 6633 }, 6634 "lines": true, 6635 "linewidth": 1, 6636 "links": [], 6637 "nullPointMode": "null", 6638 "options": { 6639 "alertThreshold": true 6640 }, 6641 "paceLength": 10, 6642 "percentage": false, 6643 "pluginVersion": "8.4.3", 6644 "pointradius": 5, 6645 "points": false, 6646 "renderer": "flot", 6647 "seriesOverrides": [], 6648 "spaceLength": 10, 6649 "stack": false, 6650 "steppedLine": false, 6651 "targets": [ 6652 { 6653 "datasource": { 6654 "type": "prometheus", 6655 "uid": "${DS_PROMETHEUS}" 6656 }, 6657 "exemplar": true, 6658 "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", 6659 "format": "time_series", 6660 "interval": "", 6661 "intervalFactor": 2, 6662 "legendFormat": "{{index}}", 6663 "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total", 6664 "refId": "A", 6665 "step": 2 6666 } 6667 ], 6668 "thresholds": [], 6669 "timeRegions": [], 6670 "title": "Push Rule Invalidations", 6671 "tooltip": { 6672 "shared": true, 6673 "sort": 0, 6674 "value_type": "individual" 6675 }, 6676 "type": "graph", 6677 "xaxis": { 6678 "mode": "time", 6679 "show": true, 6680 "values": [] 6681 }, 6682 "yaxes": [ 6683 { 6684 "format": "hertz", 6685 "label": "", 6686 "logBase": 1, 6687 "min": "0", 6688 "show": true 6689 }, 6690 { 6691 "format": "short", 6692 "logBase": 1, 6693 "show": true 6694 } 6695 ], 6696 "yaxis": { 6697 "align": false 6698 } 6699 }, 6700 { 6701 "aliasColors": {}, 6702 "bars": false, 6703 "dashLength": 10, 6704 "dashes": false, 6705 "datasource": { 6706 "type": "prometheus", 6707 "uid": "${DS_PROMETHEUS}" 6708 }, 6709 "description": "How often the \"delta optimisation\" happens.\n\nThe delta optimisation is when we update the push rules for a room incrementally after a state change where we know the delta between the old state and the new state.\n\nThis can't happen if we don't the delta or we're calculating push rules from scratch.", 6710 "fieldConfig": { 6711 "defaults": { 6712 "links": [] 6713 }, 6714 "overrides": [] 6715 }, 6716 "fill": 1, 6717 "fillGradient": 0, 6718 "gridPos": { 6719 "h": 7, 6720 "w": 12, 6721 "x": 0, 6722 "y": 40 6723 }, 6724 "hiddenSeries": false, 6725 "id": 213, 6726 "legend": { 6727 "avg": false, 6728 "current": false, 6729 "max": false, 6730 "min": false, 6731 "show": true, 6732 "total": false, 6733 "values": false 6734 }, 6735 "lines": true, 6736 "linewidth": 1, 6737 "links": [], 6738 "nullPointMode": "null", 6739 "options": { 6740 "alertThreshold": true 6741 }, 6742 "paceLength": 10, 6743 "percentage": false, 6744 "pluginVersion": "8.4.3", 6745 "pointradius": 5, 6746 "points": false, 6747 "renderer": "flot", 6748 "seriesOverrides": [ 6749 { 6750 "alias": "Number of calls", 6751 "yaxis": 2 6752 } 6753 ], 6754 "spaceLength": 10, 6755 "stack": false, 6756 "steppedLine": false, 6757 "targets": [ 6758 { 6759 "datasource": { 6760 "type": "prometheus", 6761 "uid": "${DS_PROMETHEUS}" 6762 }, 6763 "exemplar": true, 6764 "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))", 6765 "format": "time_series", 6766 "interval": "", 6767 "intervalFactor": 2, 6768 "legendFormat": "Hit Rate", 6769 "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total", 6770 "refId": "A", 6771 "step": 2 6772 }, 6773 { 6774 "datasource": { 6775 "type": "prometheus", 6776 "uid": "${DS_PROMETHEUS}" 6777 }, 6778 "exemplar": true, 6779 "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))", 6780 "format": "time_series", 6781 "interval": "", 6782 "intervalFactor": 2, 6783 "legendFormat": "Number of calls", 6784 "refId": "B", 6785 "step": 2 6786 } 6787 ], 6788 "thresholds": [], 6789 "timeRegions": [], 6790 "title": "Delta Optimisation", 6791 "tooltip": { 6792 "shared": true, 6793 "sort": 0, 6794 "value_type": "individual" 6795 }, 6796 "type": "graph", 6797 "xaxis": { 6798 "mode": "time", 6799 "show": true, 6800 "values": [] 6801 }, 6802 "yaxes": [ 6803 { 6804 "format": "percentunit", 6805 "label": "", 6806 "logBase": 1, 6807 "max": "1", 6808 "min": "0", 6809 "show": true 6810 }, 6811 { 6812 "format": "hertz", 6813 "label": "", 6814 "logBase": 1, 6815 "min": "0", 6816 "show": true 6817 } 6818 ], 6819 "yaxis": { 6820 "align": false 6821 } 6822 }, 6823 { 6824 "aliasColors": {}, 6825 "bars": false, 6826 "dashLength": 10, 6827 "dashes": false, 6828 "datasource": { 6829 "type": "prometheus", 6830 "uid": "${DS_PROMETHEUS}" 6831 }, 6832 "description": "How often we have the correct cached push rules for a room.", 6833 "fieldConfig": { 6834 "defaults": { 6835 "links": [] 6836 }, 6837 "overrides": [] 6838 }, 6839 "fill": 1, 6840 "fillGradient": 0, 6841 "gridPos": { 6842 "h": 7, 6843 "w": 12, 6844 "x": 12, 6845 "y": 40 6846 }, 6847 "hiddenSeries": false, 6848 "id": 215, 6849 "legend": { 6850 "avg": false, 6851 "current": false, 6852 "max": false, 6853 "min": false, 6854 "show": true, 6855 "total": false, 6856 "values": false 6857 }, 6858 "lines": true, 6859 "linewidth": 1, 6860 "links": [], 6861 "nullPointMode": "null", 6862 "options": { 6863 "alertThreshold": true 6864 }, 6865 "paceLength": 10, 6866 "percentage": false, 6867 "pluginVersion": "8.4.3", 6868 "pointradius": 5, 6869 "points": false, 6870 "renderer": "flot", 6871 "seriesOverrides": [ 6872 { 6873 "alias": "Number of calls", 6874 "yaxis": 2 6875 } 6876 ], 6877 "spaceLength": 10, 6878 "stack": false, 6879 "steppedLine": false, 6880 "targets": [ 6881 { 6882 "datasource": { 6883 "type": "prometheus", 6884 "uid": "${DS_PROMETHEUS}" 6885 }, 6886 "exemplar": true, 6887 "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))", 6888 "format": "time_series", 6889 "interval": "", 6890 "intervalFactor": 2, 6891 "legendFormat": "Hit Rate", 6892 "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total", 6893 "refId": "A", 6894 "step": 2 6895 }, 6896 { 6897 "datasource": { 6898 "type": "prometheus", 6899 "uid": "${DS_PROMETHEUS}" 6900 }, 6901 "exemplar": true, 6902 "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))", 6903 "format": "time_series", 6904 "interval": "", 6905 "intervalFactor": 2, 6906 "legendFormat": "Number of calls", 6907 "refId": "B", 6908 "step": 2 6909 } 6910 ], 6911 "thresholds": [], 6912 "timeRegions": [], 6913 "title": "How often we reuse existing calculated push rules", 6914 "tooltip": { 6915 "shared": true, 6916 "sort": 0, 6917 "value_type": "individual" 6918 }, 6919 "type": "graph", 6920 "xaxis": { 6921 "mode": "time", 6922 "show": true, 6923 "values": [] 6924 }, 6925 "yaxes": [ 6926 { 6927 "format": "percentunit", 6928 "label": "", 6929 "logBase": 1, 6930 "max": "1", 6931 "min": "0", 6932 "show": true 6933 }, 6934 { 6935 "format": "hertz", 6936 "logBase": 1, 6937 "show": true 6938 } 6939 ], 6940 "yaxis": { 6941 "align": false 6942 } 6943 }, 6944 { 6945 "aliasColors": {}, 6946 "bars": false, 6947 "dashLength": 10, 6948 "dashes": false, 6949 "datasource": { 6950 "type": "prometheus", 6951 "uid": "${DS_PROMETHEUS}" 6952 }, 6953 "description": "How often we have existing cached push rules for the room. \n\nNote that these might be outdated and need to be recalculated if the state has changed.", 6954 "fieldConfig": { 6955 "defaults": { 6956 "links": [] 6957 }, 6958 "overrides": [] 6959 }, 6960 "fill": 1, 6961 "fillGradient": 0, 6962 "gridPos": { 6963 "h": 7, 6964 "w": 12, 6965 "x": 0, 6966 "y": 47 6967 }, 6968 "hiddenSeries": false, 6969 "id": 217, 6970 "legend": { 6971 "avg": false, 6972 "current": false, 6973 "max": false, 6974 "min": false, 6975 "show": true, 6976 "total": false, 6977 "values": false 6978 }, 6979 "lines": true, 6980 "linewidth": 1, 6981 "links": [], 6982 "nullPointMode": "null", 6983 "options": { 6984 "alertThreshold": true 6985 }, 6986 "paceLength": 10, 6987 "percentage": false, 6988 "pluginVersion": "8.4.3", 6989 "pointradius": 5, 6990 "points": false, 6991 "renderer": "flot", 6992 "seriesOverrides": [ 6993 { 6994 "alias": "Number of calls", 6995 "yaxis": 2 6996 } 6997 ], 6998 "spaceLength": 10, 6999 "stack": false, 7000 "steppedLine": false, 7001 "targets": [ 7002 { 7003 "datasource": { 7004 "type": "prometheus", 7005 "uid": "${DS_PROMETHEUS}" 7006 }, 7007 "exemplar": true, 7008 "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))", 7009 "format": "time_series", 7010 "interval": "", 7011 "intervalFactor": 2, 7012 "legendFormat": "Hit Rate", 7013 "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total", 7014 "refId": "A", 7015 "step": 2 7016 }, 7017 { 7018 "datasource": { 7019 "type": "prometheus", 7020 "uid": "${DS_PROMETHEUS}" 7021 }, 7022 "exemplar": true, 7023 "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))", 7024 "format": "time_series", 7025 "interval": "", 7026 "intervalFactor": 2, 7027 "legendFormat": "Number of calls", 7028 "refId": "B", 7029 "step": 2 7030 } 7031 ], 7032 "thresholds": [], 7033 "timeRegions": [], 7034 "title": "How often we have the RulesForRoom cached", 7035 "tooltip": { 7036 "shared": true, 7037 "sort": 0, 7038 "value_type": "individual" 7039 }, 7040 "type": "graph", 7041 "xaxis": { 7042 "mode": "time", 7043 "show": true, 7044 "values": [] 7045 }, 7046 "yaxes": [ 7047 { 7048 "format": "percentunit", 7049 "label": "", 7050 "logBase": 1, 7051 "max": "1", 7052 "min": "0", 7053 "show": true 7054 }, 7055 { 7056 "format": "hertz", 7057 "logBase": 1, 7058 "show": true 7059 } 7060 ], 7061 "yaxis": { 7062 "align": false 7063 } 7064 } 7065 ], 7066 "targets": [ 7067 { 7068 "datasource": { 7069 "type": "prometheus", 7070 "uid": "${DS_PROMETHEUS}" 7071 }, 7072 "refId": "A" 7073 } 7074 ], 7075 "title": "Push Rule Cache", 7076 "type": "row" 7077 }, 7078 { 7079 "collapsed": true, 7080 "datasource": { 7081 "type": "prometheus", 7082 "uid": "${DS_PROMETHEUS}" 7083 }, 7084 "gridPos": { 7085 "h": 1, 7086 "w": 24, 7087 "x": 0, 7088 "y": 34 7089 }, 7090 "id": 58, 7091 "panels": [ 7092 { 7093 "datasource": { 7094 "uid": "${DS_PROMETHEUS}" 7095 }, 7096 "fieldConfig": { 7097 "defaults": { 7098 "color": { 7099 "mode": "palette-classic" 7100 }, 7101 "custom": { 7102 "axisLabel": "", 7103 "axisPlacement": "auto", 7104 "barAlignment": 0, 7105 "drawStyle": "line", 7106 "fillOpacity": 10, 7107 "gradientMode": "none", 7108 "hideFrom": { 7109 "legend": false, 7110 "tooltip": false, 7111 "viz": false 7112 }, 7113 "lineInterpolation": "linear", 7114 "lineWidth": 1, 7115 "pointSize": 5, 7116 "scaleDistribution": { 7117 "type": "linear" 7118 }, 7119 "showPoints": "never", 7120 "spanNulls": false, 7121 "stacking": { 7122 "group": "A", 7123 "mode": "none" 7124 }, 7125 "thresholdsStyle": { 7126 "mode": "off" 7127 } 7128 }, 7129 "links": [], 7130 "mappings": [], 7131 "min": 0, 7132 "thresholds": { 7133 "mode": "absolute", 7134 "steps": [ 7135 { 7136 "color": "green" 7137 }, 7138 { 7139 "color": "red", 7140 "value": 80 7141 } 7142 ] 7143 }, 7144 "unit": "s" 7145 }, 7146 "overrides": [] 7147 }, 7148 "gridPos": { 7149 "h": 7, 7150 "w": 12, 7151 "x": 0, 7152 "y": 35 7153 }, 7154 "id": 48, 7155 "links": [], 7156 "options": { 7157 "legend": { 7158 "calcs": [], 7159 "displayMode": "list", 7160 "placement": "bottom", 7161 "showLegend": true 7162 }, 7163 "tooltip": { 7164 "mode": "single", 7165 "sort": "none" 7166 } 7167 }, 7168 "pluginVersion": "9.0.4", 7169 "targets": [ 7170 { 7171 "datasource": { 7172 "uid": "${DS_PROMETHEUS}" 7173 }, 7174 "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])", 7175 "format": "time_series", 7176 "intervalFactor": 2, 7177 "legendFormat": "{{job}}-{{index}}", 7178 "refId": "A", 7179 "step": 20 7180 } 7181 ], 7182 "title": "Avg time waiting for db conn", 7183 "type": "timeseries" 7184 }, 7185 { 7186 "aliasColors": {}, 7187 "bars": false, 7188 "dashLength": 10, 7189 "dashes": false, 7190 "datasource": { 7191 "uid": "${DS_PROMETHEUS}" 7192 }, 7193 "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan", 7194 "fieldConfig": { 7195 "defaults": { 7196 "links": [] 7197 }, 7198 "overrides": [] 7199 }, 7200 "fill": 1, 7201 "fillGradient": 0, 7202 "gridPos": { 7203 "h": 7, 7204 "w": 12, 7205 "x": 12, 7206 "y": 35 7207 }, 7208 "hiddenSeries": false, 7209 "id": 104, 7210 "legend": { 7211 "alignAsTable": true, 7212 "avg": false, 7213 "current": false, 7214 "max": false, 7215 "min": false, 7216 "show": true, 7217 "total": false, 7218 "values": false 7219 }, 7220 "lines": true, 7221 "linewidth": 1, 7222 "links": [], 7223 "nullPointMode": "null", 7224 "options": { 7225 "alertThreshold": true 7226 }, 7227 "paceLength": 10, 7228 "percentage": false, 7229 "pluginVersion": "9.0.4", 7230 "pointradius": 5, 7231 "points": false, 7232 "renderer": "flot", 7233 "seriesOverrides": [], 7234 "spaceLength": 10, 7235 "stack": false, 7236 "steppedLine": false, 7237 "targets": [ 7238 { 7239 "datasource": { 7240 "uid": "${DS_PROMETHEUS}" 7241 }, 7242 "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", 7243 "format": "time_series", 7244 "hide": false, 7245 "intervalFactor": 1, 7246 "legendFormat": "{{job}} {{index}} 99%", 7247 "refId": "A", 7248 "step": 20 7249 }, 7250 { 7251 "datasource": { 7252 "uid": "${DS_PROMETHEUS}" 7253 }, 7254 "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", 7255 "format": "time_series", 7256 "intervalFactor": 1, 7257 "legendFormat": "{{job}} {{index}} 95%", 7258 "refId": "B" 7259 }, 7260 { 7261 "datasource": { 7262 "uid": "${DS_PROMETHEUS}" 7263 }, 7264 "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", 7265 "format": "time_series", 7266 "intervalFactor": 1, 7267 "legendFormat": "{{job}} {{index}} 90%", 7268 "refId": "C" 7269 }, 7270 { 7271 "datasource": { 7272 "uid": "${DS_PROMETHEUS}" 7273 }, 7274 "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 7275 "format": "time_series", 7276 "interval": "", 7277 "intervalFactor": 1, 7278 "legendFormat": "{{job}} {{index}} mean", 7279 "refId": "D" 7280 } 7281 ], 7282 "thresholds": [], 7283 "timeRegions": [], 7284 "title": "Db scheduling time quantiles", 7285 "tooltip": { 7286 "shared": false, 7287 "sort": 0, 7288 "value_type": "individual" 7289 }, 7290 "type": "graph", 7291 "xaxis": { 7292 "mode": "time", 7293 "show": true, 7294 "values": [] 7295 }, 7296 "yaxes": [ 7297 { 7298 "format": "s", 7299 "label": "", 7300 "logBase": 1, 7301 "min": "0", 7302 "show": true 7303 }, 7304 { 7305 "format": "short", 7306 "logBase": 1, 7307 "show": false 7308 } 7309 ], 7310 "yaxis": { 7311 "align": false 7312 } 7313 }, 7314 { 7315 "aliasColors": {}, 7316 "bars": false, 7317 "dashLength": 10, 7318 "dashes": false, 7319 "datasource": { 7320 "uid": "${DS_PROMETHEUS}" 7321 }, 7322 "editable": true, 7323 "error": false, 7324 "fieldConfig": { 7325 "defaults": { 7326 "links": [] 7327 }, 7328 "overrides": [] 7329 }, 7330 "fill": 0, 7331 "fillGradient": 0, 7332 "grid": {}, 7333 "gridPos": { 7334 "h": 7, 7335 "w": 12, 7336 "x": 0, 7337 "y": 42 7338 }, 7339 "hiddenSeries": false, 7340 "id": 10, 7341 "legend": { 7342 "avg": false, 7343 "current": false, 7344 "hideEmpty": true, 7345 "hideZero": true, 7346 "max": false, 7347 "min": false, 7348 "show": true, 7349 "total": false, 7350 "values": false 7351 }, 7352 "lines": true, 7353 "linewidth": 2, 7354 "links": [], 7355 "nullPointMode": "null", 7356 "options": { 7357 "alertThreshold": true 7358 }, 7359 "paceLength": 10, 7360 "percentage": false, 7361 "pluginVersion": "9.0.4", 7362 "pointradius": 5, 7363 "points": false, 7364 "renderer": "flot", 7365 "seriesOverrides": [], 7366 "spaceLength": 10, 7367 "stack": false, 7368 "steppedLine": false, 7369 "targets": [ 7370 { 7371 "datasource": { 7372 "uid": "${DS_PROMETHEUS}" 7373 }, 7374 "expr": "topk(10, rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", 7375 "format": "time_series", 7376 "interval": "", 7377 "intervalFactor": 2, 7378 "legendFormat": "{{job}}-{{index}} {{desc}}", 7379 "refId": "A", 7380 "step": 20 7381 } 7382 ], 7383 "thresholds": [], 7384 "timeRegions": [], 7385 "title": "Top DB transactions by txn rate", 7386 "tooltip": { 7387 "shared": false, 7388 "sort": 0, 7389 "value_type": "cumulative" 7390 }, 7391 "type": "graph", 7392 "xaxis": { 7393 "mode": "time", 7394 "show": true, 7395 "values": [] 7396 }, 7397 "yaxes": [ 7398 { 7399 "format": "hertz", 7400 "logBase": 1, 7401 "min": 0, 7402 "show": true 7403 }, 7404 { 7405 "format": "short", 7406 "logBase": 1, 7407 "show": true 7408 } 7409 ], 7410 "yaxis": { 7411 "align": false 7412 } 7413 }, 7414 { 7415 "aliasColors": {}, 7416 "bars": false, 7417 "dashLength": 10, 7418 "dashes": false, 7419 "datasource": { 7420 "uid": "${DS_PROMETHEUS}" 7421 }, 7422 "editable": true, 7423 "error": false, 7424 "fieldConfig": { 7425 "defaults": { 7426 "links": [] 7427 }, 7428 "overrides": [] 7429 }, 7430 "fill": 1, 7431 "fillGradient": 0, 7432 "grid": {}, 7433 "gridPos": { 7434 "h": 7, 7435 "w": 12, 7436 "x": 12, 7437 "y": 42 7438 }, 7439 "hiddenSeries": false, 7440 "id": 11, 7441 "legend": { 7442 "avg": false, 7443 "current": false, 7444 "hideEmpty": true, 7445 "hideZero": true, 7446 "max": false, 7447 "min": false, 7448 "show": true, 7449 "total": false, 7450 "values": false 7451 }, 7452 "lines": true, 7453 "linewidth": 1, 7454 "links": [], 7455 "nullPointMode": "null", 7456 "options": { 7457 "alertThreshold": true 7458 }, 7459 "paceLength": 10, 7460 "percentage": false, 7461 "pluginVersion": "9.0.4", 7462 "pointradius": 5, 7463 "points": false, 7464 "renderer": "flot", 7465 "seriesOverrides": [], 7466 "spaceLength": 10, 7467 "stack": false, 7468 "steppedLine": false, 7469 "targets": [ 7470 { 7471 "datasource": { 7472 "uid": "${DS_PROMETHEUS}" 7473 }, 7474 "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 7475 "format": "time_series", 7476 "instant": false, 7477 "interval": "", 7478 "intervalFactor": 1, 7479 "legendFormat": "{{job}}-{{index}} {{desc}}", 7480 "refId": "A", 7481 "step": 20 7482 } 7483 ], 7484 "thresholds": [], 7485 "timeRegions": [], 7486 "title": "DB transactions by total txn time", 7487 "tooltip": { 7488 "shared": false, 7489 "sort": 0, 7490 "value_type": "cumulative" 7491 }, 7492 "type": "graph", 7493 "xaxis": { 7494 "mode": "time", 7495 "show": true, 7496 "values": [] 7497 }, 7498 "yaxes": [ 7499 { 7500 "format": "percentunit", 7501 "logBase": 1, 7502 "show": true 7503 }, 7504 { 7505 "format": "short", 7506 "logBase": 1, 7507 "show": true 7508 } 7509 ], 7510 "yaxis": { 7511 "align": false 7512 } 7513 }, 7514 { 7515 "aliasColors": {}, 7516 "bars": false, 7517 "dashLength": 10, 7518 "dashes": false, 7519 "datasource": { 7520 "uid": "${DS_PROMETHEUS}" 7521 }, 7522 "editable": true, 7523 "error": false, 7524 "fieldConfig": { 7525 "defaults": { 7526 "links": [] 7527 }, 7528 "overrides": [] 7529 }, 7530 "fill": 0, 7531 "fillGradient": 0, 7532 "grid": {}, 7533 "gridPos": { 7534 "h": 7, 7535 "w": 12, 7536 "x": 0, 7537 "y": 49 7538 }, 7539 "hiddenSeries": false, 7540 "id": 180, 7541 "legend": { 7542 "avg": false, 7543 "current": false, 7544 "hideEmpty": true, 7545 "hideZero": true, 7546 "max": false, 7547 "min": false, 7548 "show": true, 7549 "total": false, 7550 "values": false 7551 }, 7552 "lines": true, 7553 "linewidth": 1, 7554 "links": [], 7555 "nullPointMode": "null", 7556 "options": { 7557 "alertThreshold": false 7558 }, 7559 "paceLength": 10, 7560 "percentage": false, 7561 "pluginVersion": "9.0.4", 7562 "pointradius": 5, 7563 "points": false, 7564 "renderer": "flot", 7565 "seriesOverrides": [], 7566 "spaceLength": 10, 7567 "stack": false, 7568 "steppedLine": false, 7569 "targets": [ 7570 { 7571 "datasource": { 7572 "uid": "${DS_PROMETHEUS}" 7573 }, 7574 "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 7575 "format": "time_series", 7576 "instant": false, 7577 "interval": "", 7578 "intervalFactor": 1, 7579 "legendFormat": "{{job}}-{{index}} {{desc}}", 7580 "refId": "A", 7581 "step": 20 7582 } 7583 ], 7584 "thresholds": [], 7585 "timeRegions": [], 7586 "title": "Average DB txn time", 7587 "tooltip": { 7588 "shared": false, 7589 "sort": 0, 7590 "value_type": "cumulative" 7591 }, 7592 "type": "graph", 7593 "xaxis": { 7594 "mode": "time", 7595 "show": true, 7596 "values": [] 7597 }, 7598 "yaxes": [ 7599 { 7600 "format": "s", 7601 "logBase": 1, 7602 "show": true 7603 }, 7604 { 7605 "format": "short", 7606 "logBase": 1, 7607 "show": true 7608 } 7609 ], 7610 "yaxis": { 7611 "align": false 7612 } 7613 }, 7614 { 7615 "aliasColors": {}, 7616 "bars": false, 7617 "dashLength": 10, 7618 "dashes": false, 7619 "datasource": { 7620 "uid": "${DS_PROMETHEUS}" 7621 }, 7622 "fieldConfig": { 7623 "defaults": { 7624 "links": [] 7625 }, 7626 "overrides": [] 7627 }, 7628 "fill": 6, 7629 "fillGradient": 0, 7630 "gridPos": { 7631 "h": 9, 7632 "w": 12, 7633 "x": 12, 7634 "y": 49 7635 }, 7636 "hiddenSeries": false, 7637 "id": 200, 7638 "legend": { 7639 "avg": false, 7640 "current": false, 7641 "max": false, 7642 "min": false, 7643 "show": true, 7644 "total": false, 7645 "values": false 7646 }, 7647 "lines": true, 7648 "linewidth": 1, 7649 "links": [], 7650 "nullPointMode": "null", 7651 "options": { 7652 "alertThreshold": true 7653 }, 7654 "percentage": false, 7655 "pluginVersion": "9.0.4", 7656 "pointradius": 5, 7657 "points": false, 7658 "renderer": "flot", 7659 "seriesOverrides": [], 7660 "spaceLength": 10, 7661 "stack": false, 7662 "steppedLine": false, 7663 "targets": [ 7664 { 7665 "datasource": { 7666 "uid": "${DS_PROMETHEUS}" 7667 }, 7668 "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", 7669 "format": "time_series", 7670 "intervalFactor": 1, 7671 "legendFormat": "99%", 7672 "refId": "D" 7673 }, 7674 { 7675 "datasource": { 7676 "uid": "${DS_PROMETHEUS}" 7677 }, 7678 "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", 7679 "format": "time_series", 7680 "intervalFactor": 1, 7681 "legendFormat": "90%", 7682 "refId": "A" 7683 }, 7684 { 7685 "datasource": { 7686 "uid": "${DS_PROMETHEUS}" 7687 }, 7688 "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", 7689 "format": "time_series", 7690 "intervalFactor": 1, 7691 "legendFormat": "75%", 7692 "refId": "C" 7693 }, 7694 { 7695 "datasource": { 7696 "uid": "${DS_PROMETHEUS}" 7697 }, 7698 "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", 7699 "format": "time_series", 7700 "intervalFactor": 1, 7701 "legendFormat": "50%", 7702 "refId": "B" 7703 } 7704 ], 7705 "thresholds": [], 7706 "timeRegions": [], 7707 "title": "Time waiting for DB connection quantiles", 7708 "tooltip": { 7709 "shared": true, 7710 "sort": 2, 7711 "value_type": "individual" 7712 }, 7713 "type": "graph", 7714 "xaxis": { 7715 "mode": "time", 7716 "show": true, 7717 "values": [] 7718 }, 7719 "yaxes": [ 7720 { 7721 "$$hashKey": "object:203", 7722 "format": "s", 7723 "label": "", 7724 "logBase": 1, 7725 "min": "0", 7726 "show": true 7727 }, 7728 { 7729 "$$hashKey": "object:204", 7730 "format": "short", 7731 "logBase": 1, 7732 "show": false 7733 } 7734 ], 7735 "yaxis": { 7736 "align": false 7737 } 7738 } 7739 ], 7740 "targets": [ 7741 { 7742 "datasource": { 7743 "type": "prometheus", 7744 "uid": "${DS_PROMETHEUS}" 7745 }, 7746 "refId": "A" 7747 } 7748 ], 7749 "title": "Database", 7750 "type": "row" 7751 }, 7752 { 7753 "collapsed": true, 7754 "datasource": { 7755 "type": "prometheus", 7756 "uid": "${DS_PROMETHEUS}" 7757 }, 7758 "gridPos": { 7759 "h": 1, 7760 "w": 24, 7761 "x": 0, 7762 "y": 35 7763 }, 7764 "id": 59, 7765 "panels": [ 7766 { 7767 "aliasColors": {}, 7768 "bars": false, 7769 "dashLength": 10, 7770 "dashes": false, 7771 "datasource": { 7772 "uid": "${DS_PROMETHEUS}" 7773 }, 7774 "editable": true, 7775 "error": false, 7776 "fieldConfig": { 7777 "defaults": { 7778 "links": [] 7779 }, 7780 "overrides": [] 7781 }, 7782 "fill": 1, 7783 "fillGradient": 0, 7784 "grid": {}, 7785 "gridPos": { 7786 "h": 13, 7787 "w": 12, 7788 "x": 0, 7789 "y": 158 7790 }, 7791 "hiddenSeries": false, 7792 "id": 12, 7793 "legend": { 7794 "alignAsTable": true, 7795 "avg": false, 7796 "current": false, 7797 "max": false, 7798 "min": false, 7799 "show": true, 7800 "total": false, 7801 "values": false 7802 }, 7803 "lines": true, 7804 "linewidth": 2, 7805 "links": [], 7806 "nullPointMode": "null", 7807 "options": { 7808 "alertThreshold": true 7809 }, 7810 "paceLength": 10, 7811 "percentage": false, 7812 "pluginVersion": "9.2.2", 7813 "pointradius": 5, 7814 "points": false, 7815 "renderer": "flot", 7816 "seriesOverrides": [], 7817 "spaceLength": 10, 7818 "stack": false, 7819 "steppedLine": false, 7820 "targets": [ 7821 { 7822 "datasource": { 7823 "uid": "${DS_PROMETHEUS}" 7824 }, 7825 "expr": "rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])", 7826 "format": "time_series", 7827 "interval": "", 7828 "intervalFactor": 2, 7829 "legendFormat": "{{job}}-{{index}} {{block_name}}", 7830 "refId": "A", 7831 "step": 20 7832 } 7833 ], 7834 "thresholds": [], 7835 "timeRegions": [], 7836 "title": "Total CPU Usage by Block", 7837 "tooltip": { 7838 "shared": true, 7839 "sort": 2, 7840 "value_type": "cumulative" 7841 }, 7842 "type": "graph", 7843 "xaxis": { 7844 "mode": "time", 7845 "show": true, 7846 "values": [] 7847 }, 7848 "yaxes": [ 7849 { 7850 "format": "percentunit", 7851 "logBase": 1, 7852 "show": true 7853 }, 7854 { 7855 "format": "short", 7856 "logBase": 1, 7857 "show": true 7858 } 7859 ], 7860 "yaxis": { 7861 "align": false 7862 } 7863 }, 7864 { 7865 "aliasColors": {}, 7866 "bars": false, 7867 "dashLength": 10, 7868 "dashes": false, 7869 "datasource": { 7870 "uid": "${DS_PROMETHEUS}" 7871 }, 7872 "editable": true, 7873 "error": false, 7874 "fieldConfig": { 7875 "defaults": { 7876 "links": [] 7877 }, 7878 "overrides": [] 7879 }, 7880 "fill": 1, 7881 "fillGradient": 0, 7882 "grid": {}, 7883 "gridPos": { 7884 "h": 13, 7885 "w": 12, 7886 "x": 12, 7887 "y": 158 7888 }, 7889 "hiddenSeries": false, 7890 "id": 26, 7891 "legend": { 7892 "alignAsTable": true, 7893 "avg": false, 7894 "current": false, 7895 "max": false, 7896 "min": false, 7897 "show": true, 7898 "total": false, 7899 "values": false 7900 }, 7901 "lines": true, 7902 "linewidth": 2, 7903 "links": [], 7904 "nullPointMode": "null", 7905 "options": { 7906 "alertThreshold": true 7907 }, 7908 "paceLength": 10, 7909 "percentage": false, 7910 "pluginVersion": "9.2.2", 7911 "pointradius": 5, 7912 "points": false, 7913 "renderer": "flot", 7914 "seriesOverrides": [], 7915 "spaceLength": 10, 7916 "stack": false, 7917 "steppedLine": false, 7918 "targets": [ 7919 { 7920 "datasource": { 7921 "uid": "${DS_PROMETHEUS}" 7922 }, 7923 "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])) / rate(synapse_util_metrics_block_count_total[$bucket_size])", 7924 "format": "time_series", 7925 "interval": "", 7926 "intervalFactor": 2, 7927 "legendFormat": "{{job}}-{{index}} {{block_name}}", 7928 "refId": "A", 7929 "step": 20 7930 } 7931 ], 7932 "thresholds": [], 7933 "timeRegions": [], 7934 "title": "Average CPU Time per Block", 7935 "tooltip": { 7936 "shared": true, 7937 "sort": 2, 7938 "value_type": "cumulative" 7939 }, 7940 "type": "graph", 7941 "xaxis": { 7942 "mode": "time", 7943 "show": true, 7944 "values": [] 7945 }, 7946 "yaxes": [ 7947 { 7948 "format": "ms", 7949 "logBase": 1, 7950 "show": true 7951 }, 7952 { 7953 "format": "short", 7954 "logBase": 1, 7955 "show": true 7956 } 7957 ], 7958 "yaxis": { 7959 "align": false 7960 } 7961 }, 7962 { 7963 "aliasColors": {}, 7964 "bars": false, 7965 "dashLength": 10, 7966 "dashes": false, 7967 "datasource": { 7968 "type": "prometheus", 7969 "uid": "${DS_PROMETHEUS}" 7970 }, 7971 "editable": true, 7972 "error": false, 7973 "fieldConfig": { 7974 "defaults": { 7975 "links": [] 7976 }, 7977 "overrides": [] 7978 }, 7979 "fill": 1, 7980 "fillGradient": 0, 7981 "grid": {}, 7982 "gridPos": { 7983 "h": 13, 7984 "w": 12, 7985 "x": 0, 7986 "y": 171 7987 }, 7988 "hiddenSeries": false, 7989 "id": 13, 7990 "legend": { 7991 "alignAsTable": true, 7992 "avg": false, 7993 "current": false, 7994 "max": false, 7995 "min": false, 7996 "show": true, 7997 "total": false, 7998 "values": false 7999 }, 8000 "lines": true, 8001 "linewidth": 2, 8002 "links": [], 8003 "nullPointMode": "null", 8004 "options": { 8005 "alertThreshold": true 8006 }, 8007 "paceLength": 10, 8008 "percentage": false, 8009 "pluginVersion": "9.2.2", 8010 "pointradius": 5, 8011 "points": false, 8012 "renderer": "flot", 8013 "seriesOverrides": [], 8014 "spaceLength": 10, 8015 "stack": false, 8016 "steppedLine": false, 8017 "targets": [ 8018 { 8019 "datasource": { 8020 "type": "prometheus", 8021 "uid": "${DS_PROMETHEUS}" 8022 }, 8023 "exemplar": true, 8024 "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 8025 "format": "time_series", 8026 "interval": "", 8027 "intervalFactor": 2, 8028 "legendFormat": "{{job}}-{{index}} {{block_name}}", 8029 "refId": "A", 8030 "step": 20 8031 } 8032 ], 8033 "thresholds": [], 8034 "timeRegions": [], 8035 "title": "Total DB Usage by Block", 8036 "tooltip": { 8037 "shared": true, 8038 "sort": 2, 8039 "value_type": "cumulative" 8040 }, 8041 "type": "graph", 8042 "xaxis": { 8043 "mode": "time", 8044 "show": true, 8045 "values": [] 8046 }, 8047 "yaxes": [ 8048 { 8049 "$$hashKey": "object:196", 8050 "format": "percentunit", 8051 "logBase": 1, 8052 "min": 0, 8053 "show": true 8054 }, 8055 { 8056 "$$hashKey": "object:197", 8057 "format": "short", 8058 "logBase": 1, 8059 "show": true 8060 } 8061 ], 8062 "yaxis": { 8063 "align": false 8064 } 8065 }, 8066 { 8067 "aliasColors": {}, 8068 "bars": false, 8069 "dashLength": 10, 8070 "dashes": false, 8071 "datasource": { 8072 "uid": "${DS_PROMETHEUS}" 8073 }, 8074 "description": "The time each database transaction takes to execute, on average, broken down by metrics block.", 8075 "editable": true, 8076 "error": false, 8077 "fieldConfig": { 8078 "defaults": { 8079 "links": [] 8080 }, 8081 "overrides": [] 8082 }, 8083 "fill": 1, 8084 "fillGradient": 0, 8085 "grid": {}, 8086 "gridPos": { 8087 "h": 13, 8088 "w": 12, 8089 "x": 12, 8090 "y": 171 8091 }, 8092 "hiddenSeries": false, 8093 "id": 27, 8094 "legend": { 8095 "alignAsTable": true, 8096 "avg": false, 8097 "current": false, 8098 "max": false, 8099 "min": false, 8100 "show": true, 8101 "total": false, 8102 "values": false 8103 }, 8104 "lines": true, 8105 "linewidth": 2, 8106 "links": [], 8107 "nullPointMode": "null", 8108 "options": { 8109 "alertThreshold": true 8110 }, 8111 "paceLength": 10, 8112 "percentage": false, 8113 "pluginVersion": "9.2.2", 8114 "pointradius": 5, 8115 "points": false, 8116 "renderer": "flot", 8117 "seriesOverrides": [], 8118 "spaceLength": 10, 8119 "stack": false, 8120 "steppedLine": false, 8121 "targets": [ 8122 { 8123 "datasource": { 8124 "uid": "${DS_PROMETHEUS}" 8125 }, 8126 "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 8127 "format": "time_series", 8128 "interval": "", 8129 "intervalFactor": 2, 8130 "legendFormat": "{{job}}-{{index}} {{block_name}}", 8131 "refId": "A", 8132 "step": 20 8133 } 8134 ], 8135 "thresholds": [], 8136 "timeRegions": [], 8137 "title": "Average Database Transaction time, by Block", 8138 "tooltip": { 8139 "shared": true, 8140 "sort": 2, 8141 "value_type": "cumulative" 8142 }, 8143 "type": "graph", 8144 "xaxis": { 8145 "mode": "time", 8146 "show": true, 8147 "values": [] 8148 }, 8149 "yaxes": [ 8150 { 8151 "format": "ms", 8152 "logBase": 1, 8153 "show": true 8154 }, 8155 { 8156 "format": "short", 8157 "logBase": 1, 8158 "show": true 8159 } 8160 ], 8161 "yaxis": { 8162 "align": false 8163 } 8164 }, 8165 { 8166 "aliasColors": {}, 8167 "bars": false, 8168 "dashLength": 10, 8169 "dashes": false, 8170 "datasource": { 8171 "uid": "${DS_PROMETHEUS}" 8172 }, 8173 "editable": true, 8174 "error": false, 8175 "fieldConfig": { 8176 "defaults": { 8177 "links": [] 8178 }, 8179 "overrides": [] 8180 }, 8181 "fill": 1, 8182 "fillGradient": 0, 8183 "grid": {}, 8184 "gridPos": { 8185 "h": 13, 8186 "w": 12, 8187 "x": 0, 8188 "y": 184 8189 }, 8190 "hiddenSeries": false, 8191 "id": 28, 8192 "legend": { 8193 "avg": false, 8194 "current": false, 8195 "max": false, 8196 "min": false, 8197 "show": false, 8198 "total": false, 8199 "values": false 8200 }, 8201 "lines": true, 8202 "linewidth": 2, 8203 "links": [], 8204 "nullPointMode": "null", 8205 "options": { 8206 "alertThreshold": true 8207 }, 8208 "paceLength": 10, 8209 "percentage": false, 8210 "pluginVersion": "9.2.2", 8211 "pointradius": 5, 8212 "points": false, 8213 "renderer": "flot", 8214 "seriesOverrides": [], 8215 "spaceLength": 10, 8216 "stack": false, 8217 "steppedLine": false, 8218 "targets": [ 8219 { 8220 "datasource": { 8221 "uid": "${DS_PROMETHEUS}" 8222 }, 8223 "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 8224 "format": "time_series", 8225 "interval": "", 8226 "intervalFactor": 2, 8227 "legendFormat": "{{job}}-{{index}} {{block_name}}", 8228 "refId": "A", 8229 "step": 20 8230 } 8231 ], 8232 "thresholds": [], 8233 "timeRegions": [], 8234 "title": "Average Transactions per Block", 8235 "tooltip": { 8236 "shared": false, 8237 "sort": 0, 8238 "value_type": "cumulative" 8239 }, 8240 "type": "graph", 8241 "xaxis": { 8242 "mode": "time", 8243 "show": true, 8244 "values": [] 8245 }, 8246 "yaxes": [ 8247 { 8248 "format": "none", 8249 "logBase": 1, 8250 "show": true 8251 }, 8252 { 8253 "format": "short", 8254 "logBase": 1, 8255 "show": true 8256 } 8257 ], 8258 "yaxis": { 8259 "align": false 8260 } 8261 }, 8262 { 8263 "aliasColors": {}, 8264 "bars": false, 8265 "dashLength": 10, 8266 "dashes": false, 8267 "datasource": { 8268 "uid": "${DS_PROMETHEUS}" 8269 }, 8270 "editable": true, 8271 "error": false, 8272 "fieldConfig": { 8273 "defaults": { 8274 "links": [] 8275 }, 8276 "overrides": [] 8277 }, 8278 "fill": 1, 8279 "fillGradient": 0, 8280 "grid": {}, 8281 "gridPos": { 8282 "h": 13, 8283 "w": 12, 8284 "x": 12, 8285 "y": 184 8286 }, 8287 "hiddenSeries": false, 8288 "id": 25, 8289 "legend": { 8290 "avg": false, 8291 "current": false, 8292 "max": false, 8293 "min": false, 8294 "show": false, 8295 "total": false, 8296 "values": false 8297 }, 8298 "lines": true, 8299 "linewidth": 2, 8300 "links": [], 8301 "nullPointMode": "null", 8302 "options": { 8303 "alertThreshold": true 8304 }, 8305 "paceLength": 10, 8306 "percentage": false, 8307 "pluginVersion": "9.2.2", 8308 "pointradius": 5, 8309 "points": false, 8310 "renderer": "flot", 8311 "seriesOverrides": [], 8312 "spaceLength": 10, 8313 "stack": false, 8314 "steppedLine": false, 8315 "targets": [ 8316 { 8317 "datasource": { 8318 "uid": "${DS_PROMETHEUS}" 8319 }, 8320 "expr": "rate(synapse_util_metrics_block_time_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 8321 "format": "time_series", 8322 "interval": "", 8323 "intervalFactor": 2, 8324 "legendFormat": "{{job}}-{{index}} {{block_name}}", 8325 "refId": "A", 8326 "step": 20 8327 } 8328 ], 8329 "thresholds": [], 8330 "timeRegions": [], 8331 "title": "Average Wallclock Time per Block", 8332 "tooltip": { 8333 "shared": false, 8334 "sort": 0, 8335 "value_type": "cumulative" 8336 }, 8337 "type": "graph", 8338 "xaxis": { 8339 "mode": "time", 8340 "show": true, 8341 "values": [] 8342 }, 8343 "yaxes": [ 8344 { 8345 "$$hashKey": "object:180", 8346 "format": "s", 8347 "logBase": 1, 8348 "show": true 8349 }, 8350 { 8351 "$$hashKey": "object:181", 8352 "format": "short", 8353 "logBase": 1, 8354 "show": true 8355 } 8356 ], 8357 "yaxis": { 8358 "align": false 8359 } 8360 }, 8361 { 8362 "aliasColors": {}, 8363 "bars": false, 8364 "dashLength": 10, 8365 "dashes": false, 8366 "datasource": { 8367 "uid": "${DS_PROMETHEUS}" 8368 }, 8369 "fill": 1, 8370 "fillGradient": 0, 8371 "gridPos": { 8372 "h": 15, 8373 "w": 12, 8374 "x": 0, 8375 "y": 197 8376 }, 8377 "hiddenSeries": false, 8378 "id": 154, 8379 "legend": { 8380 "alignAsTable": true, 8381 "avg": false, 8382 "current": false, 8383 "max": false, 8384 "min": false, 8385 "show": true, 8386 "total": false, 8387 "values": false 8388 }, 8389 "lines": true, 8390 "linewidth": 1, 8391 "nullPointMode": "null", 8392 "options": { 8393 "alertThreshold": true 8394 }, 8395 "percentage": false, 8396 "pluginVersion": "9.2.2", 8397 "pointradius": 2, 8398 "points": false, 8399 "renderer": "flot", 8400 "seriesOverrides": [], 8401 "spaceLength": 10, 8402 "stack": false, 8403 "steppedLine": false, 8404 "targets": [ 8405 { 8406 "datasource": { 8407 "uid": "${DS_PROMETHEUS}" 8408 }, 8409 "expr": "rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 8410 "interval": "", 8411 "legendFormat": "{{job}}-{{index}} {{block_name}}", 8412 "refId": "A" 8413 } 8414 ], 8415 "thresholds": [], 8416 "timeRegions": [], 8417 "title": "Block count", 8418 "tooltip": { 8419 "shared": true, 8420 "sort": 2, 8421 "value_type": "individual" 8422 }, 8423 "type": "graph", 8424 "xaxis": { 8425 "mode": "time", 8426 "show": true, 8427 "values": [] 8428 }, 8429 "yaxes": [ 8430 { 8431 "format": "hertz", 8432 "logBase": 1, 8433 "show": true 8434 }, 8435 { 8436 "format": "short", 8437 "logBase": 1, 8438 "show": true 8439 } 8440 ], 8441 "yaxis": { 8442 "align": false 8443 } 8444 } 8445 ], 8446 "targets": [ 8447 { 8448 "datasource": { 8449 "type": "prometheus", 8450 "uid": "${DS_PROMETHEUS}" 8451 }, 8452 "refId": "A" 8453 } 8454 ], 8455 "title": "Per-block metrics", 8456 "type": "row" 8457 }, 8458 { 8459 "collapsed": true, 8460 "datasource": { 8461 "type": "prometheus", 8462 "uid": "${DS_PROMETHEUS}" 8463 }, 8464 "gridPos": { 8465 "h": 1, 8466 "w": 24, 8467 "x": 0, 8468 "y": 36 8469 }, 8470 "id": 61, 8471 "panels": [ 8472 { 8473 "aliasColors": {}, 8474 "bars": false, 8475 "dashLength": 10, 8476 "dashes": false, 8477 "datasource": { 8478 "uid": "${DS_PROMETHEUS}" 8479 }, 8480 "decimals": 2, 8481 "editable": true, 8482 "error": false, 8483 "fieldConfig": { 8484 "defaults": { 8485 "links": [] 8486 }, 8487 "overrides": [] 8488 }, 8489 "fill": 0, 8490 "fillGradient": 0, 8491 "grid": {}, 8492 "gridPos": { 8493 "h": 10, 8494 "w": 12, 8495 "x": 0, 8496 "y": 36 8497 }, 8498 "hiddenSeries": false, 8499 "id": 1, 8500 "legend": { 8501 "alignAsTable": true, 8502 "avg": false, 8503 "current": false, 8504 "hideEmpty": true, 8505 "hideZero": false, 8506 "max": false, 8507 "min": false, 8508 "show": true, 8509 "total": false, 8510 "values": false 8511 }, 8512 "lines": true, 8513 "linewidth": 2, 8514 "links": [], 8515 "nullPointMode": "null", 8516 "options": { 8517 "alertThreshold": true 8518 }, 8519 "percentage": false, 8520 "pluginVersion": "9.0.4", 8521 "pointradius": 5, 8522 "points": false, 8523 "renderer": "flot", 8524 "seriesOverrides": [], 8525 "spaceLength": 10, 8526 "stack": false, 8527 "steppedLine": false, 8528 "targets": [ 8529 { 8530 "datasource": { 8531 "uid": "${DS_PROMETHEUS}" 8532 }, 8533 "expr": "rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 8534 "format": "time_series", 8535 "intervalFactor": 2, 8536 "legendFormat": "{{name}} {{job}}-{{index}}", 8537 "refId": "A", 8538 "step": 20 8539 } 8540 ], 8541 "thresholds": [], 8542 "timeRegions": [], 8543 "title": "Cache Hit Ratio", 8544 "tooltip": { 8545 "msResolution": true, 8546 "shared": false, 8547 "sort": 0, 8548 "value_type": "cumulative" 8549 }, 8550 "type": "graph", 8551 "xaxis": { 8552 "mode": "time", 8553 "show": true, 8554 "values": [] 8555 }, 8556 "yaxes": [ 8557 { 8558 "format": "percentunit", 8559 "label": "", 8560 "logBase": 1, 8561 "max": "1", 8562 "min": 0, 8563 "show": true 8564 }, 8565 { 8566 "format": "short", 8567 "logBase": 1, 8568 "show": false 8569 } 8570 ], 8571 "yaxis": { 8572 "align": false 8573 } 8574 }, 8575 { 8576 "aliasColors": {}, 8577 "bars": false, 8578 "dashLength": 10, 8579 "dashes": false, 8580 "datasource": { 8581 "uid": "${DS_PROMETHEUS}" 8582 }, 8583 "editable": true, 8584 "error": false, 8585 "fieldConfig": { 8586 "defaults": { 8587 "links": [] 8588 }, 8589 "overrides": [] 8590 }, 8591 "fill": 1, 8592 "fillGradient": 0, 8593 "grid": {}, 8594 "gridPos": { 8595 "h": 10, 8596 "w": 12, 8597 "x": 12, 8598 "y": 36 8599 }, 8600 "hiddenSeries": false, 8601 "id": 8, 8602 "legend": { 8603 "alignAsTable": true, 8604 "avg": false, 8605 "current": false, 8606 "hideZero": false, 8607 "max": false, 8608 "min": false, 8609 "show": true, 8610 "total": false, 8611 "values": false 8612 }, 8613 "lines": true, 8614 "linewidth": 2, 8615 "links": [], 8616 "nullPointMode": "connected", 8617 "options": { 8618 "alertThreshold": true 8619 }, 8620 "percentage": false, 8621 "pluginVersion": "9.0.4", 8622 "pointradius": 5, 8623 "points": false, 8624 "renderer": "flot", 8625 "seriesOverrides": [], 8626 "spaceLength": 10, 8627 "stack": false, 8628 "steppedLine": false, 8629 "targets": [ 8630 { 8631 "datasource": { 8632 "uid": "${DS_PROMETHEUS}" 8633 }, 8634 "expr": "synapse_util_caches_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 8635 "format": "time_series", 8636 "hide": false, 8637 "interval": "", 8638 "intervalFactor": 2, 8639 "legendFormat": "{{name}} {{job}}-{{index}}", 8640 "refId": "A", 8641 "step": 20 8642 } 8643 ], 8644 "thresholds": [], 8645 "timeRegions": [], 8646 "title": "Cache Size", 8647 "tooltip": { 8648 "shared": false, 8649 "sort": 0, 8650 "value_type": "cumulative" 8651 }, 8652 "type": "graph", 8653 "xaxis": { 8654 "mode": "time", 8655 "show": true, 8656 "values": [] 8657 }, 8658 "yaxes": [ 8659 { 8660 "format": "short", 8661 "logBase": 1, 8662 "min": 0, 8663 "show": true 8664 }, 8665 { 8666 "format": "short", 8667 "logBase": 1, 8668 "show": true 8669 } 8670 ], 8671 "yaxis": { 8672 "align": false 8673 } 8674 }, 8675 { 8676 "aliasColors": {}, 8677 "bars": false, 8678 "dashLength": 10, 8679 "dashes": false, 8680 "datasource": { 8681 "uid": "${DS_PROMETHEUS}" 8682 }, 8683 "editable": true, 8684 "error": false, 8685 "fieldConfig": { 8686 "defaults": { 8687 "links": [] 8688 }, 8689 "overrides": [] 8690 }, 8691 "fill": 1, 8692 "fillGradient": 0, 8693 "grid": {}, 8694 "gridPos": { 8695 "h": 10, 8696 "w": 12, 8697 "x": 0, 8698 "y": 46 8699 }, 8700 "hiddenSeries": false, 8701 "id": 38, 8702 "legend": { 8703 "alignAsTable": true, 8704 "avg": false, 8705 "current": false, 8706 "hideZero": false, 8707 "max": false, 8708 "min": false, 8709 "show": true, 8710 "total": false, 8711 "values": false 8712 }, 8713 "lines": true, 8714 "linewidth": 2, 8715 "links": [], 8716 "nullPointMode": "connected", 8717 "options": { 8718 "alertThreshold": true 8719 }, 8720 "percentage": false, 8721 "pluginVersion": "9.0.4", 8722 "pointradius": 5, 8723 "points": false, 8724 "renderer": "flot", 8725 "seriesOverrides": [], 8726 "spaceLength": 10, 8727 "stack": false, 8728 "steppedLine": false, 8729 "targets": [ 8730 { 8731 "datasource": { 8732 "uid": "${DS_PROMETHEUS}" 8733 }, 8734 "expr": "rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 8735 "format": "time_series", 8736 "interval": "", 8737 "intervalFactor": 2, 8738 "legendFormat": "{{name}} {{job}}-{{index}}", 8739 "refId": "A", 8740 "step": 20 8741 } 8742 ], 8743 "thresholds": [], 8744 "timeRegions": [], 8745 "title": "Cache request rate", 8746 "tooltip": { 8747 "shared": false, 8748 "sort": 0, 8749 "value_type": "cumulative" 8750 }, 8751 "type": "graph", 8752 "xaxis": { 8753 "mode": "time", 8754 "show": true, 8755 "values": [] 8756 }, 8757 "yaxes": [ 8758 { 8759 "format": "rps", 8760 "logBase": 1, 8761 "min": 0, 8762 "show": true 8763 }, 8764 { 8765 "format": "short", 8766 "logBase": 1, 8767 "show": true 8768 } 8769 ], 8770 "yaxis": { 8771 "align": false 8772 } 8773 }, 8774 { 8775 "aliasColors": {}, 8776 "bars": false, 8777 "dashLength": 10, 8778 "dashes": false, 8779 "datasource": { 8780 "uid": "${DS_PROMETHEUS}" 8781 }, 8782 "fieldConfig": { 8783 "defaults": { 8784 "links": [] 8785 }, 8786 "overrides": [] 8787 }, 8788 "fill": 1, 8789 "fillGradient": 0, 8790 "gridPos": { 8791 "h": 10, 8792 "w": 12, 8793 "x": 12, 8794 "y": 46 8795 }, 8796 "hiddenSeries": false, 8797 "id": 39, 8798 "legend": { 8799 "alignAsTable": true, 8800 "avg": false, 8801 "current": false, 8802 "hideEmpty": false, 8803 "hideZero": false, 8804 "max": true, 8805 "min": false, 8806 "rightSide": false, 8807 "show": true, 8808 "sort": "max", 8809 "sortDesc": true, 8810 "total": false, 8811 "values": true 8812 }, 8813 "lines": true, 8814 "linewidth": 1, 8815 "links": [], 8816 "nullPointMode": "null", 8817 "options": { 8818 "alertThreshold": true 8819 }, 8820 "percentage": false, 8821 "pluginVersion": "9.0.4", 8822 "pointradius": 5, 8823 "points": false, 8824 "renderer": "flot", 8825 "seriesOverrides": [], 8826 "spaceLength": 10, 8827 "stack": false, 8828 "steppedLine": false, 8829 "targets": [ 8830 { 8831 "datasource": { 8832 "uid": "${DS_PROMETHEUS}" 8833 }, 8834 "expr": "topk(10, rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", 8835 "format": "time_series", 8836 "interval": "", 8837 "intervalFactor": 2, 8838 "legendFormat": "{{name}} {{job}}-{{index}}", 8839 "refId": "A", 8840 "step": 20 8841 } 8842 ], 8843 "thresholds": [], 8844 "timeRegions": [], 8845 "title": "Top 10 cache misses", 8846 "tooltip": { 8847 "shared": false, 8848 "sort": 0, 8849 "value_type": "individual" 8850 }, 8851 "type": "graph", 8852 "xaxis": { 8853 "mode": "time", 8854 "show": true, 8855 "values": [] 8856 }, 8857 "yaxes": [ 8858 { 8859 "$$hashKey": "object:101", 8860 "format": "rps", 8861 "logBase": 1, 8862 "show": true 8863 }, 8864 { 8865 "$$hashKey": "object:102", 8866 "format": "short", 8867 "logBase": 1, 8868 "show": true 8869 } 8870 ], 8871 "yaxis": { 8872 "align": false 8873 } 8874 }, 8875 { 8876 "aliasColors": {}, 8877 "bars": false, 8878 "dashLength": 10, 8879 "dashes": false, 8880 "datasource": { 8881 "uid": "${DS_PROMETHEUS}" 8882 }, 8883 "fieldConfig": { 8884 "defaults": { 8885 "links": [] 8886 }, 8887 "overrides": [] 8888 }, 8889 "fill": 1, 8890 "fillGradient": 0, 8891 "gridPos": { 8892 "h": 9, 8893 "w": 12, 8894 "x": 0, 8895 "y": 56 8896 }, 8897 "hiddenSeries": false, 8898 "id": 65, 8899 "legend": { 8900 "alignAsTable": true, 8901 "avg": false, 8902 "current": false, 8903 "max": false, 8904 "min": false, 8905 "show": true, 8906 "total": false, 8907 "values": false 8908 }, 8909 "lines": true, 8910 "linewidth": 1, 8911 "links": [], 8912 "nullPointMode": "null", 8913 "options": { 8914 "alertThreshold": true 8915 }, 8916 "percentage": false, 8917 "pluginVersion": "9.0.4", 8918 "pointradius": 5, 8919 "points": false, 8920 "renderer": "flot", 8921 "seriesOverrides": [], 8922 "spaceLength": 10, 8923 "stack": false, 8924 "steppedLine": false, 8925 "targets": [ 8926 { 8927 "datasource": { 8928 "uid": "${DS_PROMETHEUS}" 8929 }, 8930 "expr": "rate(synapse_util_caches_cache_evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 8931 "format": "time_series", 8932 "interval": "", 8933 "intervalFactor": 1, 8934 "legendFormat": "{{name}} ({{reason}}) {{job}}-{{index}}", 8935 "refId": "A" 8936 } 8937 ], 8938 "thresholds": [], 8939 "timeRegions": [], 8940 "title": "Cache eviction rate", 8941 "tooltip": { 8942 "shared": false, 8943 "sort": 0, 8944 "value_type": "individual" 8945 }, 8946 "type": "graph", 8947 "xaxis": { 8948 "mode": "time", 8949 "show": true, 8950 "values": [] 8951 }, 8952 "yaxes": [ 8953 { 8954 "format": "hertz", 8955 "label": "entries / second", 8956 "logBase": 1, 8957 "show": true 8958 }, 8959 { 8960 "format": "short", 8961 "logBase": 1, 8962 "show": true 8963 } 8964 ], 8965 "yaxis": { 8966 "align": false 8967 } 8968 } 8969 ], 8970 "targets": [ 8971 { 8972 "datasource": { 8973 "type": "prometheus", 8974 "uid": "${DS_PROMETHEUS}" 8975 }, 8976 "refId": "A" 8977 } 8978 ], 8979 "title": "Caches", 8980 "type": "row" 8981 }, 8982 { 8983 "collapsed": true, 8984 "datasource": { 8985 "type": "prometheus", 8986 "uid": "${DS_PROMETHEUS}" 8987 }, 8988 "gridPos": { 8989 "h": 1, 8990 "w": 24, 8991 "x": 0, 8992 "y": 37 8993 }, 8994 "id": 148, 8995 "panels": [ 8996 { 8997 "aliasColors": {}, 8998 "bars": false, 8999 "dashLength": 10, 9000 "dashes": false, 9001 "datasource": { 9002 "uid": "${DS_PROMETHEUS}" 9003 }, 9004 "fieldConfig": { 9005 "defaults": { 9006 "custom": {}, 9007 "links": [] 9008 }, 9009 "overrides": [] 9010 }, 9011 "fill": 1, 9012 "fillGradient": 0, 9013 "gridPos": { 9014 "h": 8, 9015 "w": 12, 9016 "x": 0, 9017 "y": 29 9018 }, 9019 "hiddenSeries": false, 9020 "id": 146, 9021 "legend": { 9022 "avg": false, 9023 "current": false, 9024 "max": false, 9025 "min": false, 9026 "show": true, 9027 "total": false, 9028 "values": false 9029 }, 9030 "lines": true, 9031 "linewidth": 1, 9032 "nullPointMode": "null", 9033 "options": { 9034 "alertThreshold": true 9035 }, 9036 "percentage": false, 9037 "pluginVersion": "7.3.7", 9038 "pointradius": 2, 9039 "points": false, 9040 "renderer": "flot", 9041 "seriesOverrides": [], 9042 "spaceLength": 10, 9043 "stack": false, 9044 "steppedLine": false, 9045 "targets": [ 9046 { 9047 "datasource": { 9048 "uid": "${DS_PROMETHEUS}" 9049 }, 9050 "expr": "synapse_util_caches_response_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 9051 "interval": "", 9052 "legendFormat": "{{name}} {{job}}-{{index}}", 9053 "refId": "A" 9054 } 9055 ], 9056 "thresholds": [], 9057 "timeRegions": [], 9058 "title": "Response cache size", 9059 "tooltip": { 9060 "shared": false, 9061 "sort": 0, 9062 "value_type": "individual" 9063 }, 9064 "type": "graph", 9065 "xaxis": { 9066 "mode": "time", 9067 "show": true, 9068 "values": [] 9069 }, 9070 "yaxes": [ 9071 { 9072 "format": "short", 9073 "logBase": 1, 9074 "show": true 9075 }, 9076 { 9077 "format": "short", 9078 "logBase": 1, 9079 "show": true 9080 } 9081 ], 9082 "yaxis": { 9083 "align": false 9084 } 9085 }, 9086 { 9087 "aliasColors": {}, 9088 "bars": false, 9089 "dashLength": 10, 9090 "dashes": false, 9091 "datasource": { 9092 "uid": "${DS_PROMETHEUS}" 9093 }, 9094 "fieldConfig": { 9095 "defaults": { 9096 "custom": {}, 9097 "links": [] 9098 }, 9099 "overrides": [] 9100 }, 9101 "fill": 1, 9102 "fillGradient": 0, 9103 "gridPos": { 9104 "h": 8, 9105 "w": 12, 9106 "x": 12, 9107 "y": 29 9108 }, 9109 "hiddenSeries": false, 9110 "id": 150, 9111 "legend": { 9112 "avg": false, 9113 "current": false, 9114 "max": false, 9115 "min": false, 9116 "show": true, 9117 "total": false, 9118 "values": false 9119 }, 9120 "lines": true, 9121 "linewidth": 1, 9122 "nullPointMode": "null", 9123 "options": { 9124 "alertThreshold": true 9125 }, 9126 "percentage": false, 9127 "pluginVersion": "7.3.7", 9128 "pointradius": 2, 9129 "points": false, 9130 "renderer": "flot", 9131 "seriesOverrides": [], 9132 "spaceLength": 10, 9133 "stack": false, 9134 "steppedLine": false, 9135 "targets": [ 9136 { 9137 "datasource": { 9138 "uid": "${DS_PROMETHEUS}" 9139 }, 9140 "expr": "rate(synapse_util_caches_response_cache_hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])", 9141 "interval": "", 9142 "legendFormat": "{{name}} {{job}}-{{index}}", 9143 "refId": "A" 9144 }, 9145 { 9146 "datasource": { 9147 "uid": "${DS_PROMETHEUS}" 9148 }, 9149 "expr": "", 9150 "interval": "", 9151 "legendFormat": "", 9152 "refId": "B" 9153 } 9154 ], 9155 "thresholds": [], 9156 "timeRegions": [], 9157 "title": "Response cache hit rate", 9158 "tooltip": { 9159 "shared": false, 9160 "sort": 0, 9161 "value_type": "individual" 9162 }, 9163 "type": "graph", 9164 "xaxis": { 9165 "mode": "time", 9166 "show": true, 9167 "values": [] 9168 }, 9169 "yaxes": [ 9170 { 9171 "format": "percentunit", 9172 "logBase": 1, 9173 "max": "1", 9174 "min": "0", 9175 "show": true 9176 }, 9177 { 9178 "format": "short", 9179 "logBase": 1, 9180 "show": true 9181 } 9182 ], 9183 "yaxis": { 9184 "align": false 9185 } 9186 } 9187 ], 9188 "targets": [ 9189 { 9190 "datasource": { 9191 "type": "prometheus", 9192 "uid": "${DS_PROMETHEUS}" 9193 }, 9194 "refId": "A" 9195 } 9196 ], 9197 "title": "Response caches", 9198 "type": "row" 9199 }, 9200 { 9201 "collapsed": true, 9202 "datasource": { 9203 "type": "prometheus", 9204 "uid": "${DS_PROMETHEUS}" 9205 }, 9206 "gridPos": { 9207 "h": 1, 9208 "w": 24, 9209 "x": 0, 9210 "y": 38 9211 }, 9212 "id": 62, 9213 "panels": [ 9214 { 9215 "aliasColors": {}, 9216 "bars": false, 9217 "dashLength": 10, 9218 "dashes": false, 9219 "datasource": { 9220 "uid": "${DS_PROMETHEUS}" 9221 }, 9222 "fieldConfig": { 9223 "defaults": { 9224 "custom": {}, 9225 "links": [] 9226 }, 9227 "overrides": [] 9228 }, 9229 "fill": 1, 9230 "fillGradient": 0, 9231 "gridPos": { 9232 "h": 9, 9233 "w": 12, 9234 "x": 0, 9235 "y": 30 9236 }, 9237 "hiddenSeries": false, 9238 "id": 91, 9239 "legend": { 9240 "avg": false, 9241 "current": false, 9242 "max": false, 9243 "min": false, 9244 "show": true, 9245 "total": false, 9246 "values": false 9247 }, 9248 "lines": true, 9249 "linewidth": 1, 9250 "links": [], 9251 "nullPointMode": "null", 9252 "options": { 9253 "alertThreshold": true 9254 }, 9255 "percentage": false, 9256 "pluginVersion": "7.3.7", 9257 "pointradius": 5, 9258 "points": false, 9259 "renderer": "flot", 9260 "seriesOverrides": [], 9261 "spaceLength": 10, 9262 "stack": true, 9263 "steppedLine": false, 9264 "targets": [ 9265 { 9266 "datasource": { 9267 "uid": "${DS_PROMETHEUS}" 9268 }, 9269 "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])", 9270 "format": "time_series", 9271 "instant": false, 9272 "intervalFactor": 1, 9273 "legendFormat": "{{job}}-{{index}} gen {{gen}}", 9274 "refId": "A" 9275 } 9276 ], 9277 "thresholds": [], 9278 "timeRegions": [], 9279 "title": "Total GC time by bucket (10m smoothing)", 9280 "tooltip": { 9281 "shared": true, 9282 "sort": 0, 9283 "value_type": "individual" 9284 }, 9285 "type": "graph", 9286 "xaxis": { 9287 "mode": "time", 9288 "show": true, 9289 "values": [] 9290 }, 9291 "yaxes": [ 9292 { 9293 "format": "percentunit", 9294 "logBase": 1, 9295 "min": "0", 9296 "show": true 9297 }, 9298 { 9299 "format": "short", 9300 "logBase": 1, 9301 "show": true 9302 } 9303 ], 9304 "yaxis": { 9305 "align": false 9306 } 9307 }, 9308 { 9309 "aliasColors": {}, 9310 "bars": false, 9311 "dashLength": 10, 9312 "dashes": false, 9313 "datasource": { 9314 "uid": "${DS_PROMETHEUS}" 9315 }, 9316 "decimals": 3, 9317 "editable": true, 9318 "error": false, 9319 "fieldConfig": { 9320 "defaults": { 9321 "custom": {}, 9322 "links": [] 9323 }, 9324 "overrides": [] 9325 }, 9326 "fill": 1, 9327 "fillGradient": 0, 9328 "grid": {}, 9329 "gridPos": { 9330 "h": 9, 9331 "w": 12, 9332 "x": 12, 9333 "y": 30 9334 }, 9335 "hiddenSeries": false, 9336 "id": 21, 9337 "legend": { 9338 "alignAsTable": true, 9339 "avg": false, 9340 "current": false, 9341 "max": false, 9342 "min": false, 9343 "show": true, 9344 "total": false, 9345 "values": false 9346 }, 9347 "lines": true, 9348 "linewidth": 2, 9349 "links": [], 9350 "nullPointMode": "null as zero", 9351 "options": { 9352 "alertThreshold": true 9353 }, 9354 "percentage": false, 9355 "pluginVersion": "7.3.7", 9356 "pointradius": 5, 9357 "points": false, 9358 "renderer": "flot", 9359 "seriesOverrides": [], 9360 "spaceLength": 10, 9361 "stack": false, 9362 "steppedLine": false, 9363 "targets": [ 9364 { 9365 "datasource": { 9366 "uid": "${DS_PROMETHEUS}" 9367 }, 9368 "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])", 9369 "format": "time_series", 9370 "intervalFactor": 2, 9371 "legendFormat": "{{job}} {{index}} gen {{gen}} ", 9372 "refId": "A", 9373 "step": 20, 9374 "target": "" 9375 } 9376 ], 9377 "thresholds": [], 9378 "timeRegions": [], 9379 "title": "Average GC Time Per Collection", 9380 "tooltip": { 9381 "shared": false, 9382 "sort": 0, 9383 "value_type": "cumulative" 9384 }, 9385 "type": "graph", 9386 "xaxis": { 9387 "mode": "time", 9388 "show": true, 9389 "values": [] 9390 }, 9391 "yaxes": [ 9392 { 9393 "format": "s", 9394 "logBase": 1, 9395 "show": true 9396 }, 9397 { 9398 "format": "short", 9399 "logBase": 1, 9400 "show": true 9401 } 9402 ], 9403 "yaxis": { 9404 "align": false 9405 } 9406 }, 9407 { 9408 "aliasColors": {}, 9409 "bars": false, 9410 "dashLength": 10, 9411 "dashes": false, 9412 "datasource": { 9413 "uid": "${DS_PROMETHEUS}" 9414 }, 9415 "description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.", 9416 "fieldConfig": { 9417 "defaults": { 9418 "custom": {}, 9419 "links": [] 9420 }, 9421 "overrides": [] 9422 }, 9423 "fill": 1, 9424 "fillGradient": 0, 9425 "gridPos": { 9426 "h": 9, 9427 "w": 12, 9428 "x": 0, 9429 "y": 39 9430 }, 9431 "hiddenSeries": false, 9432 "id": 89, 9433 "legend": { 9434 "avg": false, 9435 "current": false, 9436 "hideEmpty": true, 9437 "hideZero": false, 9438 "max": false, 9439 "min": false, 9440 "show": true, 9441 "total": false, 9442 "values": false 9443 }, 9444 "lines": true, 9445 "linewidth": 1, 9446 "links": [], 9447 "nullPointMode": "null", 9448 "options": { 9449 "alertThreshold": true 9450 }, 9451 "percentage": false, 9452 "pluginVersion": "7.3.7", 9453 "pointradius": 5, 9454 "points": false, 9455 "renderer": "flot", 9456 "seriesOverrides": [ 9457 { 9458 "alias": "/gen 0$/", 9459 "yaxis": 2 9460 } 9461 ], 9462 "spaceLength": 10, 9463 "stack": false, 9464 "steppedLine": false, 9465 "targets": [ 9466 { 9467 "datasource": { 9468 "uid": "${DS_PROMETHEUS}" 9469 }, 9470 "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", 9471 "format": "time_series", 9472 "intervalFactor": 1, 9473 "legendFormat": "{{job}}-{{index}} gen {{gen}}", 9474 "refId": "A" 9475 } 9476 ], 9477 "thresholds": [], 9478 "timeRegions": [], 9479 "title": "Allocation counts", 9480 "tooltip": { 9481 "shared": false, 9482 "sort": 0, 9483 "value_type": "individual" 9484 }, 9485 "type": "graph", 9486 "xaxis": { 9487 "mode": "time", 9488 "show": true, 9489 "values": [] 9490 }, 9491 "yaxes": [ 9492 { 9493 "format": "short", 9494 "label": "Gen N-1 GCs since last Gen N GC", 9495 "logBase": 1, 9496 "show": true 9497 }, 9498 { 9499 "format": "short", 9500 "label": "Objects since last Gen 0 GC", 9501 "logBase": 1, 9502 "show": true 9503 } 9504 ], 9505 "yaxis": { 9506 "align": false 9507 } 9508 }, 9509 { 9510 "aliasColors": {}, 9511 "bars": false, 9512 "dashLength": 10, 9513 "dashes": false, 9514 "datasource": { 9515 "uid": "${DS_PROMETHEUS}" 9516 }, 9517 "fieldConfig": { 9518 "defaults": { 9519 "custom": {}, 9520 "links": [] 9521 }, 9522 "overrides": [] 9523 }, 9524 "fill": 1, 9525 "fillGradient": 0, 9526 "gridPos": { 9527 "h": 9, 9528 "w": 12, 9529 "x": 12, 9530 "y": 39 9531 }, 9532 "hiddenSeries": false, 9533 "id": 93, 9534 "legend": { 9535 "avg": false, 9536 "current": false, 9537 "max": false, 9538 "min": false, 9539 "show": true, 9540 "total": false, 9541 "values": false 9542 }, 9543 "lines": true, 9544 "linewidth": 1, 9545 "links": [], 9546 "nullPointMode": "connected", 9547 "options": { 9548 "alertThreshold": true 9549 }, 9550 "percentage": false, 9551 "pluginVersion": "7.3.7", 9552 "pointradius": 5, 9553 "points": false, 9554 "renderer": "flot", 9555 "seriesOverrides": [], 9556 "spaceLength": 10, 9557 "stack": false, 9558 "steppedLine": false, 9559 "targets": [ 9560 { 9561 "datasource": { 9562 "uid": "${DS_PROMETHEUS}" 9563 }, 9564 "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 9565 "format": "time_series", 9566 "intervalFactor": 1, 9567 "legendFormat": "{{job}}-{{index}} gen {{gen}}", 9568 "refId": "A" 9569 } 9570 ], 9571 "thresholds": [], 9572 "timeRegions": [], 9573 "title": "Object counts per collection", 9574 "tooltip": { 9575 "shared": true, 9576 "sort": 0, 9577 "value_type": "individual" 9578 }, 9579 "type": "graph", 9580 "xaxis": { 9581 "mode": "time", 9582 "show": true, 9583 "values": [] 9584 }, 9585 "yaxes": [ 9586 { 9587 "format": "short", 9588 "logBase": 1, 9589 "show": true 9590 }, 9591 { 9592 "format": "short", 9593 "logBase": 1, 9594 "show": true 9595 } 9596 ], 9597 "yaxis": { 9598 "align": false 9599 } 9600 }, 9601 { 9602 "aliasColors": {}, 9603 "bars": false, 9604 "dashLength": 10, 9605 "dashes": false, 9606 "datasource": { 9607 "uid": "${DS_PROMETHEUS}" 9608 }, 9609 "fieldConfig": { 9610 "defaults": { 9611 "custom": {}, 9612 "links": [] 9613 }, 9614 "overrides": [] 9615 }, 9616 "fill": 1, 9617 "fillGradient": 0, 9618 "gridPos": { 9619 "h": 9, 9620 "w": 12, 9621 "x": 0, 9622 "y": 48 9623 }, 9624 "hiddenSeries": false, 9625 "id": 95, 9626 "legend": { 9627 "avg": false, 9628 "current": false, 9629 "max": false, 9630 "min": false, 9631 "show": true, 9632 "total": false, 9633 "values": false 9634 }, 9635 "lines": true, 9636 "linewidth": 1, 9637 "links": [], 9638 "nullPointMode": "null", 9639 "options": { 9640 "alertThreshold": true 9641 }, 9642 "percentage": false, 9643 "pluginVersion": "7.3.7", 9644 "pointradius": 5, 9645 "points": false, 9646 "renderer": "flot", 9647 "seriesOverrides": [], 9648 "spaceLength": 10, 9649 "stack": false, 9650 "steppedLine": false, 9651 "targets": [ 9652 { 9653 "datasource": { 9654 "uid": "${DS_PROMETHEUS}" 9655 }, 9656 "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 9657 "format": "time_series", 9658 "intervalFactor": 1, 9659 "legendFormat": "{{job}}-{{index}} gen {{gen}}", 9660 "refId": "A" 9661 } 9662 ], 9663 "thresholds": [], 9664 "timeRegions": [], 9665 "title": "GC frequency", 9666 "tooltip": { 9667 "shared": true, 9668 "sort": 0, 9669 "value_type": "individual" 9670 }, 9671 "type": "graph", 9672 "xaxis": { 9673 "mode": "time", 9674 "show": true, 9675 "values": [] 9676 }, 9677 "yaxes": [ 9678 { 9679 "format": "hertz", 9680 "logBase": 1, 9681 "show": true 9682 }, 9683 { 9684 "format": "short", 9685 "logBase": 1, 9686 "show": true 9687 } 9688 ], 9689 "yaxis": { 9690 "align": false 9691 } 9692 }, 9693 { 9694 "cards": { 9695 "cardPadding": 0 9696 }, 9697 "color": { 9698 "cardColor": "#b4ff00", 9699 "colorScale": "sqrt", 9700 "colorScheme": "interpolateSpectral", 9701 "exponent": 0.5, 9702 "min": 0, 9703 "mode": "spectrum" 9704 }, 9705 "dataFormat": "tsbuckets", 9706 "datasource": { 9707 "type": "prometheus", 9708 "uid": "${DS_PROMETHEUS}" 9709 }, 9710 "fieldConfig": { 9711 "defaults": { 9712 "custom": {} 9713 }, 9714 "overrides": [] 9715 }, 9716 "gridPos": { 9717 "h": 9, 9718 "w": 12, 9719 "x": 12, 9720 "y": 48 9721 }, 9722 "heatmap": {}, 9723 "hideZeroBuckets": true, 9724 "highlightCards": true, 9725 "id": 87, 9726 "legend": { 9727 "show": true 9728 }, 9729 "links": [], 9730 "reverseYBuckets": false, 9731 "targets": [ 9732 { 9733 "datasource": { 9734 "type": "prometheus", 9735 "uid": "${DS_PROMETHEUS}" 9736 }, 9737 "expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)", 9738 "format": "heatmap", 9739 "intervalFactor": 1, 9740 "legendFormat": "{{le}}", 9741 "refId": "A" 9742 } 9743 ], 9744 "title": "GC durations", 9745 "tooltip": { 9746 "show": true, 9747 "showHistogram": false 9748 }, 9749 "type": "heatmap", 9750 "xAxis": { 9751 "show": true 9752 }, 9753 "yAxis": { 9754 "format": "s", 9755 "logBase": 1, 9756 "show": true 9757 }, 9758 "yBucketBound": "auto" 9759 } 9760 ], 9761 "targets": [ 9762 { 9763 "datasource": { 9764 "type": "prometheus", 9765 "uid": "${DS_PROMETHEUS}" 9766 }, 9767 "refId": "A" 9768 } 9769 ], 9770 "title": "GC", 9771 "type": "row" 9772 }, 9773 { 9774 "collapsed": true, 9775 "datasource": { 9776 "type": "prometheus", 9777 "uid": "${DS_PROMETHEUS}" 9778 }, 9779 "gridPos": { 9780 "h": 1, 9781 "w": 24, 9782 "x": 0, 9783 "y": 39 9784 }, 9785 "id": 63, 9786 "panels": [ 9787 { 9788 "aliasColors": {}, 9789 "bars": false, 9790 "dashLength": 10, 9791 "dashes": false, 9792 "datasource": { 9793 "type": "prometheus", 9794 "uid": "${DS_PROMETHEUS}" 9795 }, 9796 "fieldConfig": { 9797 "defaults": { 9798 "links": [] 9799 }, 9800 "overrides": [] 9801 }, 9802 "fill": 1, 9803 "fillGradient": 0, 9804 "gridPos": { 9805 "h": 7, 9806 "w": 12, 9807 "x": 0, 9808 "y": 162 9809 }, 9810 "hiddenSeries": false, 9811 "id": 43, 9812 "legend": { 9813 "avg": false, 9814 "current": false, 9815 "max": false, 9816 "min": false, 9817 "show": true, 9818 "total": false, 9819 "values": false 9820 }, 9821 "lines": true, 9822 "linewidth": 1, 9823 "links": [], 9824 "nullPointMode": "null", 9825 "options": { 9826 "alertThreshold": true 9827 }, 9828 "paceLength": 10, 9829 "percentage": false, 9830 "pluginVersion": "9.2.2", 9831 "pointradius": 5, 9832 "points": false, 9833 "renderer": "flot", 9834 "seriesOverrides": [], 9835 "spaceLength": 10, 9836 "stack": false, 9837 "steppedLine": false, 9838 "targets": [ 9839 { 9840 "datasource": { 9841 "uid": "${DS_PROMETHEUS}" 9842 }, 9843 "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)", 9844 "format": "time_series", 9845 "intervalFactor": 2, 9846 "legendFormat": "{{job}}-{{index}} {{command}}", 9847 "refId": "A", 9848 "step": 20 9849 } 9850 ], 9851 "thresholds": [], 9852 "timeRegions": [], 9853 "title": "Rate of outgoing commands", 9854 "tooltip": { 9855 "shared": false, 9856 "sort": 0, 9857 "value_type": "individual" 9858 }, 9859 "type": "graph", 9860 "xaxis": { 9861 "mode": "time", 9862 "show": true, 9863 "values": [] 9864 }, 9865 "yaxes": [ 9866 { 9867 "$$hashKey": "object:89", 9868 "format": "hertz", 9869 "logBase": 1, 9870 "show": true 9871 }, 9872 { 9873 "$$hashKey": "object:90", 9874 "format": "short", 9875 "logBase": 1, 9876 "show": true 9877 } 9878 ], 9879 "yaxis": { 9880 "align": false 9881 } 9882 }, 9883 { 9884 "datasource": { 9885 "type": "prometheus", 9886 "uid": "${DS_PROMETHEUS}" 9887 }, 9888 "fieldConfig": { 9889 "defaults": { 9890 "color": { 9891 "mode": "palette-classic" 9892 }, 9893 "custom": { 9894 "axisCenteredZero": false, 9895 "axisColorMode": "text", 9896 "axisLabel": "", 9897 "axisPlacement": "auto", 9898 "barAlignment": 0, 9899 "drawStyle": "line", 9900 "fillOpacity": 10, 9901 "gradientMode": "none", 9902 "hideFrom": { 9903 "legend": false, 9904 "tooltip": false, 9905 "viz": false 9906 }, 9907 "lineInterpolation": "linear", 9908 "lineWidth": 1, 9909 "pointSize": 5, 9910 "scaleDistribution": { 9911 "type": "linear" 9912 }, 9913 "showPoints": "never", 9914 "spanNulls": true, 9915 "stacking": { 9916 "group": "A", 9917 "mode": "none" 9918 }, 9919 "thresholdsStyle": { 9920 "mode": "off" 9921 } 9922 }, 9923 "links": [], 9924 "mappings": [], 9925 "thresholds": { 9926 "mode": "absolute", 9927 "steps": [ 9928 { 9929 "color": "green" 9930 }, 9931 { 9932 "color": "red", 9933 "value": 80 9934 } 9935 ] 9936 }, 9937 "unit": "hertz" 9938 }, 9939 "overrides": [] 9940 }, 9941 "gridPos": { 9942 "h": 7, 9943 "w": 12, 9944 "x": 12, 9945 "y": 162 9946 }, 9947 "id": 41, 9948 "links": [], 9949 "options": { 9950 "legend": { 9951 "calcs": [], 9952 "displayMode": "list", 9953 "placement": "bottom", 9954 "showLegend": true 9955 }, 9956 "tooltip": { 9957 "mode": "single", 9958 "sort": "none" 9959 } 9960 }, 9961 "pluginVersion": "8.4.3", 9962 "targets": [ 9963 { 9964 "datasource": { 9965 "type": "prometheus", 9966 "uid": "${DS_PROMETHEUS}" 9967 }, 9968 "exemplar": true, 9969 "expr": "rate(synapse_replication_tcp_resource_stream_updates_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 9970 "format": "time_series", 9971 "interval": "", 9972 "intervalFactor": 2, 9973 "legendFormat": "{{stream_name}}", 9974 "refId": "A", 9975 "step": 20 9976 } 9977 ], 9978 "title": "Rate of outgoing RDATA commands, by stream", 9979 "type": "timeseries" 9980 }, 9981 { 9982 "datasource": { 9983 "type": "prometheus", 9984 "uid": "${DS_PROMETHEUS}" 9985 }, 9986 "fieldConfig": { 9987 "defaults": { 9988 "color": { 9989 "mode": "palette-classic" 9990 }, 9991 "custom": { 9992 "axisCenteredZero": false, 9993 "axisColorMode": "text", 9994 "axisLabel": "", 9995 "axisPlacement": "auto", 9996 "barAlignment": 0, 9997 "drawStyle": "line", 9998 "fillOpacity": 10, 9999 "gradientMode": "none", 10000 "hideFrom": { 10001 "legend": false, 10002 "tooltip": false, 10003 "viz": false 10004 }, 10005 "lineInterpolation": "linear", 10006 "lineWidth": 1, 10007 "pointSize": 5, 10008 "scaleDistribution": { 10009 "type": "linear" 10010 }, 10011 "showPoints": "never", 10012 "spanNulls": true, 10013 "stacking": { 10014 "group": "A", 10015 "mode": "none" 10016 }, 10017 "thresholdsStyle": { 10018 "mode": "off" 10019 } 10020 }, 10021 "links": [], 10022 "mappings": [], 10023 "min": 0, 10024 "thresholds": { 10025 "mode": "absolute", 10026 "steps": [ 10027 { 10028 "color": "green" 10029 }, 10030 { 10031 "color": "red", 10032 "value": 80 10033 } 10034 ] 10035 }, 10036 "unit": "hertz" 10037 }, 10038 "overrides": [] 10039 }, 10040 "gridPos": { 10041 "h": 7, 10042 "w": 12, 10043 "x": 0, 10044 "y": 169 10045 }, 10046 "id": 42, 10047 "links": [], 10048 "options": { 10049 "legend": { 10050 "calcs": [], 10051 "displayMode": "list", 10052 "placement": "bottom", 10053 "showLegend": true 10054 }, 10055 "tooltip": { 10056 "mode": "single", 10057 "sort": "none" 10058 } 10059 }, 10060 "pluginVersion": "8.4.3", 10061 "targets": [ 10062 { 10063 "datasource": { 10064 "type": "prometheus", 10065 "uid": "${DS_PROMETHEUS}" 10066 }, 10067 "exemplar": true, 10068 "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)", 10069 "format": "time_series", 10070 "interval": "", 10071 "intervalFactor": 2, 10072 "legendFormat": "{{job}}-{{index}} {{command}}", 10073 "refId": "A", 10074 "step": 20 10075 } 10076 ], 10077 "title": "Rate of incoming commands (including echoes)", 10078 "type": "timeseries" 10079 }, 10080 { 10081 "datasource": { 10082 "type": "prometheus", 10083 "uid": "${DS_PROMETHEUS}" 10084 }, 10085 "fieldConfig": { 10086 "defaults": { 10087 "color": { 10088 "mode": "palette-classic" 10089 }, 10090 "custom": { 10091 "axisCenteredZero": false, 10092 "axisColorMode": "text", 10093 "axisLabel": "", 10094 "axisPlacement": "auto", 10095 "axisSoftMin": 1, 10096 "barAlignment": 0, 10097 "drawStyle": "line", 10098 "fillOpacity": 10, 10099 "gradientMode": "none", 10100 "hideFrom": { 10101 "legend": false, 10102 "tooltip": false, 10103 "viz": false 10104 }, 10105 "lineInterpolation": "linear", 10106 "lineWidth": 1, 10107 "pointSize": 5, 10108 "scaleDistribution": { 10109 "type": "linear" 10110 }, 10111 "showPoints": "never", 10112 "spanNulls": true, 10113 "stacking": { 10114 "group": "A", 10115 "mode": "none" 10116 }, 10117 "thresholdsStyle": { 10118 "mode": "off" 10119 } 10120 }, 10121 "links": [], 10122 "mappings": [], 10123 "min": 0, 10124 "thresholds": { 10125 "mode": "absolute", 10126 "steps": [ 10127 { 10128 "color": "green" 10129 }, 10130 { 10131 "color": "red", 10132 "value": 80 10133 } 10134 ] 10135 }, 10136 "unit": "hertz" 10137 }, 10138 "overrides": [] 10139 }, 10140 "gridPos": { 10141 "h": 7, 10142 "w": 12, 10143 "x": 12, 10144 "y": 169 10145 }, 10146 "id": 220, 10147 "links": [], 10148 "options": { 10149 "legend": { 10150 "calcs": [], 10151 "displayMode": "list", 10152 "placement": "bottom", 10153 "showLegend": true 10154 }, 10155 "tooltip": { 10156 "mode": "single", 10157 "sort": "none" 10158 } 10159 }, 10160 "pluginVersion": "8.4.3", 10161 "targets": [ 10162 { 10163 "datasource": { 10164 "type": "prometheus", 10165 "uid": "${DS_PROMETHEUS}" 10166 }, 10167 "exemplar": true, 10168 "expr": "rate(synapse_replication_tcp_protocol_inbound_rdata_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 10169 "format": "time_series", 10170 "interval": "", 10171 "intervalFactor": 1, 10172 "legendFormat": "{{job}}-{{index}} {{stream_name}}", 10173 "refId": "A", 10174 "step": 20 10175 } 10176 ], 10177 "title": "Rate of incoming RDATA commands (excluding echoes), by stream", 10178 "type": "timeseries" 10179 }, 10180 { 10181 "aliasColors": {}, 10182 "bars": false, 10183 "dashLength": 10, 10184 "dashes": false, 10185 "datasource": { 10186 "type": "prometheus", 10187 "uid": "${DS_PROMETHEUS}" 10188 }, 10189 "description": "", 10190 "fieldConfig": { 10191 "defaults": { 10192 "links": [] 10193 }, 10194 "overrides": [] 10195 }, 10196 "fill": 1, 10197 "fillGradient": 0, 10198 "gridPos": { 10199 "h": 7, 10200 "w": 12, 10201 "x": 0, 10202 "y": 176 10203 }, 10204 "hiddenSeries": false, 10205 "id": 144, 10206 "legend": { 10207 "avg": false, 10208 "current": false, 10209 "max": false, 10210 "min": false, 10211 "show": true, 10212 "total": false, 10213 "values": false 10214 }, 10215 "lines": true, 10216 "linewidth": 1, 10217 "nullPointMode": "null", 10218 "options": { 10219 "alertThreshold": true 10220 }, 10221 "percentage": false, 10222 "pluginVersion": "9.2.2", 10223 "pointradius": 2, 10224 "points": false, 10225 "renderer": "flot", 10226 "seriesOverrides": [], 10227 "spaceLength": 10, 10228 "stack": false, 10229 "steppedLine": false, 10230 "targets": [ 10231 { 10232 "datasource": { 10233 "type": "prometheus", 10234 "uid": "${DS_PROMETHEUS}" 10235 }, 10236 "expr": "synapse_replication_tcp_command_queue{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 10237 "interval": "", 10238 "legendFormat": "{{stream_name}} {{job}}-{{index}}", 10239 "refId": "A" 10240 } 10241 ], 10242 "thresholds": [], 10243 "timeRegions": [], 10244 "title": "Queued incoming RDATA commands, by stream", 10245 "tooltip": { 10246 "shared": false, 10247 "sort": 0, 10248 "value_type": "individual" 10249 }, 10250 "type": "graph", 10251 "xaxis": { 10252 "mode": "time", 10253 "show": true, 10254 "values": [] 10255 }, 10256 "yaxes": [ 10257 { 10258 "$$hashKey": "object:218", 10259 "format": "short", 10260 "logBase": 1, 10261 "show": true 10262 }, 10263 { 10264 "$$hashKey": "object:219", 10265 "format": "short", 10266 "logBase": 1, 10267 "show": true 10268 } 10269 ], 10270 "yaxis": { 10271 "align": false 10272 } 10273 }, 10274 { 10275 "aliasColors": {}, 10276 "bars": false, 10277 "dashLength": 10, 10278 "dashes": false, 10279 "datasource": { 10280 "type": "prometheus", 10281 "uid": "${DS_PROMETHEUS}" 10282 }, 10283 "fieldConfig": { 10284 "defaults": { 10285 "links": [] 10286 }, 10287 "overrides": [] 10288 }, 10289 "fill": 1, 10290 "fillGradient": 0, 10291 "gridPos": { 10292 "h": 7, 10293 "w": 12, 10294 "x": 12, 10295 "y": 176 10296 }, 10297 "hiddenSeries": false, 10298 "id": 115, 10299 "legend": { 10300 "avg": false, 10301 "current": false, 10302 "max": false, 10303 "min": false, 10304 "show": true, 10305 "total": false, 10306 "values": false 10307 }, 10308 "lines": true, 10309 "linewidth": 1, 10310 "links": [], 10311 "nullPointMode": "null", 10312 "options": { 10313 "alertThreshold": true 10314 }, 10315 "paceLength": 10, 10316 "percentage": false, 10317 "pluginVersion": "9.2.2", 10318 "pointradius": 5, 10319 "points": false, 10320 "renderer": "flot", 10321 "seriesOverrides": [], 10322 "spaceLength": 10, 10323 "stack": false, 10324 "steppedLine": false, 10325 "targets": [ 10326 { 10327 "datasource": { 10328 "uid": "${DS_PROMETHEUS}" 10329 }, 10330 "expr": "rate(synapse_replication_tcp_protocol_close_reason_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 10331 "format": "time_series", 10332 "intervalFactor": 1, 10333 "legendFormat": "{{job}}-{{index}} {{reason_type}}", 10334 "refId": "A" 10335 } 10336 ], 10337 "thresholds": [], 10338 "timeRegions": [], 10339 "title": "Replication connection close reasons", 10340 "tooltip": { 10341 "shared": true, 10342 "sort": 0, 10343 "value_type": "individual" 10344 }, 10345 "type": "graph", 10346 "xaxis": { 10347 "mode": "time", 10348 "show": true, 10349 "values": [] 10350 }, 10351 "yaxes": [ 10352 { 10353 "$$hashKey": "object:260", 10354 "format": "hertz", 10355 "logBase": 1, 10356 "show": true 10357 }, 10358 { 10359 "$$hashKey": "object:261", 10360 "format": "short", 10361 "logBase": 1, 10362 "show": true 10363 } 10364 ], 10365 "yaxis": { 10366 "align": false 10367 } 10368 }, 10369 { 10370 "aliasColors": {}, 10371 "bars": false, 10372 "dashLength": 10, 10373 "dashes": false, 10374 "datasource": { 10375 "uid": "${DS_PROMETHEUS}" 10376 }, 10377 "fieldConfig": { 10378 "defaults": { 10379 "links": [] 10380 }, 10381 "overrides": [] 10382 }, 10383 "fill": 1, 10384 "fillGradient": 0, 10385 "gridPos": { 10386 "h": 7, 10387 "w": 12, 10388 "x": 0, 10389 "y": 183 10390 }, 10391 "hiddenSeries": false, 10392 "id": 113, 10393 "legend": { 10394 "avg": false, 10395 "current": false, 10396 "max": false, 10397 "min": false, 10398 "show": true, 10399 "total": false, 10400 "values": false 10401 }, 10402 "lines": true, 10403 "linewidth": 1, 10404 "links": [], 10405 "nullPointMode": "null", 10406 "options": { 10407 "alertThreshold": true 10408 }, 10409 "paceLength": 10, 10410 "percentage": false, 10411 "pluginVersion": "9.2.2", 10412 "pointradius": 5, 10413 "points": false, 10414 "renderer": "flot", 10415 "seriesOverrides": [], 10416 "spaceLength": 10, 10417 "stack": false, 10418 "steppedLine": false, 10419 "targets": [ 10420 { 10421 "datasource": { 10422 "uid": "${DS_PROMETHEUS}" 10423 }, 10424 "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", 10425 "format": "time_series", 10426 "intervalFactor": 1, 10427 "legendFormat": "{{job}}-{{index}} {{stream_name}}", 10428 "refId": "A" 10429 }, 10430 { 10431 "datasource": { 10432 "uid": "${DS_PROMETHEUS}" 10433 }, 10434 "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", 10435 "format": "time_series", 10436 "intervalFactor": 1, 10437 "legendFormat": "{{job}}-{{index}}", 10438 "refId": "B" 10439 } 10440 ], 10441 "thresholds": [], 10442 "timeRegions": [], 10443 "title": "Replication connections", 10444 "tooltip": { 10445 "shared": true, 10446 "sort": 0, 10447 "value_type": "individual" 10448 }, 10449 "type": "graph", 10450 "xaxis": { 10451 "mode": "time", 10452 "show": true, 10453 "values": [] 10454 }, 10455 "yaxes": [ 10456 { 10457 "format": "short", 10458 "logBase": 1, 10459 "min": "0", 10460 "show": true 10461 }, 10462 { 10463 "format": "short", 10464 "logBase": 1, 10465 "show": true 10466 } 10467 ], 10468 "yaxis": { 10469 "align": false 10470 } 10471 } 10472 ], 10473 "targets": [ 10474 { 10475 "datasource": { 10476 "type": "prometheus", 10477 "uid": "${DS_PROMETHEUS}" 10478 }, 10479 "refId": "A" 10480 } 10481 ], 10482 "title": "Replication", 10483 "type": "row" 10484 }, 10485 { 10486 "collapsed": true, 10487 "datasource": { 10488 "type": "prometheus", 10489 "uid": "${DS_PROMETHEUS}" 10490 }, 10491 "gridPos": { 10492 "h": 1, 10493 "w": 24, 10494 "x": 0, 10495 "y": 40 10496 }, 10497 "id": 69, 10498 "panels": [ 10499 { 10500 "aliasColors": {}, 10501 "bars": false, 10502 "dashLength": 10, 10503 "dashes": false, 10504 "datasource": { 10505 "uid": "${DS_PROMETHEUS}" 10506 }, 10507 "fieldConfig": { 10508 "defaults": { 10509 "links": [] 10510 }, 10511 "overrides": [] 10512 }, 10513 "fill": 1, 10514 "fillGradient": 0, 10515 "gridPos": { 10516 "h": 9, 10517 "w": 12, 10518 "x": 0, 10519 "y": 163 10520 }, 10521 "hiddenSeries": false, 10522 "id": 67, 10523 "legend": { 10524 "avg": false, 10525 "current": false, 10526 "max": false, 10527 "min": false, 10528 "show": true, 10529 "total": false, 10530 "values": false 10531 }, 10532 "lines": true, 10533 "linewidth": 1, 10534 "links": [], 10535 "nullPointMode": "connected", 10536 "options": { 10537 "alertThreshold": true 10538 }, 10539 "paceLength": 10, 10540 "percentage": false, 10541 "pluginVersion": "9.2.2", 10542 "pointradius": 5, 10543 "points": false, 10544 "renderer": "flot", 10545 "seriesOverrides": [], 10546 "spaceLength": 10, 10547 "stack": false, 10548 "steppedLine": false, 10549 "targets": [ 10550 { 10551 "datasource": { 10552 "uid": "${DS_PROMETHEUS}" 10553 }, 10554 "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - on() group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 10555 "format": "time_series", 10556 "interval": "", 10557 "intervalFactor": 1, 10558 "legendFormat": "{{job}}-{{index}} {{name}}", 10559 "refId": "A" 10560 } 10561 ], 10562 "thresholds": [], 10563 "timeRegions": [], 10564 "title": "Event processing lag", 10565 "tooltip": { 10566 "shared": true, 10567 "sort": 0, 10568 "value_type": "individual" 10569 }, 10570 "type": "graph", 10571 "xaxis": { 10572 "mode": "time", 10573 "show": true, 10574 "values": [] 10575 }, 10576 "yaxes": [ 10577 { 10578 "format": "short", 10579 "label": "events", 10580 "logBase": 1, 10581 "min": "0", 10582 "show": true 10583 }, 10584 { 10585 "format": "short", 10586 "logBase": 1, 10587 "show": true 10588 } 10589 ], 10590 "yaxis": { 10591 "align": false 10592 } 10593 }, 10594 { 10595 "aliasColors": {}, 10596 "bars": false, 10597 "dashLength": 10, 10598 "dashes": false, 10599 "datasource": { 10600 "uid": "${DS_PROMETHEUS}" 10601 }, 10602 "fieldConfig": { 10603 "defaults": { 10604 "links": [] 10605 }, 10606 "overrides": [] 10607 }, 10608 "fill": 1, 10609 "fillGradient": 0, 10610 "gridPos": { 10611 "h": 9, 10612 "w": 12, 10613 "x": 12, 10614 "y": 163 10615 }, 10616 "hiddenSeries": false, 10617 "id": 71, 10618 "legend": { 10619 "avg": false, 10620 "current": false, 10621 "max": false, 10622 "min": false, 10623 "show": true, 10624 "total": false, 10625 "values": false 10626 }, 10627 "lines": true, 10628 "linewidth": 1, 10629 "links": [], 10630 "nullPointMode": "connected", 10631 "options": { 10632 "alertThreshold": true 10633 }, 10634 "paceLength": 10, 10635 "percentage": false, 10636 "pluginVersion": "9.2.2", 10637 "pointradius": 5, 10638 "points": false, 10639 "renderer": "flot", 10640 "seriesOverrides": [], 10641 "spaceLength": 10, 10642 "stack": false, 10643 "steppedLine": false, 10644 "targets": [ 10645 { 10646 "datasource": { 10647 "uid": "${DS_PROMETHEUS}" 10648 }, 10649 "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", 10650 "format": "time_series", 10651 "hide": false, 10652 "interval": "", 10653 "intervalFactor": 1, 10654 "legendFormat": "{{job}}-{{index}} {{name}}", 10655 "refId": "B" 10656 } 10657 ], 10658 "thresholds": [], 10659 "timeRegions": [], 10660 "title": "Age of last processed event", 10661 "tooltip": { 10662 "shared": true, 10663 "sort": 0, 10664 "value_type": "individual" 10665 }, 10666 "type": "graph", 10667 "xaxis": { 10668 "mode": "time", 10669 "show": true, 10670 "values": [] 10671 }, 10672 "yaxes": [ 10673 { 10674 "format": "ms", 10675 "logBase": 1, 10676 "min": "0", 10677 "show": true 10678 }, 10679 { 10680 "format": "short", 10681 "logBase": 1, 10682 "show": true 10683 } 10684 ], 10685 "yaxis": { 10686 "align": false 10687 } 10688 }, 10689 { 10690 "aliasColors": {}, 10691 "bars": false, 10692 "dashLength": 10, 10693 "dashes": false, 10694 "datasource": { 10695 "uid": "${DS_PROMETHEUS}" 10696 }, 10697 "fieldConfig": { 10698 "defaults": { 10699 "links": [] 10700 }, 10701 "overrides": [] 10702 }, 10703 "fill": 1, 10704 "fillGradient": 0, 10705 "gridPos": { 10706 "h": 9, 10707 "w": 12, 10708 "x": 0, 10709 "y": 172 10710 }, 10711 "hiddenSeries": false, 10712 "id": 121, 10713 "interval": "", 10714 "legend": { 10715 "avg": false, 10716 "current": false, 10717 "max": false, 10718 "min": false, 10719 "show": true, 10720 "total": false, 10721 "values": false 10722 }, 10723 "lines": true, 10724 "linewidth": 1, 10725 "links": [], 10726 "nullPointMode": "connected", 10727 "options": { 10728 "alertThreshold": true 10729 }, 10730 "paceLength": 10, 10731 "percentage": false, 10732 "pluginVersion": "9.2.2", 10733 "pointradius": 5, 10734 "points": false, 10735 "renderer": "flot", 10736 "seriesOverrides": [], 10737 "spaceLength": 10, 10738 "stack": false, 10739 "steppedLine": false, 10740 "targets": [ 10741 { 10742 "datasource": { 10743 "uid": "${DS_PROMETHEUS}" 10744 }, 10745 "expr": "deriv(synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1", 10746 "format": "time_series", 10747 "hide": false, 10748 "interval": "", 10749 "intervalFactor": 1, 10750 "legendFormat": "{{job}}-{{index}} {{name}}", 10751 "refId": "B" 10752 } 10753 ], 10754 "thresholds": [], 10755 "timeRegions": [], 10756 "title": "Event processing catchup rate", 10757 "tooltip": { 10758 "shared": true, 10759 "sort": 0, 10760 "value_type": "individual" 10761 }, 10762 "type": "graph", 10763 "xaxis": { 10764 "mode": "time", 10765 "show": true, 10766 "values": [] 10767 }, 10768 "yaxes": [ 10769 { 10770 "format": "none", 10771 "label": "fallbehind(-) / catchup(+): s/sec", 10772 "logBase": 1, 10773 "show": true 10774 }, 10775 { 10776 "format": "short", 10777 "logBase": 1, 10778 "show": true 10779 } 10780 ], 10781 "yaxis": { 10782 "align": false 10783 } 10784 } 10785 ], 10786 "targets": [ 10787 { 10788 "datasource": { 10789 "type": "prometheus", 10790 "uid": "${DS_PROMETHEUS}" 10791 }, 10792 "refId": "A" 10793 } 10794 ], 10795 "title": "Event processing loop positions", 10796 "type": "row" 10797 }, 10798 { 10799 "collapsed": true, 10800 "datasource": { 10801 "type": "prometheus", 10802 "uid": "${DS_PROMETHEUS}" 10803 }, 10804 "gridPos": { 10805 "h": 1, 10806 "w": 24, 10807 "x": 0, 10808 "y": 41 10809 }, 10810 "id": 126, 10811 "panels": [ 10812 { 10813 "cards": { 10814 "cardPadding": 0 10815 }, 10816 "color": { 10817 "cardColor": "#B877D9", 10818 "colorScale": "sqrt", 10819 "colorScheme": "interpolateInferno", 10820 "exponent": 0.5, 10821 "min": 0, 10822 "mode": "opacity" 10823 }, 10824 "dataFormat": "tsbuckets", 10825 "datasource": { 10826 "uid": "${DS_PROMETHEUS}" 10827 }, 10828 "description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.", 10829 "fieldConfig": { 10830 "defaults": { 10831 "custom": { 10832 "hideFrom": { 10833 "legend": false, 10834 "tooltip": false, 10835 "viz": false 10836 }, 10837 "scaleDistribution": { 10838 "type": "linear" 10839 } 10840 } 10841 }, 10842 "overrides": [] 10843 }, 10844 "gridPos": { 10845 "h": 8, 10846 "w": 12, 10847 "x": 0, 10848 "y": 42 10849 }, 10850 "heatmap": {}, 10851 "hideZeroBuckets": true, 10852 "highlightCards": true, 10853 "id": 122, 10854 "legend": { 10855 "show": true 10856 }, 10857 "links": [], 10858 "options": { 10859 "calculate": false, 10860 "calculation": {}, 10861 "cellGap": 1, 10862 "cellValues": {}, 10863 "color": { 10864 "exponent": 0.5, 10865 "fill": "#B877D9", 10866 "min": 0, 10867 "mode": "opacity", 10868 "reverse": false, 10869 "scale": "exponential", 10870 "scheme": "Oranges", 10871 "steps": 128 10872 }, 10873 "exemplars": { 10874 "color": "rgba(255,0,255,0.7)" 10875 }, 10876 "filterValues": { 10877 "le": 1e-9 10878 }, 10879 "legend": { 10880 "show": true 10881 }, 10882 "rowsFrame": { 10883 "layout": "auto" 10884 }, 10885 "showValue": "never", 10886 "tooltip": { 10887 "show": true, 10888 "yHistogram": true 10889 }, 10890 "yAxis": { 10891 "axisPlacement": "left", 10892 "decimals": 0, 10893 "reverse": false, 10894 "unit": "short" 10895 } 10896 }, 10897 "pluginVersion": "9.2.2", 10898 "reverseYBuckets": false, 10899 "targets": [ 10900 { 10901 "datasource": { 10902 "uid": "${DS_PROMETHEUS}" 10903 }, 10904 "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", 10905 "format": "heatmap", 10906 "intervalFactor": 1, 10907 "legendFormat": "{{le}}", 10908 "refId": "A" 10909 } 10910 ], 10911 "title": "Number of rooms, by number of forward extremities in room", 10912 "tooltip": { 10913 "show": true, 10914 "showHistogram": true 10915 }, 10916 "type": "heatmap", 10917 "xAxis": { 10918 "show": true 10919 }, 10920 "yAxis": { 10921 "decimals": 0, 10922 "format": "short", 10923 "logBase": 1, 10924 "show": true 10925 }, 10926 "yBucketBound": "auto" 10927 }, 10928 { 10929 "aliasColors": {}, 10930 "bars": false, 10931 "dashLength": 10, 10932 "dashes": false, 10933 "datasource": { 10934 "uid": "${DS_PROMETHEUS}" 10935 }, 10936 "description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.", 10937 "fieldConfig": { 10938 "defaults": { 10939 "links": [] 10940 }, 10941 "overrides": [] 10942 }, 10943 "fill": 0, 10944 "fillGradient": 0, 10945 "gridPos": { 10946 "h": 8, 10947 "w": 12, 10948 "x": 12, 10949 "y": 42 10950 }, 10951 "hiddenSeries": false, 10952 "id": 124, 10953 "interval": "", 10954 "legend": { 10955 "avg": false, 10956 "current": false, 10957 "max": false, 10958 "min": false, 10959 "show": true, 10960 "total": false, 10961 "values": false 10962 }, 10963 "lines": true, 10964 "linewidth": 1, 10965 "links": [], 10966 "nullPointMode": "connected", 10967 "options": { 10968 "alertThreshold": true 10969 }, 10970 "percentage": false, 10971 "pluginVersion": "9.2.2", 10972 "pointradius": 2, 10973 "points": false, 10974 "renderer": "flot", 10975 "seriesOverrides": [], 10976 "spaceLength": 10, 10977 "stack": false, 10978 "steppedLine": false, 10979 "targets": [ 10980 { 10981 "datasource": { 10982 "uid": "${DS_PROMETHEUS}" 10983 }, 10984 "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0", 10985 "format": "heatmap", 10986 "interval": "", 10987 "intervalFactor": 1, 10988 "legendFormat": "{{le}}", 10989 "refId": "A" 10990 } 10991 ], 10992 "thresholds": [], 10993 "timeRegions": [], 10994 "title": "Room counts, by number of extremities", 10995 "tooltip": { 10996 "shared": true, 10997 "sort": 2, 10998 "value_type": "individual" 10999 }, 11000 "type": "graph", 11001 "xaxis": { 11002 "mode": "time", 11003 "show": true, 11004 "values": [] 11005 }, 11006 "yaxes": [ 11007 { 11008 "format": "none", 11009 "label": "Number of rooms", 11010 "logBase": 10, 11011 "show": true 11012 }, 11013 { 11014 "format": "short", 11015 "logBase": 1, 11016 "show": false 11017 } 11018 ], 11019 "yaxis": { 11020 "align": false 11021 } 11022 }, 11023 { 11024 "cards": { 11025 "cardPadding": 0 11026 }, 11027 "color": { 11028 "cardColor": "#5794F2", 11029 "colorScale": "sqrt", 11030 "colorScheme": "interpolateInferno", 11031 "exponent": 0.5, 11032 "min": 0, 11033 "mode": "opacity" 11034 }, 11035 "dataFormat": "tsbuckets", 11036 "datasource": { 11037 "uid": "${DS_PROMETHEUS}" 11038 }, 11039 "description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.", 11040 "fieldConfig": { 11041 "defaults": { 11042 "custom": { 11043 "hideFrom": { 11044 "legend": false, 11045 "tooltip": false, 11046 "viz": false 11047 }, 11048 "scaleDistribution": { 11049 "type": "linear" 11050 } 11051 } 11052 }, 11053 "overrides": [] 11054 }, 11055 "gridPos": { 11056 "h": 8, 11057 "w": 12, 11058 "x": 0, 11059 "y": 50 11060 }, 11061 "heatmap": {}, 11062 "hideZeroBuckets": true, 11063 "highlightCards": true, 11064 "id": 127, 11065 "legend": { 11066 "show": true 11067 }, 11068 "links": [], 11069 "options": { 11070 "calculate": false, 11071 "calculation": {}, 11072 "cellGap": 1, 11073 "cellValues": {}, 11074 "color": { 11075 "exponent": 0.5, 11076 "fill": "#5794F2", 11077 "min": 0, 11078 "mode": "opacity", 11079 "reverse": false, 11080 "scale": "exponential", 11081 "scheme": "Oranges", 11082 "steps": 128 11083 }, 11084 "exemplars": { 11085 "color": "rgba(255,0,255,0.7)" 11086 }, 11087 "filterValues": { 11088 "le": 1e-9 11089 }, 11090 "legend": { 11091 "show": true 11092 }, 11093 "rowsFrame": { 11094 "layout": "auto" 11095 }, 11096 "showValue": "never", 11097 "tooltip": { 11098 "show": true, 11099 "yHistogram": true 11100 }, 11101 "yAxis": { 11102 "axisPlacement": "left", 11103 "decimals": 0, 11104 "reverse": false, 11105 "unit": "short" 11106 } 11107 }, 11108 "pluginVersion": "9.2.2", 11109 "reverseYBuckets": false, 11110 "targets": [ 11111 { 11112 "datasource": { 11113 "uid": "${DS_PROMETHEUS}" 11114 }, 11115 "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", 11116 "format": "heatmap", 11117 "intervalFactor": 1, 11118 "legendFormat": "{{le}}", 11119 "refId": "A" 11120 } 11121 ], 11122 "title": "Events persisted, by number of forward extremities in room (heatmap)", 11123 "tooltip": { 11124 "show": true, 11125 "showHistogram": true 11126 }, 11127 "type": "heatmap", 11128 "xAxis": { 11129 "show": true 11130 }, 11131 "yAxis": { 11132 "decimals": 0, 11133 "format": "short", 11134 "logBase": 1, 11135 "show": true 11136 }, 11137 "yBucketBound": "auto" 11138 }, 11139 { 11140 "aliasColors": {}, 11141 "bars": false, 11142 "dashLength": 10, 11143 "dashes": false, 11144 "datasource": { 11145 "uid": "${DS_PROMETHEUS}" 11146 }, 11147 "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.", 11148 "fieldConfig": { 11149 "defaults": { 11150 "links": [] 11151 }, 11152 "overrides": [] 11153 }, 11154 "fill": 1, 11155 "fillGradient": 0, 11156 "gridPos": { 11157 "h": 8, 11158 "w": 12, 11159 "x": 12, 11160 "y": 50 11161 }, 11162 "hiddenSeries": false, 11163 "id": 128, 11164 "legend": { 11165 "avg": false, 11166 "current": false, 11167 "max": false, 11168 "min": false, 11169 "show": true, 11170 "total": false, 11171 "values": false 11172 }, 11173 "lines": true, 11174 "linewidth": 1, 11175 "links": [], 11176 "nullPointMode": "null", 11177 "options": { 11178 "alertThreshold": true 11179 }, 11180 "percentage": false, 11181 "pluginVersion": "9.2.2", 11182 "pointradius": 2, 11183 "points": false, 11184 "renderer": "flot", 11185 "seriesOverrides": [], 11186 "spaceLength": 10, 11187 "stack": false, 11188 "steppedLine": false, 11189 "targets": [ 11190 { 11191 "datasource": { 11192 "uid": "${DS_PROMETHEUS}" 11193 }, 11194 "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", 11195 "format": "time_series", 11196 "intervalFactor": 1, 11197 "legendFormat": "50%", 11198 "refId": "A" 11199 }, 11200 { 11201 "datasource": { 11202 "uid": "${DS_PROMETHEUS}" 11203 }, 11204 "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", 11205 "format": "time_series", 11206 "intervalFactor": 1, 11207 "legendFormat": "75%", 11208 "refId": "B" 11209 }, 11210 { 11211 "datasource": { 11212 "uid": "${DS_PROMETHEUS}" 11213 }, 11214 "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", 11215 "format": "time_series", 11216 "intervalFactor": 1, 11217 "legendFormat": "90%", 11218 "refId": "C" 11219 }, 11220 { 11221 "datasource": { 11222 "uid": "${DS_PROMETHEUS}" 11223 }, 11224 "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", 11225 "format": "time_series", 11226 "intervalFactor": 1, 11227 "legendFormat": "99%", 11228 "refId": "D" 11229 } 11230 ], 11231 "thresholds": [], 11232 "timeRegions": [], 11233 "title": "Events persisted, by number of forward extremities in room (quantiles)", 11234 "tooltip": { 11235 "shared": true, 11236 "sort": 0, 11237 "value_type": "individual" 11238 }, 11239 "type": "graph", 11240 "xaxis": { 11241 "mode": "time", 11242 "show": true, 11243 "values": [] 11244 }, 11245 "yaxes": [ 11246 { 11247 "format": "short", 11248 "label": "Number of extremities in room", 11249 "logBase": 1, 11250 "min": "0", 11251 "show": true 11252 }, 11253 { 11254 "format": "short", 11255 "logBase": 1, 11256 "show": true 11257 } 11258 ], 11259 "yaxis": { 11260 "align": false 11261 } 11262 }, 11263 { 11264 "cards": { 11265 "cardPadding": 0 11266 }, 11267 "color": { 11268 "cardColor": "#FF9830", 11269 "colorScale": "sqrt", 11270 "colorScheme": "interpolateInferno", 11271 "exponent": 0.5, 11272 "min": 0, 11273 "mode": "opacity" 11274 }, 11275 "dataFormat": "tsbuckets", 11276 "datasource": { 11277 "uid": "${DS_PROMETHEUS}" 11278 }, 11279 "description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.", 11280 "fieldConfig": { 11281 "defaults": { 11282 "custom": { 11283 "hideFrom": { 11284 "legend": false, 11285 "tooltip": false, 11286 "viz": false 11287 }, 11288 "scaleDistribution": { 11289 "type": "linear" 11290 } 11291 } 11292 }, 11293 "overrides": [] 11294 }, 11295 "gridPos": { 11296 "h": 8, 11297 "w": 12, 11298 "x": 0, 11299 "y": 58 11300 }, 11301 "heatmap": {}, 11302 "hideZeroBuckets": true, 11303 "highlightCards": true, 11304 "id": 129, 11305 "legend": { 11306 "show": true 11307 }, 11308 "links": [], 11309 "options": { 11310 "calculate": false, 11311 "calculation": {}, 11312 "cellGap": 1, 11313 "cellValues": {}, 11314 "color": { 11315 "exponent": 0.5, 11316 "fill": "#FF9830", 11317 "min": 0, 11318 "mode": "opacity", 11319 "reverse": false, 11320 "scale": "exponential", 11321 "scheme": "Oranges", 11322 "steps": 128 11323 }, 11324 "exemplars": { 11325 "color": "rgba(255,0,255,0.7)" 11326 }, 11327 "filterValues": { 11328 "le": 1e-9 11329 }, 11330 "legend": { 11331 "show": true 11332 }, 11333 "rowsFrame": { 11334 "layout": "auto" 11335 }, 11336 "showValue": "never", 11337 "tooltip": { 11338 "show": true, 11339 "yHistogram": true 11340 }, 11341 "yAxis": { 11342 "axisPlacement": "left", 11343 "decimals": 0, 11344 "reverse": false, 11345 "unit": "short" 11346 } 11347 }, 11348 "pluginVersion": "9.2.2", 11349 "reverseYBuckets": false, 11350 "targets": [ 11351 { 11352 "datasource": { 11353 "uid": "${DS_PROMETHEUS}" 11354 }, 11355 "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", 11356 "format": "heatmap", 11357 "intervalFactor": 1, 11358 "legendFormat": "{{le}}", 11359 "refId": "A" 11360 } 11361 ], 11362 "title": "Events persisted, by number of stale forward extremities in room (heatmap)", 11363 "tooltip": { 11364 "show": true, 11365 "showHistogram": true 11366 }, 11367 "type": "heatmap", 11368 "xAxis": { 11369 "show": true 11370 }, 11371 "yAxis": { 11372 "decimals": 0, 11373 "format": "short", 11374 "logBase": 1, 11375 "show": true 11376 }, 11377 "yBucketBound": "auto" 11378 }, 11379 { 11380 "aliasColors": {}, 11381 "bars": false, 11382 "dashLength": 10, 11383 "dashes": false, 11384 "datasource": { 11385 "uid": "${DS_PROMETHEUS}" 11386 }, 11387 "description": "For given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.", 11388 "fieldConfig": { 11389 "defaults": { 11390 "links": [] 11391 }, 11392 "overrides": [] 11393 }, 11394 "fill": 1, 11395 "fillGradient": 0, 11396 "gridPos": { 11397 "h": 8, 11398 "w": 12, 11399 "x": 12, 11400 "y": 58 11401 }, 11402 "hiddenSeries": false, 11403 "id": 130, 11404 "legend": { 11405 "avg": false, 11406 "current": false, 11407 "max": false, 11408 "min": false, 11409 "show": true, 11410 "total": false, 11411 "values": false 11412 }, 11413 "lines": true, 11414 "linewidth": 1, 11415 "links": [], 11416 "nullPointMode": "null", 11417 "options": { 11418 "alertThreshold": true 11419 }, 11420 "percentage": false, 11421 "pluginVersion": "9.2.2", 11422 "pointradius": 2, 11423 "points": false, 11424 "renderer": "flot", 11425 "seriesOverrides": [], 11426 "spaceLength": 10, 11427 "stack": false, 11428 "steppedLine": false, 11429 "targets": [ 11430 { 11431 "datasource": { 11432 "uid": "${DS_PROMETHEUS}" 11433 }, 11434 "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", 11435 "format": "time_series", 11436 "intervalFactor": 1, 11437 "legendFormat": "50%", 11438 "refId": "A" 11439 }, 11440 { 11441 "datasource": { 11442 "uid": "${DS_PROMETHEUS}" 11443 }, 11444 "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", 11445 "format": "time_series", 11446 "intervalFactor": 1, 11447 "legendFormat": "75%", 11448 "refId": "B" 11449 }, 11450 { 11451 "datasource": { 11452 "uid": "${DS_PROMETHEUS}" 11453 }, 11454 "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", 11455 "format": "time_series", 11456 "intervalFactor": 1, 11457 "legendFormat": "90%", 11458 "refId": "C" 11459 }, 11460 { 11461 "datasource": { 11462 "uid": "${DS_PROMETHEUS}" 11463 }, 11464 "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", 11465 "format": "time_series", 11466 "intervalFactor": 1, 11467 "legendFormat": "99%", 11468 "refId": "D" 11469 } 11470 ], 11471 "thresholds": [], 11472 "timeRegions": [], 11473 "title": "Events persisted, by number of stale forward extremities in room (quantiles)", 11474 "tooltip": { 11475 "shared": true, 11476 "sort": 0, 11477 "value_type": "individual" 11478 }, 11479 "type": "graph", 11480 "xaxis": { 11481 "mode": "time", 11482 "show": true, 11483 "values": [] 11484 }, 11485 "yaxes": [ 11486 { 11487 "format": "short", 11488 "label": "Number of stale forward extremities in room", 11489 "logBase": 1, 11490 "min": "0", 11491 "show": true 11492 }, 11493 { 11494 "format": "short", 11495 "logBase": 1, 11496 "show": true 11497 } 11498 ], 11499 "yaxis": { 11500 "align": false 11501 } 11502 }, 11503 { 11504 "cards": { 11505 "cardPadding": 0 11506 }, 11507 "color": { 11508 "cardColor": "#73BF69", 11509 "colorScale": "sqrt", 11510 "colorScheme": "interpolateInferno", 11511 "exponent": 0.5, 11512 "min": 0, 11513 "mode": "opacity" 11514 }, 11515 "dataFormat": "tsbuckets", 11516 "datasource": { 11517 "uid": "${DS_PROMETHEUS}" 11518 }, 11519 "description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.", 11520 "fieldConfig": { 11521 "defaults": { 11522 "custom": { 11523 "hideFrom": { 11524 "legend": false, 11525 "tooltip": false, 11526 "viz": false 11527 }, 11528 "scaleDistribution": { 11529 "type": "linear" 11530 } 11531 } 11532 }, 11533 "overrides": [] 11534 }, 11535 "gridPos": { 11536 "h": 8, 11537 "w": 12, 11538 "x": 0, 11539 "y": 66 11540 }, 11541 "heatmap": {}, 11542 "hideZeroBuckets": true, 11543 "highlightCards": true, 11544 "id": 131, 11545 "legend": { 11546 "show": true 11547 }, 11548 "links": [], 11549 "options": { 11550 "calculate": false, 11551 "calculation": {}, 11552 "cellGap": 1, 11553 "cellValues": {}, 11554 "color": { 11555 "exponent": 0.5, 11556 "fill": "#73BF69", 11557 "min": 0, 11558 "mode": "opacity", 11559 "reverse": false, 11560 "scale": "exponential", 11561 "scheme": "Oranges", 11562 "steps": 128 11563 }, 11564 "exemplars": { 11565 "color": "rgba(255,0,255,0.7)" 11566 }, 11567 "filterValues": { 11568 "le": 1e-9 11569 }, 11570 "legend": { 11571 "show": true 11572 }, 11573 "rowsFrame": { 11574 "layout": "auto" 11575 }, 11576 "showValue": "never", 11577 "tooltip": { 11578 "show": true, 11579 "yHistogram": true 11580 }, 11581 "yAxis": { 11582 "axisPlacement": "left", 11583 "decimals": 0, 11584 "reverse": false, 11585 "unit": "short" 11586 } 11587 }, 11588 "pluginVersion": "9.2.2", 11589 "reverseYBuckets": false, 11590 "targets": [ 11591 { 11592 "datasource": { 11593 "uid": "${DS_PROMETHEUS}" 11594 }, 11595 "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", 11596 "format": "heatmap", 11597 "interval": "", 11598 "intervalFactor": 1, 11599 "legendFormat": "{{le}}", 11600 "refId": "A" 11601 } 11602 ], 11603 "title": "Number of state resolution performed, by number of state groups involved (heatmap)", 11604 "tooltip": { 11605 "show": true, 11606 "showHistogram": true 11607 }, 11608 "type": "heatmap", 11609 "xAxis": { 11610 "show": true 11611 }, 11612 "yAxis": { 11613 "decimals": 0, 11614 "format": "short", 11615 "logBase": 1, 11616 "show": true 11617 }, 11618 "yBucketBound": "auto" 11619 }, 11620 { 11621 "aliasColors": {}, 11622 "bars": false, 11623 "dashLength": 10, 11624 "dashes": false, 11625 "datasource": { 11626 "type": "prometheus", 11627 "uid": "${DS_PROMETHEUS}" 11628 }, 11629 "description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.", 11630 "fieldConfig": { 11631 "defaults": { 11632 "links": [] 11633 }, 11634 "overrides": [] 11635 }, 11636 "fill": 1, 11637 "fillGradient": 0, 11638 "gridPos": { 11639 "h": 8, 11640 "w": 12, 11641 "x": 12, 11642 "y": 66 11643 }, 11644 "hiddenSeries": false, 11645 "id": 132, 11646 "interval": "", 11647 "legend": { 11648 "avg": false, 11649 "current": false, 11650 "max": false, 11651 "min": false, 11652 "show": true, 11653 "total": false, 11654 "values": false 11655 }, 11656 "lines": true, 11657 "linewidth": 1, 11658 "links": [], 11659 "nullPointMode": "null", 11660 "options": { 11661 "alertThreshold": true 11662 }, 11663 "percentage": false, 11664 "pluginVersion": "9.2.2", 11665 "pointradius": 2, 11666 "points": false, 11667 "renderer": "flot", 11668 "seriesOverrides": [], 11669 "spaceLength": 10, 11670 "stack": false, 11671 "steppedLine": false, 11672 "targets": [ 11673 { 11674 "datasource": { 11675 "uid": "${DS_PROMETHEUS}" 11676 }, 11677 "editorMode": "code", 11678 "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", 11679 "format": "time_series", 11680 "interval": "", 11681 "intervalFactor": 1, 11682 "legendFormat": "50%", 11683 "range": true, 11684 "refId": "A" 11685 }, 11686 { 11687 "datasource": { 11688 "uid": "${DS_PROMETHEUS}" 11689 }, 11690 "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", 11691 "format": "time_series", 11692 "interval": "", 11693 "intervalFactor": 1, 11694 "legendFormat": "75%", 11695 "refId": "B" 11696 }, 11697 { 11698 "datasource": { 11699 "uid": "${DS_PROMETHEUS}" 11700 }, 11701 "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", 11702 "format": "time_series", 11703 "interval": "", 11704 "intervalFactor": 1, 11705 "legendFormat": "90%", 11706 "refId": "C" 11707 }, 11708 { 11709 "datasource": { 11710 "uid": "${DS_PROMETHEUS}" 11711 }, 11712 "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", 11713 "format": "time_series", 11714 "interval": "", 11715 "intervalFactor": 1, 11716 "legendFormat": "99%", 11717 "refId": "D" 11718 } 11719 ], 11720 "thresholds": [], 11721 "timeRegions": [], 11722 "title": "Number of state resolutions performed, by number of state groups involved (quantiles)", 11723 "tooltip": { 11724 "shared": true, 11725 "sort": 0, 11726 "value_type": "individual" 11727 }, 11728 "type": "graph", 11729 "xaxis": { 11730 "mode": "time", 11731 "show": true, 11732 "values": [] 11733 }, 11734 "yaxes": [ 11735 { 11736 "format": "short", 11737 "label": "Number of state groups", 11738 "logBase": 1, 11739 "min": "0", 11740 "show": true 11741 }, 11742 { 11743 "format": "short", 11744 "logBase": 1, 11745 "show": true 11746 } 11747 ], 11748 "yaxis": { 11749 "align": false 11750 } 11751 }, 11752 { 11753 "aliasColors": {}, 11754 "bars": false, 11755 "dashLength": 10, 11756 "dashes": false, 11757 "datasource": { 11758 "uid": "${DS_PROMETHEUS}" 11759 }, 11760 "description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.", 11761 "fill": 1, 11762 "fillGradient": 0, 11763 "gridPos": { 11764 "h": 8, 11765 "w": 12, 11766 "x": 0, 11767 "y": 74 11768 }, 11769 "hiddenSeries": false, 11770 "id": 179, 11771 "legend": { 11772 "avg": false, 11773 "current": false, 11774 "max": false, 11775 "min": false, 11776 "show": true, 11777 "total": false, 11778 "values": false 11779 }, 11780 "lines": true, 11781 "linewidth": 1, 11782 "nullPointMode": "null", 11783 "options": { 11784 "alertThreshold": true 11785 }, 11786 "percentage": false, 11787 "pluginVersion": "9.2.2", 11788 "pointradius": 2, 11789 "points": false, 11790 "renderer": "flot", 11791 "seriesOverrides": [], 11792 "spaceLength": 10, 11793 "stack": false, 11794 "steppedLine": false, 11795 "targets": [ 11796 { 11797 "datasource": { 11798 "uid": "${DS_PROMETHEUS}" 11799 }, 11800 "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", 11801 "interval": "", 11802 "legendFormat": "State res ", 11803 "refId": "A" 11804 }, 11805 { 11806 "datasource": { 11807 "uid": "${DS_PROMETHEUS}" 11808 }, 11809 "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", 11810 "interval": "", 11811 "legendFormat": "Potential to prune", 11812 "refId": "B" 11813 }, 11814 { 11815 "datasource": { 11816 "uid": "${DS_PROMETHEUS}" 11817 }, 11818 "expr": "sum(rate(synapse_storage_events_times_pruned_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", 11819 "interval": "", 11820 "legendFormat": "Pruned", 11821 "refId": "C" 11822 } 11823 ], 11824 "thresholds": [], 11825 "timeRegions": [], 11826 "title": "Stale extremity dropping", 11827 "tooltip": { 11828 "shared": true, 11829 "sort": 0, 11830 "value_type": "individual" 11831 }, 11832 "type": "graph", 11833 "xaxis": { 11834 "mode": "time", 11835 "show": true, 11836 "values": [] 11837 }, 11838 "yaxes": [ 11839 { 11840 "format": "hertz", 11841 "logBase": 1, 11842 "show": true 11843 }, 11844 { 11845 "format": "short", 11846 "logBase": 1, 11847 "show": true 11848 } 11849 ], 11850 "yaxis": { 11851 "align": false 11852 } 11853 } 11854 ], 11855 "targets": [ 11856 { 11857 "datasource": { 11858 "type": "prometheus", 11859 "uid": "${DS_PROMETHEUS}" 11860 }, 11861 "refId": "A" 11862 } 11863 ], 11864 "title": "Extremities", 11865 "type": "row" 11866 }, 11867 { 11868 "collapsed": true, 11869 "datasource": { 11870 "type": "prometheus", 11871 "uid": "${DS_PROMETHEUS}" 11872 }, 11873 "gridPos": { 11874 "h": 1, 11875 "w": 24, 11876 "x": 0, 11877 "y": 42 11878 }, 11879 "id": 158, 11880 "panels": [ 11881 { 11882 "aliasColors": {}, 11883 "bars": false, 11884 "dashLength": 10, 11885 "dashes": false, 11886 "datasource": { 11887 "type": "prometheus", 11888 "uid": "${DS_PROMETHEUS}" 11889 }, 11890 "fieldConfig": { 11891 "defaults": { 11892 "links": [] 11893 }, 11894 "overrides": [] 11895 }, 11896 "fill": 1, 11897 "fillGradient": 0, 11898 "gridPos": { 11899 "h": 8, 11900 "w": 12, 11901 "x": 0, 11902 "y": 43 11903 }, 11904 "hiddenSeries": false, 11905 "id": 156, 11906 "legend": { 11907 "avg": false, 11908 "current": false, 11909 "max": false, 11910 "min": false, 11911 "show": true, 11912 "total": false, 11913 "values": false 11914 }, 11915 "lines": true, 11916 "linewidth": 1, 11917 "links": [], 11918 "nullPointMode": "null", 11919 "options": { 11920 "alertThreshold": true 11921 }, 11922 "percentage": false, 11923 "pluginVersion": "9.2.2", 11924 "pointradius": 5, 11925 "points": false, 11926 "renderer": "flot", 11927 "seriesOverrides": [ 11928 { 11929 "$$hashKey": "object:632", 11930 "alias": "Max", 11931 "color": "#bf1b00", 11932 "fill": 0, 11933 "linewidth": 2 11934 } 11935 ], 11936 "spaceLength": 10, 11937 "stack": false, 11938 "steppedLine": false, 11939 "targets": [ 11940 { 11941 "datasource": { 11942 "uid": "${DS_PROMETHEUS}" 11943 }, 11944 "editorMode": "code", 11945 "expr": "max(synapse_admin_mau_max{instance=\"$instance\"})", 11946 "format": "time_series", 11947 "interval": "", 11948 "intervalFactor": 1, 11949 "legendFormat": "Max", 11950 "range": true, 11951 "refId": "B" 11952 }, 11953 { 11954 "datasource": { 11955 "type": "prometheus", 11956 "uid": "${DS_PROMETHEUS}" 11957 }, 11958 "editorMode": "code", 11959 "expr": "max(synapse_admin_mau_current{instance=\"$instance\"})", 11960 "hide": false, 11961 "legendFormat": "Current", 11962 "range": true, 11963 "refId": "C" 11964 } 11965 ], 11966 "thresholds": [], 11967 "timeRegions": [], 11968 "title": "MAU Limits", 11969 "tooltip": { 11970 "shared": true, 11971 "sort": 0, 11972 "value_type": "individual" 11973 }, 11974 "type": "graph", 11975 "xaxis": { 11976 "mode": "time", 11977 "show": true, 11978 "values": [] 11979 }, 11980 "yaxes": [ 11981 { 11982 "$$hashKey": "object:176", 11983 "format": "short", 11984 "logBase": 1, 11985 "min": "0", 11986 "show": true 11987 }, 11988 { 11989 "$$hashKey": "object:177", 11990 "format": "short", 11991 "logBase": 1, 11992 "show": true 11993 } 11994 ], 11995 "yaxis": { 11996 "align": false 11997 } 11998 }, 11999 { 12000 "aliasColors": {}, 12001 "bars": false, 12002 "dashLength": 10, 12003 "dashes": false, 12004 "datasource": { 12005 "uid": "${DS_PROMETHEUS}" 12006 }, 12007 "fill": 1, 12008 "fillGradient": 0, 12009 "gridPos": { 12010 "h": 8, 12011 "w": 12, 12012 "x": 12, 12013 "y": 43 12014 }, 12015 "hiddenSeries": false, 12016 "id": 160, 12017 "legend": { 12018 "avg": false, 12019 "current": false, 12020 "max": false, 12021 "min": false, 12022 "show": true, 12023 "total": false, 12024 "values": false 12025 }, 12026 "lines": true, 12027 "linewidth": 1, 12028 "nullPointMode": "null", 12029 "options": { 12030 "alertThreshold": true 12031 }, 12032 "percentage": false, 12033 "pluginVersion": "9.2.2", 12034 "pointradius": 2, 12035 "points": false, 12036 "renderer": "flot", 12037 "seriesOverrides": [], 12038 "spaceLength": 10, 12039 "stack": false, 12040 "steppedLine": false, 12041 "targets": [ 12042 { 12043 "datasource": { 12044 "uid": "${DS_PROMETHEUS}" 12045 }, 12046 "expr": "synapse_admin_mau_current_mau_by_service{instance=\"$instance\"}", 12047 "interval": "", 12048 "legendFormat": "{{ app_service }}", 12049 "refId": "A" 12050 } 12051 ], 12052 "thresholds": [], 12053 "timeRegions": [], 12054 "title": "MAU by Appservice", 12055 "tooltip": { 12056 "shared": true, 12057 "sort": 0, 12058 "value_type": "individual" 12059 }, 12060 "type": "graph", 12061 "xaxis": { 12062 "mode": "time", 12063 "show": true, 12064 "values": [] 12065 }, 12066 "yaxes": [ 12067 { 12068 "format": "short", 12069 "logBase": 1, 12070 "show": true 12071 }, 12072 { 12073 "format": "short", 12074 "logBase": 1, 12075 "show": true 12076 } 12077 ], 12078 "yaxis": { 12079 "align": false 12080 } 12081 } 12082 ], 12083 "targets": [ 12084 { 12085 "datasource": { 12086 "type": "prometheus", 12087 "uid": "${DS_PROMETHEUS}" 12088 }, 12089 "refId": "A" 12090 } 12091 ], 12092 "title": "MAU", 12093 "type": "row" 12094 }, 12095 { 12096 "collapsed": true, 12097 "datasource": { 12098 "type": "prometheus", 12099 "uid": "${DS_PROMETHEUS}" 12100 }, 12101 "gridPos": { 12102 "h": 1, 12103 "w": 24, 12104 "x": 0, 12105 "y": 43 12106 }, 12107 "id": 177, 12108 "panels": [ 12109 { 12110 "aliasColors": {}, 12111 "bars": false, 12112 "dashLength": 10, 12113 "dashes": false, 12114 "datasource": { 12115 "type": "prometheus", 12116 "uid": "${DS_PROMETHEUS}" 12117 }, 12118 "fieldConfig": { 12119 "defaults": { 12120 "links": [] 12121 }, 12122 "overrides": [] 12123 }, 12124 "fill": 1, 12125 "fillGradient": 0, 12126 "gridPos": { 12127 "h": 7, 12128 "w": 12, 12129 "x": 0, 12130 "y": 44 12131 }, 12132 "hiddenSeries": false, 12133 "id": 173, 12134 "legend": { 12135 "avg": false, 12136 "current": false, 12137 "max": false, 12138 "min": false, 12139 "show": true, 12140 "total": false, 12141 "values": false 12142 }, 12143 "lines": true, 12144 "linewidth": 1, 12145 "links": [], 12146 "nullPointMode": "null", 12147 "options": { 12148 "alertThreshold": true 12149 }, 12150 "percentage": false, 12151 "pluginVersion": "9.2.2", 12152 "pointradius": 5, 12153 "points": false, 12154 "renderer": "flot", 12155 "seriesOverrides": [], 12156 "spaceLength": 10, 12157 "stack": false, 12158 "steppedLine": false, 12159 "targets": [ 12160 { 12161 "datasource": { 12162 "uid": "${DS_PROMETHEUS}" 12163 }, 12164 "editorMode": "code", 12165 "expr": "rate(synapse_notifier_users_woken_by_stream_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 12166 "format": "time_series", 12167 "hide": false, 12168 "intervalFactor": 2, 12169 "legendFormat": "{{stream}} {{index}}", 12170 "metric": "synapse_notifier", 12171 "range": true, 12172 "refId": "A", 12173 "step": 2 12174 } 12175 ], 12176 "thresholds": [], 12177 "timeRegions": [], 12178 "title": "Notifier Streams Woken", 12179 "tooltip": { 12180 "shared": true, 12181 "sort": 2, 12182 "value_type": "individual" 12183 }, 12184 "type": "graph", 12185 "xaxis": { 12186 "mode": "time", 12187 "show": true, 12188 "values": [] 12189 }, 12190 "yaxes": [ 12191 { 12192 "$$hashKey": "object:734", 12193 "format": "hertz", 12194 "logBase": 1, 12195 "show": true 12196 }, 12197 { 12198 "$$hashKey": "object:735", 12199 "format": "short", 12200 "logBase": 1, 12201 "show": true 12202 } 12203 ], 12204 "yaxis": { 12205 "align": false 12206 } 12207 }, 12208 { 12209 "aliasColors": {}, 12210 "bars": false, 12211 "dashLength": 10, 12212 "dashes": false, 12213 "datasource": { 12214 "type": "prometheus", 12215 "uid": "${DS_PROMETHEUS}" 12216 }, 12217 "fieldConfig": { 12218 "defaults": { 12219 "links": [] 12220 }, 12221 "overrides": [] 12222 }, 12223 "fill": 1, 12224 "fillGradient": 0, 12225 "gridPos": { 12226 "h": 7, 12227 "w": 12, 12228 "x": 12, 12229 "y": 44 12230 }, 12231 "hiddenSeries": false, 12232 "id": 175, 12233 "legend": { 12234 "avg": false, 12235 "current": false, 12236 "max": false, 12237 "min": false, 12238 "show": true, 12239 "total": false, 12240 "values": false 12241 }, 12242 "lines": true, 12243 "linewidth": 1, 12244 "links": [], 12245 "nullPointMode": "null", 12246 "options": { 12247 "alertThreshold": true 12248 }, 12249 "percentage": false, 12250 "pluginVersion": "9.2.2", 12251 "pointradius": 5, 12252 "points": false, 12253 "renderer": "flot", 12254 "seriesOverrides": [], 12255 "spaceLength": 10, 12256 "stack": false, 12257 "steppedLine": false, 12258 "targets": [ 12259 { 12260 "datasource": { 12261 "uid": "${DS_PROMETHEUS}" 12262 }, 12263 "editorMode": "code", 12264 "expr": "rate(synapse_handler_presence_get_updates_total{job=~\"$job\",instance=\"$instance\"}[$bucket_size])", 12265 "format": "time_series", 12266 "interval": "", 12267 "intervalFactor": 2, 12268 "legendFormat": "{{type}} {{index}}", 12269 "range": true, 12270 "refId": "A", 12271 "step": 2 12272 } 12273 ], 12274 "thresholds": [], 12275 "timeRegions": [], 12276 "title": "Presence Stream Fetch Type Rates", 12277 "tooltip": { 12278 "shared": true, 12279 "sort": 2, 12280 "value_type": "individual" 12281 }, 12282 "type": "graph", 12283 "xaxis": { 12284 "mode": "time", 12285 "show": true, 12286 "values": [] 12287 }, 12288 "yaxes": [ 12289 { 12290 "$$hashKey": "object:819", 12291 "format": "hertz", 12292 "logBase": 1, 12293 "min": "0", 12294 "show": true 12295 }, 12296 { 12297 "$$hashKey": "object:820", 12298 "format": "short", 12299 "logBase": 1, 12300 "show": true 12301 } 12302 ], 12303 "yaxis": { 12304 "align": false 12305 } 12306 } 12307 ], 12308 "targets": [ 12309 { 12310 "datasource": { 12311 "type": "prometheus", 12312 "uid": "${DS_PROMETHEUS}" 12313 }, 12314 "refId": "A" 12315 } 12316 ], 12317 "title": "Notifier", 12318 "type": "row" 12319 }, 12320 { 12321 "collapsed": true, 12322 "datasource": { 12323 "type": "prometheus", 12324 "uid": "${DS_PROMETHEUS}" 12325 }, 12326 "gridPos": { 12327 "h": 1, 12328 "w": 24, 12329 "x": 0, 12330 "y": 44 12331 }, 12332 "id": 170, 12333 "panels": [ 12334 { 12335 "aliasColors": {}, 12336 "bars": false, 12337 "dashLength": 10, 12338 "dashes": false, 12339 "datasource": { 12340 "type": "prometheus", 12341 "uid": "${DS_PROMETHEUS}" 12342 }, 12343 "fill": 1, 12344 "fillGradient": 0, 12345 "gridPos": { 12346 "h": 8, 12347 "w": 12, 12348 "x": 0, 12349 "y": 45 12350 }, 12351 "hiddenSeries": false, 12352 "id": 168, 12353 "legend": { 12354 "avg": false, 12355 "current": false, 12356 "max": false, 12357 "min": false, 12358 "show": true, 12359 "total": false, 12360 "values": false 12361 }, 12362 "lines": true, 12363 "linewidth": 1, 12364 "nullPointMode": "null", 12365 "options": { 12366 "alertThreshold": true 12367 }, 12368 "percentage": false, 12369 "pluginVersion": "9.2.2", 12370 "pointradius": 2, 12371 "points": false, 12372 "renderer": "flot", 12373 "seriesOverrides": [], 12374 "spaceLength": 10, 12375 "stack": false, 12376 "steppedLine": false, 12377 "targets": [ 12378 { 12379 "datasource": { 12380 "uid": "${DS_PROMETHEUS}" 12381 }, 12382 "editorMode": "code", 12383 "expr": "rate(synapse_appservice_api_sent_events_total{instance=\"$instance\"}[$bucket_size])", 12384 "interval": "", 12385 "legendFormat": "{{service}}", 12386 "range": true, 12387 "refId": "A" 12388 } 12389 ], 12390 "thresholds": [], 12391 "timeRegions": [], 12392 "title": "Sent Events rate", 12393 "tooltip": { 12394 "shared": true, 12395 "sort": 0, 12396 "value_type": "individual" 12397 }, 12398 "type": "graph", 12399 "xaxis": { 12400 "mode": "time", 12401 "show": true, 12402 "values": [] 12403 }, 12404 "yaxes": [ 12405 { 12406 "$$hashKey": "object:177", 12407 "format": "hertz", 12408 "logBase": 1, 12409 "show": true 12410 }, 12411 { 12412 "$$hashKey": "object:178", 12413 "format": "short", 12414 "logBase": 1, 12415 "show": true 12416 } 12417 ], 12418 "yaxis": { 12419 "align": false 12420 } 12421 }, 12422 { 12423 "aliasColors": {}, 12424 "bars": false, 12425 "dashLength": 10, 12426 "dashes": false, 12427 "datasource": { 12428 "type": "prometheus", 12429 "uid": "${DS_PROMETHEUS}" 12430 }, 12431 "fill": 1, 12432 "fillGradient": 0, 12433 "gridPos": { 12434 "h": 8, 12435 "w": 12, 12436 "x": 12, 12437 "y": 45 12438 }, 12439 "hiddenSeries": false, 12440 "id": 171, 12441 "legend": { 12442 "avg": false, 12443 "current": false, 12444 "max": false, 12445 "min": false, 12446 "show": true, 12447 "total": false, 12448 "values": false 12449 }, 12450 "lines": true, 12451 "linewidth": 1, 12452 "nullPointMode": "null", 12453 "options": { 12454 "alertThreshold": true 12455 }, 12456 "percentage": false, 12457 "pluginVersion": "9.2.2", 12458 "pointradius": 2, 12459 "points": false, 12460 "renderer": "flot", 12461 "seriesOverrides": [], 12462 "spaceLength": 10, 12463 "stack": false, 12464 "steppedLine": false, 12465 "targets": [ 12466 { 12467 "datasource": { 12468 "uid": "${DS_PROMETHEUS}" 12469 }, 12470 "editorMode": "code", 12471 "expr": "rate(synapse_appservice_api_sent_transactions_total{instance=\"$instance\"}[$bucket_size])", 12472 "interval": "", 12473 "legendFormat": "{{exported_service }} {{ service }}", 12474 "range": true, 12475 "refId": "A" 12476 } 12477 ], 12478 "thresholds": [], 12479 "timeRegions": [], 12480 "title": "Transactions rate", 12481 "tooltip": { 12482 "shared": true, 12483 "sort": 0, 12484 "value_type": "individual" 12485 }, 12486 "type": "graph", 12487 "xaxis": { 12488 "mode": "time", 12489 "show": true, 12490 "values": [] 12491 }, 12492 "yaxes": [ 12493 { 12494 "$$hashKey": "object:260", 12495 "format": "hertz", 12496 "logBase": 1, 12497 "show": true 12498 }, 12499 { 12500 "$$hashKey": "object:261", 12501 "format": "short", 12502 "logBase": 1, 12503 "show": true 12504 } 12505 ], 12506 "yaxis": { 12507 "align": false 12508 } 12509 } 12510 ], 12511 "targets": [ 12512 { 12513 "datasource": { 12514 "type": "prometheus", 12515 "uid": "${DS_PROMETHEUS}" 12516 }, 12517 "refId": "A" 12518 } 12519 ], 12520 "title": "Appservices", 12521 "type": "row" 12522 }, 12523 { 12524 "collapsed": true, 12525 "datasource": { 12526 "type": "prometheus", 12527 "uid": "${DS_PROMETHEUS}" 12528 }, 12529 "gridPos": { 12530 "h": 1, 12531 "w": 24, 12532 "x": 0, 12533 "y": 45 12534 }, 12535 "id": 188, 12536 "panels": [ 12537 { 12538 "aliasColors": {}, 12539 "bars": false, 12540 "dashLength": 10, 12541 "dashes": false, 12542 "datasource": { 12543 "uid": "${DS_PROMETHEUS}" 12544 }, 12545 "fill": 1, 12546 "fillGradient": 0, 12547 "gridPos": { 12548 "h": 8, 12549 "w": 12, 12550 "x": 0, 12551 "y": 46 12552 }, 12553 "hiddenSeries": false, 12554 "id": 182, 12555 "legend": { 12556 "avg": false, 12557 "current": false, 12558 "max": false, 12559 "min": false, 12560 "show": true, 12561 "total": false, 12562 "values": false 12563 }, 12564 "lines": true, 12565 "linewidth": 1, 12566 "nullPointMode": "null", 12567 "options": { 12568 "alertThreshold": true 12569 }, 12570 "percentage": false, 12571 "pluginVersion": "9.2.2", 12572 "pointradius": 2, 12573 "points": false, 12574 "renderer": "flot", 12575 "seriesOverrides": [], 12576 "spaceLength": 10, 12577 "stack": false, 12578 "steppedLine": false, 12579 "targets": [ 12580 { 12581 "datasource": { 12582 "uid": "${DS_PROMETHEUS}" 12583 }, 12584 "expr": "rate(synapse_handler_presence_notified_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 12585 "interval": "", 12586 "legendFormat": "Notified", 12587 "refId": "A" 12588 }, 12589 { 12590 "datasource": { 12591 "uid": "${DS_PROMETHEUS}" 12592 }, 12593 "expr": "rate(synapse_handler_presence_federation_presence_out_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 12594 "interval": "", 12595 "legendFormat": "Remote ping", 12596 "refId": "B" 12597 }, 12598 { 12599 "datasource": { 12600 "uid": "${DS_PROMETHEUS}" 12601 }, 12602 "expr": "rate(synapse_handler_presence_presence_updates_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 12603 "interval": "", 12604 "legendFormat": "Total updates", 12605 "refId": "C" 12606 }, 12607 { 12608 "datasource": { 12609 "uid": "${DS_PROMETHEUS}" 12610 }, 12611 "expr": "rate(synapse_handler_presence_federation_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 12612 "interval": "", 12613 "legendFormat": "Remote updates", 12614 "refId": "D" 12615 }, 12616 { 12617 "datasource": { 12618 "uid": "${DS_PROMETHEUS}" 12619 }, 12620 "expr": "rate(synapse_handler_presence_bump_active_time_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 12621 "interval": "", 12622 "legendFormat": "Bump active time", 12623 "refId": "E" 12624 } 12625 ], 12626 "thresholds": [], 12627 "timeRegions": [], 12628 "title": "Presence", 12629 "tooltip": { 12630 "shared": true, 12631 "sort": 2, 12632 "value_type": "individual" 12633 }, 12634 "type": "graph", 12635 "xaxis": { 12636 "mode": "time", 12637 "show": true, 12638 "values": [] 12639 }, 12640 "yaxes": [ 12641 { 12642 "format": "hertz", 12643 "logBase": 1, 12644 "show": true 12645 }, 12646 { 12647 "format": "short", 12648 "logBase": 1, 12649 "show": true 12650 } 12651 ], 12652 "yaxis": { 12653 "align": false 12654 } 12655 }, 12656 { 12657 "aliasColors": {}, 12658 "bars": false, 12659 "dashLength": 10, 12660 "dashes": false, 12661 "datasource": { 12662 "type": "prometheus", 12663 "uid": "${DS_PROMETHEUS}" 12664 }, 12665 "fill": 1, 12666 "fillGradient": 0, 12667 "gridPos": { 12668 "h": 8, 12669 "w": 12, 12670 "x": 12, 12671 "y": 46 12672 }, 12673 "hiddenSeries": false, 12674 "id": 184, 12675 "legend": { 12676 "avg": false, 12677 "current": false, 12678 "max": false, 12679 "min": false, 12680 "show": true, 12681 "total": false, 12682 "values": false 12683 }, 12684 "lines": true, 12685 "linewidth": 1, 12686 "nullPointMode": "null", 12687 "options": { 12688 "alertThreshold": true 12689 }, 12690 "percentage": false, 12691 "pluginVersion": "9.2.2", 12692 "pointradius": 2, 12693 "points": false, 12694 "renderer": "flot", 12695 "seriesOverrides": [], 12696 "spaceLength": 10, 12697 "stack": false, 12698 "steppedLine": false, 12699 "targets": [ 12700 { 12701 "datasource": { 12702 "uid": "${DS_PROMETHEUS}" 12703 }, 12704 "editorMode": "code", 12705 "expr": "rate(synapse_handler_presence_state_transition_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 12706 "interval": "", 12707 "legendFormat": "{{from}} -> {{to}}", 12708 "range": true, 12709 "refId": "A" 12710 } 12711 ], 12712 "thresholds": [], 12713 "timeRegions": [], 12714 "title": "Presence state transitions", 12715 "tooltip": { 12716 "shared": true, 12717 "sort": 2, 12718 "value_type": "individual" 12719 }, 12720 "type": "graph", 12721 "xaxis": { 12722 "mode": "time", 12723 "show": true, 12724 "values": [] 12725 }, 12726 "yaxes": [ 12727 { 12728 "$$hashKey": "object:1090", 12729 "format": "hertz", 12730 "logBase": 1, 12731 "show": true 12732 }, 12733 { 12734 "$$hashKey": "object:1091", 12735 "format": "short", 12736 "logBase": 1, 12737 "show": true 12738 } 12739 ], 12740 "yaxis": { 12741 "align": false 12742 } 12743 }, 12744 { 12745 "aliasColors": {}, 12746 "bars": false, 12747 "dashLength": 10, 12748 "dashes": false, 12749 "datasource": { 12750 "type": "prometheus", 12751 "uid": "${DS_PROMETHEUS}" 12752 }, 12753 "fill": 1, 12754 "fillGradient": 0, 12755 "gridPos": { 12756 "h": 8, 12757 "w": 12, 12758 "x": 0, 12759 "y": 54 12760 }, 12761 "hiddenSeries": false, 12762 "id": 186, 12763 "legend": { 12764 "avg": false, 12765 "current": false, 12766 "max": false, 12767 "min": false, 12768 "show": true, 12769 "total": false, 12770 "values": false 12771 }, 12772 "lines": true, 12773 "linewidth": 1, 12774 "nullPointMode": "null", 12775 "options": { 12776 "alertThreshold": true 12777 }, 12778 "percentage": false, 12779 "pluginVersion": "9.2.2", 12780 "pointradius": 2, 12781 "points": false, 12782 "renderer": "flot", 12783 "seriesOverrides": [], 12784 "spaceLength": 10, 12785 "stack": false, 12786 "steppedLine": false, 12787 "targets": [ 12788 { 12789 "datasource": { 12790 "uid": "${DS_PROMETHEUS}" 12791 }, 12792 "editorMode": "code", 12793 "expr": "rate(synapse_handler_presence_notify_reason_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", 12794 "interval": "", 12795 "legendFormat": "{{reason}}", 12796 "range": true, 12797 "refId": "A" 12798 } 12799 ], 12800 "thresholds": [], 12801 "timeRegions": [], 12802 "title": "Presence notify reason", 12803 "tooltip": { 12804 "shared": true, 12805 "sort": 2, 12806 "value_type": "individual" 12807 }, 12808 "type": "graph", 12809 "xaxis": { 12810 "mode": "time", 12811 "show": true, 12812 "values": [] 12813 }, 12814 "yaxes": [ 12815 { 12816 "$$hashKey": "object:165", 12817 "format": "hertz", 12818 "logBase": 1, 12819 "show": true 12820 }, 12821 { 12822 "$$hashKey": "object:166", 12823 "format": "short", 12824 "logBase": 1, 12825 "show": true 12826 } 12827 ], 12828 "yaxis": { 12829 "align": false 12830 } 12831 } 12832 ], 12833 "targets": [ 12834 { 12835 "datasource": { 12836 "type": "prometheus", 12837 "uid": "${DS_PROMETHEUS}" 12838 }, 12839 "refId": "A" 12840 } 12841 ], 12842 "title": "Presence", 12843 "type": "row" 12844 }, 12845 { 12846 "collapsed": true, 12847 "datasource": { 12848 "type": "prometheus", 12849 "uid": "${DS_PROMETHEUS}" 12850 }, 12851 "gridPos": { 12852 "h": 1, 12853 "w": 24, 12854 "x": 0, 12855 "y": 46 12856 }, 12857 "id": 197, 12858 "panels": [ 12859 { 12860 "datasource": { 12861 "type": "prometheus", 12862 "uid": "${DS_PROMETHEUS}" 12863 }, 12864 "fieldConfig": { 12865 "defaults": { 12866 "color": { 12867 "mode": "palette-classic" 12868 }, 12869 "custom": { 12870 "axisCenteredZero": false, 12871 "axisColorMode": "text", 12872 "axisLabel": "", 12873 "axisPlacement": "auto", 12874 "barAlignment": 0, 12875 "drawStyle": "line", 12876 "fillOpacity": 10, 12877 "gradientMode": "none", 12878 "hideFrom": { 12879 "legend": false, 12880 "tooltip": false, 12881 "viz": false 12882 }, 12883 "lineInterpolation": "linear", 12884 "lineWidth": 1, 12885 "pointSize": 5, 12886 "scaleDistribution": { 12887 "type": "linear" 12888 }, 12889 "showPoints": "never", 12890 "spanNulls": false, 12891 "stacking": { 12892 "group": "A", 12893 "mode": "none" 12894 }, 12895 "thresholdsStyle": { 12896 "mode": "off" 12897 } 12898 }, 12899 "mappings": [], 12900 "thresholds": { 12901 "mode": "absolute", 12902 "steps": [ 12903 { 12904 "color": "green" 12905 }, 12906 { 12907 "color": "red", 12908 "value": 80 12909 } 12910 ] 12911 }, 12912 "unit": "hertz" 12913 }, 12914 "overrides": [] 12915 }, 12916 "gridPos": { 12917 "h": 8, 12918 "w": 12, 12919 "x": 0, 12920 "y": 47 12921 }, 12922 "id": 191, 12923 "options": { 12924 "legend": { 12925 "calcs": [], 12926 "displayMode": "list", 12927 "placement": "bottom", 12928 "showLegend": true 12929 }, 12930 "tooltip": { 12931 "mode": "multi", 12932 "sort": "desc" 12933 } 12934 }, 12935 "pluginVersion": "9.0.4", 12936 "targets": [ 12937 { 12938 "datasource": { 12939 "uid": "${DS_PROMETHEUS}" 12940 }, 12941 "editorMode": "code", 12942 "expr": "rate(synapse_external_cache_set{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])", 12943 "interval": "", 12944 "legendFormat": "{{ cache_name }} {{job}}-{{ index }}", 12945 "range": true, 12946 "refId": "A" 12947 } 12948 ], 12949 "title": "External Cache Set Rate", 12950 "type": "timeseries" 12951 }, 12952 { 12953 "aliasColors": {}, 12954 "bars": false, 12955 "dashLength": 10, 12956 "dashes": false, 12957 "datasource": { 12958 "type": "prometheus", 12959 "uid": "${DS_PROMETHEUS}" 12960 }, 12961 "description": "", 12962 "fill": 1, 12963 "fillGradient": 0, 12964 "gridPos": { 12965 "h": 8, 12966 "w": 12, 12967 "x": 12, 12968 "y": 47 12969 }, 12970 "hiddenSeries": false, 12971 "id": 193, 12972 "legend": { 12973 "avg": false, 12974 "current": false, 12975 "max": false, 12976 "min": false, 12977 "show": true, 12978 "total": false, 12979 "values": false 12980 }, 12981 "lines": true, 12982 "linewidth": 1, 12983 "nullPointMode": "null", 12984 "options": { 12985 "alertThreshold": true 12986 }, 12987 "percentage": false, 12988 "pluginVersion": "9.2.2", 12989 "pointradius": 2, 12990 "points": false, 12991 "renderer": "flot", 12992 "seriesOverrides": [], 12993 "spaceLength": 10, 12994 "stack": false, 12995 "steppedLine": false, 12996 "targets": [ 12997 { 12998 "datasource": { 12999 "uid": "${DS_PROMETHEUS}" 13000 }, 13001 "editorMode": "code", 13002 "expr": "sum without (hit) (rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size]))", 13003 "interval": "", 13004 "legendFormat": "{{ cache_name }} {{job}}-{{ index }}", 13005 "range": true, 13006 "refId": "A" 13007 } 13008 ], 13009 "thresholds": [], 13010 "timeRegions": [], 13011 "title": "External Cache Get Rate", 13012 "tooltip": { 13013 "shared": true, 13014 "sort": 2, 13015 "value_type": "individual" 13016 }, 13017 "type": "graph", 13018 "xaxis": { 13019 "mode": "time", 13020 "show": true, 13021 "values": [] 13022 }, 13023 "yaxes": [ 13024 { 13025 "$$hashKey": "object:390", 13026 "format": "hertz", 13027 "logBase": 1, 13028 "show": true 13029 }, 13030 { 13031 "$$hashKey": "object:391", 13032 "format": "short", 13033 "logBase": 1, 13034 "show": true 13035 } 13036 ], 13037 "yaxis": { 13038 "align": false 13039 } 13040 }, 13041 { 13042 "cards": { 13043 "cardPadding": -1 13044 }, 13045 "color": { 13046 "cardColor": "#b4ff00", 13047 "colorScale": "sqrt", 13048 "colorScheme": "interpolateInferno", 13049 "exponent": 0.5, 13050 "min": 0, 13051 "mode": "spectrum" 13052 }, 13053 "dataFormat": "tsbuckets", 13054 "datasource": { 13055 "type": "prometheus", 13056 "uid": "${DS_PROMETHEUS}" 13057 }, 13058 "fieldConfig": { 13059 "defaults": { 13060 "custom": { 13061 "hideFrom": { 13062 "legend": false, 13063 "tooltip": false, 13064 "viz": false 13065 }, 13066 "scaleDistribution": { 13067 "type": "linear" 13068 } 13069 } 13070 }, 13071 "overrides": [] 13072 }, 13073 "gridPos": { 13074 "h": 8, 13075 "w": 12, 13076 "x": 0, 13077 "y": 55 13078 }, 13079 "heatmap": {}, 13080 "hideZeroBuckets": false, 13081 "highlightCards": true, 13082 "id": 195, 13083 "legend": { 13084 "show": false 13085 }, 13086 "links": [], 13087 "options": { 13088 "calculate": false, 13089 "calculation": {}, 13090 "cellGap": -1, 13091 "cellValues": { 13092 "decimals": 2 13093 }, 13094 "color": { 13095 "exponent": 0.5, 13096 "fill": "#b4ff00", 13097 "min": 0, 13098 "mode": "scheme", 13099 "reverse": false, 13100 "scale": "exponential", 13101 "scheme": "Inferno", 13102 "steps": 128 13103 }, 13104 "exemplars": { 13105 "color": "rgba(255,0,255,0.7)" 13106 }, 13107 "filterValues": { 13108 "le": 1e-9 13109 }, 13110 "legend": { 13111 "show": false 13112 }, 13113 "rowsFrame": { 13114 "layout": "auto" 13115 }, 13116 "showValue": "never", 13117 "tooltip": { 13118 "show": true, 13119 "yHistogram": true 13120 }, 13121 "yAxis": { 13122 "axisPlacement": "left", 13123 "decimals": 0, 13124 "reverse": false, 13125 "unit": "s" 13126 } 13127 }, 13128 "pluginVersion": "9.2.2", 13129 "reverseYBuckets": false, 13130 "targets": [ 13131 { 13132 "datasource": { 13133 "uid": "${DS_PROMETHEUS}" 13134 }, 13135 "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])) by (le)", 13136 "format": "heatmap", 13137 "instant": false, 13138 "interval": "", 13139 "intervalFactor": 1, 13140 "legendFormat": "{{le}}", 13141 "refId": "A" 13142 } 13143 ], 13144 "title": "External Cache Response Time", 13145 "tooltip": { 13146 "show": true, 13147 "showHistogram": true 13148 }, 13149 "tooltipDecimals": 2, 13150 "type": "heatmap", 13151 "xAxis": { 13152 "show": true 13153 }, 13154 "yAxis": { 13155 "decimals": 0, 13156 "format": "s", 13157 "logBase": 1, 13158 "show": true 13159 }, 13160 "yBucketBound": "auto" 13161 }, 13162 { 13163 "datasource": { 13164 "type": "prometheus", 13165 "uid": "${DS_PROMETHEUS}" 13166 }, 13167 "description": "", 13168 "fieldConfig": { 13169 "defaults": { 13170 "color": { 13171 "mode": "palette-classic" 13172 }, 13173 "custom": { 13174 "axisCenteredZero": false, 13175 "axisColorMode": "text", 13176 "axisLabel": "", 13177 "axisPlacement": "auto", 13178 "barAlignment": 0, 13179 "drawStyle": "line", 13180 "fillOpacity": 10, 13181 "gradientMode": "none", 13182 "hideFrom": { 13183 "legend": false, 13184 "tooltip": false, 13185 "viz": false 13186 }, 13187 "lineInterpolation": "linear", 13188 "lineWidth": 1, 13189 "pointSize": 5, 13190 "scaleDistribution": { 13191 "type": "linear" 13192 }, 13193 "showPoints": "never", 13194 "spanNulls": false, 13195 "stacking": { 13196 "group": "A", 13197 "mode": "none" 13198 }, 13199 "thresholdsStyle": { 13200 "mode": "off" 13201 } 13202 }, 13203 "mappings": [], 13204 "thresholds": { 13205 "mode": "absolute", 13206 "steps": [ 13207 { 13208 "color": "green" 13209 } 13210 ] 13211 }, 13212 "unit": "hertz" 13213 }, 13214 "overrides": [] 13215 }, 13216 "gridPos": { 13217 "h": 8, 13218 "w": 12, 13219 "x": 12, 13220 "y": 55 13221 }, 13222 "id": 223, 13223 "options": { 13224 "legend": { 13225 "calcs": [], 13226 "displayMode": "list", 13227 "placement": "bottom", 13228 "showLegend": true 13229 }, 13230 "tooltip": { 13231 "mode": "multi", 13232 "sort": "desc" 13233 } 13234 }, 13235 "pluginVersion": "9.0.4", 13236 "targets": [ 13237 { 13238 "datasource": { 13239 "uid": "${DS_PROMETHEUS}" 13240 }, 13241 "editorMode": "code", 13242 "expr": "rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\", hit=\"False\"}[$bucket_size])", 13243 "interval": "", 13244 "legendFormat": "{{ cache_name }} {{job}}-{{ index }}", 13245 "range": true, 13246 "refId": "A" 13247 } 13248 ], 13249 "title": "External Cache Miss Rate", 13250 "type": "timeseries" 13251 } 13252 ], 13253 "targets": [ 13254 { 13255 "datasource": { 13256 "type": "prometheus", 13257 "uid": "${DS_PROMETHEUS}" 13258 }, 13259 "refId": "A" 13260 } 13261 ], 13262 "title": "External Cache", 13263 "type": "row" 13264 } 13265 ], 13266 "refresh": false, 13267 "schemaVersion": 37, 13268 "style": "dark", 13269 "tags": [ 13270 "matrix" 13271 ], 13272 "templating": { 13273 "list": [ 13274 { 13275 "current": { 13276 "selected": false, 13277 "text": "default", 13278 "value": "default" 13279 }, 13280 "hide": 0, 13281 "includeAll": false, 13282 "multi": false, 13283 "name": "DS_PROMETHEUS", 13284 "label": "Datasource", 13285 "options": [], 13286 "query": "prometheus", 13287 "queryValue": "", 13288 "refresh": 1, 13289 "regex": "", 13290 "skipUrlSync": false, 13291 "type": "datasource" 13292 }, 13293 { 13294 "allFormat": "glob", 13295 "auto": true, 13296 "auto_count": 100, 13297 "auto_min": "30s", 13298 "current": { 13299 "selected": false, 13300 "text": "auto", 13301 "value": "$__auto_interval_bucket_size" 13302 }, 13303 "hide": 0, 13304 "includeAll": false, 13305 "label": "Bucket Size", 13306 "multi": false, 13307 "multiFormat": "glob", 13308 "name": "bucket_size", 13309 "options": [ 13310 { 13311 "selected": true, 13312 "text": "auto", 13313 "value": "$__auto_interval_bucket_size" 13314 }, 13315 { 13316 "selected": false, 13317 "text": "30s", 13318 "value": "30s" 13319 }, 13320 { 13321 "selected": false, 13322 "text": "1m", 13323 "value": "1m" 13324 }, 13325 { 13326 "selected": false, 13327 "text": "2m", 13328 "value": "2m" 13329 }, 13330 { 13331 "selected": false, 13332 "text": "5m", 13333 "value": "5m" 13334 }, 13335 { 13336 "selected": false, 13337 "text": "10m", 13338 "value": "10m" 13339 }, 13340 { 13341 "selected": false, 13342 "text": "15m", 13343 "value": "15m" 13344 } 13345 ], 13346 "query": "30s,1m,2m,5m,10m,15m", 13347 "queryValue": "", 13348 "refresh": 2, 13349 "skipUrlSync": false, 13350 "type": "interval" 13351 }, 13352 { 13353 "current": {}, 13354 "datasource": { 13355 "uid": "${DS_PROMETHEUS}" 13356 }, 13357 "definition": "", 13358 "hide": 0, 13359 "includeAll": false, 13360 "multi": false, 13361 "name": "instance", 13362 "options": [], 13363 "query": { 13364 "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, instance)", 13365 "refId": "Prometheus-instance-Variable-Query" 13366 }, 13367 "refresh": 2, 13368 "regex": "", 13369 "skipUrlSync": false, 13370 "sort": 1, 13371 "tagValuesQuery": "", 13372 "tagsQuery": "", 13373 "type": "query", 13374 "useTags": false 13375 }, 13376 { 13377 "allFormat": "regex wildcard", 13378 "allValue": "", 13379 "current": {}, 13380 "datasource": { 13381 "uid": "${DS_PROMETHEUS}" 13382 }, 13383 "definition": "", 13384 "hide": 0, 13385 "hideLabel": false, 13386 "includeAll": true, 13387 "label": "Job", 13388 "multi": true, 13389 "multiFormat": "regex values", 13390 "name": "job", 13391 "options": [], 13392 "query": { 13393 "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, job)", 13394 "refId": "Prometheus-job-Variable-Query" 13395 }, 13396 "refresh": 2, 13397 "refresh_on_load": false, 13398 "regex": "", 13399 "skipUrlSync": false, 13400 "sort": 1, 13401 "tagValuesQuery": "", 13402 "tagsQuery": "", 13403 "type": "query", 13404 "useTags": false 13405 }, 13406 { 13407 "allFormat": "regex wildcard", 13408 "allValue": ".*", 13409 "current": {}, 13410 "datasource": { 13411 "uid": "${DS_PROMETHEUS}" 13412 }, 13413 "definition": "", 13414 "hide": 0, 13415 "hideLabel": false, 13416 "includeAll": true, 13417 "label": "", 13418 "multi": true, 13419 "multiFormat": "regex values", 13420 "name": "index", 13421 "options": [], 13422 "query": { 13423 "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, index)", 13424 "refId": "Prometheus-index-Variable-Query" 13425 }, 13426 "refresh": 2, 13427 "refresh_on_load": false, 13428 "regex": "", 13429 "skipUrlSync": false, 13430 "sort": 3, 13431 "tagValuesQuery": "", 13432 "tagsQuery": "", 13433 "type": "query", 13434 "useTags": false 13435 } 13436 ] 13437 }, 13438 "time": { 13439 "from": "now-3h", 13440 "to": "now" 13441 }, 13442 "timepicker": { 13443 "now": true, 13444 "refresh_intervals": [ 13445 "5s", 13446 "10s", 13447 "30s", 13448 "1m", 13449 "5m", 13450 "15m", 13451 "30m", 13452 "1h", 13453 "2h", 13454 "1d" 13455 ], 13456 "time_options": [ 13457 "5m", 13458 "15m", 13459 "1h", 13460 "6h", 13461 "12h", 13462 "24h", 13463 "2d", 13464 "7d", 13465 "30d" 13466 ] 13467 }, 13468 "timezone": "", 13469 "title": "Synapse", 13470 "uid": "000000012", 13471 "version": 160, 13472 "weekStart": "" 13473 }