commit 020142f7efaf752f9632a6a4e57d658cb102085b
parent 7e492128dc1090bd6e722f27d3dbd628b9ccdb40
Author: MTRNord <MTRNord@users.noreply.github.com>
Date: Sat, 4 Oct 2025 13:17:31 +0200
Meow
Diffstat:
7 files changed, 818 insertions(+), 271 deletions(-)
diff --git a/.gitleaksignore b/.gitleaksignore
@@ -0,0 +1 @@
+apps/talos_cluster/monitoring-stack/dashboards/connectivity-tester-dashboard.json:generic-api-key:328
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,5 +1,5 @@
repos:
- - repo: https://github.com/zricethezav/gitleaks
- rev: v8.16.1
+ - repo: https://github.com/gitleaks/gitleaks
+ rev: v8.28.0
hooks:
- - id: gitleaks
+ - id: gitleaks
diff --git a/apps/talos_cluster/mailserver/kustomization.yaml b/apps/talos_cluster/mailserver/kustomization.yaml
@@ -14,21 +14,21 @@ configMapGenerator:
- name: mta-sts-content
namespace: mailserver
files:
- - mta-sts.txt=mta-sts.txt
+ - mta-sts.txt=mta-sts.txt
options:
- labels:
- app: mta-sts-server
+ labels:
+ app: mta-sts-server
- name: mailman-nginx
namespace: mailserver
files:
- - nginx.conf=mailman-nginx.conf
+ - nginx.conf=mailman-nginx.conf
options:
- labels:
- app: mailman
+ labels:
+ app: mailman
- name: mailman-uwsgi
namespace: mailserver
files:
- - uwsgi.ini=mailman-uwsgi.ini
+ - uwsgi.ini=mailman-uwsgi.ini
options:
- labels:
- app: mailman
+ labels:
+ app: mailman
diff --git a/apps/talos_cluster/mailserver/mailman.yaml b/apps/talos_cluster/mailserver/mailman.yaml
@@ -1,4 +1,3 @@
----
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
@@ -48,306 +47,306 @@ spec:
hostAliases:
- ip: 0.0.0.0
hostnames:
- - mailman.mailserver.svc.cluster.local
+ - mailman.mailserver.svc.cluster.local
- ip: 127.0.0.1
hostnames:
- - mailman-web
+ - mailman-web
containers:
- name: core
image: ghcr.io/mtrnord/mailman-core:rolling
imagePullPolicy: Always
ports:
- - name: api
- containerPort: 8001
- - name: lmtp
- containerPort: 8024
+ - name: api
+ containerPort: 8001
+ - name: lmtp
+ containerPort: 8024
resources:
- requests:
- memory: "512Mi"
- cpu: "100m"
- limits:
- memory: "1Gi"
- cpu: "500m"
+ requests:
+ memory: "512Mi"
+ cpu: "100m"
+ limits:
+ memory: "1Gi"
+ cpu: "500m"
env:
- - name: DATABASE_URL
- valueFrom:
- secretKeyRef:
- name: mailmancore
- key: url
- - name: DATABASE_TYPE
- value: postgres
- - name: DATABASE_CLASS
- value: mailman.database.postgresql.PostgreSQLDatabase
- - name: HYPERKITTY_API_KEY
- valueFrom:
- secretKeyRef:
- name: mailman-passwords
- key: apikey
- - name: MAILMAN_REST_USER
- value: restadm
- - name: MAILMAN_REST_PASSWORD
- valueFrom:
- secretKeyRef:
- name: mailman-passwords
- key: restpassword
- - name: SMTP_HOST
- value: mail.midnightthoughts.space
- - name: SMTP_PORT
- value: "465"
- - name: SMTP_USE_SSL
- value: "true"
- - name: SMTP_SECURE_MODE
- value: "smtps"
- - name: SMTP_HOST_USER
- valueFrom:
- secretKeyRef:
- name: mailman-passwords
- key: smtp_user
- - name: SMTP_HOST_PASSWORD
- valueFrom:
- secretKeyRef:
- name: mailman-passwords
- key: smtp_password
- - name: MTA
- value: postfix
- - name: MM_HOSTNAME
- value: mailman.mailserver.svc.cluster.local
- - name: HYPERKITTY_URL
- value: http://localhost:8000/hyperkitty
+ - name: DATABASE_URL
+ valueFrom:
+ secretKeyRef:
+ name: mailmancore
+ key: url
+ - name: DATABASE_TYPE
+ value: postgres
+ - name: DATABASE_CLASS
+ value: mailman.database.postgresql.PostgreSQLDatabase
+ - name: HYPERKITTY_API_KEY
+ valueFrom:
+ secretKeyRef:
+ name: mailman-passwords
+ key: apikey
+ - name: MAILMAN_REST_USER
+ value: restadm
+ - name: MAILMAN_REST_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: mailman-passwords
+ key: restpassword
+ - name: SMTP_HOST
+ value: mail.midnightthoughts.space
+ - name: SMTP_PORT
+ value: "465"
+ - name: SMTP_USE_SSL
+ value: "true"
+ - name: SMTP_SECURE_MODE
+ value: "smtps"
+ - name: SMTP_HOST_USER
+ valueFrom:
+ secretKeyRef:
+ name: mailman-passwords
+ key: smtp_user
+ - name: SMTP_HOST_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: mailman-passwords
+ key: smtp_password
+ - name: MTA
+ value: postfix
+ - name: MM_HOSTNAME
+ value: mailman.mailserver.svc.cluster.local
+ - name: HYPERKITTY_URL
+ value: http://localhost:8000/hyperkitty
volumeMounts:
- - name: mailman-opt
- mountPath: /opt/mailman
- - name: mailman-extra
- mountPath: /opt/mailman/mailman-extra.cfg
- subPath: mailman-extra.cfg
- - name: mailman-extra
- mountPath: /usr/bin/chown
- subPath: chown
- - name: mailman-uwsgi
- mountPath: /opt/mailman-web/uwsgi.ini
- subPath: uwsgi.ini
+ - name: mailman-opt
+ mountPath: /opt/mailman
+ - name: mailman-extra
+ mountPath: /opt/mailman/mailman-extra.cfg
+ subPath: mailman-extra.cfg
+ - name: mailman-extra
+ mountPath: /usr/bin/chown
+ subPath: chown
+ - name: mailman-uwsgi
+ mountPath: /opt/mailman-web/uwsgi.ini
+ subPath: uwsgi.ini
- name: web
image: ghcr.io/mtrnord/mailman-web:rolling
imagePullPolicy: Always
ports:
- - name: http
- containerPort: 8000
- - name: uwsgi
- containerPort: 8080
+ - name: http
+ containerPort: 8000
+ - name: uwsgi
+ containerPort: 8080
resources:
- requests:
- memory: "768Mi"
- cpu: "200m"
- limits:
- memory: "2Gi"
- cpu: "1000m"
+ requests:
+ memory: "768Mi"
+ cpu: "200m"
+ limits:
+ memory: "2Gi"
+ cpu: "1000m"
#command:
# - tail
#args:
# - -f
# - /dev/null
env:
- - name: DATABASE_URL
- valueFrom:
- secretKeyRef:
- name: mailmanweb
- key: url
- - name: DATABASE_TYPE
- value: postgres
- - name: DATABASE_CLASS
- value: mailman.database.postgresql.PostgreSQLDatabase
- - name: HYPERKITTY_API_KEY
- valueFrom:
- secretKeyRef:
- name: mailman-passwords
- key: apikey
- - name: MAILMAN_REST_USER
- value: restadm
- - name: MAILMAN_REST_PASSWORD
- valueFrom:
- secretKeyRef:
- name: mailman-passwords
- key: restpassword
- - name: SECRET_KEY
- valueFrom:
- secretKeyRef:
- name: mailman-passwords
- key: websecretkey
- - name: POSTORIUS_TEMPLATE_BASE_URL
- value: http://localhost:8000/
- # serving static files by uwsgi
- - name: UWSGI_STATIC_MAP
- value: /static=/opt/mailman-web-data/static
- - name: MAILMAN_ADMIN_USER
- value: admin
- - name: MAILMAN_ADMIN_EMAIL
- value: mtrnord@nordgedanken.dev
- - name: MAILMAN_HOST_IP
- value: 127.0.0.1
- - name: MAILMAN_HOSTNAME
- value: localhost
- - name: SERVE_FROM_DOMAIN
- value: lists.midnightthoughts.space
- - name: MAILMAN_REST_URL # MAILMAN_REST_API_URL is set from this variable in settings.
- value: http://127.0.0.1:8001
- - name: MAILMAN_REST_API_USER
- value: restadm
- - name: MAILMAN_REST_API_PASS
- valueFrom:
- secretKeyRef:
- name: mailman-passwords
- key: restpassword
- - name: SMTP_HOST
- value: mail.midnightthoughts.space
- - name: SMTP_PORT
- value: "465"
- - name: SMTP_USE_SSL
- value: "true"
- - name: SMTP_HOST_USER
- valueFrom:
- secretKeyRef:
- name: mailman-passwords
- key: smtp_user
- - name: SMTP_HOST_PASSWORD
- valueFrom:
- secretKeyRef:
- name: mailman-passwords
- key: smtp_password
+ - name: DATABASE_URL
+ valueFrom:
+ secretKeyRef:
+ name: mailmanweb
+ key: url
+ - name: DATABASE_TYPE
+ value: postgres
+ - name: DATABASE_CLASS
+ value: mailman.database.postgresql.PostgreSQLDatabase
+ - name: HYPERKITTY_API_KEY
+ valueFrom:
+ secretKeyRef:
+ name: mailman-passwords
+ key: apikey
+ - name: MAILMAN_REST_USER
+ value: restadm
+ - name: MAILMAN_REST_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: mailman-passwords
+ key: restpassword
+ - name: SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ name: mailman-passwords
+ key: websecretkey
+ - name: POSTORIUS_TEMPLATE_BASE_URL
+ value: http://localhost:8000/
+ # serving static files by uwsgi
+ - name: UWSGI_STATIC_MAP
+ value: /static=/opt/mailman-web-data/static
+ - name: MAILMAN_ADMIN_USER
+ value: admin
+ - name: MAILMAN_ADMIN_EMAIL
+ value: mtrnord@nordgedanken.dev
+ - name: MAILMAN_HOST_IP
+ value: 127.0.0.1
+ - name: MAILMAN_HOSTNAME
+ value: localhost
+ - name: SERVE_FROM_DOMAIN
+ value: lists.midnightthoughts.space
+ - name: MAILMAN_REST_URL # MAILMAN_REST_API_URL is set from this variable in settings.
+ value: http://127.0.0.1:8001
+ - name: MAILMAN_REST_API_USER
+ value: restadm
+ - name: MAILMAN_REST_API_PASS
+ valueFrom:
+ secretKeyRef:
+ name: mailman-passwords
+ key: restpassword
+ - name: SMTP_HOST
+ value: mail.midnightthoughts.space
+ - name: SMTP_PORT
+ value: "465"
+ - name: SMTP_USE_SSL
+ value: "true"
+ - name: SMTP_HOST_USER
+ valueFrom:
+ secretKeyRef:
+ name: mailman-passwords
+ key: smtp_user
+ - name: SMTP_HOST_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: mailman-passwords
+ key: smtp_password
# otherwise django cannot find the mysql driver.
#- name: DYLD_LIBRARY_PATH
# value: /usr/local/mysql/lib/
volumeMounts:
- - name: mailman-web
- mountPath: /opt/mailman-web-data
- - name: mailman-extra
- mountPath: /opt/mailman-web-data/settings_local.py
- subPath: settings_local.py
- - name: mailman-extra
- mountPath: /usr/bin/chown
- subPath: chown
- - name: django-cache
- mountPath: /opt/mailman-web-data/diskcache
- - name: static-cache
- mountPath: /opt/mailman-web-data/static/CACHE
+ - name: mailman-web
+ mountPath: /opt/mailman-web-data
+ - name: mailman-extra
+ mountPath: /opt/mailman-web-data/settings_local.py
+ subPath: settings_local.py
+ - name: mailman-extra
+ mountPath: /usr/bin/chown
+ subPath: chown
+ - name: django-cache
+ mountPath: /opt/mailman-web-data/diskcache
+ - name: static-cache
+ mountPath: /opt/mailman-web-data/static/CACHE
- name: redis
image: redis:7-alpine
imagePullPolicy: IfNotPresent
ports:
- - name: redis
- containerPort: 6379
+ - name: redis
+ containerPort: 6379
command:
- - redis-server
- - --appendonly
- - "yes"
- - --maxmemory
- - "256mb"
- - --maxmemory-policy
- - "allkeys-lru"
+ - redis-server
+ - --appendonly
+ - "yes"
+ - --maxmemory
+ - "256mb"
+ - --maxmemory-policy
+ - "allkeys-lru"
resources:
- requests:
- memory: "128Mi"
- cpu: "50m"
- limits:
- memory: "512Mi"
- cpu: "200m"
+ requests:
+ memory: "128Mi"
+ cpu: "50m"
+ limits:
+ memory: "512Mi"
+ cpu: "200m"
livenessProbe:
- tcpSocket:
- port: 6379
- initialDelaySeconds: 30
- periodSeconds: 10
+ tcpSocket:
+ port: 6379
+ initialDelaySeconds: 30
+ periodSeconds: 10
readinessProbe:
- tcpSocket:
- port: 6379
- initialDelaySeconds: 5
- periodSeconds: 5
+ tcpSocket:
+ port: 6379
+ initialDelaySeconds: 5
+ periodSeconds: 5
volumeMounts:
- - name: redis-data
- mountPath: /data
+ - name: redis-data
+ mountPath: /data
- name: nginx
image: ghcr.io/nginx/nginx-unprivileged:stable-alpine
imagePullPolicy: IfNotPresent
ports:
- - name: http
- containerPort: 9090
+ - name: http
+ containerPort: 9090
resources:
- requests:
- memory: "128Mi"
- cpu: "50m"
- limits:
- memory: "512Mi"
- cpu: "200m"
+ requests:
+ memory: "128Mi"
+ cpu: "50m"
+ limits:
+ memory: "512Mi"
+ cpu: "200m"
startupProbe:
- httpGet:
- path: /mailman3/lists/
- port: 9090
- httpHeaders:
- - name: Host
- value: lists.midnightthoughts.space
- initialDelaySeconds: 30
- periodSeconds: 10
- timeoutSeconds: 5
- failureThreshold: 24
+ httpGet:
+ path: /mailman3/lists/
+ port: 9090
+ httpHeaders:
+ - name: Host
+ value: lists.midnightthoughts.space
+ initialDelaySeconds: 30
+ periodSeconds: 10
+ timeoutSeconds: 5
+ failureThreshold: 24
livenessProbe:
- httpGet:
- path: /mailman3/lists/
- port: 9090
- httpHeaders:
- - name: Host
- value: lists.midnightthoughts.space
- periodSeconds: 30
- timeoutSeconds: 5
- failureThreshold: 3
+ httpGet:
+ path: /mailman3/lists/
+ port: 9090
+ httpHeaders:
+ - name: Host
+ value: lists.midnightthoughts.space
+ periodSeconds: 30
+ timeoutSeconds: 5
+ failureThreshold: 3
readinessProbe:
- httpGet:
- path: /mailman3/lists/
- port: 9090
- httpHeaders:
- - name: Host
- value: lists.midnightthoughts.space
- periodSeconds: 10
- timeoutSeconds: 3
- failureThreshold: 3
+ httpGet:
+ path: /mailman3/lists/
+ port: 9090
+ httpHeaders:
+ - name: Host
+ value: lists.midnightthoughts.space
+ periodSeconds: 10
+ timeoutSeconds: 3
+ failureThreshold: 3
volumeMounts:
- - name: mailman-web
- mountPath: /opt/mailman-web-data
- readOnly: true
- - name: static-cache
- mountPath: /opt/mailman-web-data/static/CACHE
- readOnly: true
- - name: nginx-config
- mountPath: /etc/nginx/nginx.conf
- subPath: nginx.conf
- readOnly: true
- - name: tmp
- mountPath: /tmp
- readOnly: false
- - name: tmp-client-body
- mountPath: /tmp/nginx/client_body
- readOnly: false
- - name: tmp-cgi
- mountPath: /tmp/nginx/fastcgi
- readOnly: false
- - name: tmp-cgi
- mountPath: /tmp/nginx/uwsgi
- readOnly: false
- - name: tmp-cgi
- mountPath: /tmp/nginx/scgi
- readOnly: false
- - name: tmp-proxy
- mountPath: /tmp/nginx/proxy
- readOnly: false
+ - name: mailman-web
+ mountPath: /opt/mailman-web-data
+ readOnly: true
+ - name: static-cache
+ mountPath: /opt/mailman-web-data/static/CACHE
+ readOnly: true
+ - name: nginx-config
+ mountPath: /etc/nginx/nginx.conf
+ subPath: nginx.conf
+ readOnly: true
+ - name: tmp
+ mountPath: /tmp
+ readOnly: false
+ - name: tmp-client-body
+ mountPath: /tmp/nginx/client_body
+ readOnly: false
+ - name: tmp-cgi
+ mountPath: /tmp/nginx/fastcgi
+ readOnly: false
+ - name: tmp-cgi
+ mountPath: /tmp/nginx/uwsgi
+ readOnly: false
+ - name: tmp-cgi
+ mountPath: /tmp/nginx/scgi
+ readOnly: false
+ - name: tmp-proxy
+ mountPath: /tmp/nginx/proxy
+ readOnly: false
volumes:
- name: mailman-extra
secret:
- secretName: mailman-extra
- defaultMode: 0555
+ secretName: mailman-extra
+ defaultMode: 0555
- name: nginx-config
configMap:
- name: mailman-nginx
+ name: mailman-nginx
- name: mailman-uwsgi
configMap:
- name: mailman-uwsgi
+ name: mailman-uwsgi
- name: tmp
emptyDir: {}
- name: tmp-client-body
@@ -364,10 +363,10 @@ spec:
emptyDir: {}
- name: mailman-opt
persistentVolumeClaim:
- claimName: mailman-opt-mailman-0 # Existing PVC from StatefulSet
+ claimName: mailman-opt-mailman-0 # Existing PVC from StatefulSet
- name: mailman-web
persistentVolumeClaim:
- claimName: mailman-web-mailman-0 # Existing PVC from StatefulSet
+ claimName: mailman-web-mailman-0 # Existing PVC from StatefulSet
---
apiVersion: v1
kind: Service
@@ -405,8 +404,8 @@ spec:
- lists.midnightthoughts.space
rules:
- backendRefs:
- - name: mailman
- port: 9090
+ - name: mailman
+ port: 9090
timeouts:
- request: 240s
- backendRequest: 0s
+ request: 240s
+ backendRequest: 0s
diff --git a/apps/talos_cluster/monitoring-stack/dashboards/connectivity-tester-dashboard.json b/apps/talos_cluster/monitoring-stack/dashboards/connectivity-tester-dashboard.json
@@ -0,0 +1,542 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 40,
+ "links": [],
+ "panels": [
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 5,
+ "panels": [],
+ "title": "Alert Statistics",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "grafana-postgresql-datasource",
+ "uid": "postgres-connectivity-tester"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 0,
+ "y": 1
+ },
+ "id": 6,
+ "options": {
+ "colorMode": "background_solid",
+ "graphMode": "none",
+ "justifyMode": "center",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "12.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "grafana-postgresql-datasource",
+ "uid": "postgres-connectivity-tester"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "SELECT\n COUNT(*) AS value\nFROM\n alert\nWHERE\n is_currently_failing = true\n AND verified = true;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Active alerts",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "grafana-postgresql-datasource",
+ "uid": "postgres-connectivity-tester"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 3,
+ "y": 1
+ },
+ "id": 7,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "grafana-postgresql-datasource",
+ "uid": "postgres-connectivity-tester"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "SELECT\n $__timeGroup(sent_at, '1h') AS time,\n COUNT(*) FILTER (WHERE email_type = 'failure') AS failure_emails,\n COUNT(*) FILTER (WHERE email_type = 'recovery') AS recovery_emails,\n COUNT(*) AS total_emails\nFROM\n email_log\nWHERE\n $__timeFilter(sent_at)\nGROUP BY\n time\nORDER BY\n time ASC;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Email volume over time",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 9
+ },
+ "id": 2,
+ "panels": [],
+ "title": "Federation Statistics",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "grafana-postgresql-datasource",
+ "uid": "postgres-connectivity-tester"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "fillOpacity": 80,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineWidth": 1,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 10
+ },
+ "id": 1,
+ "options": {
+ "barRadius": 0,
+ "barWidth": 0.25,
+ "fullHighlight": false,
+ "groupWidth": 0.7,
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "orientation": "auto",
+ "showValue": "auto",
+ "stacking": "normal",
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ },
+ "xTickLabelRotation": 0,
+ "xTickLabelSpacing": 100
+ },
+ "pluginVersion": "12.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "grafana-postgresql-datasource",
+ "uid": "postgres-connectivity-tester"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "key": "Q-774d1582-40cf-46af-9f49-9b146da67c67-0",
+ "rawQuery": true,
+ "rawSql": "SELECT\n DATE_TRUNC('hour', ts) AS time,\n COUNT(*) FILTER (WHERE federation_ok = true) AS successful_checks,\n COUNT(*) FILTER (WHERE federation_ok = false) AS failed_checks\nFROM\n federation_stat_raw\nWHERE\n $__timeFilter(ts) \nGROUP BY\n time\nORDER BY\n time ASC;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Federation checks over time (success/failure volume)",
+ "type": "barchart"
+ },
+ {
+ "datasource": {
+ "type": "grafana-postgresql-datasource",
+ "uid": "postgres-connectivity-tester"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 10
+ },
+ "id": 3,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true
+ },
+ "pluginVersion": "12.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "grafana-postgresql-datasource",
+ "uid": "postgres-connectivity-tester"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "SELECT\n server_name,\n COUNT(*) AS total_checks,\n SUM(CASE WHEN federation_ok THEN 1 ELSE 0 END) AS success_count,\n SUM(CASE WHEN NOT federation_ok THEN 1 ELSE 0 END) AS failure_count\nFROM\n federation_stat_raw\nGROUP BY\n server_name\nORDER BY\n total_checks DESC\nLIMIT 20;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Top servers by check volume (All Time)",
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "grafana-postgresql-datasource",
+ "uid": "postgres-connectivity-tester"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 18
+ },
+ "id": 4,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true
+ },
+ "pluginVersion": "12.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "grafana-postgresql-datasource",
+ "uid": "postgres-connectivity-tester"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "SELECT\n server_name,\n last_seen_at,\n failure_count,\n (failure_count::float / NULLIF(req_count, 0) * 100) AS failure_rate\nFROM\n federation_stat_aggregate\nWHERE\n last_seen_at >= $__timeFrom()\n AND failure_count > 0\nORDER BY\n failure_rate DESC\nLIMIT 20;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Recently failing servers",
+ "type": "table"
+ }
+ ],
+ "preload": false,
+ "schemaVersion": 41,
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-30d",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "browser",
+ "title": "Connectivity Tester",
+ "uid": "c6527508-9d56-4edb-8fc2-4df0b28a2de1",
+ "version": 5
+}
diff --git a/apps/talos_cluster/monitoring-stack/dashboards/kustomization.yaml b/apps/talos_cluster/monitoring-stack/dashboards/kustomization.yaml
@@ -8,6 +8,7 @@ configMapGenerator:
- draupnir4all_status.json=draupnir4all_status.json
- d4all_pg.json=d4all_pg.json
- synapse.json=synapse.json
+ - connectivity-tester-dashboard.json=connectivity-tester-dashboard.json
options:
labels:
grafana_dashboard: "1"
diff --git a/apps/talos_cluster/monitoring-stack/release.yaml b/apps/talos_cluster/monitoring-stack/release.yaml
@@ -159,7 +159,9 @@ spec:
annotations:
__dashboardUid__: tbO9LAiZK
__panelId__: "7"
- description: One or more OSDs have reached the NEARFULL threshold. Use 'ceph health detail' and 'ceph osd df' to identify the problem. To resolve, add capacity to the affected OSD's failure domain, restore down/out OSDs, or delete unwanted data.
+ description: One or more OSDs have reached the NEARFULL threshold. Use 'ceph health detail'
+ and 'ceph osd df' to identify the problem. To resolve, add capacity to the affected OSD's
+ failure domain, restore down/out OSDs, or delete unwanted data.
runbook_url: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-nearfull
summary: OSD(s) running low on free space (NEARFULL)
isPaused: false
@@ -220,7 +222,8 @@ spec:
execErrState: Error
for: 5m
annotations:
- description: Depending on the rate limit, cert-manager may be unable to generate certificates for up to a week.
+ description: Depending on the rate limit, cert-manager may be unable to generate certificates
+ for up to a week.
summary: Cert manager hitting LetsEncrypt rate limits.
labels: {}
isPaused: false
@@ -696,7 +699,8 @@ spec:
auth_url: https://auth.midnightthoughts.space/application/o/authorize/
token_url: https://auth.midnightthoughts.space/application/o/token/
api_url: https://auth.midnightthoughts.space/application/o/userinfo/
- role_attribute_path: contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors') && 'Editor' || 'Viewer'
+ role_attribute_path: contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors')
+ && 'Editor' || 'Viewer'
imageRenderer:
enabled: true
kubeProxy:
