commit 9b7995a5f28b6987aca8fe4b86067fcb07cb6f67 parent 06de4dc7c97fa430eaa09da4c68bc9624e699b4a Author: MTRNord <MTRNord@users.noreply.github.com> Date: Sun, 22 Mar 2026 01:01:23 +0100 set up k6 Signed-off-by: MTRNord <MTRNord@users.noreply.github.com> Diffstat:
15 files changed, 2839 insertions(+), 73 deletions(-)
diff --git a/apps/talos_cluster/k6-operator/cronjobs.yaml b/apps/talos_cluster/k6-operator/cronjobs.yaml @@ -0,0 +1,258 @@ +# Common env vars injected into every k6 TestRun runner via the CronJob shell scripts: +# K6_PROMETHEUS_RW_SERVER_URL — pushes metrics to Prometheus remote-write receiver +# K6_PROMETHEUS_RW_STALE_MARKERS — marks stale series when the test ends +# BASE_URL — connectivity-tester-stage internal cluster address +# +# SERVER_NAMES is intentionally NOT set; config.js default (matrix.org,maunium.net,mtrnord.blog) is used. +# +# cleanup: "post" on every TestRun tells the operator to delete the TestRun object +# after all pods complete, so stale objects don't accumulate even if the CronJob +# pod crashes before it can run its own kubectl delete at the end. +--- +# ── Smoke: every 5 minutes, ~10 s, sanity check all endpoints ────────────── +apiVersion: batch/v1 +kind: CronJob +metadata: + name: k6-smoke + namespace: k6-operator +spec: + schedule: "*/5 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + ttlSecondsAfterFinished: 3600 + template: + spec: + serviceAccountName: k6-cronjob + restartPolicy: OnFailure + containers: + - name: runner + image: bitnami/kubectl:1.32 + command: + - /bin/sh + - -c + - | + set -e + NAME="k6-smoke-$(date +%s)" + kubectl -n matrix apply -f - << EOF + apiVersion: k6.io/v1alpha1 + kind: TestRun + metadata: + name: ${NAME} + namespace: matrix + spec: + parallelism: 1 + cleanup: "post" + script: + configMap: + name: k6-scripts + file: smoke.js + arguments: --out prometheus-rw + runner: + env: + - name: K6_PROMETHEUS_RW_SERVER_URL + value: http://prometheus-operated.monitoring.svc.cluster.local:9090/api/v1/write + - name: K6_PROMETHEUS_RW_STALE_MARKERS + value: "true" + - name: BASE_URL + value: http://connectivity-tester-stage.matrix.svc.cluster.local:8080 + EOF + TIMEOUT=300 + START=$(date +%s) + while true; do + STAGE=$(kubectl -n matrix get testrun/"${NAME}" -o jsonpath='{.status.stage}' 2>/dev/null || true) + echo "Stage: ${STAGE}" + case "${STAGE}" in finished|error) break ;; esac + [ $(($(date +%s) - START)) -ge ${TIMEOUT} ] && { echo "Timed out"; break; } + sleep 10 + done + kubectl -n matrix delete testrun/"${NAME}" --ignore-not-found +--- +# ── Health: every 30 minutes, 30 s, /healthz load baseline ───────────────── +apiVersion: batch/v1 +kind: CronJob +metadata: + name: k6-health + namespace: k6-operator +spec: + schedule: "*/30 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + ttlSecondsAfterFinished: 3600 + template: + spec: + serviceAccountName: k6-cronjob + restartPolicy: OnFailure + containers: + - name: runner + image: bitnami/kubectl:1.32 + command: + - /bin/sh + - -c + - | + set -e + NAME="k6-health-$(date +%s)" + kubectl -n matrix apply -f - << EOF + apiVersion: k6.io/v1alpha1 + kind: TestRun + metadata: + name: ${NAME} + namespace: matrix + spec: + parallelism: 1 + cleanup: "post" + script: + configMap: + name: k6-scripts + file: health.js + arguments: --out prometheus-rw + runner: + env: + - name: K6_PROMETHEUS_RW_SERVER_URL + value: http://prometheus-operated.monitoring.svc.cluster.local:9090/api/v1/write + - name: K6_PROMETHEUS_RW_STALE_MARKERS + value: "true" + - name: BASE_URL + value: http://connectivity-tester-stage.matrix.svc.cluster.local:8080 + EOF + TIMEOUT=300 + START=$(date +%s) + while true; do + STAGE=$(kubectl -n matrix get testrun/"${NAME}" -o jsonpath='{.status.stage}' 2>/dev/null || true) + echo "Stage: ${STAGE}" + case "${STAGE}" in finished|error) break ;; esac + [ $(($(date +%s) - START)) -ge ${TIMEOUT} ] && { echo "Timed out"; break; } + sleep 10 + done + kubectl -n matrix delete testrun/"${NAME}" --ignore-not-found +--- +# ── Federation: every 30 minutes, ~60 s, 3+5 VUs across two scenarios ────── +apiVersion: batch/v1 +kind: CronJob +metadata: + name: k6-federation + namespace: k6-operator +spec: + schedule: "15,45 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + ttlSecondsAfterFinished: 3600 + template: + spec: + serviceAccountName: k6-cronjob + restartPolicy: OnFailure + containers: + - name: runner + image: bitnami/kubectl:1.32 + command: + - /bin/sh + - -c + - | + set -e + NAME="k6-federation-$(date +%s)" + kubectl -n matrix apply -f - << EOF + apiVersion: k6.io/v1alpha1 + kind: TestRun + metadata: + name: ${NAME} + namespace: matrix + spec: + parallelism: 1 + cleanup: "post" + script: + configMap: + name: k6-scripts + file: federation.js + arguments: --out prometheus-rw + runner: + env: + - name: K6_PROMETHEUS_RW_SERVER_URL + value: http://prometheus-operated.monitoring.svc.cluster.local:9090/api/v1/write + - name: K6_PROMETHEUS_RW_STALE_MARKERS + value: "true" + - name: BASE_URL + value: http://connectivity-tester-stage.matrix.svc.cluster.local:8080 + EOF + TIMEOUT=600 + START=$(date +%s) + while true; do + STAGE=$(kubectl -n matrix get testrun/"${NAME}" -o jsonpath='{.status.stage}' 2>/dev/null || true) + echo "Stage: ${STAGE}" + case "${STAGE}" in finished|error) break ;; esac + [ $(($(date +%s) - START)) -ge ${TIMEOUT} ] && { echo "Timed out"; break; } + sleep 10 + done + kubectl -n matrix delete testrun/"${NAME}" --ignore-not-found +--- +# ── Soak: every 6 hours (on the hour), 30 min, memory leak detection ──────── +apiVersion: batch/v1 +kind: CronJob +metadata: + name: k6-soak + namespace: k6-operator +spec: + schedule: "0 */6 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + ttlSecondsAfterFinished: 3600 + template: + spec: + serviceAccountName: k6-cronjob + restartPolicy: OnFailure + containers: + - name: runner + image: bitnami/kubectl:1.32 + command: + - /bin/sh + - -c + - | + set -e + NAME="k6-soak-$(date +%s)" + kubectl -n matrix apply -f - << EOF + apiVersion: k6.io/v1alpha1 + kind: TestRun + metadata: + name: ${NAME} + namespace: matrix + spec: + parallelism: 1 + cleanup: "post" + script: + configMap: + name: k6-scripts + file: soak-with-memory-check.js + arguments: --out prometheus-rw + runner: + env: + - name: K6_PROMETHEUS_RW_SERVER_URL + value: http://prometheus-operated.monitoring.svc.cluster.local:9090/api/v1/write + - name: K6_PROMETHEUS_RW_STALE_MARKERS + value: "true" + - name: BASE_URL + value: http://connectivity-tester-stage.matrix.svc.cluster.local:8080 + - name: PROMETHEUS_URL + value: http://prometheus-operated.monitoring.svc.cluster.local:9090 + - name: MEMORY_GROWTH_THRESHOLD + value: "0.30" + EOF + TIMEOUT=2700 + START=$(date +%s) + while true; do + STAGE=$(kubectl -n matrix get testrun/"${NAME}" -o jsonpath='{.status.stage}' 2>/dev/null || true) + echo "Stage: ${STAGE}" + case "${STAGE}" in finished|error) break ;; esac + [ $(($(date +%s) - START)) -ge ${TIMEOUT} ] && { echo "Timed out"; break; } + sleep 30 + done + kubectl -n matrix delete testrun/"${NAME}" --ignore-not-found diff --git a/apps/talos_cluster/k6-operator/kustomization.yaml b/apps/talos_cluster/k6-operator/kustomization.yaml @@ -0,0 +1,20 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - repository.yaml + - release.yaml + - rbac.yaml + - cronjobs.yaml +# k6 test scripts — mounted by k6 runner pods in the matrix namespace +configMapGenerator: + - name: k6-scripts + namespace: matrix + files: + - scripts/config.js + - scripts/smoke.js + - scripts/health.js + - scripts/federation.js + - scripts/soak-with-memory-check.js + options: + disableNameSuffixHash: true diff --git a/apps/talos_cluster/k6-operator/namespace.yaml b/apps/talos_cluster/k6-operator/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: k6-operator diff --git a/apps/talos_cluster/k6-operator/rbac.yaml b/apps/talos_cluster/k6-operator/rbac.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k6-cronjob + namespace: k6-operator +--- +# Role in the matrix namespace so CronJob pods can manage TestRun objects there +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: k6-testrun-manager + namespace: matrix +rules: + - apiGroups: ["k6.io"] + resources: ["testruns"] + verbs: ["create", "get", "list", "watch", "delete"] +--- +# Bind the k6-operator SA to the matrix-namespace role +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: k6-cronjob-testrun-manager + namespace: matrix +subjects: + - kind: ServiceAccount + name: k6-cronjob + namespace: k6-operator +roleRef: + kind: Role + name: k6-testrun-manager + apiGroup: rbac.authorization.k8s.io diff --git a/apps/talos_cluster/k6-operator/release.yaml b/apps/talos_cluster/k6-operator/release.yaml @@ -0,0 +1,21 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: k6-operator + namespace: k6-operator +spec: + interval: 5m + chart: + spec: + chart: k6-operator + version: "4.3.x" + sourceRef: + kind: HelmRepository + name: grafana + namespace: k6-operator + interval: 60m + install: + crds: CreateReplace + upgrade: + crds: CreateReplace + values: {} diff --git a/apps/talos_cluster/k6-operator/repository.yaml b/apps/talos_cluster/k6-operator/repository.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: grafana + namespace: k6-operator +spec: + interval: 120m + url: https://grafana.github.io/helm-charts diff --git a/apps/talos_cluster/k6-operator/scripts/config.js b/apps/talos_cluster/k6-operator/scripts/config.js @@ -0,0 +1,29 @@ +/** + * Shared configuration for k6 tests. + * + * Override via environment variables: + * BASE_URL=http://prod:8080 k6 run federation.js + * SERVER_NAMES=matrix.org,maunium.net k6 run federation.js + */ + +export const BASE_URL = __ENV.BASE_URL || 'http://localhost:8080'; + +// Comma-separated list of Matrix server names to test against +export const SERVER_NAMES = (__ENV.SERVER_NAMES || 'matrix.org,maunium.net,mtrnord.blog').split(','); + +/** Pick a random entry from an array */ +export function randomItem(arr) { + return arr[Math.floor(Math.random() * arr.length)]; +} + +/** + * Default thresholds used across load tests. + * + * Federation checks are inherently slow (DNS + TLS + HTTP to external servers), + * so the p95 threshold is intentionally generous at 10s. + * Adjust per-test as needed. + */ +export const DEFAULT_THRESHOLDS = { + http_req_failed: ['rate<0.05'], // fewer than 5% errors + http_req_duration: ['p(95)<10000'], // 95th percentile under 10s +}; diff --git a/apps/talos_cluster/k6-operator/scripts/federation.js b/apps/talos_cluster/k6-operator/scripts/federation.js @@ -0,0 +1,76 @@ +/** + * Federation endpoint load test. + * + * Tests both /api/federation/report (full check) and /api/federation/federation-ok + * (lightweight status) under concurrent load. Uses a randomised pool of server names + * to avoid hitting the connection cache for a single target on every request. + * + * Run: + * k6 run k6/federation.js + * BASE_URL=http://prod:8080 SERVER_NAMES=matrix.org,maunium.net k6 run k6/federation.js + * + * Tune VU counts and duration via env vars: + * REPORT_VUS=3 OK_VUS=10 DURATION=120s k6 run k6/federation.js + */ + +import http from 'k6/http'; +import { check, sleep } from 'k6'; +import { BASE_URL, SERVER_NAMES, DEFAULT_THRESHOLDS, randomItem } from './config.js'; + +const REPORT_VUS = parseInt(__ENV.REPORT_VUS || '3', 10); +const OK_VUS = parseInt(__ENV.OK_VUS || '5', 10); +const DURATION = __ENV.DURATION || '60s'; + +export const options = { + scenarios: { + // Full federation report — expensive (does DNS + TLS + HTTP to external servers) + federation_report: { + executor: 'constant-vus', + vus: REPORT_VUS, + duration: DURATION, + exec: 'fullReport', + }, + // Lightweight status check — much cheaper + federation_ok: { + executor: 'constant-vus', + vus: OK_VUS, + duration: DURATION, + exec: 'federationOk', + }, + }, + thresholds: DEFAULT_THRESHOLDS, +}; + +export function fullReport() { + const server = randomItem(SERVER_NAMES); + const res = http.get( + `${BASE_URL}/api/federation/report?server_name=${encodeURIComponent(server)}&stats_opt_in=false`, + { timeout: '30s' }, + ); + check(res, { + 'report: status 200': (r) => r.status === 200, + 'report: has FederationOK': (r) => { + try { + return typeof JSON.parse(r.body).FederationOK === 'boolean'; + } catch { + return false; + } + }, + }); + // Small pause — federation checks are slow enough without hammering continuously + sleep(1); +} + +export function federationOk() { + const server = randomItem(SERVER_NAMES); + const res = http.get( + `${BASE_URL}/api/federation/federation-ok?server_name=${encodeURIComponent(server)}`, + { timeout: '15s' }, + ); + check(res, { + 'federation-ok: status 200': (r) => r.status === 200, + 'federation-ok: GOOD or BAD': (r) => + r.body.trim() === 'GOOD' || r.body.trim() === 'BAD', + }); + sleep(0.5); +} diff --git a/apps/talos_cluster/k6-operator/scripts/health.js b/apps/talos_cluster/k6-operator/scripts/health.js @@ -0,0 +1,36 @@ +/** + * Health endpoint load test — establishes a baseline for server throughput + * separate from federation logic (which is network-bound). + * + * Run: + * k6 run k6/health.js + * BASE_URL=http://prod:8080 k6 run k6/health.js + */ + +import http from 'k6/http'; +import { check, sleep } from 'k6'; +import { BASE_URL, DEFAULT_THRESHOLDS } from './config.js'; + +export const options = { + scenarios: { + health_check: { + executor: 'constant-vus', + vus: 20, + duration: '30s', + }, + }, + thresholds: { + ...DEFAULT_THRESHOLDS, + // Health endpoint should be much faster than federation checks + http_req_duration: ['p(95)<500'], + }, +}; + +export default function () { + const res = http.get(`${BASE_URL}/healthz`); + check(res, { + 'status 200': (r) => r.status === 200, + 'body is ok': (r) => r.body.trim() === 'ok', + }); + sleep(0.1); +} diff --git a/apps/talos_cluster/k6-operator/scripts/smoke.js b/apps/talos_cluster/k6-operator/scripts/smoke.js @@ -0,0 +1,66 @@ +/** + * Smoke test — single VU, single iteration, verifies all key endpoints respond correctly. + * + * Run: + * k6 run k6/smoke.js + * BASE_URL=http://staging:8080 k6 run k6/smoke.js + */ + +import http from 'k6/http'; +import { check } from 'k6'; +import { BASE_URL } from './config.js'; + +export const options = { + vus: 1, + iterations: 1, + thresholds: { + checks: ['rate==1.0'], // every check must pass in smoke mode + }, +}; + +export default function () { + // 1. Health check + { + const res = http.get(`${BASE_URL}/healthz`); + check(res, { + 'healthz: status 200': (r) => r.status === 200, + 'healthz: body is ok': (r) => r.body.trim() === 'ok', + }); + } + + // 2. Federation-ok (simple status endpoint) + { + const res = http.get(`${BASE_URL}/api/federation/federation-ok?server_name=matrix.org`); + check(res, { + 'federation-ok: status 200': (r) => r.status === 200, + 'federation-ok: body is GOOD or BAD': (r) => + r.body.trim() === 'GOOD' || r.body.trim() === 'BAD', + }); + } + + // 3. Full federation report + { + const res = http.get(`${BASE_URL}/api/federation/report?server_name=matrix.org`, { + timeout: '30s', + }); + check(res, { + 'federation/report: status 200': (r) => r.status === 200, + 'federation/report: has FederationOK field': (r) => { + try { + const body = JSON.parse(r.body); + return typeof body.FederationOK === 'boolean'; + } catch { + return false; + } + }, + }); + } + + // 4. Metrics endpoint (if Prometheus is enabled; gracefully skip if 404) + { + const res = http.get(`${BASE_URL}/metrics`); + check(res, { + 'metrics: status 200 or 404': (r) => r.status === 200 || r.status === 404, + }); + } +} diff --git a/apps/talos_cluster/k6-operator/scripts/soak-with-memory-check.js b/apps/talos_cluster/k6-operator/scripts/soak-with-memory-check.js @@ -0,0 +1,111 @@ +/** + * Soak test with memory leak detection. + * + * Runs the same 15 rps steady-state load as soak.js. At setup() time, queries + * Prometheus for a baseline of the connectivity-tester-stage container memory. + * After the soak, teardown() re-queries and fails the `checks` threshold if + * memory grew more than MEMORY_GROWTH_THRESHOLD (default 30%). + * + * Env vars: + * SOAK_RPS — requests/sec (default: 15) + * SOAK_DURATION — total test duration (default: 30m) + * PROMETHEUS_URL — Prometheus HTTP API base URL + * MEMORY_GROWTH_THRESHOLD — fractional threshold, e.g. 0.30 = 30% (default: 0.30) + */ + +import http from 'k6/http'; +import { check } from 'k6'; +import { BASE_URL, SERVER_NAMES, randomItem } from './config.js'; + +const SOAK_RPS = parseInt(__ENV.SOAK_RPS || '15', 10); +const SOAK_DURATION = __ENV.SOAK_DURATION || '30m'; +const PROMETHEUS_URL = __ENV.PROMETHEUS_URL || 'http://prometheus-operated.monitoring.svc.cluster.local:9090'; +const MEMORY_GROWTH_THRESHOLD = parseFloat(__ENV.MEMORY_GROWTH_THRESHOLD || '0.30'); + +// cAdvisor metric — excludes page cache, reflects true resident memory +const MEMORY_QUERY = 'avg(container_memory_working_set_bytes{namespace="matrix",pod=~"connectivity-tester-stage-.*",container="federation-tester-api"})'; + +export const options = { + scenarios: { + soak: { + executor: 'constant-arrival-rate', + exec: 'federationOk', + rate: SOAK_RPS, + timeUnit: '1s', + duration: SOAK_DURATION, + // At 15 rps × ~1.5s avg = ~23 VUs needed; allocate headroom + preAllocatedVUs: 40, + maxVUs: 80, + }, + }, + thresholds: { + http_req_failed: ['rate<0.05'], + // If p95 climbs above 5s during a soak that was fine at 1.2s, something is leaking + http_req_duration: ['p(95)<5000'], + // Memory growth check — reported via teardown() check() + checks: ['rate==1.0'], + }, +}; + +function queryMemoryBytes() { + const url = `${PROMETHEUS_URL}/api/v1/query?query=${encodeURIComponent(MEMORY_QUERY)}`; + const res = http.get(url, { timeout: '10s', tags: { name: 'prometheus_memory_query' } }); + if (res.status !== 200) { + console.warn(`Prometheus query failed: HTTP ${res.status}`); + return null; + } + try { + const body = JSON.parse(res.body); + if (body.status === 'success' && body.data.result.length > 0) { + return parseFloat(body.data.result[0].value[1]); + } + console.warn('Prometheus returned no results for memory query'); + } catch (e) { + console.warn(`Failed to parse Prometheus response: ${e}`); + } + return null; +} + +export function setup() { + const baseline = queryMemoryBytes(); + if (baseline !== null) { + console.log(`Baseline memory: ${(baseline / 1024 / 1024).toFixed(1)} MiB`); + } else { + console.warn('Could not establish baseline memory; memory growth check will be skipped in teardown'); + } + return { baselineMemory: baseline }; +} + +export function federationOk() { + const server = randomItem(SERVER_NAMES); + const res = http.get( + `${BASE_URL}/api/federation/federation-ok?server_name=${encodeURIComponent(server)}`, + { timeout: '15s' }, + ); + check(res, { + 'federation-ok: status 200': (r) => r.status === 200, + 'federation-ok: GOOD or BAD': (r) => + r.body.trim() === 'GOOD' || r.body.trim() === 'BAD', + }); +} + +export function teardown(data) { + if (data.baselineMemory === null) { + console.warn('Baseline memory was unavailable; skipping memory growth check'); + return; + } + const current = queryMemoryBytes(); + if (current === null) { + console.warn('Post-soak memory unavailable; skipping memory growth check'); + return; + } + + const growth = (current - data.baselineMemory) / data.baselineMemory; + const baseMiB = (data.baselineMemory / 1024 / 1024).toFixed(1); + const currentMiB = (current / 1024 / 1024).toFixed(1); + console.log(`Memory: baseline=${baseMiB} MiB → current=${currentMiB} MiB (${(growth * 100).toFixed(1)}% growth)`); + + check(growth, { + [`memory growth <= ${(MEMORY_GROWTH_THRESHOLD * 100).toFixed(0)}%`]: (g) => g <= MEMORY_GROWTH_THRESHOLD, + }); +} diff --git a/apps/talos_cluster/kustomization.yaml b/apps/talos_cluster/kustomization.yaml @@ -37,4 +37,5 @@ resources: - ./persephone - ./codeberg - ./opencost + - ./k6-operator #- ./proxmox-ccm diff --git a/apps/talos_cluster/monitoring-stack/dashboards/k6-prometheus.json b/apps/talos_cluster/monitoring-stack/dashboards/k6-prometheus.json @@ -0,0 +1,2079 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Visualize k6 OSS results stored in Prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 19665, + "graphTooltip": 2, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Grafana k6 OSS Docs: Prometheus Remote Write", + "tooltip": "Open docs in a new tab", + "type": "link", + "url": "https://k6.io/docs/results-output/real-time/prometheus-remote-write/" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http_req_s_errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "reqps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "http_req_s" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "vus" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed" + } + }, + { + "id": "unit", + "value": "VUs" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(k6_vus{testid=~\"$testid\"})", + "instant": false, + "legendFormat": "vus", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_duration_$quantile_stat", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http_req_s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(round(k6_http_req_failed_rate{testid=~\"$testid\"}, 0.1)*100)", + "hide": true, + "instant": false, + "legendFormat": "http_req_failed", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\", expected_response=\"false\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http_req_s_errors", + "range": true, + "refId": "D" + } + ], + "title": "Performance Overview", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 1, + "panels": [], + "title": "Performance Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(k6_http_reqs_total{testid=~\"$testid\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "HTTP requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(k6_http_reqs_total{testid=~\"$testid\", expected_response=\"false\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "HTTP request failures", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Peak RPS", + "transformations": [ + { + "id": "reduce", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Select a different Stat to change the query", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Request Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(irate(k6_data_sent_total{testid=~\"$testid\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "data_sent", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(irate(k6_data_received_total{testid=~\"$testid\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "data_received", + "range": true, + "refId": "B" + } + ], + "title": "Transfer Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "dropped_iterations" + }, + "properties": [ + { + "id": "unit", + "value": "none" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_iteration_duration_$quantile_stat{testid=~\"$testid\"})", + "instant": false, + "legendFormat": "iteration_duration_$quantile_stat", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(k6_dropped_iterations_total{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "dropped_iterations", + "range": true, + "refId": "B" + } + ], + "title": "Iterations", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 16, + "panels": [], + "title": "HTTP", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Select a different Stat to change the query\n\n<a href=\"https://k6.io/docs/using-k6/metrics/reference/#http\" target=\"_blank\">HTTP-specific built-in metrics</a>", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_blocked_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_blocked_$quantile_stat", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_tls_handshaking_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_tls_handshaking_$quantile_stat", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_sending_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_sending_$quantile_stat", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_waiting_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_waiting_$quantile_stat", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_receiving_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_receiving_$quantile_stat", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_duration_$quantile_stat", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Latency Timings", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Select a different Stat to change the query", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "errors_http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "success_http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 24 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_duration_$quantile_stat", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\", expected_response=\"true\"})", + "instant": false, + "legendFormat": "success_http_req_duration_$quantile_stat", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\", expected_response=\"false\"})", + "hide": false, + "instant": false, + "legendFormat": "errors_http_req_duration_$quantile_stat", + "range": true, + "refId": "B" + } + ], + "title": "HTTP Latency Stats", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http_req_s_errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "http_req_s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "http_req_s_success" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 24 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "http_req_s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\", expected_response=\"false\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http_req_s_errors", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\", expected_response=\"true\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http_req_s_success", + "range": true, + "refId": "C" + } + ], + "title": "HTTP Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "min/max/p95/p99 depends on the available Quantile Stats", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "method" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "min" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 17, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 2, + "showHeader": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(name, method, status) (k6_http_req_duration_min{testid=~\"$testid\"})", + "format": "table", + "hide": false, + "instant": false, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(name, method, status) (k6_http_req_duration_max{testid=~\"$testid\"})", + "format": "table", + "hide": false, + "instant": false, + "legendFormat": "max", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(name, method, status) (k6_http_req_duration_p95{testid=~\"$testid\"})", + "format": "table", + "hide": false, + "instant": false, + "legendFormat": "p95", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(name, method, status) (k6_http_req_duration_p99{testid=~\"$testid\"})", + "format": "table", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "E" + } + ], + "title": "Requests by URL", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "groupBy", + "options": { + "fields": { + "Value #B": { + "aggregations": [ + "min" + ], + "operation": "aggregate" + }, + "Value #C": { + "aggregations": [ + "max" + ], + "operation": "aggregate" + }, + "Value #D": { + "aggregations": [ + "mean" + ], + "operation": "aggregate" + }, + "Value #E": { + "aggregations": [ + "mean" + ], + "operation": "aggregate" + }, + "method": { + "aggregations": [], + "operation": "groupby" + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "status": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "Time": 0, + "Value #B": 4, + "Value #C": 5, + "Value #D": 6, + "Value #E": 7, + "method": 2, + "name": 1, + "status": 3 + }, + "renameByName": { + "Value #B": "min", + "Value #B (min)": "min", + "Value #C": "max", + "Value #C (max)": "max", + "Value #D": "p95", + "Value #D (mean)": "p95", + "Value #E": "p99", + "Value #E (mean)": "p99" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 11, + "panels": [], + "title": "Checks", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Success Rate" + }, + "properties": [ + { + "id": "custom.hidden", + "value": false + }, + { + "id": "unit", + "value": "%" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value (mean)" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "check" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 2, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value (count)" + } + ] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "round(k6_checks_rate{testid=~\"$testid\"}, 0.1)", + "format": "table", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Checks list", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "keepLabels": [ + "__name__", + "check" + ], + "mode": "columns" + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": [ + "mean" + ], + "operation": "aggregate" + }, + "check": { + "aggregations": [], + "operation": "groupby" + }, + "k6_checks_rate": { + "aggregations": [ + "sum", + "count" + ], + "operation": "aggregate" + } + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Success Rate", + "binary": { + "left": "Value (mean)", + "operator": "*", + "reducer": "sum", + "right": "100" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [], + "fields": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Filter by check name to query a particular check", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "axisSoftMin": 0, + "barAlignment": -1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "%" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(round(k6_checks_rate{testid=~\"$testid\"}, 0.1)*100)", + "instant": false, + "legendFormat": "k6_checks_rate", + "range": true, + "refId": "A" + } + ], + "title": "Checks Success Rate (aggregate individual checks)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 23, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "### Visualize other k6 results \n\nAt the top of the dashboard, click `Add` and select `Visualization` from the dropdown menu. Choose the visualization type and input the PromQL queries for the `k6_` metric(s).\n\nAlternatively, click on the `Explore` icon on the menu bar and input the queries for the `k6_` metric(s). From `Explore`, you can add new Panels to this dashboard. \n\nNote that all <a href=\"https://k6.io/docs/using-k6/metrics/\" target=\"_blank\">k6 metrics</a> are prefixed with the `k6_` namespace when sent to Prometheus.", + "mode": "markdown" + }, + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "prometheus", + "k6" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "prometheus", + "value": "${DS_PROMETHEUS}" + }, + "description": "Choose a Prometheus Data Source", + "hide": 0, + "includeAll": false, + "label": "Prometheus DS", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(testid)", + "description": "Filter by \"testid\" tag. Define it by tagging: k6 run --tag testid=xyz", + "hide": 0, + "includeAll": true, + "label": "Test ID", + "multi": true, + "name": "testid", + "options": [], + "query": { + "query": "label_values(testid)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "metrics(k6_http_req_duration_)", + "description": "Statistic for Trend Metrics Queries. The available options depend on the values of the K6_PROMETHEUS_RW_TREND_STATS setting.", + "hide": 0, + "includeAll": false, + "label": "Trend Metrics Query", + "multi": false, + "name": "quantile_stat", + "options": [], + "query": { + "query": "metrics(k6_http_req_duration_)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "/http_req_duration_(min|max|count|sum|avg|med|p[0-9]+)/g", + "skipUrlSync": false, + "sort": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Adhoc filters are applied to all panels. To enable it, go to Dashboard Settings / Variables / adhoc_filter and select the target Prometheus data source.", + "filters": [], + "hide": 0, + "label": "AdhocFilter", + "name": "adhoc_filter", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "", + "title": "k6 Prometheus", + "uid": "ccbb2351-2ae2-462f-ae0e-f2c893ad1028", + "version": 3, + "weekStart": "" +} +\ No newline at end of file diff --git a/apps/talos_cluster/monitoring-stack/dashboards/kustomization.yaml b/apps/talos_cluster/monitoring-stack/dashboards/kustomization.yaml @@ -78,3 +78,12 @@ configMapGenerator: grafana_dashboard: "1" annotations: grafana_folder: Postgres + - name: grafana-dashboards-k6 + namespace: monitoring + files: + - k6-prometheus.json=k6-prometheus.json + options: + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: k6 diff --git a/apps/talos_cluster/monitoring-stack/prometheus-release.yaml b/apps/talos_cluster/monitoring-stack/prometheus-release.yaml @@ -32,8 +32,8 @@ spec: resolve_timeout: 5m smtp_smarthost: smtp.fastmail.com:587 smtp_from: ops@nordgedanken.dev - smtp_auth_username: ENC[AES256_GCM,data:Mo3P1MGqkvBqnAnyRD2H8n5mR2pFa5Q0,iv:c0dttME9mN+r0eBN1rp5pb1mKL4GP/TjvFt0XEwkdbU=,tag:5lTvltnsrsWKqpLWkijipg==,type:str] - smtp_auth_password: ENC[AES256_GCM,data:RWBcINgdx9S2JnahnbOCCQ==,iv:bghGHoRVueN4pC4HaUJjSaQe0dsnbpX4hOL7NlXm+U4=,tag:gEm5dAPs6pxCm2P9zajxMA==,type:str] + smtp_auth_username: ENC[AES256_GCM,data:Z5LWLrdB0O3GuE+9KQ+PAim41veCw2ES,iv:WkwDGwWj2587aLjlwymZMilnB9RqoY0oiiBBZ4Lhh0Q=,tag:hk7K1qgdmP/j8C8lXXcnvg==,type:str] + smtp_auth_password: ENC[AES256_GCM,data:02+CFavijb9rjNjKAwlSsQ==,iv:6oMt99sXo1nsR+0CpgRqdvJ167FTwMjMJyvQRfm5sTw=,tag:wmWeAGIu+l0vV3Gdb1tLZQ==,type:str] smtp_require_tls: true route: receiver: email @@ -88,6 +88,7 @@ spec: prometheus: prometheusSpec: retention: 4h + enableRemoteWriteReceiver: true externalUrl: https://prometheus.midnightthoughts.space externalLabels: k8s_cluster: midnightthoughts @@ -114,7 +115,7 @@ spec: objectStorageConfig: existingSecret: name: thanos-objstore-secret - key: ENC[AES256_GCM,data:yuiOo2EB,iv:00CEd3P+QL3mIVpIe15agmcN/27bpEf0R3CYyFdVoyQ=,tag:aMmhJ857MaBL8vEVHX9CeA==,type:str] + key: ENC[AES256_GCM,data:GEWtKmnn,iv:N5qFNjzi1PqmC4JJYfMPN2VsAyezukLHVk8R2S2y8qY=,tag:C2dqJyVECpWy7nGLcJkurA==,type:str] storageSpec: volumeClaimTemplate: spec: @@ -125,7 +126,7 @@ spec: requests: storage: 10Gi grafana: - adminPassword: ENC[AES256_GCM,data:ifzzu7pWFlcxiaYJoly7vN44V8jBIfoq8Tm4ZRTNgZKWpfFFkqPHDacRd7LdGOVXVKHbsA+IC1qsmJrJR9bJuQ==,iv:tZgpLFkV1ck5f1c+z2k7oZni8HzkFGaXYEJ2MaKZXH8=,tag:uPc+/y+HE+xF0bovEuKCRA==,type:str] + adminPassword: ENC[AES256_GCM,data:B+t9wnxmvNWEWlxHROGU/vj++xP8UP5Gz825pKRWUIag3bTyHtM97RmJLNUNy79ESLyRfZ8vB1LJMp+Xx07gyA==,iv:wvjgKGpvrHHJ4JPNL46AHrHl920q8xuOPHdG/NU9IsI=,tag:SziF6UhoL38xPYqONPf6XA==,type:str] sidecar: datasources: # Disable the auto-generated default Prometheus datasource so it @@ -185,7 +186,7 @@ spec: database: draupnir_synapse user: grafana-ro secureJsonData: - password: ENC[AES256_GCM,data:Z73wFEe/UXXxUPZXet2IN7i4AWaiehyR707NyIHW1Sg+NzmxSag77tVVJWVPAVG75zHD+kU4gVPcknHuWMmMQQ==,iv:6Jer5PTNH6gekBHEbraHlib5qOnmDoTATYdWD0s9w/0=,tag:DfHs20BVPUuG/Kaw6cbpwA==,type:str] + password: ENC[AES256_GCM,data:s4/wgtItvv9hX64MU//4gokgvjHQpTHeHQ8PMY8DM9pyP4eqzH/XguwfVEJe/yXYgQKIXVGIF7A8uzGyUbDiBQ==,iv:nXDzG9sTcI+IMHsxXnHA5dCSkTfLwJ625uOUTsBePSA=,tag:hMOr3hk/xUlbs9TUI07npQ==,type:str] isDefault: false jsonData: sslmode: disable @@ -209,7 +210,7 @@ spec: database: connectivity-tester user: connectivity-tester secureJsonData: - password: ENC[AES256_GCM,data:xhjpBtTSTjsb1C8y/mxNV4fnIGJBN6PswA1mR5XLQ4BMjnEvNrmjPDb60QXuzhxBkjL18c4vFlLBtT9afhgxng==,iv:p655YSlOnpTQuXPCLuL7Lvi1buly/nBYy06Axim8/G8=,tag:+oEEWy6r3/WqNxJcew75bA==,type:str] + password: ENC[AES256_GCM,data:epwM0/QtgJfTmRCtXYA9riBj1wf2LCaZVGeyT8rcTUN0Q3EUg83RBli7+pm//tLszYg2AwivAT40NUCFb/NMQA==,iv:yOusHEl+dOFhH2MlVIKJ70orTkIG7esTpkksPZ5c3T0=,tag:LQk6izNz5RWMOfLBfywZXw==,type:str] isDefault: false jsonData: sslmode: disable @@ -227,7 +228,7 @@ spec: host: smtp.fastmail.com:465 from_address: ops@nordgedanken.dev paths: - data: ENC[AES256_GCM,data:VICfKHgMqgcjrkQxPodNygo=,iv:/VGqAik9B1TYQnGxyN44G1yYsBn0Jxbe/oW5sGaAqJU=,tag:ZccuYrycS4WZt8wVqG6wKQ==,type:str] + data: ENC[AES256_GCM,data:1+toTwruaOuD2YVLi4JCX90=,iv:7OR1ElU0GIV8hihDytclZDZIeT/KokHnceJWWHmnaJY=,tag:KjtaS6NmBzzFx/GQF4xTpA==,type:str] logs: /var/log/grafana plugins: /var/lib/grafana/plugins provisioning: /etc/grafana/provisioning @@ -243,8 +244,8 @@ spec: auth.generic_oauth: name: authentik enabled: true - client_id: ENC[AES256_GCM,data:vC7wSpZmpFKnU931t3EAjcKVfjmPILD+PIKlHEa7qB6JTbPxfrhO/00upkPgeg1z23Q=,iv:dvdA07VK5mhUoR/x5kbXY9nsDiEsyzSOQj19zz7p13E=,tag:0WhdmAj2X6qm7jSDnzRztQ==,type:str] - client_secret: ENC[AES256_GCM,data:8JY04aMe8E36XuJCWXwc9wCyOx+Y1dg4K7W4KW7o4NLkbbSEDIkzfd+vcbLJfvI/oCFrBUHA,iv:t9JH5HI1+9h05nEk+HFR79tAdU+5NSyYgPkBGKCnboY=,tag:5zRXlDqnJ4zLZTchZfR9/g==,type:str] + client_id: ENC[AES256_GCM,data:hf2A+rUoHBdStvsdVkrOKFEpcjnO9xkx+ou0VL4tKQ+qEuxL3exXAmzqSUDC5RprfzQ=,iv:vcUMWwji4O434wvREPmFpnK45seoUR56IxAz0GVPdao=,tag:fL71dp9SMGhfBjDl95R6hg==,type:str] + client_secret: ENC[AES256_GCM,data:3a+l9Dteo2iiquZjVPWKtLwEy5CcL+49soiu5yTHaZPy5dTKzN0kAvtrBrD7cLLA7kgmtaJn,iv:oKcSbUXEVWJI24NdZ3rulT6N9qC7SYeg3h0uv1cWQEc=,tag:meaXmbh6YDhyao6gRC6cCg==,type:str] scopes: openid email profile auth_url: https://auth.midnightthoughts.space/application/o/authorize/ token_url: https://auth.midnightthoughts.space/application/o/token/ @@ -265,48 +266,48 @@ spec: title: CertManagerHittingRateLimits condition: B data: - - refId: ENC[AES256_GCM,data:AA==,iv:GSk+Aw8EBOmiZMiLmoqGocFGUzyHcuosZUYMAt/YT9s=,tag:I1XQL+z6GriwrVwSIUAEHw==,type:str] + - refId: ENC[AES256_GCM,data:jw==,iv:DHSM6jHEY0idpXjELHo9lyJ+ljX2BKprqqmdrfst75Y=,tag:DBqb84zMONGSXBC+3Rt50w==,type:str] relativeTimeRange: - from: ENC[AES256_GCM,data:Y9gd,iv:KPCuYi9gLbErZcmd2nDp74Qs2PUYleiDUiL3vNJV8ls=,tag:fUz4eJNBFA0VVaG24AVOyA==,type:int] - to: ENC[AES256_GCM,data:mA==,iv:73MRODil02ix25ej3ZBD9cqgvHfnpeeOaeKWqlbJsBY=,tag:upjDBHKD062a8xGKcMI7kA==,type:int] - datasourceUid: ENC[AES256_GCM,data:XC+KlbR5hHMMiQ==,iv:/wQ3YKASGQkDli6IuqUsi3ACvhny1c9MOrobXYNLO90=,tag:+c/8yHxMQXb4iPn8xTkd7A==,type:str] + from: ENC[AES256_GCM,data:38AE,iv:6qJdaGrKrqq6T0InAEhDp69Hvi77IXl6gzVLoAsMyG4=,tag:BgY5ZR9ZZF4O4oeEB0YIFA==,type:int] + to: ENC[AES256_GCM,data:rg==,iv:uYh3LXV04tzuBXqTNBqyfZ9sp7aNluu88hy+LpzoNsI=,tag:OdByi5PTct/7sJCGqT+Jlg==,type:int] + datasourceUid: ENC[AES256_GCM,data:cPqSL/FYFBr6ow==,iv:XhT0H+0No9OPJSj+4tRrK4Fwo1J0YRCMRjWzxh9cCmI=,tag:pjLZ5ghs/M899ONWp+9PWQ==,type:str] model: - editorMode: ENC[AES256_GCM,data:q3w5HA==,iv:u61ll9KvSqwihx9paihVdA7m5W2RmWr5yIAMIoZUT6k=,tag:ZnIb+gfK67smzyuBaU3tfA==,type:str] - expr: ENC[AES256_GCM,data:zKnx9WrKizU+lItToV+OEG0wbO1rvsGXN+quTfv9w2PsJ3lgkwrQuSs2/2IoZfjCenk9MlLaHW1Y3FzYvYn+i/NLz90N2bOhFC9RXuBnWzBtL9XgR3f3+FmURb33Sd4rv8zSGHyJGRL/BoXcp8yPs2IP,iv:hk4V3X1PzCxmxjRtvuXF0Vlg63CEjD/5StC/uZY9jUs=,tag:lWJ6UPP4qNgKESsFSDW6jg==,type:str] - instant: ENC[AES256_GCM,data:roWCLQ==,iv:Ia6sqsKBvTNqeoNtVKonNKXWJo5/7jyZUO7TYHBD9Gk=,tag:X/zqNmKo+RXDe6rqKNHIoQ==,type:bool] - intervalMs: ENC[AES256_GCM,data:OfMPgg==,iv:CaOX6ee8v/L9B7G1Dd05i6i4zGyuDit9HBdwyySrk4A=,tag:Ke9sJNl90qbvch3ZKBqoaA==,type:int] - legendFormat: ENC[AES256_GCM,data:EAKeoRct,iv:agCwtoOM3sGrQfFhtGz6xaUL4HEK32QRB91ZSd3UhZU=,tag:2d3lpLxWy3YVqHxibRBCUQ==,type:str] - maxDataPoints: ENC[AES256_GCM,data:Cxs8tu0=,iv:gFIztmoRE+vyFTcoNCy8VHZ5TY74RXoiHbXL6bdAX40=,tag:8hwnlNx6pGwtZYp+rTuy6w==,type:int] - range: ENC[AES256_GCM,data:Av94/3Q=,iv:N7iQaLwWQ6TvT3M/t8ejMENGD67syKf2Jx5FnYpEmng=,tag:mOn+VPU2uX3saJRr97fUXw==,type:bool] - refId: ENC[AES256_GCM,data:bg==,iv:Gs2O40DbhIVvcqTLwTG20SnXjxj1ao5IkqkWUCbhJcc=,tag:it+dePmJGkg0x8pPz2cAMg==,type:str] - - refId: ENC[AES256_GCM,data:+g==,iv:TG3qK266T02AIgKJR4SsCVe1dJRybb94JAG3thK2O4o=,tag:KYfw4VC//nvmjWu//DGwxA==,type:str] + editorMode: ENC[AES256_GCM,data:bidhbQ==,iv:Z8FwUQxN+bvShsXgM+i6OfAdgTXhU0Fus4by6FhykCI=,tag:JbXkVSC630++wBOkaYn+Eg==,type:str] + expr: ENC[AES256_GCM,data:ifIm1Av6mJ3ir9ZsyCtiMLQ6RUOEnhrgIEYJWHY7bnBXipXyzIEkR4YtJWq1kdi+lH7P1SAHkZ9Pu40zcgI+8S+h0ri5Qpi8e3B0jV6ewy3S0BIpK9qtfQUPhzo6RW47RZKtjUdtnzTX76McCAubukTk,iv:om9zAho2DcLUoT7R6a8mdDXKtQYkGQQT+nLRpZ4uwAo=,tag:34BJpRRxRQS+xIevBDmt8g==,type:str] + instant: ENC[AES256_GCM,data:BNYXcw==,iv:oDhSWGSpj1Kympzck1OEoRBYrFBsZd/ftP1pzQAxuuE=,tag:AmHdsvQcBTTLvMXtPvuXPA==,type:bool] + intervalMs: ENC[AES256_GCM,data:41Muhw==,iv:lk3L0POB5mQ+rsBEIH2OmqheXVLjtNQtnl2JDVjZDWg=,tag:/I2Ni4nVsLc3rJaskYCBYg==,type:int] + legendFormat: ENC[AES256_GCM,data:pahJA8IP,iv:yolVmh1nnq6/fX4QNhD0AeiIF2llVy9313UFDUrZiaQ=,tag:CYh6GPokjl1at9FSd6PzTg==,type:str] + maxDataPoints: ENC[AES256_GCM,data:NMOPdK4=,iv:xpoabeSrGrsWYkZ3VtFysPFcpzqVeSv6IYMclIngn5A=,tag:OdwEFxPH7Z3nH6apmLRuYQ==,type:int] + range: ENC[AES256_GCM,data:CQw1764=,iv:jco6zgYKBfzEhNNqkyLQLuVCXz4AMaLcHSzn+CQfI20=,tag:3cGHNBpxswxlnK0sCJoMfA==,type:bool] + refId: ENC[AES256_GCM,data:hA==,iv:lpddiegFu0FSphil9CMwDXLWfNgl4c7J8wV7ZN3PJOg=,tag:A8lykdGEZ2eO5Wbqv4ZaJA==,type:str] + - refId: ENC[AES256_GCM,data:Hw==,iv:RmvIoNr2xBQoldW40NvByP30JF7VRgWew6TM8HhR4dI=,tag:1hHMqMwjODNWWJcnB92L0g==,type:str] relativeTimeRange: - from: ENC[AES256_GCM,data:xig6,iv:WnPWVKahA2+ZsUt0nDtwwZx8EKfaqGWRKfsJ+yDZfi4=,tag:gVb1frEQPY3OSJenr5PX/w==,type:int] - to: ENC[AES256_GCM,data:CA==,iv:OPxGXQ0EAhar4xpwde1s5VuucIMJc4XHYAlc/3QP+Jo=,tag:ctyZsqFZuG12F8ggPvjPww==,type:int] - datasourceUid: ENC[AES256_GCM,data:p3ZEMOsF9ZY=,iv:RnBq/8T28ymsUZU2kxHLY0yQSO7w671HnLrFxCaFw8M=,tag:EXCqtGCw0cuvXjo5pEu8Yw==,type:str] + from: ENC[AES256_GCM,data:oZx1,iv:NQrsFxBNPw1EeFE52TjbwnKWBRXh3t+7SrDZfa93cuA=,tag:H87fa8VSZNcPw5Mxj18zRg==,type:int] + to: ENC[AES256_GCM,data:tg==,iv:0bcocqb9t0/5wNNiey8GgsZqmkc82+notfjATw9jG6I=,tag:hEnui7q60fiWXM129Ezn5g==,type:int] + datasourceUid: ENC[AES256_GCM,data:xzJ6u/ZAh24=,iv:/Oyp21ABZ7HqJ3DlQ5fB7Jeb2hRjHiWh3IWw1s6ZPq0=,tag:RRvuiuCPkpVv7liKGQxlyg==,type:str] model: conditions: - evaluator: params: [] - type: ENC[AES256_GCM,data:BKI=,iv:8H53p5jkFZryaLSUEEKbCNqJcptO/XstsuoVrYmj8o0=,tag:qnnOMHpsH1N1sa264ZIrpA==,type:str] + type: ENC[AES256_GCM,data:m1U=,iv:JVTp3IWGGbxgCLF5soSVzuaceq6e8t8eAv9mcP1ZsSs=,tag:xF33DBU5zBnrMN/p3Ql1PQ==,type:str] operator: - type: ENC[AES256_GCM,data:fxob,iv:1DWf+lUmiKB6KUMYIBRd34dR55Ub79MFuLb8NSnOuMQ=,tag:g+50+4/YE6PHlHcZAliLIw==,type:str] + type: ENC[AES256_GCM,data:sjjw,iv:8N/rkHz8+Y5yKhzfoLwx+VZZ5zQ3H+MYBmLQ6aealSs=,tag:ucUbrqO8L0M+euLXB7WIhw==,type:str] query: params: - - ENC[AES256_GCM,data:AQ==,iv:2DlpmllbOBQrf0zrSnOeTcttwRG0vCVFfjuuKAzr+1M=,tag:aBfzuuvXrRB23hiyRoKHWQ==,type:str] + - ENC[AES256_GCM,data:ww==,iv:JAXJfI0p5JDgQCncQ9d7MNpOq4Yz/RRCL6jj/VYVGvE=,tag:/IiZo3837+377qwcASGu7w==,type:str] reducer: params: [] - type: ENC[AES256_GCM,data:YVimDA==,iv:nRc60xVbKngJBzS/XwklWUQjP3GeTOWutsYYe/oRx3s=,tag:9itiAvex0eNkkN5IlrCznQ==,type:str] - type: ENC[AES256_GCM,data:dO8I9I4=,iv:PRKnUZt21RTrIPj40cVZ+AQitfScwg8biN5zTs5SPNU=,tag:wcEcwbe2xz9FVdBSEiS2Qg==,type:str] + type: ENC[AES256_GCM,data:sUeZfA==,iv:d32KFGvvixroebeARG95WIuwDyRDeRWbceTBZXv0mZM=,tag:HGJeXnrJKrkXujtAbGjnHg==,type:str] + type: ENC[AES256_GCM,data:JHlhkYQ=,iv:mHd+rxyI7jE/0niXh2LYf34zTrn2gzvjhtJReZFbIMk=,tag:ELOIfrnrQcoY1QUpzBRkQQ==,type:str] datasource: - type: ENC[AES256_GCM,data:Zk4sV+5nN4Q=,iv:tMbxMJvYoOhK+sX1GAPCyMkrBUGU32RDBU6l+uYIAmc=,tag:WkY/FVqMA8VF5J7cqR39aw==,type:str] - uid: ENC[AES256_GCM,data:G8f+0B4zff4=,iv:DXwvjOeLaK/oWB3zoNb+J7L+h6qrnT25AU1XTuYx05A=,tag:cQprGwrJ5+x2pi/V2XRybA==,type:str] - expression: ENC[AES256_GCM,data:Sg==,iv:0vgIyqp6UG1bAMlL4lFxtVNmdNp0QwN9GoV0yLS+/Wo=,tag:+CnyjSCtBbrVOETFbKeLWw==,type:str] - intervalMs: ENC[AES256_GCM,data:shIwpA==,iv:rAnBso3b5QbAjr36LxIVgXspD+TqoZMaxyXaE3nC/ew=,tag:HZaNL0Tv8iHs4V68DtBgxQ==,type:int] - maxDataPoints: ENC[AES256_GCM,data:rxBXNCE=,iv:tDT+f8ZmsqVBXhdNFUulgEz5Kwh+0UBRAQPyo62BpYY=,tag:h5Bn8WRiePCgY/k3qwgulQ==,type:int] - reducer: ENC[AES256_GCM,data:LhanyA==,iv:tOuk7GlKS0Csk/LebhKx0RdgwDqBTfgElw4FY6lsx2I=,tag:RFuuE8klH19VyI/in3QFVA==,type:str] - refId: ENC[AES256_GCM,data:yQ==,iv:WVmzRTbItAYXJZ5a9XSthuCggGJP8n9qyc5Z+tuTZ4A=,tag:tpyGVQMp/N/NXylPUawnPA==,type:str] - type: ENC[AES256_GCM,data:2+Z15fuC,iv:7r7A/XjPaV2l0X5bh7XIp14dR1T2fyzM/i1oTEbZ6G8=,tag:Di7q8Jm3CuiDcQ4GXGK7cA==,type:str] + type: ENC[AES256_GCM,data:bCeG+VxApQk=,iv:r33lsIDnjVyChoWpZPN1rwB+6Kj7Xk/Ot9RAdvoI4Fg=,tag:WDuS2Mb9ZYGTrRLdz7J4fg==,type:str] + uid: ENC[AES256_GCM,data:TLucN/lr/LM=,iv:aDTkk6kJbBtX2dAj44nsbjeugnI1jj5GOGL+3BX27lw=,tag:V+vNcogvZFEsx8qNm9Jn+w==,type:str] + expression: ENC[AES256_GCM,data:9A==,iv:jfQ3/A6lAPTMuPhVd6ap49L+a+PFJbneXO7d27LUQSA=,tag:/6Zjyrz5nuQPy4kpXteUSw==,type:str] + intervalMs: ENC[AES256_GCM,data:dIB2kg==,iv:n+9cxU2iMrdHlcCHwQwi8aonzHnjVAEKNsxT06WsQ8I=,tag:7wtZaMVJYs8oG7RGdvEpsQ==,type:int] + maxDataPoints: ENC[AES256_GCM,data:eQLRJUQ=,iv:oN+mQNcbffiMmejpmI1PBVgu6sS18HHzGp8wceeYBe8=,tag:GoC5sBXgL/tfUcO0tg8EIg==,type:int] + reducer: ENC[AES256_GCM,data:nOutoQ==,iv:2IS8pZ0peDF8SAHAAtpI7QHLP8dcIvCQhy2YNCT/U70=,tag:g63SP5QV60ER4GMrdcyyuQ==,type:str] + refId: ENC[AES256_GCM,data:qA==,iv:pVpN6UApUcN9gAHR5Ieg4C/Ao/qVTR0Bg3Vd5CSqiOQ=,tag:ynfqigeLZPqG6qU2UPToSA==,type:str] + type: ENC[AES256_GCM,data:zKQ6SnK4,iv:cVkxMdDAbvyuVymUnj8zYzUqja3vatXaoQjw8JRkhdU=,tag:JvaEiAXbT9GQlDrkL8jq7Q==,type:str] noDataState: OK execErrState: Error for: 5m @@ -329,7 +330,7 @@ spec: sendReminder: true frequency: 1m settings: - addresses: ENC[AES256_GCM,data:91pqyrAAjDs7ihPYlsl15JiaL3AzdxLIiZAOu1eZhBwaeZwwdf6nGA4ZNQ==,iv:kDqpe6nOjeICG0gYzu3a9n81uaHFFOwdAyvHxQtUS0M=,tag:HQSIktvZcjsEC0RWFBI2Ww==,type:str] + addresses: ENC[AES256_GCM,data:RLR81dEvRkak2neUWD+N94qnEjjy80WV9bcumNjd9m9osYNtSOJ7PqX2Ng==,iv:IHQsWAoLG0qYxifueU78Ni3allyKPMZwR+6JqufVfoo=,tag:8ZZdNr4TVJAvBlrcqe3Xiw==,type:str] policies.yaml: apiVersion: 1 policies: @@ -343,9 +344,13 @@ spec: - nodes=[topology.kubernetes.io/zone] rbac: extraRules: - - apiGroups: ["postgresql.cnpg.io"] - resources: ["backups"] - verbs: ["list", "watch"] + - apiGroups: + - postgresql.cnpg.io + resources: + - backups + verbs: + - list + - watch customResourceState: enabled: true config: @@ -357,24 +362,35 @@ spec: version: v1 kind: Backup labelsFromPath: - namespace: [metadata, namespace] - cluster: [metadata, labels, "cnpg.io/cluster"] - backup_name: [metadata, name] + namespace: + - metadata + - namespace + cluster: + - metadata + - labels + - cnpg.io/cluster + backup_name: + - metadata + - name metrics: - - name: cnpg_backup_stopped_at - help: "Unix timestamp when the CNPG backup completed (status.stoppedAt)" - each: - type: Gauge - gauge: - path: [status, stoppedAt] - nilIsZero: true - - name: cnpg_backup_started_at - help: "Unix timestamp when the CNPG backup started (status.startedAt)" - each: - type: Gauge - gauge: - path: [status, startedAt] - nilIsZero: true + - name: cnpg_backup_stopped_at + help: Unix timestamp when the CNPG backup completed (status.stoppedAt) + each: + type: Gauge + gauge: + path: + - status + - stoppedAt + nilIsZero: true + - name: cnpg_backup_started_at + help: Unix timestamp when the CNPG backup started (status.startedAt) + each: + type: Gauge + gauge: + path: + - status + - startedAt + nilIsZero: true postRenderers: - kustomize: patches: @@ -402,14 +418,14 @@ sops: - recipient: age1esjyg2qfy49awv0ptkzvpk425adczjr38m37w2mmcahzc4p8n54sll2nzh enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxQyttNWppWW1xdUlJZU11 - eENJWXd3dEpEQThsZVV3SUhYR1NNR3YyS3djCnFNaFNkd3ZxL3J3aXVPcjVKalZ4 - S2ptLytrcTVuOUN0MGlUalFmWis4UjAKLS0tIDUwS3lWcTN6YXRQVzBKUngwdmZX - dzVPTjU3TVZxZnYxNEZBNkdndUE1V1EK+UpftDTPskISciWDmQVxX2aJLEW0EBjQ - YniYqfOSjJNDcHeXL5BlS/iBG/pxFeov3dbHzCirZs7NAhOlNL/AkA== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHQ1NodStPMWhxbzdhOUpK + azN6VkpUL001ZnZJUjUzSllyTHc2cHJDbFJNCkluckZsbUZUS21PWEFqV0RhbXZY + SjNFRlNFLzRId3RaZ2FGaUpHQTZyOEUKLS0tIDQ0UldZaitFbUQvK3ZtQm1CYkFF + V2wvWkdtNFdyTHhkQ0JJZ1REYldMTWMKM0c5fsGrnZudOqQdSsEDVAEMCjyt2fks + ggm0qZv5rdj7XDlhNmkFsetnyd/mkfHv+IlfJMqQn8dzLw/MMeMJEg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-03-14T15:06:44Z" - mac: ENC[AES256_GCM,data:DCBZ+GJ6R/4NFr4StbrSUAYf7NURoQIT0uCYSQ1+P1VRh+ubcHiaF0zZqahrFhbE/gAYs7zcVsOSPA2AuDenkUW4EZbcHzc6Itl2P/h/TKZlSkN8/E3+Nq4qPrLk42hk/ax8irnKMA5z0VfePcBhXnn6VzNEi3d3pWjW3XKZFzs=,iv:pIsNYw7B4Ojs2yDUAesIztfl8pqnP1PUYF2cKNvwc3s=,tag:UNGXqkUZgmq0HIxjJbpu3g==,type:str] + lastmodified: "2026-03-22T00:01:12Z" + mac: ENC[AES256_GCM,data:nVpuP5+n8fpl+OGabWM8/aPScOs4gVDulwVHivuWXOojbnM7G5N/0zhr2c3AQUSO1EHRQKcxTtdODr329tmU0AEia2uxM19qIOr0FSeS6HQStKA5FPt2A4Yqsm7mAysN9eVfpBwOjJlJlj/jsOersMGGKX0yj09OuIlgeyiAbkM=,iv:opd9HxlrQYhsiauXse4eWeu9pZIt3WuXw6CRWCmK0aA=,tag:+5EsVY30l0tF340202BNQA==,type:str] pgp: [] encrypted_regex: ^(apiKey|appUserPassword|otelUserPassword|harborAdminPassword|totpVaultKey|kimaiAppSecret|kimaiAdminPassword|GITHUB_CLIENT_ID|GITHUB_CLIENT_SECRET|GITHUB_PRIVATE_KEY|woosh|root_password|rspamd_password|pgdb_password|matrix_access_token|pgdb_remote_url|hmac_secret_key|adminPassword|adminEmail|jenkinsAdminEmail|securityRealm|gerrit.config|routing_key|DATABASE_URL|SMTP_PASSWORD|SECRET_KEY_BASE|admin_password|extraCommands|key|clickhouseDatabaseURL|databaseURL|client_id|client_secret|secret_key_base|otp_secret|private_key|public_key|primaryKey|deterministicKey|keyDerivationSalt|token|clientId|secretKey|installationId|installationKey|uriOverride|adminToken.value|password.value|sql_password|erlangCookie|AUTHENTICATION_PASSWORD|ROOM_API_SECRET_KEY|adminPassword|configPassword|adminUser|configUser|MAIL_PASSWORD|APP_KEY|api_key|api_secret|keys|livekit_key|livekit_secret|secret_key|admin_pass|admin_email|mariadbPassword|mariadbRootPassword|privateKey|data|stringData|PASSWD|password|pass|postgresPassword|smtp_auth_password|addresses|smtp_auth_username|authorization_credentials|postgresqlPassword|redminePassword|smtpPassword|registration_shared_secret|shared_secret|secret|admin_token|integrationKey|integration_key|rootPassword|adminPassword|adminUser|adminEmail|emailPassword|secretKey|appId|clientSecret|webhookSecret)$ version: 3.9.1 @@ -441,14 +457,14 @@ sops: - recipient: age1esjyg2qfy49awv0ptkzvpk425adczjr38m37w2mmcahzc4p8n54sll2nzh enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxQyttNWppWW1xdUlJZU11 - eENJWXd3dEpEQThsZVV3SUhYR1NNR3YyS3djCnFNaFNkd3ZxL3J3aXVPcjVKalZ4 - S2ptLytrcTVuOUN0MGlUalFmWis4UjAKLS0tIDUwS3lWcTN6YXRQVzBKUngwdmZX - dzVPTjU3TVZxZnYxNEZBNkdndUE1V1EK+UpftDTPskISciWDmQVxX2aJLEW0EBjQ - YniYqfOSjJNDcHeXL5BlS/iBG/pxFeov3dbHzCirZs7NAhOlNL/AkA== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHQ1NodStPMWhxbzdhOUpK + azN6VkpUL001ZnZJUjUzSllyTHc2cHJDbFJNCkluckZsbUZUS21PWEFqV0RhbXZY + SjNFRlNFLzRId3RaZ2FGaUpHQTZyOEUKLS0tIDQ0UldZaitFbUQvK3ZtQm1CYkFF + V2wvWkdtNFdyTHhkQ0JJZ1REYldMTWMKM0c5fsGrnZudOqQdSsEDVAEMCjyt2fks + ggm0qZv5rdj7XDlhNmkFsetnyd/mkfHv+IlfJMqQn8dzLw/MMeMJEg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-03-14T15:06:44Z" - mac: ENC[AES256_GCM,data:DCBZ+GJ6R/4NFr4StbrSUAYf7NURoQIT0uCYSQ1+P1VRh+ubcHiaF0zZqahrFhbE/gAYs7zcVsOSPA2AuDenkUW4EZbcHzc6Itl2P/h/TKZlSkN8/E3+Nq4qPrLk42hk/ax8irnKMA5z0VfePcBhXnn6VzNEi3d3pWjW3XKZFzs=,iv:pIsNYw7B4Ojs2yDUAesIztfl8pqnP1PUYF2cKNvwc3s=,tag:UNGXqkUZgmq0HIxjJbpu3g==,type:str] + lastmodified: "2026-03-22T00:01:12Z" + mac: ENC[AES256_GCM,data:nVpuP5+n8fpl+OGabWM8/aPScOs4gVDulwVHivuWXOojbnM7G5N/0zhr2c3AQUSO1EHRQKcxTtdODr329tmU0AEia2uxM19qIOr0FSeS6HQStKA5FPt2A4Yqsm7mAysN9eVfpBwOjJlJlj/jsOersMGGKX0yj09OuIlgeyiAbkM=,iv:opd9HxlrQYhsiauXse4eWeu9pZIt3WuXw6CRWCmK0aA=,tag:+5EsVY30l0tF340202BNQA==,type:str] pgp: [] encrypted_regex: ^(apiKey|appUserPassword|otelUserPassword|harborAdminPassword|totpVaultKey|kimaiAppSecret|kimaiAdminPassword|GITHUB_CLIENT_ID|GITHUB_CLIENT_SECRET|GITHUB_PRIVATE_KEY|woosh|root_password|rspamd_password|pgdb_password|matrix_access_token|pgdb_remote_url|hmac_secret_key|adminPassword|adminEmail|jenkinsAdminEmail|securityRealm|gerrit.config|routing_key|DATABASE_URL|SMTP_PASSWORD|SECRET_KEY_BASE|admin_password|extraCommands|key|clickhouseDatabaseURL|databaseURL|client_id|client_secret|secret_key_base|otp_secret|private_key|public_key|primaryKey|deterministicKey|keyDerivationSalt|token|clientId|secretKey|installationId|installationKey|uriOverride|adminToken.value|password.value|sql_password|erlangCookie|AUTHENTICATION_PASSWORD|ROOM_API_SECRET_KEY|adminPassword|configPassword|adminUser|configUser|MAIL_PASSWORD|APP_KEY|api_key|api_secret|keys|livekit_key|livekit_secret|secret_key|admin_pass|admin_email|mariadbPassword|mariadbRootPassword|privateKey|data|stringData|PASSWD|password|pass|postgresPassword|smtp_auth_password|addresses|smtp_auth_username|authorization_credentials|postgresqlPassword|redminePassword|smtpPassword|registration_shared_secret|shared_secret|secret|admin_token|integrationKey|integration_key|rootPassword|adminPassword|adminUser|adminEmail|emailPassword|secretKey|appId|clientSecret|webhookSecret)$ version: 3.9.1