cluster

Infrastructure files for Nordgedanken and Midnightthoughts.
git clone git://archive.git.mtrnord.blog/MTRNord/cluster.git
Log | Files | Refs | README

prometheus-release.yaml (28630B)


      1 apiVersion: helm.toolkit.fluxcd.io/v2
      2 kind: HelmRelease
      3 metadata:
      4     name: prometheus-stack
      5     namespace: monitoring
      6 spec:
      7     interval: 5m
      8     chart:
      9         spec:
     10             version: 83.x
     11             chart: kube-prometheus-stack
     12             sourceRef:
     13                 kind: HelmRepository
     14                 name: prometheus-community
     15             interval: 60m
     16     install:
     17         crds: CreateReplace
     18     upgrade:
     19         crds: CreateReplace
     20     # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml
     21     values:
     22         defaultRules:
     23             disabled:
     24                 NodeMemoryMajorPagesFaults: true
     25         kubeProxy:
     26             enabled: false
     27         alertmanager:
     28             alertmanagerSpec:
     29                 externalUrl: https://alertmanager.midnightthoughts.space
     30             config:
     31                 global:
     32                     resolve_timeout: 5m
     33                     smtp_smarthost: smtp.fastmail.com:587
     34                     smtp_from: ops@nordgedanken.dev
     35                     smtp_auth_username: ENC[AES256_GCM,data:dWNgHEu+ECkHEKMLZypPCY/lkM1OQGfE,iv:iHYwgtA6yaTYgmvWd8cyIltyxhRZn14pu+xMmBZ/0KY=,tag:n/q44Rja9CA5MViG9iEYbw==,type:str]
     36                     smtp_auth_password: ENC[AES256_GCM,data:AKStfCjEoNf4nX4EoGvCew==,iv:6leXdKz4S1baX58IHE0xjDdSVZ6ew7GdvCL+MCXL1qo=,tag:hiyR2OkNHa7ymXDd/qd18A==,type:str]
     37                     smtp_require_tls: true
     38                 route:
     39                     receiver: email
     40                     group_by:
     41                         - alertname
     42                         - namespace
     43                     group_wait: 30s
     44                     group_interval: 5m
     45                     repeat_interval: 4h
     46                     routes:
     47                         - receiver: "null"
     48                           matchers:
     49                             - alertname = "Watchdog"
     50                         - receiver: "null"
     51                           matchers:
     52                             - alertname = "InfoInhibitor"
     53                         - receiver: "null"
     54                           matchers:
     55                             - severity = "info"
     56                         - receiver: "null"
     57                           matchers:
     58                             - alertname = "NodeMemoryHighUtilization"
     59                         - receiver: "null"
     60                           matchers:
     61                             - alertname = "KubePersistentVolumeFillingUp"
     62                             - persistentvolumeclaim = "data-plausible-clickhouse-shard0-0"
     63                         - receiver: "null"
     64                           matchers:
     65                             - alertname = "KubePersistentVolumeFillingUp"
     66                             - persistentvolumeclaim = "data-thanos-compact-0"
     67                         - receiver: "null"
     68                           matchers:
     69                             - alertname = "KubePersistentVolumeFillingUp"
     70                             - persistentvolumeclaim = "storage-loki-0"
     71                         - receiver: email
     72                           matchers:
     73                             - severity = "critical"
     74                 receivers:
     75                     - name: "null"
     76                     - name: email
     77                       email_configs:
     78                         - to: mtrnord@nordgedanken.dev
     79                           send_resolved: true
     80                 inhibit_rules:
     81                     - source_matchers:
     82                         - severity = "critical"
     83                       target_matchers:
     84                         - severity = "warning"
     85                       equal:
     86                         - namespace
     87                         - alertname
     88         prometheus:
     89             prometheusSpec:
     90                 retention: 4h
     91                 enableRemoteWriteReceiver: true
     92                 externalUrl: https://prometheus.midnightthoughts.space
     93                 externalLabels:
     94                     k8s_cluster: midnightthoughts
     95                 # Pick up all ServiceMonitors/PodMonitors/etc. regardless of labels
     96                 serviceMonitorSelectorNilUsesHelmValues: false
     97                 serviceMonitorSelector: {}
     98                 serviceMonitorNamespaceSelector: {}
     99                 podMonitorSelectorNilUsesHelmValues: false
    100                 podMonitorSelector: {}
    101                 podMonitorNamespaceSelector: {}
    102                 probeSelectorNilUsesHelmValues: false
    103                 probeSelector: {}
    104                 probeNamespaceSelector: {}
    105                 ruleSelectorNilUsesHelmValues: false
    106                 ruleSelector: {}
    107                 ruleNamespaceSelector: {}
    108                 scrapeConfigSelectorNilUsesHelmValues: false
    109                 scrapeConfigSelector: {}
    110                 scrapeConfigNamespaceSelector: {}
    111                 # Thanos sidecar uploads completed TSDB blocks to Hetzner S3.
    112                 # First block is cut after ~3h (Prometheus head span must exceed chunkRange × 1.5 = 3h).
    113                 # Subsequent blocks are cut every 2h once the head is warm.
    114                 thanos:
    115                     objectStorageConfig:
    116                         existingSecret:
    117                             name: thanos-objstore-secret
    118                             key: ENC[AES256_GCM,data:sDr7TGbh,iv:Eusb+NJFD0+729ZbLJsfBGQPHGR6mtZs8SE82+t7OXk=,tag:ofVye8ZUN2ltquET7yQzuA==,type:str]
    119                 storageSpec:
    120                     volumeClaimTemplate:
    121                         spec:
    122                             storageClassName: longhorn-2replicas
    123                             accessModes:
    124                                 - ReadWriteOnce
    125                             resources:
    126                                 requests:
    127                                     storage: 10Gi
    128         grafana:
    129             adminPassword: ENC[AES256_GCM,data:7jNegIl3sDUGSr4S4BWe6VAQXtX/IucAW9qGl13/JeIf3KzaOGE382v2Inh/0aFTYp3Rc/Aa9edfCLGRh10CAQ==,iv:HyrR6dXMCQ/AY7AKzw5HaEQzzyC2eo/XGYj9aWvfk+A=,tag:PMtd4C5CHqT8jJCOI/KIqQ==,type:str]
    130             sidecar:
    131                 datasources:
    132                     # Disable the auto-generated default Prometheus datasource so it
    133                     # doesn't conflict with our custom datasources.yaml (which points to Thanos)
    134                     defaultDatasourceEnabled: false
    135                 dashboards:
    136                     folderAnnotation: grafana_folder
    137                     provider:
    138                         foldersFromFilesStructure: true
    139                     logLevel: WARN
    140                     watchServerTimeout: 3600
    141                     enabled: true
    142             plugins:
    143                 - grafana-piechart-panel
    144                 - grafana-worldmap-panel
    145                 - isovalent-hubble-datasource
    146                 - isovalent-hubbleprocessancestry-panel
    147                 - chaosmeshorg-datasource
    148             smtp:
    149                 existingSecret: smtp-auth-secret
    150             extraSecretMounts:
    151                 - name: auth-generic-oauth-secret-mount
    152                   secretName: auth-generic-oauth-secret
    153                   defaultMode: 288
    154                   mountPath: /etc/secrets/auth_generic_oauth
    155                   readOnly: true
    156             defaultDashboardsTimezone: Europe/Berlin
    157             datasources:
    158                 datasources.yaml:
    159                     apiVersion: 1
    160                     datasources:
    161                         - name: Prometheus
    162                           type: prometheus
    163                           uid: prometheus
    164                           url: http://thanos-query-frontend.monitoring.svc.cluster.local:10902
    165                           access: proxy
    166                         - name: ChaosMesh
    167                           type: chaosmeshorg-datasource
    168                           url: http://chaos-dashboard.chaosmesh.svc.cluster.local:2333
    169                           access: proxy
    170                           isDefault: false
    171                         - name: Hubble
    172                           type: isovalent-hubble-datasource
    173                           uid: hubble
    174                           access: proxy
    175                           url: http://hubble-relay.kube-system.svc.cluster.local:80
    176                           isDefault: false
    177                           jsonData:
    178                             serviceMap:
    179                                 datasourceUid: prometheus
    180                             prometheusDatasourceUid: prometheus
    181                         - name: Postgres-d4all
    182                           type: postgres
    183                           uid: postgres-d4all
    184                           access: proxy
    185                           url: pg-cluster-v2-ro.postgres-cluster.svc.cluster.local:5432
    186                           database: draupnir_synapse
    187                           user: grafana-ro
    188                           secureJsonData:
    189                             password: ENC[AES256_GCM,data:aC4xiZ+hkAjog7Oi27Vg2QNdwV3NAcMKyhxaYD/NWrcv59pvIHrsMrSGUUA6VsvMk8bZ17J+Z4Y35TV7sUM/Lw==,iv:sdAwh6ae6T7QPsGEZPOapFxJN+64/3EHPopklTSqkEg=,tag:wpeD0jchSxWVUkxh7f0bxA==,type:str]
    190                           isDefault: false
    191                           jsonData:
    192                             sslmode: disable
    193                             maxIdleConnsAuto: true
    194                             timescaledb: false
    195                             maxOpenConns: 10
    196                             maxIdleConns: 10
    197                         - name: Loki
    198                           type: loki
    199                           uid: loki
    200                           url: http://loki.loki.svc.cluster.local:3100
    201                           access: proxy
    202                           isDefault: false
    203                           jsonData:
    204                             maxLines: 1000
    205                         - name: Postgres-connectivity-tester
    206                           type: postgres
    207                           uid: postgres-connectivity-tester
    208                           access: proxy
    209                           url: pg-cluster-v2-ro.postgres-cluster.svc.cluster.local:5432
    210                           database: connectivity-tester
    211                           user: connectivity-tester
    212                           secureJsonData:
    213                             password: ENC[AES256_GCM,data:gBjF+yULX8kvTSRvPKvz+/P6qxz4aCoOGCxKahbCLVGPTx+Ehjlqk1QGNyqjpKNJOAPwmHYuW7GxWxGcKe+S1A==,iv:IThe6eOiMnpr//gljfUx7o/EWEOXIaOlPWXt8kK44ps=,tag:2V9cITo4gTlXE1QOLkN+cQ==,type:str]
    214                           isDefault: false
    215                           jsonData:
    216                             sslmode: disable
    217                             maxIdleConnsAuto: true
    218                             timescaledb: false
    219                             maxOpenConns: 10
    220                             maxIdleConns: 10
    221             ingress:
    222                 enabled: false
    223             grafana.ini:
    224                 server:
    225                     root_url: https://grafana.midnightthoughts.space
    226                 smtp:
    227                     enabled: true
    228                     host: smtp.fastmail.com:465
    229                     from_address: ops@nordgedanken.dev
    230                 paths:
    231                     data: ENC[AES256_GCM,data:A376g74PLQSg00L506CfWq4=,iv:XjJXd1hXvnG2pk26RuPNI/nHaB+h0pi1SgfpIjk4aj8=,tag:Gtk84zKiZTivRDdfc6RXzw==,type:str]
    232                     logs: /var/log/grafana
    233                     plugins: /var/lib/grafana/plugins
    234                     provisioning: /etc/grafana/provisioning
    235                 analytics:
    236                     check_for_updates: true
    237                 log:
    238                     mode: console
    239                 grafana_net:
    240                     url: https://grafana.net
    241                 auth:
    242                     signout_redirect_url: https://auth.midnightthoughts.space/application/o/grafana/end-session/
    243                     oauth_auto_login: true
    244                 auth.generic_oauth:
    245                     name: authentik
    246                     enabled: true
    247                     client_id: ENC[AES256_GCM,data:j1w+rs2UIvMLziGrxjcsTHJTAt2YEqss+cj1nGuD99Rx26xhib6o6gw9O0JwAH1gO9s=,iv:Zz425UOToBzsYokEtW3v/KDks+L/vqiqUMDIm+60eQo=,tag:DuYGA2jAQEMWq0gKqvj5Lg==,type:str]
    248                     client_secret: ENC[AES256_GCM,data:eIDG94pds3IEVdDGwfe4/2pZdlTOcu1p4s2+SJi/YSZF9v8VOndhXsKNJ3ev7Zs5AgrqYiub,iv:D310pyD6czsrgQVo0kl3/Hl1VbcHcq8plXiwrKK0+RE=,tag:T6bbKvg5yef3RIgGWigvcQ==,type:str]
    249                     scopes: openid email profile
    250                     auth_url: https://auth.midnightthoughts.space/application/o/authorize/
    251                     token_url: https://auth.midnightthoughts.space/application/o/token/
    252                     api_url: https://auth.midnightthoughts.space/application/o/userinfo/
    253                     role_attribute_path: contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors') && 'Editor' || 'Viewer'
    254             imageRenderer:
    255                 enabled: true
    256             alerting:
    257                 rules.yaml:
    258                     apiVersion: 1
    259                     groups:
    260                         - orgId: 1
    261                           name: CertManager
    262                           folder: Cluster
    263                           interval: 5m
    264                           rules:
    265                             - uid: ddz3zj028wb28b
    266                               title: CertManagerHittingRateLimits
    267                               condition: B
    268                               data:
    269                                 - refId: ENC[AES256_GCM,data:eQ==,iv:IwmV/SafvMWiF+ghamjGlkdzljYJLAq4Qu/mr/RNGRo=,tag:Uu7+gmGFCgmaNeBh3SnTwA==,type:str]
    270                                   relativeTimeRange:
    271                                     from: ENC[AES256_GCM,data:iwEe,iv:xamSpUUlDgyIUcKSGgaYCOh8uKTTy82HhBkyMs88Ngs=,tag:lNKcMtWhfwjPOJcoV8iceg==,type:int]
    272                                     to: ENC[AES256_GCM,data:Sw==,iv:15vVABzloiJIEm1DyISytBU6mgPnbchU+1wf959Wn04=,tag:dS5hwG9TlV29B86sBYoYjw==,type:int]
    273                                   datasourceUid: ENC[AES256_GCM,data:00YD53NlC4iSUA==,iv:A+5OKL0bRbcEmDnVyESTBJYMkJsOrpuKmUpSIiYISMs=,tag:rrpzF0StsNmSPKY7IdwAoA==,type:str]
    274                                   model:
    275                                     editorMode: ENC[AES256_GCM,data:C7NTcw==,iv:8bFbBiAhOgKuA3v9/a8+h8QzlPoOBiRNYWofJPvGZMg=,tag:kIecd4xpHJy7S02VXmo5Xg==,type:str]
    276                                     expr: ENC[AES256_GCM,data:Y80g1pCh2xOLgkj52eP+N0RG0C7fp+MQ+qTvQLxnb3G2Vc6YZy5/JLa5YveAfkg+MmhCvm93UzXj+N1CbSKuR2xREWQP65d4g7iBvwcwtJLm7XXpCOhcmV4sSJl6GSY179W6z5s4PANete5SH/gOF3Th,iv:q5pBnIEVJMDEHYAz9vx1rSkBt/5XhISytLD1vdEt7y8=,tag:c4VQ+n6aOQbPtBtUNzglEw==,type:str]
    277                                     instant: ENC[AES256_GCM,data:M+tOHg==,iv:gsgDb8r+06E6G5O+Dyw5IUOG2KyfRObwDgfIrel/QLA=,tag:X+V40ptlf0yYLAtPMyHq+w==,type:bool]
    278                                     intervalMs: ENC[AES256_GCM,data:DgzugA==,iv:Q3weq35b1/BrFqojCFn9ZBZWlhvOhsx0bExGNU00lps=,tag:lMPEsCSa7PlT0jM5MqdWSA==,type:int]
    279                                     legendFormat: ENC[AES256_GCM,data:dJrTBWj5,iv:OLBqP2ZTSBvcu4y6JNnLtqRxcsJI1JJ2NaF2Ha7Oj5w=,tag:tkMY0I0TzPeg3anopWyNZg==,type:str]
    280                                     maxDataPoints: ENC[AES256_GCM,data:PDz/Ixk=,iv:BNiZWBcuiZgZtWeo4PkAl2p+hBjF2Myb8at5kNluEbQ=,tag:k3wrOBP4eEHvhn1RTAtNmg==,type:int]
    281                                     range: ENC[AES256_GCM,data:9BAVOyY=,iv:lcRK+jP4PGzkQ6ot1KOaiJ4wkQdhy2Xjl3JcpVPykXM=,tag:fwyi5ITrdHboWjDbkcYSWA==,type:bool]
    282                                     refId: ENC[AES256_GCM,data:Yg==,iv:kuX4tTN4JwHOfDv8AqyMKE8/jKO4VNNEq0BS6LzvfYY=,tag:hMZEQi8sUBv96QbVl8vlhQ==,type:str]
    283                                 - refId: ENC[AES256_GCM,data:pw==,iv:QZco98ZH81zcM5AJEg4gU9o81F/nI3Q1oB6C6G5UvQk=,tag:6LfapaUUVPjBvMeTKBSGkw==,type:str]
    284                                   relativeTimeRange:
    285                                     from: ENC[AES256_GCM,data:Mxsj,iv:WeLiWy3saMKZ8j+FGOiSAgBtEUZIwwEwWRPcMlOFFWc=,tag:Ox5lQeePsDQp+I5z7q+XFw==,type:int]
    286                                     to: ENC[AES256_GCM,data:EA==,iv:9rIX4wigE/VoJRr/04UqKwgYOTI7YzpZu4bShEtdydU=,tag:rh0rbiXBoYyqXtp0FMGfMA==,type:int]
    287                                   datasourceUid: ENC[AES256_GCM,data:su1lPSyaTaU=,iv:nqKdey3fKR1KqxyM1+w7ev+GLh7pvoag917ikNFHSVA=,tag:9xFJ3teCmk5K5q5LIWMwrg==,type:str]
    288                                   model:
    289                                     conditions:
    290                                         - evaluator:
    291                                             params: []
    292                                             type: ENC[AES256_GCM,data:DzI=,iv:hbKYErI7Z1nfIrcDt/P3g/WkxwBQTndA8hS8fPntgnU=,tag:R/Bm4cK3ZNz2ojMFxZY4cw==,type:str]
    293                                           operator:
    294                                             type: ENC[AES256_GCM,data:/5oY,iv:UQn8j5yegwiK4afD9vJFmA/t2Ad4PjajMJP3/eZAebg=,tag:BAifwGpPXcFGlw0M4SMR7A==,type:str]
    295                                           query:
    296                                             params:
    297                                                 - ENC[AES256_GCM,data:KQ==,iv:85lBe6LyMeaTRzntkuPhIh8Ro2YI+MXZIF4AjD2sgT8=,tag:LDxOhQAfw+PyWpnfyo5D1g==,type:str]
    298                                           reducer:
    299                                             params: []
    300                                             type: ENC[AES256_GCM,data:s9HjJg==,iv:cB1K/huDl/IbLQy+xGz53XwJssfelA+wSjkRculaMD0=,tag:tj8x/Al9MupV8i45eUn8kA==,type:str]
    301                                           type: ENC[AES256_GCM,data:6B3RqTM=,iv:vkex9D+kJsS9fZJPujf18e4em8Ox39c9k7ACJAj5o7Y=,tag:DHJeaFhg27oLRnjZuu7EiQ==,type:str]
    302                                     datasource:
    303                                         type: ENC[AES256_GCM,data:dMhRkvAZu6k=,iv:LObXOYNdwfHlYvI0iafyihf4AXjug+xkqayqVm4yBSk=,tag:GU5ORy9NN7qYZn3dhv4gZA==,type:str]
    304                                         uid: ENC[AES256_GCM,data:3/FXcRLk/04=,iv:58P/mviD3HY7qHER5uyGDzs/7+8hzAJmzZqbUBW6d/c=,tag:oslVrcz151yqPGpUgUTw2w==,type:str]
    305                                     expression: ENC[AES256_GCM,data:6Q==,iv:5GxqzZWzKtEHikCZLV/GJhea4jBI9vhz8WHa1TBKWhU=,tag:wViKCJ4k3ThJ5pJd55vNqw==,type:str]
    306                                     intervalMs: ENC[AES256_GCM,data:yl3o5g==,iv:yGicM9bPANeN/ISujvcrZjFeVA6dFREWbC1umld8698=,tag:mV6KaxbiG8HKmR+7kyfgWQ==,type:int]
    307                                     maxDataPoints: ENC[AES256_GCM,data:MDVjYkU=,iv:JehwlaeBUhqeX7r7wz3s6gEpoafV3/tw5u/JPb/usZ8=,tag:jPzJEBvu4DgJ5VghVe6+zw==,type:int]
    308                                     reducer: ENC[AES256_GCM,data:2N631w==,iv:k1G20L+kDpuk23pLeMLD9orbWzpn5ERpTqd9qnoeDyc=,tag:lnTFFWXzQFNAIt3Qm/XPgw==,type:str]
    309                                     refId: ENC[AES256_GCM,data:fw==,iv:97D+UIh2NfUibXMVYX7lmtuSZpCUSx3z8tevmm0MxM8=,tag:XeAUKvurGdVlgfRvlz7eYA==,type:str]
    310                                     type: ENC[AES256_GCM,data:WKjVfsSU,iv:g4ROHEWE9edRiEy8+cibWXSxN2AB6RAqimpa2OLb1uM=,tag:nYMFI2yGVThspa1Cc6Iy9A==,type:str]
    311                               noDataState: OK
    312                               execErrState: Error
    313                               for: 5m
    314                               annotations:
    315                                 description: Depending on the rate limit, cert-manager may be unable to generate certificates for up to a week.
    316                                 summary: Cert manager hitting LetsEncrypt rate limits.
    317                               labels: {}
    318                               isPaused: false
    319                               notification_settings:
    320                                 receiver: MidnightThoughts Ops
    321                 contactpoints.yaml:
    322                     apiVersion: 1
    323                     contactPoints:
    324                         - orgId: 1
    325                           name: MidnightThoughts Ops
    326                           receivers:
    327                             - uid: "1"
    328                               name: MidnightThoughts Ops
    329                               type: email
    330                               sendReminder: true
    331                               frequency: 1m
    332                               settings:
    333                                 addresses: ENC[AES256_GCM,data:MgCMCMAnF7LD7RuzdQY6VBQP6RB4GoX3sZ5j/EAAJUFZ624qt6PsXHvseg==,iv:JqiDzLiaePlJ4NyzcSHQPLz9a+vT031QoUhJ48QLGr0=,tag:CIkxW1K1gbJ7kVaKqIqelg==,type:str]
    334                 policies.yaml:
    335                     apiVersion: 1
    336                     policies:
    337                         - orgId: 1
    338                           receiver: MidnightThoughts Ops
    339                           group_by:
    340                             - grafana_folder
    341                             - alertname
    342         kube-state-metrics:
    343             metricLabelsAllowlist:
    344                 - nodes=[topology.kubernetes.io/zone]
    345             rbac:
    346                 extraRules:
    347                     - apiGroups:
    348                         - postgresql.cnpg.io
    349                       resources:
    350                         - backups
    351                       verbs:
    352                         - list
    353                         - watch
    354             customResourceState:
    355                 enabled: true
    356                 config:
    357                     kind: CustomResourceStateMetrics
    358                     spec:
    359                         resources:
    360                             - groupVersionKind:
    361                                 group: postgresql.cnpg.io
    362                                 version: v1
    363                                 kind: Backup
    364                               labelsFromPath:
    365                                 namespace:
    366                                     - metadata
    367                                     - namespace
    368                                 cluster:
    369                                     - metadata
    370                                     - labels
    371                                     - cnpg.io/cluster
    372                                 backup_name:
    373                                     - metadata
    374                                     - name
    375                               metrics:
    376                                 - name: cnpg_backup_stopped_at
    377                                   help: Unix timestamp when the CNPG backup completed (status.stoppedAt)
    378                                   each:
    379                                     type: Gauge
    380                                     gauge:
    381                                         path:
    382                                             - status
    383                                             - stoppedAt
    384                                         nilIsZero: true
    385                                 - name: cnpg_backup_started_at
    386                                   help: Unix timestamp when the CNPG backup started (status.startedAt)
    387                                   each:
    388                                     type: Gauge
    389                                     gauge:
    390                                         path:
    391                                             - status
    392                                             - startedAt
    393                                         nilIsZero: true
    394     postRenderers:
    395         - kustomize:
    396             patches:
    397                 - target:
    398                     # Ignore these objects from Flux diff as they are mutated from chart hooks
    399                     kind: (ValidatingWebhookConfiguration|MutatingWebhookConfiguration)
    400                     name: kube-prometheus-stack-admission
    401                   patch: |
    402                     - op: add
    403                       path: /metadata/annotations/helm.toolkit.fluxcd.io~1driftDetection
    404                       value: disabled
    405                 - target:
    406                     # Ignore these objects from Flux diff as they are mutated at apply time but not at dry-run time
    407                     kind: PrometheusRule
    408                   patch: |
    409                     - op: add
    410                       path: /metadata/annotations/helm.toolkit.fluxcd.io~1driftDetection
    411                       value: disabled
    412 sops:
    413     kms: []
    414     gcp_kms: []
    415     azure_kv: []
    416     hc_vault: []
    417     age:
    418         - recipient: age1esjyg2qfy49awv0ptkzvpk425adczjr38m37w2mmcahzc4p8n54sll2nzh
    419           enc: |
    420             -----BEGIN AGE ENCRYPTED FILE-----
    421             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUM2p0QTBBYzhEeDZDOXRp
    422             ZHpKcGRJY3pIdnNvY2svYTZLNTVFVStacDBJCmx4WUtZMlp3THZoeGY0Uk9raUtT
    423             NUFiYkxlT3NXSE14QVVYZ2pWbHZ4OHMKLS0tIGcveHdMT2VxSWJSUzNNR1dKRlJW
    424             SFBvS3piME1GSlBmdDZCZmc5R25BbEEK7jY2Pdip5MqUhXYfxxl9Ponz5P2TAPlq
    425             ioVtacSqZFaqKIMOZW1OGF9UKcvBNSRblhHyzcC3g4pQ7wZJxCZAFw==
    426             -----END AGE ENCRYPTED FILE-----
    427     lastmodified: "2026-03-27T09:38:27Z"
    428     mac: ENC[AES256_GCM,data:q4TKx+R4zxPdEvhped1zEDGSuP+f0Y/XpsPxJV1SxgVzfeeakr7DD4EgF1ZXOai2DWRsfq4hsiisaoEVjqIaGZ2qHNypo3VmTP0QWoQ+tSkuu7ITq2DfJ3o45ScOO3TzrhUMrvGdiVilSRXSo8d2+rNdHpg5yEVjSTJSHXr5lVI=,iv:PhPs3zO9TmzyizMmPSwuDyeELwqjQbWTPxTIGo8yGwA=,tag:aY1JwBgkSx4CIZMHo8hoTQ==,type:str]
    429     pgp: []
    430     encrypted_regex: ^(apiKey|appUserPassword|otelUserPassword|harborAdminPassword|totpVaultKey|kimaiAppSecret|kimaiAdminPassword|GITHUB_CLIENT_ID|GITHUB_CLIENT_SECRET|GITHUB_PRIVATE_KEY|woosh|root_password|rspamd_password|pgdb_password|matrix_access_token|pgdb_remote_url|hmac_secret_key|adminPassword|adminEmail|jenkinsAdminEmail|securityRealm|gerrit.config|routing_key|DATABASE_URL|SMTP_PASSWORD|SECRET_KEY_BASE|admin_password|extraCommands|key|clickhouseDatabaseURL|databaseURL|client_id|client_secret|secret_key_base|otp_secret|private_key|public_key|primaryKey|deterministicKey|keyDerivationSalt|token|clientId|secretKey|installationId|installationKey|uriOverride|adminToken.value|password.value|sql_password|erlangCookie|AUTHENTICATION_PASSWORD|ROOM_API_SECRET_KEY|adminPassword|configPassword|adminUser|configUser|MAIL_PASSWORD|APP_KEY|api_key|api_secret|keys|livekit_key|livekit_secret|secret_key|admin_pass|admin_email|mariadbPassword|mariadbRootPassword|privateKey|data|stringData|PASSWD|password|pass|postgresPassword|smtp_auth_password|addresses|smtp_auth_username|authorization_credentials|postgresqlPassword|redminePassword|smtpPassword|registration_shared_secret|shared_secret|secret|admin_token|integrationKey|integration_key|rootPassword|adminPassword|adminUser|adminEmail|emailPassword|secretKey|appId|clientSecret|webhookSecret)$
    431     version: 3.9.1
    432 ---
    433 apiVersion: gateway.networking.k8s.io/v1
    434 kind: HTTPRoute
    435 metadata:
    436     name: grafana
    437     namespace: monitoring
    438 spec:
    439     parentRefs:
    440         - name: envoy-gateway
    441           namespace: envoy-gateway
    442     hostnames:
    443         - grafana.midnightthoughts.space
    444     rules:
    445         - backendRefs:
    446             - name: prometheus-stack-grafana
    447               port: 80
    448           timeouts:
    449             request: 240s
    450             backendRequest: 0s
    451 sops:
    452     kms: []
    453     gcp_kms: []
    454     azure_kv: []
    455     hc_vault: []
    456     age:
    457         - recipient: age1esjyg2qfy49awv0ptkzvpk425adczjr38m37w2mmcahzc4p8n54sll2nzh
    458           enc: |
    459             -----BEGIN AGE ENCRYPTED FILE-----
    460             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUM2p0QTBBYzhEeDZDOXRp
    461             ZHpKcGRJY3pIdnNvY2svYTZLNTVFVStacDBJCmx4WUtZMlp3THZoeGY0Uk9raUtT
    462             NUFiYkxlT3NXSE14QVVYZ2pWbHZ4OHMKLS0tIGcveHdMT2VxSWJSUzNNR1dKRlJW
    463             SFBvS3piME1GSlBmdDZCZmc5R25BbEEK7jY2Pdip5MqUhXYfxxl9Ponz5P2TAPlq
    464             ioVtacSqZFaqKIMOZW1OGF9UKcvBNSRblhHyzcC3g4pQ7wZJxCZAFw==
    465             -----END AGE ENCRYPTED FILE-----
    466     lastmodified: "2026-03-27T09:38:27Z"
    467     mac: ENC[AES256_GCM,data:q4TKx+R4zxPdEvhped1zEDGSuP+f0Y/XpsPxJV1SxgVzfeeakr7DD4EgF1ZXOai2DWRsfq4hsiisaoEVjqIaGZ2qHNypo3VmTP0QWoQ+tSkuu7ITq2DfJ3o45ScOO3TzrhUMrvGdiVilSRXSo8d2+rNdHpg5yEVjSTJSHXr5lVI=,iv:PhPs3zO9TmzyizMmPSwuDyeELwqjQbWTPxTIGo8yGwA=,tag:aY1JwBgkSx4CIZMHo8hoTQ==,type:str]
    468     pgp: []
    469     encrypted_regex: ^(apiKey|appUserPassword|otelUserPassword|harborAdminPassword|totpVaultKey|kimaiAppSecret|kimaiAdminPassword|GITHUB_CLIENT_ID|GITHUB_CLIENT_SECRET|GITHUB_PRIVATE_KEY|woosh|root_password|rspamd_password|pgdb_password|matrix_access_token|pgdb_remote_url|hmac_secret_key|adminPassword|adminEmail|jenkinsAdminEmail|securityRealm|gerrit.config|routing_key|DATABASE_URL|SMTP_PASSWORD|SECRET_KEY_BASE|admin_password|extraCommands|key|clickhouseDatabaseURL|databaseURL|client_id|client_secret|secret_key_base|otp_secret|private_key|public_key|primaryKey|deterministicKey|keyDerivationSalt|token|clientId|secretKey|installationId|installationKey|uriOverride|adminToken.value|password.value|sql_password|erlangCookie|AUTHENTICATION_PASSWORD|ROOM_API_SECRET_KEY|adminPassword|configPassword|adminUser|configUser|MAIL_PASSWORD|APP_KEY|api_key|api_secret|keys|livekit_key|livekit_secret|secret_key|admin_pass|admin_email|mariadbPassword|mariadbRootPassword|privateKey|data|stringData|PASSWD|password|pass|postgresPassword|smtp_auth_password|addresses|smtp_auth_username|authorization_credentials|postgresqlPassword|redminePassword|smtpPassword|registration_shared_secret|shared_secret|secret|admin_token|integrationKey|integration_key|rootPassword|adminPassword|adminUser|adminEmail|emailPassword|secretKey|appId|clientSecret|webhookSecret)$
    470     version: 3.9.1