commit fe6ec93cf55999b2fb5fe9da8e6b08247bab1985
parent 225fc80cf390642f8c4d0c7627138f00d3fdc46b
Author: MTRNord <MTRNord@users.noreply.github.com>
Date: Mon, 23 Mar 2026 11:00:40 +0100
make inline docs happen
Signed-off-by: MTRNord <MTRNord@users.noreply.github.com>
Diffstat:
12 files changed, 284 insertions(+), 90 deletions(-)
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -0,0 +1,35 @@
+name: Deploy Documentation
+
+on:
+ push:
+ branches: [main]
+ paths:
+ - "**/*.md"
+ - "mkdocs.yml"
+ - "requirements-docs.txt"
+ - ".pages"
+ - "**/.pages"
+ workflow_dispatch:
+
+permissions:
+ contents: write
+
+jobs:
+ deploy:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ cache: pip
+ cache-dependency-path: requirements-docs.txt
+
+ - name: Install dependencies
+ run: pip install -r requirements-docs.txt
+
+ - name: Deploy to GitHub Pages
+ run: mkdocs gh-deploy --force
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
age.agekey
+site/
bak_synapse.txt
audit.txt
homeserver_config.yaml
diff --git a/DISASTER_RECOVERY.md b/DISASTER_RECOVERY.md
@@ -1,4 +1,4 @@
-# Disaster Recovery Runbook — cluster-2025
+# Disaster Recovery Runbook
Last updated: 2026-03-21
@@ -8,12 +8,12 @@ Last updated: 2026-03-21
This cluster has three layers of backup:
-| Layer | Tool | Scope | Frequency | Retention | Storage |
-|---|---|---|---|---|---|
-| Application + Volumes | Velero (Kopia) | All namespaces + hcloud-volumes | 4× daily (0,6,12,18 UTC) | 14 days | `mtrnord-talos-velero` S3 bucket |
-| Longhorn Volumes | Longhorn backup (incremental, full every 14d) | All Longhorn volumes (group: default) | Daily 02:00 UTC | 14 backups | `mtrnord-longhorn-backup` S3 bucket |
-| Longhorn Config | Longhorn system-backup | Longhorn settings + metadata | Daily 05:00 UTC | 7 backups | `mtrnord-longhorn-backup` S3 bucket |
-| PostgreSQL WAL | CNPG barman-cloud | postgres-cluster | Continuous + scheduled | 30 days | `mtrnord-talos-pg-backup` S3 bucket |
+| Layer | Tool | Scope | Frequency | Retention | Storage |
+| --------------------- | --------------------------------------------- | ------------------------------------- | ------------------------ | ---------- | ----------------------------------- |
+| Application + Volumes | Velero (Kopia) | All namespaces + hcloud-volumes | 4× daily (0,6,12,18 UTC) | 14 days | `mtrnord-talos-velero` S3 bucket |
+| Longhorn Volumes | Longhorn backup (incremental, full every 14d) | All Longhorn volumes (group: default) | Daily 02:00 UTC | 14 backups | `mtrnord-longhorn-backup` S3 bucket |
+| Longhorn Config | Longhorn system-backup | Longhorn settings + metadata | Daily 05:00 UTC | 7 backups | `mtrnord-longhorn-backup` S3 bucket |
+| PostgreSQL WAL | CNPG barman-cloud | postgres-cluster | Continuous + scheduled | 30 days | `mtrnord-talos-pg-backup` S3 bucket |
> **Key constraint**: hcloud-volumes CSI does NOT support snapshots. Velero uses Kopia (file-system copy into S3) for those volumes, which requires applications to be quiesced or tolerant of slightly inconsistent snapshots.
@@ -50,6 +50,7 @@ velero restore create --from-backup cluster-backup-0600-20260228060000 \
### Critical: how Velero Kopia volume restore actually works
Kopia does NOT restore data directly into a PVC. Instead:
+
1. Velero restores the **pod** (with an injected `restore-wait` init container)
2. The node-agent writes backup data into the PVC while the init container waits
3. Only after kopia completes does the main container start
@@ -123,6 +124,7 @@ This is safe when the restored pod was deleted immediately (kopia never started)
### Backup consistency: databases being written during backup
Velero Kopia takes a file-system snapshot of a live volume. For databases like RocksDB (continuwuity/conduwuit), the backup may capture an inconsistent state if:
+
- The database was in recovery mode during backup
- The CURRENT file was written pointing to a MANIFEST that was later renamed/replaced before the snapshot completed
@@ -174,7 +176,7 @@ bootstrap:
recovery:
source: pg-s3-backup
recoveryTarget:
- targetTime: "2026-02-27T20:00:00Z" # RFC 3339, adjust as needed
+ targetTime: "2026-02-27T20:00:00Z" # RFC 3339, adjust as needed
```
### 2c. Important lessons from past migrations
@@ -224,30 +226,34 @@ Backup target: `s3://mtrnord-longhorn-backup` (Hetzner Object Storage HEL1)
**Current configuration:**
-| Job name | Type | Schedule | Group | Retain | Concurrency | Notes |
-|---|---|---|---|---|---|---|
-| `volume-backup` | `backup` | `0 2 * * *` (02:00) | default | 14 | 2 | Volume data → S3. Primary recovery source. Incremental, full every 14 days. |
-| `post-backup-cleanup` | `snapshot-delete` | `0 3 * * *` (03:00) | default | 2 | 2 | Enforces max 2 snapshots per volume after backup runs. Prevents copy/move failures. |
-| `system-backup` | `system-backup` | `0 5 * * *` (05:00) | — | 7 | — | Longhorn config/metadata backup. `volume-backup-policy: if-not-present`. |
-| `filesystem-trim` | `filesystem-trim` | `0 4 * * *` (04:00) | default | — | 2 | Reclaim space from deleted files. Runs between backup (02:00) and system-backup (05:00). |
+| Job name | Type | Schedule | Group | Retain | Concurrency | Notes |
+| --------------------- | ----------------- | ------------------- | ------- | ------ | ----------- | ---------------------------------------------------------------------------------------- |
+| `volume-backup` | `backup` | `0 2 * * *` (02:00) | default | 14 | 2 | Volume data → S3. Primary recovery source. Incremental, full every 14 days. |
+| `post-backup-cleanup` | `snapshot-delete` | `0 3 * * *` (03:00) | default | 2 | 2 | Enforces max 2 snapshots per volume after backup runs. Prevents copy/move failures. |
+| `system-backup` | `system-backup` | `0 5 * * *` (05:00) | — | 7 | — | Longhorn config/metadata backup. `volume-backup-policy: if-not-present`. |
+| `filesystem-trim` | `filesystem-trim` | `0 4 * * *` (04:00) | default | — | 2 | Reclaim space from deleted files. Runs between backup (02:00) and system-backup (05:00). |
**Global Longhorn settings:**
+
- Max snapshots per volume: **5** (hard ceiling, monitoring before raising — `snapshot-delete` retain=2 enforces the soft limit, leaving 3 slots for system snapshots during replica rebuilds)
- Backup target: `s3://mtrnord-longhorn-backup` (Hetzner Object Storage HEL1)
**Why no `snapshot` or `snapshot-cleanup` job:**
+
- No `snapshot` job: with a low global snapshot limit, an hourly retain=24 would immediately hit the ceiling. Velero (6h) + Longhorn backup (daily) provide sufficient recovery points without in-cluster snapshots.
- No `snapshot-cleanup`: redundant when `backup` job does pre-backup cleanup and `snapshot-delete` enforces the count hard limit.
### 3a. Restore a single Longhorn volume from backup
**Via Longhorn UI (easiest):**
+
1. Go to Longhorn UI → Backup
2. Find the volume backup
3. Click Restore → enter a name for the restored volume
4. Once restored, create a PVC pointing to the new volume or update the app's PVC
**Via kubectl:**
+
```bash
# List available backups
kubectl -n longhorn-system get backups.longhorn.io
@@ -308,12 +314,12 @@ Velero recreates the VolumeSnapshot and Longhorn restores the volume from it aut
### 3d. Which method to use?
-| Situation | Best method |
-|---|---|
-| Single volume, recent data loss | Longhorn UI restore (3a) |
-| Need data from >4 days ago | Velero restore (3c) — 14-day retention |
-| Total Longhorn state loss | Longhorn system-backup restore (3b) |
-| App namespace fully deleted | Velero namespace restore (Scenario 1) |
+| Situation | Best method |
+| ------------------------------- | -------------------------------------- |
+| Single volume, recent data loss | Longhorn UI restore (3a) |
+| Need data from >4 days ago | Velero restore (3c) — 14-day retention |
+| Total Longhorn state loss | Longhorn system-backup restore (3b) |
+| App namespace fully deleted | Velero namespace restore (Scenario 1) |
---
@@ -322,6 +328,7 @@ Velero recreates the VolumeSnapshot and Longhorn restores the volume from it aut
Use this when the entire cluster is gone and you need to rebuild from scratch.
### Prerequisites
+
- Terraform state intact (in Hetzner Cloud or backed up)
- Access to S3 buckets: `mtrnord-talos-velero` and `mtrnord-talos-pg-backup`
- Age private key (stored separately — see below)
@@ -409,16 +416,16 @@ flux get ks apps
## Key Backup Locations
-| What | Where | Path |
-|---|---|---|
-| Velero backups (all namespaces + volumes) | Hetzner Object Storage HEL1 | `mtrnord-talos-velero` bucket |
-| Postgres WAL archives | Hetzner Object Storage HEL1 | `mtrnord-talos-pg-backup/pg-base-backup/pg-cluster-v2/` |
-| Postgres scheduled base backups | same bucket | `mtrnord-talos-pg-backup/pg-base-backup/pg-cluster-v2/base/` |
-| Longhorn volume backups | Hetzner Object Storage HEL1 | `mtrnord-longhorn-backup` bucket |
-| Longhorn system backups | same bucket | `mtrnord-longhorn-backup` bucket |
-| GitOps repo | GitHub | MTRNord/gitops |
-| Terraform state | Hetzner Cloud S3 / local | cluster2025-talos/cloud/terraform.tfstate |
-| Age private key | Local machine | `~/.config/sops/age/keys.txt` or `age.agekey` |
+| What | Where | Path |
+| ----------------------------------------- | --------------------------- | ------------------------------------------------------------ |
+| Velero backups (all namespaces + volumes) | Hetzner Object Storage HEL1 | `mtrnord-talos-velero` bucket |
+| Postgres WAL archives | Hetzner Object Storage HEL1 | `mtrnord-talos-pg-backup/pg-base-backup/pg-cluster-v2/` |
+| Postgres scheduled base backups | same bucket | `mtrnord-talos-pg-backup/pg-base-backup/pg-cluster-v2/base/` |
+| Longhorn volume backups | Hetzner Object Storage HEL1 | `mtrnord-longhorn-backup` bucket |
+| Longhorn system backups | same bucket | `mtrnord-longhorn-backup` bucket |
+| GitOps repo | GitHub | MTRNord/gitops |
+| Terraform state | Hetzner Cloud S3 / local | cluster2025-talos/cloud/terraform.tfstate |
+| Age private key | Local machine | `~/.config/sops/age/keys.txt` or `age.agekey` |
> **CRITICAL**: The age private key is the master key for all cluster secrets. Store it in a password manager (Bitwarden, etc.) in addition to the local file. Without it you cannot decrypt any secret in the cluster.
diff --git a/Makefile b/Makefile
@@ -0,0 +1,23 @@
+export DISABLE_MKDOCS_2_WARNING := true
+
+.PHONY: docs-install docs docs-build docs-deploy docs-clean
+
+## Install documentation dependencies (run once)
+docs-install:
+ pip install -r requirements-docs.txt
+
+## Live preview with auto-reload at http://127.0.0.1:8000
+docs:
+ mkdocs serve
+
+## Build static site into site/ (check for errors without deploying)
+docs-build:
+ mkdocs build --strict
+
+## Manually push to GitHub Pages (same as CI does)
+docs-deploy:
+ mkdocs gh-deploy --force
+
+## Remove built site output
+docs-clean:
+ rm -rf site/
diff --git a/README.md b/README.md
@@ -1,29 +1,43 @@
-# Talos Kubernetes GitOps Repository
+# Midnightthoughts GitOps
GitOps configuration for Kubernetes on [Talos Linux](https://www.talos.dev/) using [Flux CD](https://fluxcd.io/).
## Stack
-- **OS**: Talos Linux
-- **GitOps**: Flux CD
-- **Secrets**: SOPS + age
-- **CNI**: Cilium
-- **Updates**: Renovate
-
-## Structure
+| Component | Technology |
+|---|---|
+| OS | [Talos Linux](https://www.talos.dev/) |
+| GitOps | [Flux CD](https://fluxcd.io/) |
+| Cloud | [Hetzner Cloud](https://www.hetzner.com/cloud) (CAX nodes) + Hetzner LBs |
+| CNI | [Cilium](https://cilium.io/) |
+| Ingress | [Envoy Gateway](https://gateway.envoyproxy.io/) |
+| Storage | [Longhorn](https://longhorn.io/) (block) + Hetzner Object Storage (S3) |
+| Database | [CloudNativePG](https://cloudnative-pg.io/) (PostgreSQL 18) |
+| Secrets | [SOPS](https://getsops.io/) + [age](https://age-encryption.org/) |
+| Backups | [Velero](https://velero.io/) (4× daily) |
+| Monitoring | VictoriaMetrics + Grafana + Loki |
+| Auth | [Authentik](https://goauthentik.io/) |
+| Updates | [Renovate](https://docs.renovatebot.com/) |
+| TLS | [cert-manager](https://cert-manager.io/) + Let's Encrypt |
+
+## Repository Structure
```
-├── clusters/talos_cluster/ # Flux bootstrap
-├── infrastructure_talos/ # Controllers, monitoring
-├── apps/ # Applications
-└── .github/workflows/ # CI (validation, renovate)
+├── clusters/talos_cluster/ # Flux bootstrap & Kustomizations
+├── infrastructure_talos/ # Controllers (CNI, storage, cert-manager, monitoring)
+│ ├── controllers/ # Longhorn, Velero, CloudNativePG, etc.
+│ └── configs/ # Cluster config (CNPG cluster, issuers, secrets)
+├── apps/
+│ └── talos_cluster/ # All deployed applications
+├── .github/workflows/ # CI: validation, security scans, docs deployment
+└── scripts/ # Helper scripts
```
## Quick Start
```bash
-# Bootstrap Talos
-talosctl gen config my-cluster https://CONTROL_PLANE_IP:6443
+# Generate Talos config
+talosctl gen config cluster-2025 https://CONTROL_PLANE_IP:6443
talosctl apply-config --insecure --nodes CONTROL_PLANE_IP --file controlplane.yaml
talosctl bootstrap --nodes CONTROL_PLANE_IP
talosctl kubeconfig --nodes CONTROL_PLANE_IP
@@ -32,84 +46,103 @@ talosctl kubeconfig --nodes CONTROL_PLANE_IP
flux check --pre
kubectl apply -k clusters/talos_cluster/flux-system
-# Setup SOPS (generate NEW key, never use the one in repo!)
+# Setup SOPS (generate a NEW key — never reuse an existing one)
age-keygen -o age.key
kubectl create secret generic sops-age --namespace=flux-system --from-file=age.agekey=age.key
-# Update .sops.yaml with your public key, store age.key securely
+# Update .sops.yaml with your public key
+# Store age.key somewhere secure (password manager, not in this repo)
-# Deploy
+# Trigger reconciliation
flux reconcile kustomization flux-system --with-source
```
-## Secrets
+## Working with Secrets
```bash
-# Encrypt
-sops --encrypt --encrypted-regex '^(data|stringData)$' secret.yaml > secret.enc.yaml
+# Encrypt a new secret
+sops -e -i secret.yaml
+
+# Edit an encrypted secret
+sops secret.yaml
-# Edit
-sops secret.enc.yaml
+# View decrypted (don't commit output)
+sops -d secret.yaml
```
-**Never commit:** `age.key`, `age.agekey`, decrypted secrets
+The `encrypted_regex` in `.sops.yaml` controls which fields are encrypted. All secrets are encrypted with age using a shared cluster key stored in the `sops-age` Kubernetes secret.
-## Common Tasks
+## Common Operations
```bash
-# Deploy changes
-git commit -am "update" && git push
-
-# Force reconcile
+# Force reconcile after a push
flux reconcile kustomization flux-system --with-source
-# Check status
+# Check status of all Flux resources
flux get all -A
-flux logs --level=error
-# Rollback
-git revert COMMIT && git push
+# Show recent Flux errors
+flux logs --level=error --all-namespaces
+
+# Rollback: revert the commit and push
+git revert HEAD && git push
```
-## CI/CD
+## Backups
+
+Velero runs 4× daily (00:00, 06:00, 12:00, 18:00 UTC) backing up all cluster resources and Longhorn volumes to Hetzner Object Storage. CloudNativePG WAL archiving provides continuous PostgreSQL backup to a separate S3 bucket.
-- **validate.yaml**: Validates manifests, runs security scans (gitleaks, trivy, kubeaudit)
-- **renovate.yaml**: Automated dependency updates (daily 2 AM UTC)
+```bash
+# Check backup status
+kubectl get backup.velero.io -n velero --sort-by='.metadata.creationTimestamp'
-Run locally: `./scripts/validate.sh`
+# Trigger manual backup
+velero backup create manual-$(date +%Y%m%d-%H%M) --include-namespaces '*'
+```
-## Troubleshooting
+## CI/CD
+
+| Workflow | Trigger | Purpose |
+|---|---|---|
+| `validate.yaml` | PR / push to main | Manifest validation, security scans (gitleaks, trivy, kubescape) |
+| `docs.yml` | Push to main (*.md changes) | Build and deploy MkDocs to GitHub Pages |
+| `build-continuwuity.yaml` | Manual | Custom Continuwuity image build |
+Run validation locally:
```bash
-# Flux
-flux check
-flux logs --all-namespaces
+./scripts/validate.sh
+```
-# SOPS
-kubectl get secret sops-age -n flux-system
+## Documentation
-# Apps
-kubectl describe pod POD -n NAMESPACE
-kubectl logs POD -n NAMESPACE
+```bash
+# Install doc dependencies (once)
+make docs-install
-# Talos
-talosctl health --nodes NODE_IP
-talosctl logs -n NODE_IP
-```
+# Preview locally at http://127.0.0.1:8000
+make docs
-## Security Notes
+# Build to verify (outputs to site/)
+make docs-build
+```
-**Critical**: The `age.agekey` in this repo is exposed and must be rotated immediately.
+Add a `README.md` to any app directory under `apps/talos_cluster/<app>/` and it will automatically appear in the docs navigation.
-Remove sensitive files: `age.agekey`, `gerrit_key*`, `*.log`, `audit.txt`, `bak_*`
+## Troubleshooting
-Apply security policies:
+```bash
+# Flux
+flux check
+flux logs --all-namespaces
-- Pod Security Standards (restricted mode)
-- LimitRanges for resource defaults
-- ResourceQuotas for namespace limits
-- NetworkPolicies (default-deny)
+# App not reconciling
+kubectl describe kustomization <name> -n flux-system
+kubectl describe helmrelease <name> -n <namespace>
-## Resources
+# Check pod
+kubectl describe pod <pod> -n <namespace>
+kubectl logs <pod> -n <namespace> --previous
-- [Flux Docs](https://fluxcd.io/docs/)
-- [Talos Docs](https://www.talos.dev/docs/)
+# Talos node health
+talosctl health --nodes <NODE_IP>
+talosctl logs -n <NODE_IP>
+```
diff --git a/apps/.pages b/apps/.pages
@@ -0,0 +1 @@
+title: Applications
diff --git a/apps/talos_cluster/.pages b/apps/talos_cluster/.pages
@@ -0,0 +1 @@
+title: Talos Cluster
diff --git a/apps/talos_cluster/k6-operator/README.md b/apps/talos_cluster/k6-operator/README.md
@@ -0,0 +1 @@
+# K6 Operator
diff --git a/docs/gen_pages.py b/docs/gen_pages.py
@@ -0,0 +1,38 @@
+"""
+Auto-discover markdown files from the repo and expose them as virtual MkDocs pages.
+
+README.md files in any directory are included as the page for that directory.
+DISASTER_RECOVERY.md and other root-level .md files are included as top-level pages.
+PLAN.md files are excluded (internal planning docs, not committed).
+.pages files are copied so mkdocs-awesome-pages-plugin picks up nav labels.
+
+Run by mkdocs-gen-files during `mkdocs serve` / `mkdocs build`.
+"""
+
+from pathlib import Path
+
+import mkdocs_gen_files
+
+ROOT = Path(".")
+EXCLUDE_DIRS = {".git", ".github", "docs", "site", "security-policies"}
+
+
+def is_excluded(path: Path) -> bool:
+ return any(part in EXCLUDE_DIRS for part in path.parts)
+
+
+# Copy markdown files into the virtual docs directory
+for md_path in sorted(ROOT.rglob("*.md")):
+ if is_excluded(md_path):
+ continue
+ if md_path.name == "PLAN.md":
+ continue
+ with mkdocs_gen_files.open(str(md_path), "w") as f:
+ f.write(md_path.read_text())
+
+# Copy .pages files so mkdocs-awesome-pages-plugin picks up nav labels and ordering
+for pages_path in sorted(ROOT.rglob(".pages")):
+ if is_excluded(pages_path):
+ continue
+ with mkdocs_gen_files.open(str(pages_path), "w") as f:
+ f.write(pages_path.read_text())
diff --git a/infrastructure_talos/.pages b/infrastructure_talos/.pages
@@ -0,0 +1 @@
+title: Infrastructure
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -0,0 +1,50 @@
+site_name: Midnightthoughts GitOps
+site_description: Kubernetes GitOps configuration for the Midnightthoughts infrastructure
+
+docs_dir: docs
+
+watch:
+ - apps
+ - infrastructure_talos
+ - clusters
+ - scripts
+ - README.md
+ - DISASTER_RECOVERY.md
+
+theme:
+ name: material
+ features:
+ - navigation.tabs
+ - navigation.sections
+ - navigation.indexes
+ - navigation.top
+ - search.highlight
+ - content.code.copy
+ palette:
+ - scheme: default
+ toggle:
+ icon: material/brightness-7
+ name: Switch to dark mode
+ - scheme: slate
+ toggle:
+ icon: material/brightness-4
+ name: Switch to light mode
+
+plugins:
+ - search
+ - gen-files:
+ scripts:
+ - docs/gen_pages.py
+ - awesome-pages:
+ filename: .pages
+
+markdown_extensions:
+ - admonition
+ - pymdownx.details
+ - pymdownx.superfences
+ - pymdownx.highlight:
+ anchor_linenums: true
+ - pymdownx.inlinehilite
+ - pymdownx.snippets
+ - toc:
+ permalink: true
diff --git a/requirements-docs.txt b/requirements-docs.txt
@@ -0,0 +1,3 @@
+mkdocs-material>=9.5,<10
+mkdocs-gen-files>=0.5
+mkdocs-awesome-pages-plugin>=2.9
