cluster

Infrastructure files for Nordgedanken and Midnightthoughts.
git clone git://archive.git.mtrnord.blog/MTRNord/cluster.git
Log | Files | Refs | README

commit cf376b3df9bd7f0c5f61073233f3bf9ba1dc68fd
parent 9dbbf95c77bccb6731954023dec7760e0e23d5e6
Author: MTRNord <MTRNord@users.noreply.github.com>
Date:   Fri, 10 Apr 2026 00:53:01 +0200

more jenkins fixes

Signed-off-by: MTRNord <MTRNord@users.noreply.github.com>

Diffstat:
DJenkinsfile | 304-------------------------------------------------------------------------------
Aapps/talos_cluster/jenkins/instance/backup-pvc.yaml | 12++++++++++++
Mapps/talos_cluster/jenkins/instance/jenkins-instance.yaml | 51+++++++++++++++++++++++++++++++++++++++++----------
Mapps/talos_cluster/jenkins/instance/kustomization.yaml | 1+
Aapps/talos_cluster/jenkins/seeds/jobs/gitops-multiarch-builds.groovy | 330+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 384 insertions(+), 314 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile @@ -1,304 +0,0 @@ -pipeline { - agent any - - options { - timestamps() - timeout(time: 4, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '5')) - } - - parameters { - choice( - name: 'BUILD_IMAGE', - choices: ['matrix-backup', 'continuwuity', 'blog', 'bookwyrm', 'all'], - description: 'Which image(s) to build' - ) - } - - environment { - REGISTRY = 'registry.midnightthoughts.space' - REGISTRY_URL = 'https://registry.midnightthoughts.space' - DOCKER_CONFIG = '/var/run/secrets/docker.io' - COSIGN_KEY_PATH = '/var/run/secrets/cosign/key' - } - - stages { - stage('Prepare Build Environment') { - steps { - script { - echo "=== Preparing multi-arch build environment ===" - sh ''' - # Register QEMU handlers for multi-arch builds - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes || true - - # Verify QEMU is registered - ls -la /proc/sys/fs/binfmt_misc/ || echo "binfmt_misc not available" - - # Enable buildx for multi-arch support - docker buildx create --use --name multiarch-builder || docker buildx use multiarch-builder - docker buildx inspect --bootstrap - ''' - } - } - } - - stage('Clone Repository') { - steps { - checkout scm - } - } - - stage('Build Images') { - parallel { - stage('matrix-backup') { - when { - expression { params.BUILD_IMAGE == 'matrix-backup' || params.BUILD_IMAGE == 'all' } - } - steps { - script { - build_matrix_backup() - } - } - } - - stage('continuwuity') { - when { - expression { params.BUILD_IMAGE == 'continuwuity' || params.BUILD_IMAGE == 'all' } - } - steps { - script { - build_continuwuity() - } - } - } - - stage('blog') { - when { - expression { params.BUILD_IMAGE == 'blog' || params.BUILD_IMAGE == 'all' } - } - steps { - script { - build_blog() - } - } - } - - stage('bookwyrm') { - when { - expression { params.BUILD_IMAGE == 'bookwyrm' || params.BUILD_IMAGE == 'all' } - } - steps { - script { - build_bookwyrm() - } - } - } - } - } - } - - post { - always { - cleanWs() - } - success { - echo "✓ Build pipeline completed successfully!" - } - failure { - echo "✗ Build pipeline failed!" - } - } -} - -def build_matrix_backup() { - echo "=== Building matrix-backup ===" - docker.image('docker:latest').inside('--privileged') { - sh ''' - set -eu - - # Configure Docker login - mkdir -p ~/.docker - cp ${DOCKER_CONFIG}/config.json ~/.docker/config.json 2>/dev/null || true - - # Get image metadata - TAG_TS=$(date -u +%Y%m%d-%H%M%S) - TAG_SHA="sha-$(git rev-parse --short HEAD)" - IMAGE="${REGISTRY}/mtrnord/cluster/matrix-backup" - - echo "Building multi-arch image: ${IMAGE}" - echo "Tags: ${TAG_TS}, main, ${TAG_SHA}" - - # Build and push multi-arch image - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag "${IMAGE}:${TAG_TS}" \ - --tag "${IMAGE}:main" \ - --tag "${IMAGE}:${TAG_SHA}" \ - --push \ - -f apps/talos_cluster/image-builder/matrix-backup/Dockerfile \ - apps/talos_cluster/image-builder/matrix-backup/ - - # Sign all pushed digests with cosign - docker run --rm \ - -v ${COSIGN_KEY_PATH}:/cosign/key:ro \ - -v ~/.docker:/root/.docker:ro \ - gcr.io/projectsigstore/cosign:latest \ - sign --key /cosign/key "${IMAGE}:${TAG_TS}" - - docker run --rm \ - -v ${COSIGN_KEY_PATH}:/cosign/key:ro \ - -v ~/.docker:/root/.docker:ro \ - gcr.io/projectsigstore/cosign:latest \ - sign --key /cosign/key "${IMAGE}:main" - - docker run --rm \ - -v ${COSIGN_KEY_PATH}:/cosign/key:ro \ - -v ~/.docker:/root/.docker:ro \ - gcr.io/projectsigstore/cosign:latest \ - sign --key /cosign/key "${IMAGE}:${TAG_SHA}" - ''' - } -} - -def build_continuwuity() { - echo "=== Building continuwuity ===" - docker.image('docker:latest').inside('--privileged') { - sh ''' - set -eu - - # Configure Docker login - mkdir -p ~/.docker - cp ${DOCKER_CONFIG}/config.json ~/.docker/config.json 2>/dev/null || true - - # Get image metadata - TAG_SHA="sha-$(git rev-parse --short HEAD)" - IMAGE="${REGISTRY}/mtrnord/cluster/continuwuity" - - echo "Building multi-arch image: ${IMAGE}" - echo "Tags: main, ${TAG_SHA}" - - # Build and push multi-arch image - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag "${IMAGE}:main" \ - --tag "${IMAGE}:${TAG_SHA}" \ - --push \ - -f apps/talos_cluster/image-builder/continuwuity/Dockerfile \ - apps/talos_cluster/image-builder/continuwuity/ - - # Sign all pushed digests with cosign - docker run --rm \ - -v ${COSIGN_KEY_PATH}:/cosign/key:ro \ - -v ~/.docker:/root/.docker:ro \ - gcr.io/projectsigstore/cosign:latest \ - sign --key /cosign/key "${IMAGE}:main" - - docker run --rm \ - -v ${COSIGN_KEY_PATH}:/cosign/key:ro \ - -v ~/.docker:/root/.docker:ro \ - gcr.io/projectsigstore/cosign:latest \ - sign --key /cosign/key "${IMAGE}:${TAG_SHA}" - ''' - } -} - -def build_blog() { - echo "=== Building blog ===" - docker.image('docker:latest').inside('--privileged') { - sh ''' - set -eu - - # Configure Docker login - mkdir -p ~/.docker - cp ${DOCKER_CONFIG}/config.json ~/.docker/config.json 2>/dev/null || true - - # Get image metadata - TAG_TS=$(date -u +%Y%m%d-%H%M%S) - TAG_SHA="sha-$(git rev-parse --short HEAD)" - IMAGE="${REGISTRY}/mtrnord/blog" - - echo "Building multi-arch image: ${IMAGE}" - echo "Tags: ${TAG_TS}, latest, ${TAG_SHA}" - - # Build and push multi-arch image - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag "${IMAGE}:${TAG_TS}" \ - --tag "${IMAGE}:latest" \ - --tag "${IMAGE}:${TAG_SHA}" \ - --push \ - -f apps/talos_cluster/image-builder/blog/Dockerfile \ - apps/talos_cluster/image-builder/blog/ - - # Sign all pushed digests with cosign - docker run --rm \ - -v ${COSIGN_KEY_PATH}:/cosign/key:ro \ - -v ~/.docker:/root/.docker:ro \ - gcr.io/projectsigstore/cosign:latest \ - sign --key /cosign/key "${IMAGE}:${TAG_TS}" - - docker run --rm \ - -v ${COSIGN_KEY_PATH}:/cosign/key:ro \ - -v ~/.docker:/root/.docker:ro \ - gcr.io/projectsigstore/cosign:latest \ - sign --key /cosign/key "${IMAGE}:latest" - - docker run --rm \ - -v ${COSIGN_KEY_PATH}:/cosign/key:ro \ - -v ~/.docker:/root/.docker:ro \ - gcr.io/projectsigstore/cosign:latest \ - sign --key /cosign/key "${IMAGE}:${TAG_SHA}" - ''' - } -} - -def build_bookwyrm() { - echo "=== Building bookwyrm ===" - docker.image('docker:latest').inside('--privileged') { - sh ''' - set -eu - - # Configure Docker login - mkdir -p ~/.docker - cp ${DOCKER_CONFIG}/config.json ~/.docker/config.json 2>/dev/null || true - - # Get upstream version - VERSION=$(docker run --rm curlimages/curl:latest \ - curl -sf "https://api.github.com/repos/bookwyrm-social/bookwyrm/releases/latest" | \ - grep '"tag_name"' | head -1 | cut -d'"' -f4) - - if [ -z "$VERSION" ]; then - echo "ERROR: could not determine upstream version" - exit 1 - fi - - IMAGE="${REGISTRY}/mtrnord/cluster/bookwyrm" - - echo "Building multi-arch image: ${IMAGE}" - echo "Tags: ${VERSION}, latest" - - # Build and push multi-arch image - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --build-arg VERSION="${VERSION}" \ - --tag "${IMAGE}:${VERSION}" \ - --tag "${IMAGE}:latest" \ - --push \ - -f apps/talos_cluster/image-builder/bookwyrm/Dockerfile \ - apps/talos_cluster/image-builder/bookwyrm/ - - # Sign all pushed digests with cosign - docker run --rm \ - -v ${COSIGN_KEY_PATH}:/cosign/key:ro \ - -v ~/.docker:/root/.docker:ro \ - gcr.io/projectsigstore/cosign:latest \ - sign --key /cosign/key "${IMAGE}:${VERSION}" - - docker run --rm \ - -v ${COSIGN_KEY_PATH}:/cosign/key:ro \ - -v ~/.docker:/root/.docker:ro \ - gcr.io/projectsigstore/cosign:latest \ - sign --key /cosign/key "${IMAGE}:latest" - ''' - } -} diff --git a/apps/talos_cluster/jenkins/instance/backup-pvc.yaml b/apps/talos_cluster/jenkins/instance/backup-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: jenkins-backup + namespace: jenkins +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn-2replicas diff --git a/apps/talos_cluster/jenkins/instance/jenkins-instance.yaml b/apps/talos_cluster/jenkins/instance/jenkins-instance.yaml @@ -20,6 +20,21 @@ spec: # API settings - IMPORTANT: jenkins-operator user must have admin rights jenkinsAPISettings: authorizationStrategy: createUser + # Seed jobs - auto-create pipelines from gitops repo + seedJobs: + - id: gitops-pipelines + targets: "apps/talos_cluster/jenkins/seeds/jobs/*.groovy" + description: "GitOps pipelines and jobs" + repositoryBranch: main + repositoryUrl: https://github.com/mtrnord/gitops.git + # Backup and restore for build history + backup: + enabled: true + destination: /backup + interval: 30 + restore: + enabled: true + recoveryOnce: true # Service configuration service: type: ClusterIP @@ -74,10 +89,6 @@ spec: - name: jenkins-master image: jenkins/jenkins:2.541.3-lts imagePullPolicy: IfNotPresent - # Environment variables - env: - - name: CASC_JENKINS_CONFIG - value: /var/lib/jenkins/casc_configs # Security context for container securityContext: # Allow privileged for Docker socket access @@ -123,20 +134,37 @@ spec: protocol: TCP # Volume mounts volumeMounts: - - name: casc-config - mountPath: /var/lib/jenkins/casc_configs - readOnly: true - name: registry-secret mountPath: /var/run/secrets/docker.io readOnly: true - name: cosign-key mountPath: /var/run/secrets/cosign readOnly: true + # Backup container sidecar + - name: backup + image: quay.io/jenkins-kubernetes-operator/backup-pvc:v0.4.4 + imagePullPolicy: IfNotPresent + env: + - name: BACKUP_DIR + value: /backup + - name: JENKINS_HOME + value: /var/lib/jenkins + - name: BACKUP_COUNT + value: "3" + volumeMounts: + - name: jenkins-home + mountPath: /var/lib/jenkins + - name: backup + mountPath: /backup + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 100m + memory: 500Mi # Volumes volumes: - - name: casc-config - configMap: - name: jenkins-casc - name: registry-secret secret: secretName: registry-credentials @@ -145,6 +173,9 @@ spec: secret: secretName: cosign-signing-key defaultMode: 256 + - name: backup + persistentVolumeClaim: + claimName: jenkins-backup --- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute diff --git a/apps/talos_cluster/jenkins/instance/kustomization.yaml b/apps/talos_cluster/jenkins/instance/kustomization.yaml @@ -8,4 +8,5 @@ resources: - jenkins-casc-configmap.yaml - registry-secret.yaml - cosign-secret.yaml + - backup-pvc.yaml - jenkins-instance.yaml diff --git a/apps/talos_cluster/jenkins/seeds/jobs/gitops-multiarch-builds.groovy b/apps/talos_cluster/jenkins/seeds/jobs/gitops-multiarch-builds.groovy @@ -0,0 +1,330 @@ +// Seed job: GitOps multi-arch build pipeline +// Auto-creates the image build pipeline job from this definition + +pipelineJob('gitops-multiarch-builds') { + description('Multi-arch container image builds for GitOps infrastructure') + + triggers { + githubPush() + // Check for updates daily at 3 AM UTC + cron('H 3 * * *') + } + + parameters { + choice( + name: 'BUILD_IMAGE', + choices: ['matrix-backup', 'continuwuity', 'blog', 'bookwyrm', 'all'], + description: 'Which image(s) to build' + ) + } + + definition { + cps { + script(''' + pipeline { + agent any + + options { + timestamps() + timeout(time: 4, unit: 'HOURS') + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '5')) + } + + triggers { + githubPush() + // Nightly check for upstream updates (3 AM UTC) + cron('H 3 * * *') + } + + environment { + REGISTRY = 'registry.midnightthoughts.space' + REGISTRY_URL = 'https://registry.midnightthoughts.space' + DOCKER_CONFIG = '/var/run/secrets/docker.io' + COSIGN_KEY_PATH = '/var/run/secrets/cosign/key' + } + + stages { + stage('Prepare Build Environment') { + steps { + script { + echo "=== Preparing multi-arch build environment ===" + sh ''' + # Register QEMU handlers for multi-arch builds + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes || true + + # Verify QEMU is registered + ls -la /proc/sys/fs/binfmt_misc/ || echo "binfmt_misc not available" + + # Enable buildx for multi-arch support + docker buildx create --use --name multiarch-builder || docker buildx use multiarch-builder + docker buildx inspect --bootstrap + ''' + } + } + } + + stage('Clone Repository') { + steps { + checkout scm + } + } + + stage('Build Images') { + parallel { + stage('matrix-backup') { + when { + expression { params.BUILD_IMAGE == 'matrix-backup' || params.BUILD_IMAGE == 'all' } + } + steps { + script { + build_matrix_backup() + } + } + } + + stage('continuwuity') { + when { + expression { params.BUILD_IMAGE == 'continuwuity' || params.BUILD_IMAGE == 'all' } + } + steps { + script { + build_continuwuity() + } + } + } + + stage('blog') { + when { + expression { params.BUILD_IMAGE == 'blog' || params.BUILD_IMAGE == 'all' } + } + steps { + script { + build_blog() + } + } + } + + stage('bookwyrm') { + when { + expression { params.BUILD_IMAGE == 'bookwyrm' || params.BUILD_IMAGE == 'all' } + } + steps { + script { + build_bookwyrm() + } + } + } + } + } + } + + post { + always { + cleanWs() + } + success { + echo "✓ Build pipeline completed successfully!" + } + failure { + echo "✗ Build pipeline failed!" + } + } + } + '''.stripIndent()) + sandbox(true) + } + } +} + +def build_matrix_backup() { + echo "=== Building matrix-backup ===" + docker.image('docker:latest').inside('--privileged') { + sh ''' + set -eu + + # Configure Docker login + mkdir -p ~/.docker + cp ${DOCKER_CONFIG}/config.json ~/.docker/config.json 2>/dev/null || true + + # Get image metadata + TAG_TS=$(date -u +%Y%m%d-%H%M%S) + TAG_SHA="sha-$(git rev-parse --short HEAD)" + IMAGE="${REGISTRY}/mtrnord/cluster/matrix-backup" + + echo "Building multi-arch image: ${IMAGE}" + echo "Tags: ${TAG_TS}, main, ${TAG_SHA}" + + # Build and push multi-arch image + docker buildx build \\ + --platform linux/amd64,linux/arm64 \\ + --tag "${IMAGE}:${TAG_TS}" \\ + --tag "${IMAGE}:main" \\ + --tag "${IMAGE}:${TAG_SHA}" \\ + --push \\ + -f apps/talos_cluster/image-builder/matrix-backup/Dockerfile \\ + apps/talos_cluster/image-builder/matrix-backup/ + + # Sign all pushed digests with cosign + docker run --rm \\ + -v ${COSIGN_KEY_PATH}:/cosign/key:ro \\ + -v ~/.docker:/root/.docker:ro \\ + gcr.io/projectsigstore/cosign:latest \\ + sign --key /cosign/key "${IMAGE}:${TAG_TS}" + + docker run --rm \\ + -v ${COSIGN_KEY_PATH}:/cosign/key:ro \\ + -v ~/.docker:/root/.docker:ro \\ + gcr.io/projectsigstore/cosign:latest \\ + sign --key /cosign/key "${IMAGE}:main" + + docker run --rm \\ + -v ${COSIGN_KEY_PATH}:/cosign/key:ro \\ + -v ~/.docker:/root/.docker:ro \\ + gcr.io/projectsigstore/cosign:latest \\ + sign --key /cosign/key "${IMAGE}:${TAG_SHA}" + ''' + } +} + +def build_continuwuity() { + echo "=== Building continuwuity ===" + docker.image('docker:latest').inside('--privileged') { + sh ''' + set -eu + + # Configure Docker login + mkdir -p ~/.docker + cp ${DOCKER_CONFIG}/config.json ~/.docker/config.json 2>/dev/null || true + + # Get image metadata + TAG_SHA="sha-$(git rev-parse --short HEAD)" + IMAGE="${REGISTRY}/mtrnord/cluster/continuwuity" + + echo "Building multi-arch image: ${IMAGE}" + echo "Tags: main, ${TAG_SHA}" + + # Build and push multi-arch image + docker buildx build \\ + --platform linux/amd64,linux/arm64 \\ + --tag "${IMAGE}:main" \\ + --tag "${IMAGE}:${TAG_SHA}" \\ + --push \\ + -f apps/talos_cluster/image-builder/continuwuity/Dockerfile \\ + apps/talos_cluster/image-builder/continuwuity/ + + # Sign all pushed digests with cosign + docker run --rm \\ + -v ${COSIGN_KEY_PATH}:/cosign/key:ro \\ + -v ~/.docker:/root/.docker:ro \\ + gcr.io/projectsigstore/cosign:latest \\ + sign --key /cosign/key "${IMAGE}:main" + + docker run --rm \\ + -v ${COSIGN_KEY_PATH}:/cosign/key:ro \\ + -v ~/.docker:/root/.docker:ro \\ + gcr.io/projectsigstore/cosign:latest \\ + sign --key /cosign/key "${IMAGE}:${TAG_SHA}" + ''' + } +} + +def build_blog() { + echo "=== Building blog ===" + docker.image('docker:latest').inside('--privileged') { + sh ''' + set -eu + + # Configure Docker login + mkdir -p ~/.docker + cp ${DOCKER_CONFIG}/config.json ~/.docker/config.json 2>/dev/null || true + + # Get image metadata + TAG_TS=$(date -u +%Y%m%d-%H%M%S) + TAG_SHA="sha-$(git rev-parse --short HEAD)" + IMAGE="${REGISTRY}/mtrnord/blog" + + echo "Building multi-arch image: ${IMAGE}" + echo "Tags: ${TAG_TS}, latest, ${TAG_SHA}" + + # Build and push multi-arch image + docker buildx build \\ + --platform linux/amd64,linux/arm64 \\ + --tag "${IMAGE}:${TAG_TS}" \\ + --tag "${IMAGE}:latest" \\ + --tag "${IMAGE}:${TAG_SHA}" \\ + --push \\ + -f apps/talos_cluster/image-builder/blog/Dockerfile \\ + apps/talos_cluster/image-builder/blog/ + + # Sign all pushed digests with cosign + docker run --rm \\ + -v ${COSIGN_KEY_PATH}:/cosign/key:ro \\ + -v ~/.docker:/root/.docker:ro \\ + gcr.io/projectsigstore/cosign:latest \\ + sign --key /cosign/key "${IMAGE}:${TAG_TS}" + + docker run --rm \\ + -v ${COSIGN_KEY_PATH}:/cosign/key:ro \\ + -v ~/.docker:/root/.docker:ro \\ + gcr.io/projectsigstore/cosign:latest \\ + sign --key /cosign/key "${IMAGE}:latest" + + docker run --rm \\ + -v ${COSIGN_KEY_PATH}:/cosign/key:ro \\ + -v ~/.docker:/root/.docker:ro \\ + gcr.io/projectsigstore/cosign:latest \\ + sign --key /cosign/key "${IMAGE}:${TAG_SHA}" + ''' + } +} + +def build_bookwyrm() { + echo "=== Building bookwyrm ===" + docker.image('docker:latest').inside('--privileged') { + sh ''' + set -eu + + # Configure Docker login + mkdir -p ~/.docker + cp ${DOCKER_CONFIG}/config.json ~/.docker/config.json 2>/dev/null || true + + # Get upstream version + VERSION=$(docker run --rm curlimages/curl:latest \\ + curl -sf "https://api.github.com/repos/bookwyrm-social/bookwyrm/releases/latest" | \\ + grep '"tag_name"' | head -1 | cut -d'"' -f4) + + if [ -z "$VERSION" ]; then + echo "ERROR: could not determine upstream version" + exit 1 + fi + + IMAGE="${REGISTRY}/mtrnord/cluster/bookwyrm" + + echo "Building multi-arch image: ${IMAGE}" + echo "Tags: ${VERSION}, latest" + + # Build and push multi-arch image + docker buildx build \\ + --platform linux/amd64,linux/arm64 \\ + --build-arg VERSION="${VERSION}" \\ + --tag "${IMAGE}:${VERSION}" \\ + --tag "${IMAGE}:latest" \\ + --push \\ + -f apps/talos_cluster/image-builder/bookwyrm/Dockerfile \\ + apps/talos_cluster/image-builder/bookwyrm/ + + # Sign all pushed digests with cosign + docker run --rm \\ + -v ${COSIGN_KEY_PATH}:/cosign/key:ro \\ + -v ~/.docker:/root/.docker:ro \\ + gcr.io/projectsigstore/cosign:latest \\ + sign --key /cosign/key "${IMAGE}:${VERSION}" + + docker run --rm \\ + -v ${COSIGN_KEY_PATH}:/cosign/key:ro \\ + -v ~/.docker:/root/.docker:ro \\ + gcr.io/projectsigstore/cosign:latest \\ + sign --key /cosign/key "${IMAGE}:latest" + ''' + } +}