commit d9ecb0240d4bfd5023ed3684ec50f04f89200696
parent 4f6c5285832253656a03d303f06443cdaa10c998
Author: MTRNord <mtrnord1@gmail.com>
Date: Wed, 19 Jul 2023 11:06:12 +0200
nfs csi
Diffstat:
2 files changed, 369 insertions(+), 0 deletions(-)
diff --git a/apps/production/kustomization.yaml b/apps/production/kustomization.yaml
@@ -15,6 +15,8 @@ resources:
- ../base/zammad
- sliding-proxy-ingress.yaml
- pvcs-mjolnir.yaml
+ - nfs-server.yaml
+ - nfs-csi.yaml
patchesStrategicMerge:
- cosign-values.yaml
- vaultwarden-values.yaml
diff --git a/apps/production/nfs-csi.yaml b/apps/production/nfs-csi.yaml
@@ -0,0 +1,367 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: csi-nfs-controller-sa
+ namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: csi-nfs-node-sa
+ namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: nfs-external-provisioner-role
+rules:
+ - apiGroups: [""]
+ resources: ["persistentvolumes"]
+ verbs: ["get", "list", "watch", "create", "delete"]
+ - apiGroups: [""]
+ resources: ["persistentvolumeclaims"]
+ verbs: ["get", "list", "watch", "update"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["storageclasses"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshotclasses", "volumesnapshots"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshotcontents"]
+ verbs: ["get", "list", "watch", "update", "patch"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshotcontents/status"]
+ verbs: ["get", "update", "patch"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["get", "list", "watch", "create", "update", "patch"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["csinodes"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["nodes"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["get", "list", "watch", "create", "update", "patch"]
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["get"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: nfs-csi-provisioner-binding
+subjects:
+ - kind: ServiceAccount
+ name: csi-nfs-controller-sa
+ namespace: kube-system
+roleRef:
+ kind: ClusterRole
+ name: nfs-external-provisioner-role
+ apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+ name: nfs.csi.k8s.io
+spec:
+ attachRequired: false
+ volumeLifecycleModes:
+ - Persistent
+ fsGroupPolicy: File
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+ name: csi-nfs-controller
+ namespace: kube-system
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: csi-nfs-controller
+ template:
+ metadata:
+ labels:
+ app: csi-nfs-controller
+ spec:
+ hostNetwork: true # controller also needs to mount nfs to create dir
+ dnsPolicy: ClusterFirstWithHostNet # available values: Default, ClusterFirstWithHostNet, ClusterFirst
+ serviceAccountName: csi-nfs-controller-sa
+ nodeSelector:
+ kubernetes.io/os: linux # add "kubernetes.io/role: master" to run controller on master node
+ priorityClassName: system-cluster-critical
+ securityContext:
+ seccompProfile:
+ type: RuntimeDefault
+ tolerations:
+ - key: "node-role.kubernetes.io/master"
+ operator: "Exists"
+ effect: "NoSchedule"
+ - key: "node-role.kubernetes.io/controlplane"
+ operator: "Exists"
+ effect: "NoSchedule"
+ - key: "node-role.kubernetes.io/control-plane"
+ operator: "Exists"
+ effect: "NoSchedule"
+ containers:
+ - name: csi-provisioner
+ image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
+ args:
+ - "-v=2"
+ - "--csi-address=$(ADDRESS)"
+ - "--leader-election"
+ - "--leader-election-namespace=kube-system"
+ - "--extra-create-metadata=true"
+ env:
+ - name: ADDRESS
+ value: /csi/csi.sock
+ volumeMounts:
+ - mountPath: /csi
+ name: socket-dir
+ resources:
+ limits:
+ memory: 400Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: csi-snapshotter
+ image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2
+ args:
+ - "--v=2"
+ - "--csi-address=$(ADDRESS)"
+ - "--leader-election-namespace=kube-system"
+ - "--leader-election"
+ env:
+ - name: ADDRESS
+ value: /csi/csi.sock
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ resources:
+ limits:
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: liveness-probe
+ image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0
+ args:
+ - --csi-address=/csi/csi.sock
+ - --probe-timeout=3s
+ - --health-port=29652
+ - --v=2
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ resources:
+ limits:
+ memory: 100Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: nfs
+ image: registry.k8s.io/sig-storage/nfsplugin:v4.4.0
+ securityContext:
+ privileged: true
+ capabilities:
+ add: ["SYS_ADMIN"]
+ allowPrivilegeEscalation: true
+ imagePullPolicy: IfNotPresent
+ args:
+ - "-v=5"
+ - "--nodeid=$(NODE_ID)"
+ - "--endpoint=$(CSI_ENDPOINT)"
+ env:
+ - name: NODE_ID
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ - name: CSI_ENDPOINT
+ value: unix:///csi/csi.sock
+ ports:
+ - containerPort: 29652
+ name: healthz
+ protocol: TCP
+ livenessProbe:
+ failureThreshold: 5
+ httpGet:
+ path: /healthz
+ port: healthz
+ initialDelaySeconds: 30
+ timeoutSeconds: 10
+ periodSeconds: 30
+ volumeMounts:
+ - name: pods-mount-dir
+ mountPath: /var/lib/kubelet/pods
+ mountPropagation: "Bidirectional"
+ - mountPath: /csi
+ name: socket-dir
+ resources:
+ limits:
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ volumes:
+ - name: pods-mount-dir
+ hostPath:
+ path: /var/lib/kubelet/pods
+ type: Directory
+ - name: socket-dir
+ emptyDir: {}
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+ name: csi-nfs-node
+ namespace: kube-system
+spec:
+ updateStrategy:
+ rollingUpdate:
+ maxUnavailable: 1
+ type: RollingUpdate
+ selector:
+ matchLabels:
+ app: csi-nfs-node
+ template:
+ metadata:
+ labels:
+ app: csi-nfs-node
+ spec:
+ hostNetwork: true # original nfs connection would be broken without hostNetwork setting
+ dnsPolicy: ClusterFirstWithHostNet # available values: Default, ClusterFirstWithHostNet, ClusterFirst
+ serviceAccountName: csi-nfs-node-sa
+ priorityClassName: system-node-critical
+ securityContext:
+ seccompProfile:
+ type: RuntimeDefault
+ nodeSelector:
+ kubernetes.io/os: linux
+ tolerations:
+ - operator: "Exists"
+ containers:
+ - name: liveness-probe
+ image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0
+ args:
+ - --csi-address=/csi/csi.sock
+ - --probe-timeout=3s
+ - --health-port=29653
+ - --v=2
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ resources:
+ limits:
+ memory: 100Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: node-driver-registrar
+ image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0
+ args:
+ - --v=2
+ - --csi-address=/csi/csi.sock
+ - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+ livenessProbe:
+ exec:
+ command:
+ - /csi-node-driver-registrar
+ - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+ - --mode=kubelet-registration-probe
+ initialDelaySeconds: 30
+ timeoutSeconds: 15
+ env:
+ - name: DRIVER_REG_SOCK_PATH
+ value: /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
+ - name: KUBE_NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ - name: registration-dir
+ mountPath: /registration
+ resources:
+ limits:
+ memory: 100Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: nfs
+ securityContext:
+ privileged: true
+ capabilities:
+ add: ["SYS_ADMIN"]
+ allowPrivilegeEscalation: true
+ image: registry.k8s.io/sig-storage/nfsplugin:v4.4.0
+ args:
+ - "-v=5"
+ - "--nodeid=$(NODE_ID)"
+ - "--endpoint=$(CSI_ENDPOINT)"
+ env:
+ - name: NODE_ID
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ - name: CSI_ENDPOINT
+ value: unix:///csi/csi.sock
+ ports:
+ - containerPort: 29653
+ name: healthz
+ protocol: TCP
+ livenessProbe:
+ failureThreshold: 5
+ httpGet:
+ path: /healthz
+ port: healthz
+ initialDelaySeconds: 30
+ timeoutSeconds: 10
+ periodSeconds: 30
+ imagePullPolicy: "IfNotPresent"
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ - name: pods-mount-dir
+ mountPath: /var/lib/kubelet/pods
+ mountPropagation: "Bidirectional"
+ resources:
+ limits:
+ memory: 300Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ volumes:
+ - name: socket-dir
+ hostPath:
+ path: /var/lib/kubelet/plugins/csi-nfsplugin
+ type: DirectoryOrCreate
+ - name: pods-mount-dir
+ hostPath:
+ path: /var/lib/kubelet/pods
+ type: Directory
+ - hostPath:
+ path: /var/lib/kubelet/plugins_registry
+ type: Directory
+ name: registration-dir
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ name: nfs-csi
+provisioner: nfs.csi.k8s.io
+parameters:
+ server: nfs-server.default.svc.cluster.local
+ share: /
+ # csi.storage.k8s.io/provisioner-secret is only needed for providing mountOptions in DeleteVolume
+ # csi.storage.k8s.io/provisioner-secret-name: "mount-options"
+ # csi.storage.k8s.io/provisioner-secret-namespace: "default"
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+mountOptions:
+ - nfsvers=4.1
