commit 61dd92758556ec1ae2e83067e3acc637e36fad77
parent 6ef6562365d044f4d6b27e70b816f349b630d943
Author: MTRNord <mtrnord1@gmail.com>
Date: Sun, 16 Apr 2023 23:09:49 +0200
Use rook ceph without helm for better control
Diffstat:
3 files changed, 788 insertions(+), 255 deletions(-)
diff --git a/apps/base/rook/filesystem.yaml b/apps/base/rook/filesystem.yaml
@@ -0,0 +1,532 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+ name: rook-ceph
+ namespace: rook-ceph # namespace:cluster
+spec:
+ cephVersion:
+ # The container image used to launch the Ceph daemon pods (mon, mgr, osd, mds, rgw).
+ # v16 is Pacific, and v17 is Quincy.
+ # RECOMMENDATION: In production, use a specific version tag instead of the general v17 flag, which pulls the latest release and could result in different
+ # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
+ # If you want to be more precise, you can always use a timestamp tag such quay.io/ceph/ceph:v17.2.6-20230410
+ # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
+ image: quay.io/ceph/ceph:v17.2.6
+ # Whether to allow unsupported versions of Ceph. Currently `pacific` and `quincy` are supported.
+ # Future versions such as `reef` (v18) would require this to be set to `true`.
+ # Do not set to true in production.
+ allowUnsupported: false
+ # The path on the host where configuration files will be persisted. Must be specified.
+ # Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster.
+ # In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment.
+ dataDirHostPath: /var/lib/rook
+ # Whether or not upgrade should continue even if a check fails
+ # This means Ceph's status could be degraded and we don't recommend upgrading but you might decide otherwise
+ # Use at your OWN risk
+ # To understand Rook's upgrade process of Ceph, read https://rook.io/docs/rook/latest/ceph-upgrade.html#ceph-version-upgrades
+ skipUpgradeChecks: false
+ # Whether or not continue if PGs are not clean during an upgrade
+ continueUpgradeAfterChecksEvenIfNotHealthy: false
+ # WaitTimeoutForHealthyOSDInMinutes defines the time (in minutes) the operator would wait before an OSD can be stopped for upgrade or restart.
+ # If the timeout exceeds and OSD is not ok to stop, then the operator would skip upgrade for the current OSD and proceed with the next one
+ # if `continueUpgradeAfterChecksEvenIfNotHealthy` is `false`. If `continueUpgradeAfterChecksEvenIfNotHealthy` is `true`, then operator would
+ # continue with the upgrade of an OSD even if its not ok to stop after the timeout. This timeout won't be applied if `skipUpgradeChecks` is `true`.
+ # The default wait timeout is 10 minutes.
+ waitTimeoutForHealthyOSDInMinutes: 10
+ mon:
+ # Set the number of mons to be started. Generally recommended to be 3.
+ # For highest availability, an odd number of mons should be specified.
+ count: 3
+ # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason.
+ # Mons should only be allowed on the same node for test environments where data loss is acceptable.
+ allowMultiplePerNode: false
+ mgr:
+ # When higher availability of the mgr is needed, increase the count to 2.
+ # In that case, one mgr will be active and one in standby. When Ceph updates which
+ # mgr is active, Rook will update the mgr services to match the active mgr.
+ count: 2
+ allowMultiplePerNode: false
+ modules:
+ # Several modules should not need to be included in this list. The "dashboard" and "monitoring" modules
+ # are already enabled by other settings in the cluster CR.
+ - name: pg_autoscaler
+ enabled: true
+ # enable the ceph dashboard for viewing cluster status
+ dashboard:
+ enabled: true
+ # serve the dashboard under a subpath (useful when you are accessing the dashboard via a reverse proxy)
+ # urlPrefix: /ceph-dashboard
+ # serve the dashboard at the given port.
+ # port: 8443
+ # serve the dashboard using SSL
+ ssl: false
+ # enable prometheus alerting for cluster
+ monitoring:
+ # requires Prometheus to be pre-installed
+ enabled: true
+ network:
+ connections:
+ # Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network.
+ # The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted.
+ # When encryption is not enabled, clients still establish a strong initial authentication and data integrity is still validated with a crc check.
+ # IMPORTANT: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only,
+ # you can set the "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class.
+ # The nbd and fuse drivers are *not* recommended in production since restarting the csi driver pod will disconnect the volumes.
+ encryption:
+ enabled: false
+ # Whether to compress the data in transit across the wire. The default is false.
+ # Requires Ceph Quincy (v17) or newer. Also see the kernel requirements above for encryption.
+ compression:
+ enabled: false
+ # Whether to require communication over msgr2. If true, the msgr v1 port (6789) will be disabled
+ # and clients will be required to connect to the Ceph cluster with the v2 port (3300).
+ # Requires a kernel that supports msgr v2 (kernel 5.11 or CentOS 8.4 or newer).
+ requireMsgr2: false
+ # enable host networking
+ #provider: host
+ # enable the Multus network provider
+ #provider: multus
+ #selectors:
+ # The selector keys are required to be `public` and `cluster`.
+ # Based on the configuration, the operator will do the following:
+ # 1. if only the `public` selector key is specified both public_network and cluster_network Ceph settings will listen on that interface
+ # 2. if both `public` and `cluster` selector keys are specified the first one will point to 'public_network' flag and the second one to 'cluster_network'
+ #
+ # In order to work, each selector value must match a NetworkAttachmentDefinition object in Multus
+ #
+ #public: public-conf --> NetworkAttachmentDefinition object name in Multus
+ #cluster: cluster-conf --> NetworkAttachmentDefinition object name in Multus
+ # Provide internet protocol version. IPv6, IPv4 or empty string are valid options. Empty string would mean IPv4
+ #ipFamily: "IPv6"
+ # Ceph daemons to listen on both IPv4 and Ipv6 networks
+ #dualStack: false
+ # Enable multiClusterService to export the mon and OSD services to peer cluster.
+ # This is useful to support RBD mirroring between two clusters having overlapping CIDRs.
+ # Ensure that peer clusters are connected using an MCS API compatible application, like Globalnet Submariner.
+ #multiClusterService:
+ # enabled: false
+
+ # enable the crash collector for ceph daemon crash collection
+ crashCollector:
+ disable: false
+ # Uncomment daysToRetain to prune ceph crash entries older than the
+ # specified number of days.
+ #daysToRetain: 30
+ # enable log collector, daemons will log on files and rotate
+ logCollector:
+ enabled: true
+ periodicity: daily # one of: hourly, daily, weekly, monthly
+ maxLogSize: 500M # SUFFIX may be 'M' or 'G'. Must be at least 1M.
+ # automate [data cleanup process](https://github.com/rook/rook/blob/master/Documentation/Storage-Configuration/ceph-teardown.md#delete-the-data-on-hosts) in cluster destruction.
+ cleanupPolicy:
+ # Since cluster cleanup is destructive to data, confirmation is required.
+ # To destroy all Rook data on hosts during uninstall, confirmation must be set to "yes-really-destroy-data".
+ # This value should only be set when the cluster is about to be deleted. After the confirmation is set,
+ # Rook will immediately stop configuring the cluster and only wait for the delete command.
+ # If the empty string is set, Rook will not destroy any data on hosts during uninstall.
+ confirmation: ""
+ # sanitizeDisks represents settings for sanitizing OSD disks on cluster deletion
+ sanitizeDisks:
+ # method indicates if the entire disk should be sanitized or simply ceph's metadata
+ # in both case, re-install is possible
+ # possible choices are 'complete' or 'quick' (default)
+ method: quick
+ # dataSource indicate where to get random bytes from to write on the disk
+ # possible choices are 'zero' (default) or 'random'
+ # using random sources will consume entropy from the system and will take much more time then the zero source
+ dataSource: zero
+ # iteration overwrite N times instead of the default (1)
+ # takes an integer value
+ iteration: 1
+ # allowUninstallWithVolumes defines how the uninstall should be performed
+ # If set to true, cephCluster deletion does not wait for the PVs to be deleted.
+ allowUninstallWithVolumes: false
+ # To control where various services will be scheduled by kubernetes, use the placement configuration sections below.
+ # The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage-node' and
+ # tolerate taints with a key of 'storage-node'.
+ placement:
+ all:
+ nodeAffinity: {}
+ tolerations:
+ - effect: NoSchedule
+ key: node-role.kubernetes.io/control-plane
+ operator: Exists
+
+ # nodeAffinity:
+ # requiredDuringSchedulingIgnoredDuringExecution:
+ # nodeSelectorTerms:
+ # - matchExpressions:
+ # - key: role
+ # operator: In
+ # values:
+ # - storage-node
+ # podAffinity:
+ # podAntiAffinity:
+ # topologySpreadConstraints:
+ # tolerations:
+ # - key: storage-node
+ # operator: Exists
+ # The above placement information can also be specified for mon, osd, and mgr components
+ # mon:
+ # Monitor deployments may contain an anti-affinity rule for avoiding monitor
+ # collocation on the same node. This is a required rule when host network is used
+ # or when AllowMultiplePerNode is false. Otherwise this anti-affinity rule is a
+ # preferred rule with weight: 50.
+ # osd:
+ # prepareosd:
+ # mgr:
+ # cleanup:
+ annotations: {}
+ # all:
+ # mon:
+ # osd:
+ # cleanup:
+ # prepareosd:
+ # clusterMetadata annotations will be applied to only `rook-ceph-mon-endpoints` configmap and the `rook-ceph-mon` and `rook-ceph-admin-keyring` secrets.
+ # And clusterMetadata annotations will not be merged with `all` annotations.
+ # clusterMetadata:
+ # kubed.appscode.com/sync: "true"
+ # If no mgr annotations are set, prometheus scrape annotations will be set by default.
+ # mgr:
+ labels: {}
+ # all:
+ # mon:
+ # osd:
+ # cleanup:
+ # mgr:
+ # prepareosd:
+ # monitoring is a list of key-value pairs. It is injected into all the monitoring resources created by operator.
+ # These labels can be passed as LabelSelector to Prometheus
+ # monitoring:
+ # crashcollector:
+ resources:
+ #The requests and limits set here, allow the mgr pod to use half of one CPU core and 1 gigabyte of memory
+ mgr:
+ limits:
+ cpu: "1000m"
+ memory: "1024Mi"
+ requests:
+ cpu: "0m"
+ memory: "512Mi"
+ # The above example requests/limits can also be added to the other components
+ mon:
+ limits:
+ cpu: "2000m"
+ memory: "2Gi"
+ requests:
+ cpu: "0m"
+ memory: "1024Mi"
+ osd:
+ limits:
+ cpu: "2000m"
+ memory: "4Gi"
+ requests:
+ cpu: "0m"
+ memory: "2Gi"
+ # For OSD it also is a possible to specify requests/limits based on device class
+ osd-hdd:
+ limits:
+ cpu: "500m"
+ memory: "60Mi"
+ requests:
+ cpu: "0m"
+ memory: "60Mi"
+ osd-ssd:
+ limits:
+ cpu: "500m"
+ memory: "60Mi"
+ requests:
+ cpu: "0m"
+ memory: "60Mi"
+ osd-nvme:
+ limits:
+ cpu: "500m"
+ memory: "60Mi"
+ requests:
+ cpu: "0m"
+ memory: "60Mi"
+ prepareosd:
+ requests:
+ cpu: "0m"
+ memory: "50Mi"
+ mgr-sidecar:
+ limits:
+ cpu: "500m"
+ memory: "100Mi"
+ requests:
+ cpu: "0m"
+ memory: "40Mi"
+ crashcollector:
+ limits:
+ cpu: "500m"
+ memory: "60Mi"
+ requests:
+ cpu: "0m"
+ memory: "60Mi"
+ logcollector:
+ limits:
+ cpu: "500m"
+ memory: "1Gi"
+ requests:
+ cpu: "0m"
+ memory: "100Mi"
+ cleanup:
+ limits:
+ cpu: "500m"
+ memory: "1Gi"
+ requests:
+ cpu: "0m"
+ memory: "100Mi"
+ # The option to automatically remove OSDs that are out and are safe to destroy.
+ removeOSDsIfOutAndSafeToRemove: false
+ priorityClassNames:
+ #all: rook-ceph-default-priority-class
+ mon: system-node-critical
+ osd: system-node-critical
+ mgr: system-cluster-critical
+ #crashcollector: rook-ceph-crashcollector-priority-class
+ storage: # cluster level storage configuration and selection
+ useAllNodes: true
+ useAllDevices: true
+ #deviceFilter:
+ config: {}
+ # crushRoot: "custom-root" # specify a non-default root label for the CRUSH map
+ # metadataDevice: "md0" # specify a non-rotational storage so ceph-volume will use it as block db device of bluestore.
+ # databaseSizeMB: "1024" # uncomment if the disks are smaller than 100 GB
+ # journalSizeMB: "1024" # uncomment if the disks are 20 GB or smaller
+ # osdsPerDevice: "1" # this value can be overridden at the node or device level
+ # encryptedDevice: "true" # the default value for this option is "false"
+ # Individual nodes and their config can be specified as well, but 'useAllNodes' above must be set to false. Then, only the named
+ # nodes below will be used as storage resources. Each node's 'name' field should match their 'kubernetes.io/hostname' label.
+ nodes:
+ - name: "control-plane-2"
+ devices: # specific devices to use for storage can be specified for each node
+ - name: "sdb"
+ - name: "control-plane-3"
+ devices: # specific devices to use for storage can be specified for each node
+ - name: "sdb"
+ # - name: "nvme01" # multiple osds can be created on high performance devices
+ # config:
+ # osdsPerDevice: "5"
+ # - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths
+ # config: # configuration can be specified at the node level which overrides the cluster level config
+ # - name: "172.17.4.301"
+ # deviceFilter: "^sd."
+ # when onlyApplyOSDPlacement is false, will merge both placement.All() and placement.osd
+ onlyApplyOSDPlacement: false
+ # The section for configuring management of daemon disruptions during upgrade or fencing.
+ disruptionManagement:
+ # If true, the operator will create and manage PodDisruptionBudgets for OSD, Mon, RGW, and MDS daemons. OSD PDBs are managed dynamically
+ # via the strategy outlined in the [design](https://github.com/rook/rook/blob/master/design/ceph/ceph-managed-disruptionbudgets.md). The operator will
+ # block eviction of OSDs by default and unblock them safely when drains are detected.
+ managePodBudgets: true
+ # A duration in minutes that determines how long an entire failureDomain like `region/zone/host` will be held in `noout` (in addition to the
+ # default DOWN/OUT interval) when it is draining. This is only relevant when `managePodBudgets` is `true`. The default value is `30` minutes.
+ osdMaintenanceTimeout: 30
+ # A duration in minutes that the operator will wait for the placement groups to become healthy (active+clean) after a drain was completed and OSDs came back up.
+ # Operator will continue with the next drain if the timeout exceeds. It only works if `managePodBudgets` is `true`.
+ # No values or 0 means that the operator will wait until the placement groups are healthy before unblocking the next drain.
+ pgHealthCheckTimeout: 0
+
+ # healthChecks
+ # Valid values for daemons are 'mon', 'osd', 'status'
+ healthCheck:
+ daemonHealth:
+ mon:
+ disabled: false
+ interval: 45s
+ osd:
+ disabled: false
+ interval: 60s
+ status:
+ disabled: false
+ interval: 60s
+ # Change pod liveness probe timing or threshold values. Works for all mon,mgr,osd daemons.
+ livenessProbe:
+ mon:
+ disabled: false
+ mgr:
+ disabled: false
+ osd:
+ disabled: false
+ # Change pod startup probe timing or threshold values. Works for all mon,mgr,osd daemons.
+ startupProbe:
+ mon:
+ disabled: false
+ mgr:
+ disabled: false
+ osd:
+ disabled: false
+---
+apiVersion: ceph.rook.io/v1
+kind: CephFilesystem
+metadata:
+ name: main-fs
+ namespace: rook-ceph
+spec:
+ # The metadata pool spec. Must use replication.
+ metadataPool:
+ replicated:
+ size: 3
+ requireSafeReplicaSize: true
+ parameters:
+ # Inline compression mode for the data pool
+ # Further reference: https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#inline-compression
+ compression_mode:
+ none
+ # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
+ # for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
+ #target_size_ratio: ".5"
+ # The list of data pool specs. Can use replication or erasure coding.
+ dataPools:
+ - name: replicated
+ failureDomain: host
+ replicated:
+ size: 3
+ # Disallow setting pool with replica 1, this could lead to data loss without recovery.
+ # Make sure you're *ABSOLUTELY CERTAIN* that is what you want
+ requireSafeReplicaSize: true
+ parameters:
+ # Inline compression mode for the data pool
+ # Further reference: https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#inline-compression
+ compression_mode:
+ none
+ # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
+ # for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
+ #target_size_ratio: ".5"
+ # Whether to preserve filesystem after CephFilesystem CRD deletion
+ preserveFilesystemOnDelete: true
+ # The metadata service (mds) configuration
+ metadataServer:
+ # The number of active MDS instances
+ activeCount: 1
+ # Whether each active MDS instance will have an active standby with a warm metadata cache for faster failover.
+ # If false, standbys will be available, but will not have a warm cache.
+ activeStandby: true
+ # The affinity rules to apply to the mds deployment
+ placement:
+ # nodeAffinity:
+ # requiredDuringSchedulingIgnoredDuringExecution:
+ # nodeSelectorTerms:
+ # - matchExpressions:
+ # - key: role
+ # operator: In
+ # values:
+ # - mds-node
+ # topologySpreadConstraints:
+ # tolerations:
+ # - key: mds-node
+ # operator: Exists
+ # podAffinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: app
+ operator: In
+ values:
+ - rook-ceph-mds
+ ## Add this if you want to allow mds daemons for different filesystems to run on one
+ ## node. The value in "values" must match .metadata.name.
+ # - key: rook_file_system
+ # operator: In
+ # values:
+ # - myfs
+ # topologyKey: kubernetes.io/hostname will place MDS across different hosts
+ topologyKey: kubernetes.io/hostname
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ podAffinityTerm:
+ labelSelector:
+ matchExpressions:
+ - key: app
+ operator: In
+ values:
+ - rook-ceph-mds
+ # topologyKey: */zone can be used to spread MDS across different AZ
+ # Use <topologyKey: failure-domain.beta.kubernetes.io/zone> in k8s cluster if your cluster is v1.16 or lower
+ # Use <topologyKey: topology.kubernetes.io/zone> in k8s cluster is v1.17 or upper
+ topologyKey: topology.kubernetes.io/zone
+ # A key/value list of annotations
+ # annotations:
+ # key: value
+ # A key/value list of labels
+ # labels:
+ # key: value
+ resources:
+ # The requests and limits set here, allow the filesystem MDS Pod(s) to use half of one CPU core and 1 gigabyte of memory
+ limits:
+ cpu: "500m"
+ memory: "1024Mi"
+ requests:
+ cpu: "00m"
+ memory: "1024Mi"
+ priorityClassName: system-cluster-critical
+ livenessProbe:
+ disabled: false
+ startupProbe:
+ disabled: false
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ name: rook-cephfs
+ namespace: rook-cephfs
+# Change "rook-ceph" provisioner prefix to match the operator namespace if needed
+provisioner: rook-ceph.cephfs.csi.ceph.com # driver:namespace:operator
+parameters:
+ # clusterID is the namespace where the rook cluster is running
+ # If you change this namespace, also change the namespace below where the secret namespaces are defined
+ clusterID: rook-ceph # namespace:cluster
+
+ # CephFS filesystem name into which the volume shall be created
+ fsName: main-fs
+
+ # Ceph pool into which the volume shall be created
+ # Required for provisionVolume: "true"
+ pool: main-fs-replicated
+
+ # The secrets contain Ceph admin credentials. These are generated automatically by the operator
+ # in the same namespace as the cluster.
+ csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+ csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph # namespace:cluster
+ csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+ csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph # namespace:cluster
+ csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+ csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster
+
+ # (optional) The driver can use either ceph-fuse (fuse) or ceph kernel client (kernel)
+ # If omitted, default volume mounter will be used - this is determined by probing for ceph-fuse
+ # or by setting the default mounter explicitly via --volumemounter command-line argument.
+ # mounter: kernel
+reclaimPolicy: Retain
+allowVolumeExpansion: true
+mountOptions: []
+ # uncomment the following line for debugging
+ #- debug
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: rook-ceph-mgr-dashboard
+ namespace: rook-ceph
+ annotations:
+ traefik.ingress.kubernetes.io/router.middlewares: traefik-ingress-sso@kubernetescrd
+ cert-manager.io/cluster-issuer: letsencrypt-dns
+spec:
+ tls:
+ - hosts:
+ - rook.ceph.midnightthoughts.space
+ secretName: rook.ceph.midnightthoughts.space-tls
+ rules:
+ - host: rook.ceph.midnightthoughts.space
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: rook-ceph-mgr-dashboard
+ port:
+ name: https-dashboard
diff --git a/apps/base/rook/kustomization.yaml b/apps/base/rook/kustomization.yaml
@@ -4,3 +4,4 @@ namespace: rook-ceph
resources:
- repository.yaml
- release.yaml
+ - filesystem.yaml
diff --git a/apps/base/rook/release.yaml b/apps/base/rook/release.yaml
@@ -23,263 +23,263 @@ spec:
enabled: false
monitoring:
enabled: true
----
-apiVersion: helm.toolkit.fluxcd.io/v2beta1
-kind: HelmRelease
-metadata:
- name: rook-ceph-cluster
- namespace: rook-ceph
-spec:
- interval: 5m
- chart:
- spec:
- version: "1.11.x"
- chart: rook-ceph-cluster
- sourceRef:
- kind: HelmRepository
- name: rook-ceph
- interval: 60m
- install:
- crds: Create
- upgrade:
- crds: CreateReplace
- values:
- toolbox:
- enabled: false
- monitoring:
- enabled: true
- createPrometheusRules: true
- cephClusterSpec:
- cephVersion:
- image: quay.io/ceph/ceph:v17.2.6
- mon:
- # Set the number of mons to be started. Generally recommended to be 3.
- # For highest availability, an odd number of mons should be specified.
- count: 3
- # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason.
- # Mons should only be allowed on the same node for test environments where data loss is acceptable.
- allowMultiplePerNode: false
- mgr:
- # When higher availability of the mgr is needed, increase the count to 2.
- # In that case, one mgr will be active and one in standby. When Ceph updates which
- # mgr is active, Rook will update the mgr services to match the active mgr.
- count: 2
- allowMultiplePerNode: false
- modules:
- # Several modules should not need to be included in this list. The "dashboard" and "monitoring" modules
- # are already enabled by other settings in the cluster CR.
- - name: pg_autoscaler
- enabled: true
+# ---
+# apiVersion: helm.toolkit.fluxcd.io/v2beta1
+# kind: HelmRelease
+# metadata:
+# name: rook-ceph-cluster
+# namespace: rook-ceph
+# spec:
+# interval: 5m
+# chart:
+# spec:
+# version: "1.11.x"
+# chart: rook-ceph-cluster
+# sourceRef:
+# kind: HelmRepository
+# name: rook-ceph
+# interval: 60m
+# install:
+# crds: Create
+# upgrade:
+# crds: CreateReplace
+# values:
+# toolbox:
+# enabled: false
+# monitoring:
+# enabled: true
+# createPrometheusRules: true
+# cephClusterSpec:
+# cephVersion:
+# image: quay.io/ceph/ceph:v17.2.6
+# mon:
+# # Set the number of mons to be started. Generally recommended to be 3.
+# # For highest availability, an odd number of mons should be specified.
+# count: 3
+# # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason.
+# # Mons should only be allowed on the same node for test environments where data loss is acceptable.
+# allowMultiplePerNode: false
+# mgr:
+# # When higher availability of the mgr is needed, increase the count to 2.
+# # In that case, one mgr will be active and one in standby. When Ceph updates which
+# # mgr is active, Rook will update the mgr services to match the active mgr.
+# count: 2
+# allowMultiplePerNode: false
+# modules:
+# # Several modules should not need to be included in this list. The "dashboard" and "monitoring" modules
+# # are already enabled by other settings in the cluster CR.
+# - name: pg_autoscaler
+# enabled: true
- # enable the ceph dashboard for viewing cluster status
- dashboard:
- enabled: true
- # serve the dashboard under a subpath (useful when you are accessing the dashboard via a reverse proxy)
- # urlPrefix: /ceph-dashboard
- # serve the dashboard at the given port.
- # port: 8443
- # Serve the dashboard using SSL (if using ingress to expose the dashboard and `ssl: true` you need to set
- # the corresponding "backend protocol" annotation(s) for your ingress controller of choice)
- ssl: false
- storage:
- useAllNodes: false
- nodes:
- - name: "control-plane-2"
- devices:
- - name: "sdb"
- - name: "control-plane-3"
- devices:
- - name: "sdb"
- placement:
- all:
- nodeAffinity: null
- tolerations:
- - effect: NoSchedule
- key: node-role.kubernetes.io/control-plane
- operator: Exists
- resources:
- mgr:
- limits:
- cpu: "1000m"
- memory: "1Gi"
- requests:
- cpu: "0m"
- memory: "512Mi"
- mon:
- limits:
- cpu: "2000m"
- memory: "2Gi"
- requests:
- cpu: "0m"
- memory: "1Gi"
- osd:
- limits:
- cpu: "2000m"
- memory: "4Gi"
- requests:
- cpu: "0m"
- memory: "4Gi"
- prepareosd:
- # limits: It is not recommended to set limits on the OSD prepare job
- # since it's a one-time burst for memory that must be allowed to
- # complete without an OOM kill. Note however that if a k8s
- # limitRange guardrail is defined external to Rook, the lack of
- # a limit here may result in a sync failure, in which case a
- # limit should be added. 1200Mi may suffice for up to 15Ti
- # OSDs ; for larger devices 2Gi may be required.
- # cf. https://github.com/rook/rook/pull/11103
- requests:
- cpu: "0m"
- memory: "50Mi"
- mgr-sidecar:
- limits:
- cpu: "500m"
- memory: "100Mi"
- requests:
- cpu: "0m"
- memory: "40Mi"
- crashcollector:
- limits:
- cpu: "500m"
- memory: "60Mi"
- requests:
- cpu: "0m"
- memory: "60Mi"
- logcollector:
- limits:
- cpu: "500m"
- memory: "1Gi"
- requests:
- cpu: "0m"
- memory: "100Mi"
- cleanup:
- limits:
- cpu: "500m"
- memory: "1Gi"
- requests:
- cpu: "0m"
- memory: "100Mi"
- ingress:
- # -- Enable an ingress for the ceph-dashboard
- dashboard:
- annotations:
- traefik.ingress.kubernetes.io/router.middlewares: traefik-ingress-sso@kubernetescrd
- cert-manager.io/cluster-issuer: letsencrypt-dns
- host:
- name: dashboard.ceph.midnightthoughts.space
- #path: "/ceph-dashboard(/|$)(.*)"
- path: /
- tls:
- - hosts:
- - dashboard.ceph.midnightthoughts.space
- secretName: dashboard.ceph.midnightthoughts.space-tls
- cephBlockPools: []
- # - name: ceph-blockpool
- # # see https://github.com/rook/rook/blob/master/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md#spec for available configuration
- # spec:
- # failureDomain: host
- # replicated:
- # size: 2
- # # Enables collecting RBD per-image IO statistics by enabling dynamic OSD performance counters. Defaults to false.
- # # For reference: https://docs.ceph.com/docs/master/mgr/prometheus/#rbd-io-statistics
- # # enableRBDStats: true
- # storageClass:
- # enabled: true
- # name: ceph-block
- # isDefault: false
- # reclaimPolicy: Retain
- # allowVolumeExpansion: true
- # volumeBindingMode: "Immediate"
- # mountOptions: []
- # # see https://kubernetes.io/docs/concepts/storage/storage-classes/#allowed-topologies
- # allowedTopologies: []
- # # - matchLabelExpressions:
- # # - key: rook-ceph-role
- # # values:
- # # - storage-node
- # # see https://github.com/rook/rook/blob/master/Documentation/ceph-block.md#provision-storage for available configuration
- # parameters:
- # # (optional) mapOptions is a comma-separated list of map options.
- # # For krbd options refer
- # # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
- # # For nbd options refer
- # # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
- # # mapOptions: lock_on_read,queue_depth=1024
+# # enable the ceph dashboard for viewing cluster status
+# dashboard:
+# enabled: true
+# # serve the dashboard under a subpath (useful when you are accessing the dashboard via a reverse proxy)
+# # urlPrefix: /ceph-dashboard
+# # serve the dashboard at the given port.
+# # port: 8443
+# # Serve the dashboard using SSL (if using ingress to expose the dashboard and `ssl: true` you need to set
+# # the corresponding "backend protocol" annotation(s) for your ingress controller of choice)
+# ssl: false
+# storage:
+# useAllNodes: false
+# nodes:
+# - name: "control-plane-2"
+# devices:
+# - name: "sdb"
+# - name: "control-plane-3"
+# devices:
+# - name: "sdb"
+# placement:
+# all:
+# nodeAffinity: null
+# tolerations:
+# - effect: NoSchedule
+# key: node-role.kubernetes.io/control-plane
+# operator: Exists
+# resources:
+# mgr:
+# limits:
+# cpu: "1000m"
+# memory: "1Gi"
+# requests:
+# cpu: "0m"
+# memory: "512Mi"
+# mon:
+# limits:
+# cpu: "2000m"
+# memory: "2Gi"
+# requests:
+# cpu: "0m"
+# memory: "1Gi"
+# osd:
+# limits:
+# cpu: "2000m"
+# memory: "4Gi"
+# requests:
+# cpu: "0m"
+# memory: "4Gi"
+# prepareosd:
+# # limits: It is not recommended to set limits on the OSD prepare job
+# # since it's a one-time burst for memory that must be allowed to
+# # complete without an OOM kill. Note however that if a k8s
+# # limitRange guardrail is defined external to Rook, the lack of
+# # a limit here may result in a sync failure, in which case a
+# # limit should be added. 1200Mi may suffice for up to 15Ti
+# # OSDs ; for larger devices 2Gi may be required.
+# # cf. https://github.com/rook/rook/pull/11103
+# requests:
+# cpu: "0m"
+# memory: "50Mi"
+# mgr-sidecar:
+# limits:
+# cpu: "500m"
+# memory: "100Mi"
+# requests:
+# cpu: "0m"
+# memory: "40Mi"
+# crashcollector:
+# limits:
+# cpu: "500m"
+# memory: "60Mi"
+# requests:
+# cpu: "0m"
+# memory: "60Mi"
+# logcollector:
+# limits:
+# cpu: "500m"
+# memory: "1Gi"
+# requests:
+# cpu: "0m"
+# memory: "100Mi"
+# cleanup:
+# limits:
+# cpu: "500m"
+# memory: "1Gi"
+# requests:
+# cpu: "0m"
+# memory: "100Mi"
+# ingress:
+# # -- Enable an ingress for the ceph-dashboard
+# dashboard:
+# annotations:
+# traefik.ingress.kubernetes.io/router.middlewares: traefik-ingress-sso@kubernetescrd
+# cert-manager.io/cluster-issuer: letsencrypt-dns
+# host:
+# name: dashboard.ceph.midnightthoughts.space
+# #path: "/ceph-dashboard(/|$)(.*)"
+# path: /
+# tls:
+# - hosts:
+# - dashboard.ceph.midnightthoughts.space
+# secretName: dashboard.ceph.midnightthoughts.space-tls
+# cephBlockPools: []
+# # - name: ceph-blockpool
+# # # see https://github.com/rook/rook/blob/master/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md#spec for available configuration
+# # spec:
+# # failureDomain: host
+# # replicated:
+# # size: 2
+# # # Enables collecting RBD per-image IO statistics by enabling dynamic OSD performance counters. Defaults to false.
+# # # For reference: https://docs.ceph.com/docs/master/mgr/prometheus/#rbd-io-statistics
+# # # enableRBDStats: true
+# # storageClass:
+# # enabled: true
+# # name: ceph-block
+# # isDefault: false
+# # reclaimPolicy: Retain
+# # allowVolumeExpansion: true
+# # volumeBindingMode: "Immediate"
+# # mountOptions: []
+# # # see https://kubernetes.io/docs/concepts/storage/storage-classes/#allowed-topologies
+# # allowedTopologies: []
+# # # - matchLabelExpressions:
+# # # - key: rook-ceph-role
+# # # values:
+# # # - storage-node
+# # # see https://github.com/rook/rook/blob/master/Documentation/ceph-block.md#provision-storage for available configuration
+# # parameters:
+# # # (optional) mapOptions is a comma-separated list of map options.
+# # # For krbd options refer
+# # # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
+# # # For nbd options refer
+# # # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
+# # # mapOptions: lock_on_read,queue_depth=1024
- # # (optional) unmapOptions is a comma-separated list of unmap options.
- # # For krbd options refer
- # # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
- # # For nbd options refer
- # # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
- # # unmapOptions: force
+# # # (optional) unmapOptions is a comma-separated list of unmap options.
+# # # For krbd options refer
+# # # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
+# # # For nbd options refer
+# # # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
+# # # unmapOptions: force
- # # RBD image format. Defaults to "2".
- # imageFormat: "2"
+# # # RBD image format. Defaults to "2".
+# # imageFormat: "2"
- # # RBD image features, equivalent to OR'd bitfield value: 63
- # # Available for imageFormat: "2". Older releases of CSI RBD
- # # support only the `layering` feature. The Linux kernel (KRBD) supports the
- # # full feature complement as of 5.4
- # imageFeatures: layering
+# # # RBD image features, equivalent to OR'd bitfield value: 63
+# # # Available for imageFormat: "2". Older releases of CSI RBD
+# # # support only the `layering` feature. The Linux kernel (KRBD) supports the
+# # # full feature complement as of 5.4
+# # imageFeatures: layering
- # # These secrets contain Ceph admin credentials.
- # csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
- # csi.storage.k8s.io/provisioner-secret-namespace: "{{ .Release.Namespace }}"
- # csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
- # csi.storage.k8s.io/controller-expand-secret-namespace: "{{ .Release.Namespace }}"
- # csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
- # csi.storage.k8s.io/node-stage-secret-namespace: "{{ .Release.Namespace }}"
- # # Specify the filesystem type of the volume. If not specified, csi-provisioner
- # # will set default as `ext4`. Note that `xfs` is not recommended due to potential deadlock
- # # in hyperconverged settings where the volume is mounted on the same node as the osds.
- # csi.storage.k8s.io/fstype: ext4
- # -- A list of CephFileSystem configurations to deploy
- # @default -- See [below](#ceph-file-systems)
- cephFileSystems:
- - name: ceph-filesystem
- # see https://github.com/rook/rook/blob/master/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md#filesystem-settings for available configuration
- spec:
- metadataPool:
- replicated:
- size: 2
- dataPools:
- - failureDomain: host
- replicated:
- size: 2
- # Optional and highly recommended, 'data0' by default, see https://github.com/rook/rook/blob/master/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md#pools
- name: data0
- metadataServer:
- activeCount: 1
- activeStandby: true
- resources:
- limits:
- cpu: "2000m"
- memory: "4Gi"
- requests:
- cpu: "0m"
- memory: "4Gi"
- priorityClassName: system-cluster-critical
- storageClass:
- enabled: true
- isDefault: false
- name: ceph-filesystem
- # (Optional) specify a data pool to use, must be the name of one of the data pools above, 'data0' by default
- pool: data0
- reclaimPolicy: Retain
- allowVolumeExpansion: true
- volumeBindingMode: "Immediate"
- mountOptions: []
- # see https://github.com/rook/rook/blob/master/Documentation/ceph-filesystem.md#provision-storage for available configuration
- parameters:
- # The secrets contain Ceph admin credentials.
- csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
- csi.storage.k8s.io/provisioner-secret-namespace: "{{ .Release.Namespace }}"
- csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
- csi.storage.k8s.io/controller-expand-secret-namespace: "{{ .Release.Namespace }}"
- csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
- csi.storage.k8s.io/node-stage-secret-namespace: "{{ .Release.Namespace }}"
- # Specify the filesystem type of the volume. If not specified, csi-provisioner
- # will set default as `ext4`. Note that `xfs` is not recommended due to potential deadlock
- # in hyperconverged settings where the volume is mounted on the same node as the osds.
- csi.storage.k8s.io/fstype: ext4
- cephObjectStores: []
+# # # These secrets contain Ceph admin credentials.
+# # csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
+# # csi.storage.k8s.io/provisioner-secret-namespace: "{{ .Release.Namespace }}"
+# # csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
+# # csi.storage.k8s.io/controller-expand-secret-namespace: "{{ .Release.Namespace }}"
+# # csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
+# # csi.storage.k8s.io/node-stage-secret-namespace: "{{ .Release.Namespace }}"
+# # # Specify the filesystem type of the volume. If not specified, csi-provisioner
+# # # will set default as `ext4`. Note that `xfs` is not recommended due to potential deadlock
+# # # in hyperconverged settings where the volume is mounted on the same node as the osds.
+# # csi.storage.k8s.io/fstype: ext4
+# # -- A list of CephFileSystem configurations to deploy
+# # @default -- See [below](#ceph-file-systems)
+# cephFileSystems:
+# - name: ceph-filesystem
+# # see https://github.com/rook/rook/blob/master/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md#filesystem-settings for available configuration
+# spec:
+# metadataPool:
+# replicated:
+# size: 2
+# dataPools:
+# - failureDomain: host
+# replicated:
+# size: 2
+# # Optional and highly recommended, 'data0' by default, see https://github.com/rook/rook/blob/master/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md#pools
+# name: data0
+# metadataServer:
+# activeCount: 1
+# activeStandby: true
+# resources:
+# limits:
+# cpu: "2000m"
+# memory: "4Gi"
+# requests:
+# cpu: "0m"
+# memory: "4Gi"
+# priorityClassName: system-cluster-critical
+# storageClass:
+# enabled: true
+# isDefault: false
+# name: ceph-filesystem
+# # (Optional) specify a data pool to use, must be the name of one of the data pools above, 'data0' by default
+# pool: data0
+# reclaimPolicy: Retain
+# allowVolumeExpansion: true
+# volumeBindingMode: "Immediate"
+# mountOptions: []
+# # see https://github.com/rook/rook/blob/master/Documentation/ceph-filesystem.md#provision-storage for available configuration
+# parameters:
+# # The secrets contain Ceph admin credentials.
+# csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+# csi.storage.k8s.io/provisioner-secret-namespace: "{{ .Release.Namespace }}"
+# csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+# csi.storage.k8s.io/controller-expand-secret-namespace: "{{ .Release.Namespace }}"
+# csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+# csi.storage.k8s.io/node-stage-secret-namespace: "{{ .Release.Namespace }}"
+# # Specify the filesystem type of the volume. If not specified, csi-provisioner
+# # will set default as `ext4`. Note that `xfs` is not recommended due to potential deadlock
+# # in hyperconverged settings where the volume is mounted on the same node as the osds.
+# csi.storage.k8s.io/fstype: ext4
+# cephObjectStores: []
