Add dynamic overrides in rook-ceph app

This change add new dynamic overrides and enable/disable services based
on storage-backend.

Dynamic overrides added:
  Overrides based on how many hosts have host-fs ceph:
    - mds replicas size
    - mon count
    - mgr count
  Overrides based on host-stor
   - nodes
     - devices (osds)

Services that can be enabled:
 - CephFS (filesystem)
 - RBD (block or ecblock)
 - RGW (object)

Test Plan:
 - PASS: Load the rook-ceph app and check system-overrides for each
         chart
 - PASS: Apply the rook-ceph app and check if system-overrides have
         changed, only if something has changed before applying the app
 - PASS: Check if the services are enabled correctly based on the
         storage-backend services column
 - PASS: Check if the ceph is in HEALTH_OK status

Depends-On: https://review.opendev.org/c/starlingx/config/+/921801

Story: 2011066
Task: 50298

Change-Id: Ib245b0f1195d4c6437ed45346fe00cf16a69f67f
Signed-off-by: Gustavo Ornaghi Antunes <gustavo.ornaghiantunes@windriver.com>

helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/templates/job-ceph-mon-audit.yaml

@@ -82,6 +82,13 @@ spec:
             app: ceph-mon-audit
             app.starlingx.io/component: platform
         spec:
+          tolerations:
+          - effect: NoSchedule
+            operator: Exists
+            key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            operator: Exists
+            key: node-role.kubernetes.io/control-plane
           serviceAccountName: {{ .Values.rbac.serviceAccount }}
           restartPolicy: OnFailure
           hostNetwork: true

python3-k8sapp-rook-ceph/k8sapp_rook_ceph/k8sapp_rook_ceph/common/constants.py

@@ -22,18 +22,53 @@ FLUXCD_HELMRELEASE_ROOK_CEPH = 'rook-ceph'
 FLUXCD_HELMRELEASE_ROOK_CEPH_CLUSTER = 'rook-ceph-cluster'
 FLUXCD_HELMRELEASE_ROOK_CEPH_PROVISIONER = 'rook-ceph-provisioner'
 
-ROOK_CEPH_CLUSTER_SECRET_NAMESPACE = 'rook-ceph'
+SECRET_NAMESPACE = 'rook-ceph'
 
-ROOK_CEPH_RDB_SECRET_NAME = 'rook-csi-rbd-provisioner'
+RBD_SECRET_NAME = 'rook-csi-rbd-provisioner'
-ROOK_CEPH_RDB_NODE_SECRET_NAME = 'rook-csi-rbd-node'
+RBD_NODE_SECRET_NAME = 'rook-csi-rbd-node'
 
-ROOK_CEPH_FS_SECRET_NAME = 'rook-csi-cephfs-provisioner'
+FS_SECRET_NAME = 'rook-csi-cephfs-provisioner'
-ROOK_CEPH_FS_NODE_SECRET_NAME = 'rook-csi-cephfs-node'
+FS_NODE_SECRET_NAME = 'rook-csi-cephfs-node'
-
-ROOK_CEPH_CLUSTER_RDB_STORAGE_CLASS_NAME = 'general'
-ROOK_CEPH_CLUSTER_CEPHFS_STORAGE_CLASS_NAME = 'cephfs'
-
-ROOK_CEPH_CLUSTER_CEPHFS_FILE_SYSTEM_NAME = 'kube-cephfs'
 
 # Storage Backend Name
 SB_NAME = constants.SB_DEFAULT_NAMES[constants.SB_TYPE_CEPH_ROOK]
+
+# Node labels
+LABEL_PLACEMENT_MON = "ceph-mon-placement"
+LABEL_PLACEMENT_MGR = "ceph-mgr-placement"
+LABEL_PLACEMENT_MDS = LABEL_PLACEMENT_MON
+LABEL_PLACEMENT_OSD = "ceph-osd-placement"
+
+# Deployment Models (values used in the storage backend)
+DEP_MODEL_CONTROLLER = constants.CEPH_ROOK_DEPLOYMENT_CONTROLLER
+DEP_MODEL_DEDICATED = constants.CEPH_ROOK_DEPLOYMENT_DEDICATED
+DEP_MODEL_OPEN = constants.CEPH_ROOK_DEPLOYMENT_OPEN
+
+# Services (values used in the storage backend)
+SVC_BLOCK = constants.SB_SVC_CEPH_ROOK_BLOCK
+SVC_ECBLOCK = constants.SB_SVC_CEPH_ROOK_ECBLOCK
+SVC_FS = constants.SB_SVC_CEPH_ROOK_FILESYSTEM
+SVC_OBJ = constants.SB_SVC_CEPH_ROOK_OBJECT
+
+BLOCK_STORAGE_CLASS_NAME = 'general'
+ECBLOCK_STORAGE_CLASS_NAME = 'general'
+CEPHFS_STORAGE_CLASS_NAME = 'cephfs'
+RGW_STORAGE_CLASS_NAME = 'cephrgw'
+
+STORAGE_CLASS_NAMES = {
+    SVC_BLOCK: BLOCK_STORAGE_CLASS_NAME,
+    SVC_ECBLOCK: ECBLOCK_STORAGE_CLASS_NAME,
+    SVC_FS: CEPHFS_STORAGE_CLASS_NAME,
+    SVC_OBJ: RGW_STORAGE_CLASS_NAME
+}
+
+BLOCK_NAME = 'kube-rbd'
+ECBLOCK_NAME = 'kube-rbd'
+RGW_NAME = 'kube-rgw'
+CEPHFS_NAME = 'kube-cephfs'
+
+
+# Chart specific dynamic overrides
+# -> HELM_CHART_ROOK_CEPH_CLUSTER
+CEPH_CLUSTER_HOST_FAIL_DOMAIN = 'host'
+CEPH_CLUSTER_OSD_FAIL_DOMAIN = 'osd'

python3-k8sapp-rook-ceph/k8sapp_rook_ceph/k8sapp_rook_ceph/helm/rook_ceph.py

@@ -12,14 +12,26 @@ from sysinv.common import exception
 
 class RookCephHelm(storage.StorageBaseHelm):
     """Class to encapsulate helm operations for the rook-operator chart"""
+
     CHART = app_constants.HELM_CHART_ROOK_CEPH
     HELM_RELEASE = app_constants.FLUXCD_HELMRELEASE_ROOK_CEPH
 
+    def _get_csi_overrides(self):
+        csi_config = {
+            'enableRbdDriver': True,
+            'enableRBDSnapshotter': True,
+            'enableCephfsDriver': self._is_service_enabled(app_constants.SVC_FS),
+            'enableCephfsSnapshotter': self._is_service_enabled(app_constants.SVC_FS)
+        }
+
+        return csi_config
+
     def get_overrides(self, namespace=None):
         secrets = [{"name": "default-registry-key"}]
         overrides = {
             app_constants.HELM_NS_ROOK_CEPH: {
                 'imagePullSecrets': secrets,
+                'csi': self._get_csi_overrides()
             }
         }
 

python3-k8sapp-rook-ceph/k8sapp_rook_ceph/k8sapp_rook_ceph/helm/rook_ceph_cluster.py

@@ -11,9 +11,9 @@ from k8sapp_rook_ceph.helm import storage
 
 from sysinv.common import constants
 from sysinv.common import exception
-from sysinv.common import utils as cutils
+from sysinv.common import utils
 
-import socket
+import math
 
 
 class RookCephClusterHelm(storage.StorageBaseHelm):
@@ -22,43 +22,84 @@ class RookCephClusterHelm(storage.StorageBaseHelm):
     CHART = app_constants.HELM_CHART_ROOK_CEPH_CLUSTER
     HELM_RELEASE = app_constants.FLUXCD_HELMRELEASE_ROOK_CEPH_CLUSTER
 
-    def get_overrides(self, namespace=None):
+    FLOATING_MON_ALLOWED = False
-        overrides = {
+
-            app_constants.HELM_NS_ROOK_CEPH: {
+    def _get_mon_count(self):
-                'cephClusterSpec': self._get_cluster_override(),
+        labeled_hosts = self._get_host_count_with_label(
-                'cephBlockPools': self._get_rdb_override(),
+            app_constants.LABEL_PLACEMENT_MON)
-                'cephFileSystems': self._get_cephfs_override(),
+        if utils.is_aio_duplex_system(self.dbapi):
-                'hook': self._get_hook_override(),
+            if self.dbapi.count_hosts_matching_criteria() == 2:
-            }
+                # Bump for floating monitor on a AIO-DX only.
+                if self.FLOATING_MON_ALLOWED:
+                    labeled_hosts += 1
+
+        return labeled_hosts
+
+    def _get_mds_count(self):
+        labeled_hosts = self._get_mon_count()
+
+        if not utils.is_aio_simplex_system(self.dbapi):
+            labeled_hosts = math.floor(labeled_hosts / 2)
+
+        return labeled_hosts
+
+    def _get_failure_domain(self):
+        # based on deployment model and installation type
+        if utils.is_aio_simplex_system(self.dbapi):
+            return app_constants.CEPH_CLUSTER_OSD_FAIL_DOMAIN
+        elif self._get_deployment_model() in [app_constants.DEP_MODEL_CONTROLLER,
+                                              app_constants.DEP_MODEL_DEDICATED]:
+            return app_constants.CEPH_CLUSTER_HOST_FAIL_DOMAIN
+        else:
+            return app_constants.CEPH_CLUSTER_OSD_FAIL_DOMAIN
+
+    def _get_mon_hostname_list(self):
+        return [h.hostname.encode('utf8', 'strict')
+                for h in
+                self._get_hosts_with_label(app_constants.LABEL_PLACEMENT_MON)]
+
+    def _get_duplex_preparation(self):
+        duplex = {
+            'enable': self.FLOATING_MON_ALLOWED
         }
 
-        if namespace in self.SUPPORTED_NAMESPACES:
+        if utils.is_aio_duplex_system(self.dbapi) and self.FLOATING_MON_ALLOWED:
-            return overrides[namespace]
+            # This can code only be executed on the active controller
-        elif namespace:
+            duplex.update({'activeController':
-            raise exception.InvalidHelmNamespace(chart=self.CHART,
+                           utils.get_local_controller_hostname().encode(
-                                                 namespace=namespace)
+                               'utf8', 'strict')})
-        else:
-            return overrides
 
-    def _get_cephfs_override(self):
+            cluster_host_addr_name = utils.format_address_name(
-        if cutils.is_aio_simplex_system(self.dbapi):
+                constants.CONTROLLER_HOSTNAME, constants.NETWORK_TYPE_CLUSTER_HOST)
-            replica = 1
+            address = utils.get_primary_address_by_name(
-        else:
+                self.dbapi, cluster_host_addr_name,
-            replica = 2
+                constants.NETWORK_TYPE_CLUSTER_HOST, True)
+
+            duplex.update({'floatIP': utils.format_url_address(address.address)})
+
+        return duplex
+
+    def _get_cephfs_overrides(self):
 
         parameters = {
-            'csi.storage.k8s.io/provisioner-secret-name': app_constants.ROOK_CEPH_FS_SECRET_NAME,
+            'csi.storage.k8s.io/provisioner-secret-name':
-            'csi.storage.k8s.io/provisioner-secret-namespace': app_constants.ROOK_CEPH_CLUSTER_SECRET_NAMESPACE,
+                app_constants.FS_SECRET_NAME,
-            'csi.storage.k8s.io/controller-expand-secret-name': app_constants.ROOK_CEPH_FS_SECRET_NAME,
+            'csi.storage.k8s.io/provisioner-secret-namespace':
-            'csi.storage.k8s.io/controller-expand-secret-namespace': app_constants.ROOK_CEPH_CLUSTER_SECRET_NAMESPACE,
+                app_constants.SECRET_NAMESPACE,
-            'csi.storage.k8s.io/node-stage-secret-name': app_constants.ROOK_CEPH_FS_NODE_SECRET_NAME,
+            'csi.storage.k8s.io/controller-expand-secret-name':
-            'csi.storage.k8s.io/node-stage-secret-namespace': app_constants.ROOK_CEPH_CLUSTER_SECRET_NAMESPACE,
+                app_constants.FS_SECRET_NAME,
+            'csi.storage.k8s.io/controller-expand-secret-namespace':
+                app_constants.SECRET_NAMESPACE,
+            'csi.storage.k8s.io/node-stage-secret-name':
+                app_constants.FS_NODE_SECRET_NAME,
+            'csi.storage.k8s.io/node-stage-secret-namespace':
+                app_constants.SECRET_NAMESPACE,
             'csi.storage.k8s.io/fstype': 'ext4'
         }
 
         storage_class = {
             'enabled': True,
-            'name': app_constants.ROOK_CEPH_CLUSTER_CEPHFS_STORAGE_CLASS_NAME,
+            'name': app_constants.STORAGE_CLASS_NAMES[app_constants.SVC_FS],
             'isDefault': False,
             'pool': 'data',
             'allowVolumeExpansion': True,
@@ -66,6 +107,22 @@ class RookCephClusterHelm(storage.StorageBaseHelm):
             'parameters': parameters
         }
 
+        anti_affinity = {
+            'requiredDuringSchedulingIgnoredDuringExecution': [{
+                'labelSelector': {
+                    'matchExpressions': [{
+                        'key': 'app',
+                        'operator': 'In',
+                        'values': ['rook-ceph-mds']
+                    }]
+                },
+                'topologyKey': 'kubernetes.io/hostname'
+            }]
+        }
+
+        if utils.is_aio_simplex_system(self.dbapi):
+            anti_affinity = {}
+
         placement = {
             'nodeAffinity': {
                 'requiredDuringSchedulingIgnoredDuringExecution': {
@@ -78,29 +135,29 @@ class RookCephClusterHelm(storage.StorageBaseHelm):
                     }]
                 }
             },
+            'podAntiAffinity': anti_affinity,
             'tolerations': [{
-                            'effect': 'NoSchedule',
+                'effect': 'NoSchedule',
-                            'operator': 'Exists',
+                'operator': 'Exists',
-                            'key': 'node-role.kubernetes.io/master'
+                'key': 'node-role.kubernetes.io/master'
-                            },
+            }, {
-                            {
+                'effect': 'NoSchedule',
-                            'effect': 'NoSchedule',
+                'operator': 'Exists',
-                            'operator': 'Exists',
+                'key': 'node-role.kubernetes.io/control-plane'
-                            'key': 'node-role.kubernetes.io/control-plane'
+            }]
-                            }]
         }
 
-        ceph_fs_config = [{
+        fs_config = [{
-            'name': app_constants.ROOK_CEPH_CLUSTER_CEPHFS_FILE_SYSTEM_NAME,
+            'name': app_constants.CEPHFS_NAME,
             'spec': {
                 'metadataPool': {
                     'replicated':
-                        {'size': replica}},
+                        {'size': self._get_data_replication_factor()}},
                 'metadataServer': {
                     'labels': {
-                        'app.starlingx.io/component': "platform"
+                        'app.starlingx.io/component': 'platform'
                     },
-                    'activeCount': 1,
+                    'activeCount': self._get_mds_count(),
                     'activeStandby': True,
                     'placement': placement,
                     'resources': {
@@ -111,37 +168,39 @@ class RookCephClusterHelm(storage.StorageBaseHelm):
                             'cpu': '0'}},
                     'priorityClassName': 'system-cluster-critical'},
                 'dataPools': [{
-                    'failureDomain': 'host',
+                    'failureDomain': self._get_failure_domain(),
                     'name': 'data',
                     'replicated':
-                        {'size': replica}}],
+                        {'size': self._get_data_replication_factor()}}],
             },
             'storageClass': storage_class
         }]
 
-        return ceph_fs_config
+        return fs_config
 
-    def _get_rdb_override(self):
+    def _get_block_overrides(self):
-        if cutils.is_aio_simplex_system(self.dbapi):
-            replica = 1
-        else:
-            replica = 2
 
         parameters = {
             'imageFormat': '2',
             'imageFeatures': 'layering',
-            'csi.storage.k8s.io/provisioner-secret-name': app_constants.ROOK_CEPH_RDB_SECRET_NAME,
+            'csi.storage.k8s.io/provisioner-secret-name':
-            'csi.storage.k8s.io/provisioner-secret-namespace': app_constants.ROOK_CEPH_CLUSTER_SECRET_NAMESPACE,
+                app_constants.RBD_SECRET_NAME,
-            'csi.storage.k8s.io/controller-expand-secret-name': app_constants.ROOK_CEPH_RDB_SECRET_NAME,
+            'csi.storage.k8s.io/provisioner-secret-namespace':
-            'csi.storage.k8s.io/controller-expand-secret-namespace': app_constants.ROOK_CEPH_CLUSTER_SECRET_NAMESPACE,
+                app_constants.SECRET_NAMESPACE,
-            'csi.storage.k8s.io/node-stage-secret-name': app_constants.ROOK_CEPH_RDB_NODE_SECRET_NAME,
+            'csi.storage.k8s.io/controller-expand-secret-name':
-            'csi.storage.k8s.io/node-stage-secret-namespace': app_constants.ROOK_CEPH_CLUSTER_SECRET_NAMESPACE,
+                app_constants.RBD_SECRET_NAME,
+            'csi.storage.k8s.io/controller-expand-secret-namespace':
+                app_constants.SECRET_NAMESPACE,
+            'csi.storage.k8s.io/node-stage-secret-name':
+                app_constants.RBD_NODE_SECRET_NAME,
+            'csi.storage.k8s.io/node-stage-secret-namespace':
+                app_constants.SECRET_NAMESPACE,
             'csi.storage.k8s.io/fstype': 'ext4'
         }
 
         storage_class = {
             'enabled': True,
-            'name': app_constants.ROOK_CEPH_CLUSTER_RDB_STORAGE_CLASS_NAME,
+            'name': app_constants.STORAGE_CLASS_NAMES[app_constants.SVC_BLOCK],
             'isDefault': True,
             'allowVolumeExpansion': True,
             'reclaimPolicy': 'Delete',
@@ -149,93 +208,224 @@ class RookCephClusterHelm(storage.StorageBaseHelm):
             'parameters': parameters
         }
 
-        rdb_config = [{
+        block_config = [{
-            'name': 'kube-rbd',
+            'name': app_constants.BLOCK_NAME,
             'spec': {
-                'failureDomain': 'host',
+                'failureDomain': self._get_failure_domain(),
-                'replicated': {'size': replica}
+                'replicated': {'size': self._get_data_replication_factor()}
             },
             'storageClass': storage_class
         }]
 
-        return rdb_config
+        return block_config
 
-    def _get_cluster_override(self):
+    def _get_ecblock_overrides(self):
+
+        ec_block_config = [{
+            'name': app_constants.ECBLOCK_NAME,
+            'spec': {
+                'failureDomain': self._get_failure_domain(),
+                'replicated': {
+                    'size': self._get_data_replication_factor()
+                }
+            }
+        }, {
+            'name': 'ec-data-pool',
+            'spec': {
+                'failureDomain': self._get_failure_domain(),
+                'replicated': {
+                    'size': self._get_data_replication_factor()
+                },
+                'deviceClass': 'hdd'
+            }
+        }]
+
+        return ec_block_config
+
+    def _get_ecblocksc_overrides(self):
+
+        parameters = {
+            'clusterID': app_constants.STORAGE_CLASS_NAMES[app_constants.SVC_ECBLOCK],
+            'dataPool': 'ec-data-pool',
+            'pool': app_constants.ECBLOCK_NAME,
+            'imageFormat': '2',
+            'imageFeatures': 'layering',
+            'csi.storage.k8s.io/provisioner-secret-name':
+                app_constants.RBD_SECRET_NAME,
+            'csi.storage.k8s.io/provisioner-secret-namespace':
+                app_constants.SECRET_NAMESPACE,
+            'csi.storage.k8s.io/controller-expand-secret-name':
+                app_constants.RBD_SECRET_NAME,
+            'csi.storage.k8s.io/controller-expand-secret-namespace':
+                app_constants.SECRET_NAMESPACE,
+            'csi.storage.k8s.io/node-stage-secret-name':
+                app_constants.RBD_NODE_SECRET_NAME,
+            'csi.storage.k8s.io/node-stage-secret-namespace':
+                app_constants.SECRET_NAMESPACE,
+            'csi.storage.k8s.io/fstype': 'ext4'
+        }
+
+        ec_block_sc_config = {
+            'name': app_constants.STORAGE_CLASS_NAMES[app_constants.SVC_ECBLOCK],
+            'isDefault': True,
+            'parameters': parameters,
+            'allowVolumeExpansion': True,
+            'reclaimPolicy': 'Delete',
+            'mountOptions': []
+        }
+
+        return ec_block_sc_config
+
+    def _get_rgw_overrides(self):
+
+        metadataPool = {
+            'failureDomain': self._get_failure_domain(),
+            'replicated': {
+                'size': self._get_data_replication_factor()
+            }
+        }
+
+        dataPool = {
+            'failureDomain': self._get_failure_domain(),
+            'replicated': {
+                'size': self._get_data_replication_factor()
+            }
+        }
+
+        gateway = {
+            'port': 9800,
+            'hostNetwork': True,
+            'resources': {
+                'limits': {
+                    'memory': '4Gi'
+                },
+                'requests': {
+                    'cpu': 0,
+                    'memory': 0
+                }
+            },
+            'instances': 1,
+            'priorityClassName': 'system-cluster-critical'
+        }
+
+        parameters = {
+            'imageFormat': '2',
+            'imageFeatures': 'layering',
+            'csi.storage.k8s.io/provisioner-secret-name':
+                app_constants.RBD_SECRET_NAME,
+            'csi.storage.k8s.io/provisioner-secret-namespace':
+                app_constants.SECRET_NAMESPACE,
+            'csi.storage.k8s.io/controller-expand-secret-name':
+                app_constants.RBD_SECRET_NAME,
+            'csi.storage.k8s.io/controller-expand-secret-namespace':
+                app_constants.SECRET_NAMESPACE,
+            'csi.storage.k8s.io/node-stage-secret-name':
+                app_constants.RBD_NODE_SECRET_NAME,
+            'csi.storage.k8s.io/node-stage-secret-namespace':
+                app_constants.SECRET_NAMESPACE,
+            'csi.storage.k8s.io/fstype': 'ext4'
+        }
+
+        storage_class = {
+            'enabled': True,
+            'name': app_constants.STORAGE_CLASS_NAMES[app_constants.SVC_OBJ],
+            'isDefault': False,
+            'allowVolumeExpansion': True,
+            'volumeBindingMode': 'Immediate',
+            'reclaimPolicy': 'Delete',
+            'mountOptions': [],
+            'parameters': parameters
+        }
+
+        rgw_config = [{
+            'name': app_constants.RGW_NAME,
+            'spec': {
+                'metadataPool': metadataPool,
+                'dataPool': dataPool,
+                'preservePoolsOnDelete': True,
+                'gateway': gateway
+            },
+            'storageClass': storage_class,
+            'ingress': {
+                'enabled': False
+            }
+        }]
+
+        return rgw_config
+
+    def _get_cluster_overrides(self):
+
+        cluster_host_addr_name = utils.format_address_name(
+            constants.CONTROLLER_HOSTNAME, constants.NETWORK_TYPE_CLUSTER_HOST)
+        address = utils.get_primary_address_by_name(
+            self.dbapi, cluster_host_addr_name,
+            constants.NETWORK_TYPE_CLUSTER_HOST, True)
 
-        cluster_host_addr_name = cutils.format_address_name(constants.CONTROLLER_HOSTNAME,
-                                                    constants.NETWORK_TYPE_CLUSTER_HOST)
-        address = cutils.get_primary_address_by_name(self.dbapi, cluster_host_addr_name,
-                                                    constants.NETWORK_TYPE_CLUSTER_HOST, True)
         cluster = {
             'mon': {
                 'count': self._get_mon_count(),
             },
+            'mgr': {
+                'count': self._get_mon_count(),
+            },
             'network': {
                 'ipFamily': 'IPv' + str(address.family)
             },
+            'storage': {
+                'nodes': self._get_nodes_osds(),
+            },
         }
 
         return cluster
 
-    def _get_mon_count(self):
+    def _get_hook_overrides(self):
-        # change it with deployment configs:
-        # AIO simplex/duplex have 1 mon, multi-node has 3 mons,
-        # 2 controllers + first mon (and cannot reconfig)
-        if cutils.is_aio_system(self.dbapi):
-            return 1
-        else:
-            return 3
-
-    def _get_mds_override(self):
-        if cutils.is_aio_simplex_system(self.dbapi):
-            replica = 1
-        else:
-            replica = 2
-
-        mds = {
-            'replica': replica,
-        }
-
-        return mds
-
-    def _get_hook_override(self):
         hook = {
             'cleanup': {
-                'mon_hosts': self._get_mon_hosts(),
+                'mon_hosts': self._get_mon_hostname_list(),
             },
             'duplexPreparation': self._get_duplex_preparation(),
         }
         return hook
 
-    def _get_mon_hosts(self):
+    def get_overrides(self, namespace=None):
-        ceph_mon_label = "ceph-mon-placement=enabled"
-        mon_hosts = []
 
-        hosts = self.dbapi.ihost_get_list()
+        overrides = {
-        for h in hosts:
+            app_constants.HELM_NS_ROOK_CEPH: {
-            labels = self.dbapi.label_get_by_host(h.uuid)
+                'cephClusterSpec': self._get_cluster_overrides(),
-            for label in labels:
+                'hook': self._get_hook_overrides(),
-                if (ceph_mon_label == str(label.label_key) + '=' + str(label.label_value)):
+            }
-                    mon_hosts.append(h.hostname.encode('utf8', 'strict'))
-
-        return mon_hosts
-
-    def _get_duplex_preparation(self):
-        duplex = {
-            'enable': cutils.is_aio_duplex_system(self.dbapi)
         }
 
-        if cutils.is_aio_duplex_system(self.dbapi):
+        # One of the block pools is required
-            hosts = self.dbapi.ihost_get_by_personality(
+        # Get overrides based on the enabled block pool
-                constants.CONTROLLER)
+        if self._is_service_enabled(app_constants.SVC_BLOCK):
-            for host in hosts:
+            overrides[app_constants.HELM_NS_ROOK_CEPH].update(
-                if host['hostname'] == socket.gethostname():
+                {'cephBlockPools': self._get_block_overrides()})
-                    duplex.update({'activeController': host['hostname'].encode('utf8', 'strict')})
+        else:
+            overrides[app_constants.HELM_NS_ROOK_CEPH].update(
+                {'cephECBlockPools': self._get_ecblock_overrides(),
+                 'cephECStorageClass': self._get_ecblocksc_overrides()})
 
-            cluster_host_addr_name = cutils.format_address_name(constants.CONTROLLER_HOSTNAME,
+        # Enable optional filesystem store
-                                                        constants.NETWORK_TYPE_CLUSTER_HOST)
+        if self._is_service_enabled(app_constants.SVC_FS):
-            address = cutils.get_primary_address_by_name(self.dbapi, cluster_host_addr_name,
+            overrides[app_constants.HELM_NS_ROOK_CEPH].update(
-                                                        constants.NETWORK_TYPE_CLUSTER_HOST, True)
+                {'cephFileSystems': self._get_cephfs_overrides()})
-            duplex.update({'floatIP': cutils.format_url_address(address.address)})
+        else:
+            overrides[app_constants.HELM_NS_ROOK_CEPH].update(
+                {'cephFileSystems': []})
 
-        return duplex
+        # Enable optional object stores
+        if self._is_service_enabled(app_constants.SVC_OBJ):
+            overrides[app_constants.HELM_NS_ROOK_CEPH].update(
+                {'cephObjectStores': self._get_rgw_overrides()})
+        else:
+            overrides[app_constants.HELM_NS_ROOK_CEPH].update(
+                {'cephObjectStores': []})
+
+        if namespace in self.SUPPORTED_NAMESPACES:
+            return overrides[namespace]
+        elif namespace:
+            raise exception.InvalidHelmNamespace(chart=self.CHART,
+                                                 namespace=namespace)
+        else:
+            return overrides

python3-k8sapp-rook-ceph/k8sapp_rook_ceph/k8sapp_rook_ceph/helm/rook_ceph_provisioner.py

@@ -8,11 +8,9 @@
 from k8sapp_rook_ceph.common import constants as app_constants
 from k8sapp_rook_ceph.helm import storage
 
-from kubernetes.client.rest import ApiException
 from oslo_log import log as logging
 from sysinv.common import constants
 from sysinv.common import exception
-from sysinv.common import kubernetes
 from sysinv.common import utils as cutils
 
 LOG = logging.getLogger(__name__)
@@ -24,34 +22,76 @@ class RookCephClusterProvisionerHelm(storage.StorageBaseHelm):
     CHART = app_constants.HELM_CHART_ROOK_CEPH_PROVISIONER
     HELM_RELEASE = app_constants.FLUXCD_HELMRELEASE_ROOK_CEPH_PROVISIONER
 
+    def _get_controller_hosts(self):
+        controller_hosts = []
+
+        hosts = self.dbapi.ihost_get_by_personality(constants.CONTROLLER)
+        for h in hosts:
+            controller_hosts.append(h.hostname.encode('utf8', 'strict'))
+
+        return controller_hosts
+
+    def _get_cluster_host_ip(self, hostname):
+        addr_name = cutils.format_address_name(
+            hostname, constants.NETWORK_TYPE_CLUSTER_HOST)
+
+        address = cutils.get_primary_address_by_name(
+            self.dbapi, addr_name, constants.NETWORK_TYPE_CLUSTER_HOST, True)
+
+        return address.address
+
+    def _get_monitor_ips(self):
+        monitor_ips = []
+
+        # Get the IPs from the labeled monitors
+        hosts = self._get_hosts_with_label(app_constants.LABEL_PLACEMENT_MON)
+        for host in hosts:
+            monitor_ips.append(self._get_cluster_host_ip(host.hostname))
+
+        # Add the floating monitor
+        if cutils.is_aio_duplex_system(self.dbapi):
+            if (self.dbapi.count_hosts_matching_criteria() == 2 and
+                    self._is_ceph_controllerfs_provisioned()):
+                monitor_ips.append(self._get_cluster_host_ip(
+                    constants.CONTROLLER_HOSTNAME))
+        return ','.join(monitor_ips)
+
+    def _get_ceph_audit_jobs_overrides(self):
+        audit = {}
+
+        if cutils.is_aio_duplex_system(self.dbapi):
+            if (self.dbapi.count_hosts_matching_criteria() == 2 and
+                    self._is_ceph_controllerfs_provisioned()):
+                audit.update({'floatIP': cutils.format_url_address(
+                    self._get_cluster_host_ip(constants.CONTROLLER_HOSTNAME))})
+
+        return audit
+
     def get_overrides(self, namespace=None):
         base_name = 'ceph-pool'
         secret_name = base_name + '-' + constants.CEPH_POOL_KUBE_NAME
 
-        if cutils.is_aio_simplex_system(self.dbapi):
-            replica = 1
-        else:
-            replica = 2
-
-        audit = cutils.is_aio_duplex_system(self.dbapi)
-
         overrides = {
             app_constants.HELM_NS_ROOK_CEPH: {
                 "global": {
-                    "job_ceph_mon_audit": audit,
+                    "job_ceph_mon_audit":
+                        True if self._get_host_count_with_label(
+                            app_constants.LABEL_PLACEMENT_MON) > 1 else False
                 },
                 "provisionStorage": {
-                    "defaultStorageClass": constants.K8S_RBD_PROV_STOR_CLASS_NAME,
+                    "defaultStorageClass":
+                        constants.K8S_RBD_PROV_STOR_CLASS_NAME,
                     "classdefaults": {
-                        "monitors": self._get_monitors(),
+                        "monitors": self._get_monitor_ips(),
                         "adminId": constants.K8S_RBD_PROV_USER_NAME,
-                        "adminSecretName": constants.K8S_RBD_PROV_ADMIN_SECRET_NAME,
+                        "adminSecretName":
+                            constants.K8S_RBD_PROV_ADMIN_SECRET_NAME,
                     },
                     "classes": {
                         "name": constants.K8S_RBD_PROV_STOR_CLASS_NAME,
                         "pool": {
                             "pool_name": constants.CEPH_POOL_KUBE_NAME,
-                            "replication": replica,
+                            "replication": self._get_data_replication_factor(),
                             "crush_rule_name": "storage_tier_ruleset",
                             "chunk_size": 64,
                         },
@@ -64,7 +104,7 @@ class RookCephClusterProvisionerHelm(storage.StorageBaseHelm):
                 "host_provision": {
                     "controller_hosts": self._get_controller_hosts(),
                 },
-                "ceph_audit_jobs": self._get_ceph_audit(),
+                "ceph_audit_jobs": self._get_ceph_audit_jobs_overrides(),
             }
         }
 
@@ -75,68 +115,3 @@ class RookCephClusterProvisionerHelm(storage.StorageBaseHelm):
                                                  namespace=namespace)
         else:
             return overrides
-
-    def _get_rook_mon_ip(self):
-        try:
-            kube = kubernetes.KubeOperator()
-            mon_ip_name = 'rook-ceph-mon-endpoints'
-
-            configmap = kube.kube_read_config_map(mon_ip_name,
-                                                  app_constants.HELM_NS_ROOK_CEPH)
-            if configmap is not None:
-                data = configmap.data['data']
-                LOG.info('rook configmap data is %s' % data)
-                mons = data.split(',')
-                lists = []
-                for mon in mons:
-                    mon = mon.split('=')
-                    lists.append(mon[1])
-                ip_str = ','.join(lists)
-                LOG.info('rook mon ip is %s' % ip_str)
-                return ip_str
-
-        except Exception as e:
-            LOG.error("Kubernetes exception in rook mon ip: %s" % e)
-            raise
-        return ''
-
-    def _is_rook_ceph(self):
-        try:
-            label = "mon_cluster=" + app_constants.HELM_NS_ROOK_CEPH
-            kube = kubernetes.KubeOperator()
-            pods = kube.kube_get_pods_by_selector(app_constants.HELM_NS_ROOK_CEPH, label, "")
-            if len(pods) > 0:
-                return True
-        except ApiException as ae:
-            LOG.error("get monitor pod exception: %s" % ae)
-        except exception.SysinvException as se:
-            LOG.error("get sysinv exception: %s" % se)
-
-        return False
-
-    def _get_monitors(self):
-        if self._is_rook_ceph():
-            return self._get_rook_mon_ip()
-        else:
-            return ''
-
-    def _get_controller_hosts(self):
-        controller_hosts = []
-
-        hosts = self.dbapi.ihost_get_by_personality(constants.CONTROLLER)
-        for h in hosts:
-            controller_hosts.append(h.hostname.encode('utf8', 'strict'))
-
-        return controller_hosts
-
-    def _get_ceph_audit(self):
-        audit = {}
-
-        if cutils.is_aio_duplex_system(self.dbapi):
-            mgmt_addr_name = cutils.format_address_name(constants.CONTROLLER_HOSTNAME,
-                                                        constants.NETWORK_TYPE_CLUSTER_HOST)
-            address = cutils.get_primary_address_by_name(self.dbapi, mgmt_addr_name,
-                                                        constants.NETWORK_TYPE_CLUSTER_HOST, True)
-            audit.update({'floatIP': cutils.format_url_address(address.address)})
-
-        return audit

python3-k8sapp-rook-ceph/k8sapp_rook_ceph/k8sapp_rook_ceph/helm/storage.py

@@ -4,9 +4,13 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-from sysinv.helm import base
 from k8sapp_rook_ceph.common import constants as app_constants
 
+from sysinv.common import constants
+from sysinv.common import exception
+from sysinv.common import utils
+from sysinv.helm import base
+
 
 class BaseHelm(base.FluxCDBaseHelm):
     """Class to encapsulate storage related service operations for helm"""
@@ -51,3 +55,110 @@ class StorageBaseHelm(BaseHelm):
         if not self._is_enabled(operator.APP, self.CHART,
                                 app_constants.HELM_NS_ROOK_CEPH):
             operator.helm_release_resource_delete(self.HELM_RELEASE)
+
+    def _get_hosts_with_label(self, label):
+        return self.dbapi.ihosts_get_by_label(label)
+
+    def _get_host_count_with_label(self, label):
+        return self.dbapi.count_hosts_by_label(label)
+
+    def _get_ceph_rook_backend(self):
+        try:
+            sb = self.dbapi.storage_backend_get_by_name(app_constants.SB_NAME)
+        except exception.StorageBackendNotFoundByName:
+            return None
+        return sb
+
+    def _get_deployment_model(self):
+        sb = self._get_ceph_rook_backend()
+        deployment_model = None
+
+        if sb:
+            deployment_model = sb.capabilities.get(constants.CEPH_ROOK_BACKEND_DEPLOYMENT_CAP, None)
+            if not deployment_model:
+                raise exception.SysinvException(
+                    '{} missing from storage backend {}'.format(
+                        constants.CEPH_ROOK_BACKEND_DEPLOYMENT_CAP,
+                        app_constants.SB_NAME))
+
+        return deployment_model
+
+    def _get_hosts(self):
+        dm = self._get_deployment_model()
+        if dm == app_constants.DEP_MODEL_CONTROLLER:
+            hosts = self.dbapi.ihost_get_by_personality(constants.CONTROLLER)
+        elif dm == app_constants.DEP_MODEL_DEDICATED:
+            hosts = self.dbapi.ihost_get_by_personality(constants.WORKER)
+        else:
+            hosts = self.dbapi.ihost_get_list()
+        return hosts
+
+    def _get_nodes_osds(self):
+        hosts = self._get_hosts()
+
+        nodes = []
+        for host in hosts:
+            new_node = {
+                'name': host.hostname,
+                'devices': []
+            }
+            istors = self.dbapi.istor_get_by_ihost(host.uuid)
+            for stor in istors:
+                if (stor.function == constants.STOR_FUNCTION_OSD and
+                        (stor.state == constants.SB_STATE_CONFIGURED or
+                         stor.state == constants.SB_STATE_CONFIGURING_WITH_APP)):
+                    idisk = self.dbapi.idisk_get(stor.idisk_uuid)
+                    new_node['devices'].append({
+                        'name': idisk.device_path
+                    })
+            nodes.append(new_node)
+
+        return nodes
+
+    def _get_data_replication_factor(self):
+        sb = self._get_ceph_rook_backend()
+        if not sb:
+            if utils.is_aio_simplex_system(self.dbapi):
+                return constants.AIO_SX_CEPH_REPLICATION_FACTOR_DEFAULT
+            return constants.CEPH_REPLICATION_FACTOR_DEFAULT
+
+        replication = sb.capabilities.get(constants.CEPH_BACKEND_REPLICATION_CAP, None)
+        if not replication:
+            raise exception.SysinvException(
+                '{} missing from storage backend {}.'.format(
+                    constants.CEPH_BACKEND_REPLICATION_CAP,
+                    app_constants.SB_NAME))
+
+        try:
+            replication = int(replication)
+        except ValueError:
+            raise exception.SysinvException(
+                '{} from storage backend {} must be a integer.'.format(
+                    constants.CEPH_BACKEND_REPLICATION_CAP,
+                    app_constants.SB_NAME))
+
+        return replication
+
+    def _is_ceph_controllerfs_provisioned(self):
+        try:
+            self.dbapi.controller_fs_get_by_name(
+                constants.FILESYSTEM_NAME_CEPH_DRBD)
+        except exception.ControllerFSNameNotFound:
+            return False
+
+        return True
+
+    def _get_services(self):
+        services_list = []
+        sb = self._get_ceph_rook_backend()
+
+        if sb:
+            services_list = sb.services.split(',')
+
+        return services_list
+
+    def _is_service_enabled(self, service):
+        services = self._get_services()
+        if services and service in services:
+            return True
+        return False

python3-k8sapp-rook-ceph/k8sapp_rook_ceph/k8sapp_rook_ceph/tests/test_rook.py

@@ -11,15 +11,36 @@ from sysinv.tests.db import base as dbbase
 from sysinv.tests.db import utils as dbutils
 from sysinv.tests.helm import base
 
+from sysinv.common import constants
+
 
 class RookTestCase(test_plugins.K8SAppRookAppMixin,
                    base.HelmTestCaseMixin):
 
     def setUp(self):
         super(RookTestCase, self).setUp()
-        self.app = dbutils.create_test_app(name=app_constants.HELM_APP_ROOK_CEPH)
         self.dbapi = dbapi.get_instance()
 
+        # Create storage backend
+        self.backend = dbutils.create_ceph_rook_storage_backend()
+
+        # Create Ceph controllerfs
+        self.controller_fs = dbutils.create_test_controller_fs(
+            name=constants.FILESYSTEM_NAME_CEPH_DRBD)
+
+        # Create app
+        self.app = dbutils.create_test_app(name=app_constants.HELM_APP_ROOK_CEPH)
+
+        self.replication = int(self.backend.capabilities.get(constants.CEPH_BACKEND_REPLICATION_CAP, 1))
+
+        # Create Label hosts
+        hosts = self.dbapi.ihost_get_list()
+        for h in hosts:
+            dbutils.create_test_label(
+                host_id=h.id,
+                label_key=app_constants.LABEL_PLACEMENT_MON,
+                label_value="enabled")
+
 
 class RookIPv4ControllerHostTestCase(RookTestCase,
                                      dbbase.ProvisionedControllerHostTestCase):
@@ -40,7 +61,7 @@ class RookIPv4ControllerHostTestCase(RookTestCase,
             cnamespace=app_constants.HELM_NS_ROOK_CEPH)
 
         self.assertOverridesParameters(e_overrides.get('cephFileSystems')[0].get('spec').
-                                       get('metadataPool').get('replicated').get('size'), 2)
+                                       get('metadataPool').get('replicated').get('size'), self.replication)
 
     def test_rook_ceph_provisioner_overrides(self):
         f_overrides = self.operator.get_helm_chart_overrides(
@@ -74,7 +95,7 @@ class RookIPv6AIODuplexSystemTestCase(RookTestCase,
             cnamespace=app_constants.HELM_NS_ROOK_CEPH)
 
         self.assertOverridesParameters(b_overrides.get('cephFileSystems')[0].get('spec').
-                                       get('metadataPool').get('replicated').get('size'), 2)
+                                       get('metadataPool').get('replicated').get('size'), self.replication)
 
     def test_rook_ceph_provisioner_overrides(self):
         c_overrides = self.operator.get_helm_chart_overrides(
@@ -107,7 +128,7 @@ class RookDualStackControllerIPv4TestCase(RookTestCase,
             cnamespace=app_constants.HELM_NS_ROOK_CEPH)
 
         self.assertOverridesParameters(h_overrides.get('cephFileSystems')[0].get('spec').
-                                       get('metadataPool').get('replicated').get('size'), 2)
+                                       get('metadataPool').get('replicated').get('size'), self.replication)
 
     def test_rook_ceph_provisioner_overrides(self):
         i_overrides = self.operator.get_helm_chart_overrides(

stx-rook-ceph-helm/stx-rook-ceph-helm/fluxcd-manifests/rook-ceph-cluster/rook-ceph-cluster-static-overrides.yaml

@@ -12,6 +12,9 @@ configOverride: |
   [global]
   osd_pool_default_size = 1
   osd_pool_default_min_size = 1
+  auth_cluster_required = cephx
+  auth_service_required = cephx
+  auth_client_required = cephx
 
   [osd]
   osd_mkfs_type = xfs
@@ -147,6 +150,9 @@ cephClusterSpec:
       - effect: NoSchedule
         operator: Exists
         key: node-role.kubernetes.io/control-plane
+      - effect: NoExecute
+        operator: Exists
+        key: services
     mon:
       nodeAffinity:
         requiredDuringSchedulingIgnoredDuringExecution:
@@ -166,7 +172,6 @@ cephClusterSpec:
               values:
               - enabled
 
-
 toolbox:
   enabled: true
   image: quay.io/ceph/ceph:v18.2.2
@@ -191,169 +196,18 @@ monitoring:
 
 
 cephFileSystems:
-  - name: cephfs
-    # see https://github.com/rook/rook/blob/master/Documentation/ceph-filesystem-crd.md#filesystem-settings for available configuration
-    spec:
-      metadataPool:
-        replicated:
-          size: 1
-      dataPools:
-        - failureDomain: osd # TODO
-          name: data
-          replicated:
-            size: 1
-      metadataServer:
-        labels:
-          app.starlingx.io/component: "platform"
-        activeCount: 1
-        activeStandby: true
-        resources:
-          limits:
-            memory: "4Gi"
-          requests:
-            cpu: 0
-            memory: 0
-        placement:
-          nodeAffinity:
-            requiredDuringSchedulingIgnoredDuringExecution:
-              nodeSelectorTerms:
-              - matchExpressions:
-                - key: ceph-mon-placement
-                  operator: In
-                  values:
-                  - enabled
-          podAntiAffinity:
-            tolerations:
-            - effect: NoSchedule
-              operator: Exists
-              key: node-role.kubernetes.io/master
-            - effect: NoSchedule
-              operator: Exists
-              key: node-role.kubernetes.io/control-plane
-        priorityClassName: system-cluster-critical
-    storageClass:
-      enabled: true
-      isDefault: false
-      name: cephfs
-      pool: data
-      reclaimPolicy: Delete
-      allowVolumeExpansion: true
-      volumeBindingMode: "Immediate"
-      mountOptions: []
-      # see https://github.com/rook/rook/blob/master/Documentation/ceph-filesystem.md#provision-storage for available configuration
-      parameters:
-        # The secrets contain Ceph admin credentials.
-        csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
-        csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
-        csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
-        csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
-        csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
-        csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
-        # Specify the filesystem type of the volume. If not specified, csi-provisioner
-        # will set default as `ext4`. Note that `xfs` is not recommended due to potential deadlock
-        # in hyperconverged settings where the volume is mounted on the same node as the osds.
-        csi.storage.k8s.io/fstype: ext4
-
-
 
 cephBlockPools:
-  - name: kube-rbd
-    # see https://github.com/rook/rook/blob/master/Documentation/ceph-pool-crd.md#spec for available configuration
-    spec:
-      failureDomain: osd
-      replicated:
-        size: 1
-    storageClass:
-      enabled: true
-      name: general
-      isDefault: true
-      reclaimPolicy: Delete
-      allowVolumeExpansion: true
-      volumeBindingMode: "Immediate"
-      mountOptions: []
-      allowedTopologies: []
-      # see https://github.com/rook/rook/blob/master/Documentation/ceph-block.md#provision-storage for available configuration
-      parameters:
-        # (optional) mapOptions is a comma-separated list of map options.
-        # For krbd options refer
-        # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
-        # For nbd options refer
-        # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
-        # mapOptions: lock_on_read,queue_depth=1024
 
-        # (optional) unmapOptions is a comma-separated list of unmap options.
-        # For krbd options refer
-        # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
-        # For nbd options refer
-        # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
-        # unmapOptions: force
-
-        # RBD image format. Defaults to "2".
-        imageFormat: "2"
-        # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature.
-        imageFeatures: layering
-        # The secrets contain Ceph admin credentials.
-        csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
-        csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
-        csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
-        csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
-        csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
-        csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
-        # Specify the filesystem type of the volume. If not specified, csi-provisioner
-        # will set default as `ext4`. Note that `xfs` is not recommended due to potential deadlock
-        # in hyperconverged settings where the volume is mounted on the same node as the osds.
-        csi.storage.k8s.io/fstype: ext4
 # -- A list of CephObjectStore configurations to deploy
 # @default -- See [below](#ceph-object-stores)
 cephObjectStores:
-  - name: ceph-objectstore
-    # see https://github.com/rook/rook/blob/master/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md#object-store-settings for available configuration
-    spec:
-      metadataPool:
-        failureDomain: osd
-        replicated:
-          size: 0
-      dataPool:
-        failureDomain: osd
-        erasureCoded:
-          dataChunks: 0
-          codingChunks: 0
-      preservePoolsOnDelete: true
-      gateway:
-        port: 80
-        resources:
-          limits:
-            memory: "4Gi"
-          requests:
-        cpu: 0
-        memory: 0
-        # securePort: 443
-        # sslCertificateRef:
-        instances: 1
-        priorityClassName: system-cluster-critical
-    storageClass:
-      enabled: false
-      name: ceph-bucket
-      reclaimPolicy: Delete
-      volumeBindingMode: "Immediate"
-      # see https://github.com/rook/rook/blob/master/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md#storageclass for available configuration
-      parameters:
-        # note: objectStoreNamespace and objectStoreName are configured by the chart
-        region: us-east-1
-    ingress:
-      # Enable an ingress for the ceph-objectstore
-      enabled: false
-      # annotations: {}
-      # host:
-      #   name: objectstore.example.com
-      #   path: /
-      # tls:
-      # - hosts:
-      #     - objectstore.example.com
-      #   secretName: ceph-objectstore-tls
-      # ingressClassName: nginx
 
+cephECBlockPools:
 
+# cephECStorageClass also is disabled by default, please remove the comments and set desired values to enable it
+# if cephECBlockPools are uncommented you must remove the comments of cephEcStorageClass as well
+cephECStorageClass:
 
 imagePullSecrets:
   - name: default-registry-key