Uplift rook components to latest v1.7.5

* upgraded rook-operator to v1.7.5 * upgraded ceph to v16.2.6 Relates-To: #194 Change-Id: I23fa69cfc2e03db77df26dd4adab53447f4e8c9a
2021-10-19 17:17:03 +00:00 · 2021-10-19 17:17:03 +00:00 · e55e960caf
commit e55e960caf
parent f819f6c510
20 changed files with 1797 additions and 351 deletions
--- a/manifests/composite/storage-cluster/Kptfile
+++ b/manifests/composite/storage-cluster/Kptfile
@ -5,5 +5,5 @@ dependencies:
  git:
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph
-    ref: v1.6.3
+    ref: v1.7.5
  updateStrategy: force-delete-replace
--- a/manifests/composite/storage-cluster/upstream/Kptfile
+++ b/manifests/composite/storage-cluster/upstream/Kptfile
@ -5,7 +5,7 @@ metadata:
 upstream:
  type: git
  git:
-    commit: 69591248f69e23964734f0192944ef2442bc7885
+    commit: 16d2ffbb2d1ff8c36f3735c96f309c1cca0d9308
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph
-    ref: v1.6.3
+    ref: v1.7.5
--- a/manifests/composite/storage-cluster/upstream/cluster.yaml
+++ b/manifests/composite/storage-cluster/upstream/cluster.yaml
@ -16,13 +16,13 @@ metadata:
 spec:
  cephVersion:
    # The container image used to launch the Ceph daemon pods (mon, mgr, osd, mds, rgw).
-    # v13 is mimic, v14 is nautilus, and v15 is octopus.
+    # v14 is nautilus, v15 is octopus, and v16 is pacific.
    # RECOMMENDATION: In production, use a specific version tag instead of the general v14 flag, which pulls the latest release and could result in different
    # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
-    # If you want to be more precise, you can always use a timestamp tag such ceph/ceph:v15.2.11-20200419
+    # If you want to be more precise, you can always use a timestamp tag such quay.io/ceph/ceph:v16.2.6-20210918
    # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
-    image: ceph/ceph:v15.2.11
-    # Whether to allow unsupported versions of Ceph. Currently `nautilus` and `octopus` are supported.
+    image: quay.io/ceph/ceph:v16.2.6
+    # Whether to allow unsupported versions of Ceph. Currently `nautilus`, `octopus`, and `pacific` are supported.
    # Future versions such as `pacific` would require this to be set to `true`.
    # Do not set to true in production.
    allowUnsupported: false
@ -44,7 +44,8 @@ spec:
  # The default wait timeout is 10 minutes.
  waitTimeoutForHealthyOSDInMinutes: 10
  mon:
-    # Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3.
+    # Set the number of mons to be started. Generally recommended to be 3.
+    # For highest availability, an odd number of mons should be specified.
    count: 3
    # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason.
    # Mons should only be allowed on the same node for test environments where data loss is acceptable.
@ -81,7 +82,7 @@ spec:
  network:
    # enable host networking
    #provider: host
-    # EXPERIMENTAL: enable the Multus network provider
+    # enable the Multus network provider
    #provider: multus
    #selectors:
      # The selector keys are required to be `public` and `cluster`.
@ -189,6 +190,10 @@ spec:
 # The above example requests/limits can also be added to the other components
 #    mon:
 #    osd:
+# For OSD it also is a possible to specify requests/limits based on device class
+#    osd-hdd:
+#    osd-ssd:
+#    osd-nvme:
 #    prepareosd:
 #    mgr-sidecar:
 #    crashcollector:
@ -214,17 +219,19 @@ spec:
      # encryptedDevice: "true" # the default value for this option is "false"
 # Individual nodes and their config can be specified as well, but 'useAllNodes' above must be set to false. Then, only the named
 # nodes below will be used as storage resources.  Each node's 'name' field should match their 'kubernetes.io/hostname' label.
-#    nodes:
-#    - name: "172.17.4.201"
-#      devices: # specific devices to use for storage can be specified for each node
-#      - name: "sdb"
-#      - name: "nvme01" # multiple osds can be created on high performance devices
-#        config:
-#          osdsPerDevice: "5"
-#      - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths
-#      config: # configuration can be specified at the node level which overrides the cluster level config
-#    - name: "172.17.4.301"
-#      deviceFilter: "^sd."
+    # nodes:
+    #   - name: "172.17.4.201"
+    #     devices: # specific devices to use for storage can be specified for each node
+    #       - name: "sdb"
+    #       - name: "nvme01" # multiple osds can be created on high performance devices
+    #         config:
+    #           osdsPerDevice: "5"
+    #       - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths
+    #     config: # configuration can be specified at the node level which overrides the cluster level config
+    #   - name: "172.17.4.301"
+    #     deviceFilter: "^sd."
+    # when onlyApplyOSDPlacement is false, will merge both placement.All() and placement.osd
+    onlyApplyOSDPlacement: false
  # The section for configuring management of daemon disruptions during upgrade or fencing.
  disruptionManagement:
    # If true, the operator will create and manage PodDisruptionBudgets for OSD, Mon, RGW, and MDS daemons. OSD PDBs are managed dynamically
--- a/manifests/composite/storage-cluster/upstream/toolbox.yaml
+++ b/manifests/composite/storage-cluster/upstream/toolbox.yaml
@ -18,7 +18,7 @@ spec:
      dnsPolicy: ClusterFirstWithHostNet
      containers:
        - name: rook-ceph-tools
-          image: rook/ceph:v1.6.3
+          image: rook/ceph:v1.7.5
          command: ["/tini"]
          args: ["-g", "--", "/usr/local/bin/toolbox.sh"]
          imagePullPolicy: IfNotPresent
--- a/manifests/function/rook-cluster/Kptfile
+++ b/manifests/function/rook-cluster/Kptfile
@ -5,29 +5,29 @@ dependencies:
  git:
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph
-    ref: v1.6.3
+    ref: v1.7.5
  updateStrategy: force-delete-replace
 - name: dashboard/base/upstream
  git:
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph
-    ref: v1.6.3
+    ref: v1.7.5
  updateStrategy: force-delete-replace
 - name: pools/base/upstream
  git:
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph
-    ref: v1.6.3
+    ref: v1.7.5
  updateStrategy: force-delete-replace
 - name: storageclasses/block/upstream
  git:
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph/csi/rbd
-    ref: v1.6.3
+    ref: v1.7.5
  updateStrategy: force-delete-replace
 - name: storageclasses/file/upstream
  git:
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph/csi/cephfs
-    ref: v1.6.3
+    ref: v1.7.5
  updateStrategy: force-delete-replace
--- a/manifests/function/rook-cluster/cephfs/base/upstream/Kptfile
+++ b/manifests/function/rook-cluster/cephfs/base/upstream/Kptfile
@ -5,7 +5,7 @@ metadata:
 upstream:
  type: git
  git:
-    commit: 69591248f69e23964734f0192944ef2442bc7885
+    commit: 16d2ffbb2d1ff8c36f3735c96f309c1cca0d9308
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph
-    ref: v1.6.3
+    ref: v1.7.5
--- a/manifests/function/rook-cluster/cephfs/base/upstream/filesystem.yaml
+++ b/manifests/function/rook-cluster/cephfs/base/upstream/filesystem.yaml
@ -17,7 +17,7 @@ spec:
      requireSafeReplicaSize: true
    parameters:
      # Inline compression mode for the data pool
-      # Further reference: https://docs.ceph.com/docs/nautilus/rados/configuration/bluestore-config-ref/#inline-compression
+      # Further reference: https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#inline-compression
      compression_mode:
        none
        # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
@ -33,7 +33,7 @@ spec:
        requireSafeReplicaSize: true
      parameters:
        # Inline compression mode for the data pool
-        # Further reference: https://docs.ceph.com/docs/nautilus/rados/configuration/bluestore-config-ref/#inline-compression
+        # Further reference: https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#inline-compression
        compression_mode:
          none
          # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
@ -101,5 +101,22 @@ spec:
    #    cpu: "500m"
    #    memory: "1024Mi"
    # priorityClassName: my-priority-class
-  mirroring:
-    enabled: false
+  # Filesystem mirroring settings
+  # mirroring:
+    # enabled: true
+    # list of Kubernetes Secrets containing the peer token
+    # for more details see: https://docs.ceph.com/en/latest/dev/cephfs-mirroring/#bootstrap-peers
+    # peers:
+      #secretNames:
+        #- secondary-cluster-peer
+    # specify the schedule(s) on which snapshots should be taken
+    # see the official syntax here https://docs.ceph.com/en/latest/cephfs/snap-schedule/#add-and-remove-schedules
+    # snapshotSchedules:
+    #   - path: /
+    #     interval: 24h # daily snapshots
+    #     startTime: 11:55
+    # manage retention policies
+    # see syntax duration here https://docs.ceph.com/en/latest/cephfs/snap-schedule/#add-and-remove-retention-policies
+    # snapshotRetention:
+    #   - path: /
+    #     duration: "h 24"
--- a/manifests/function/rook-cluster/dashboard/base/upstream/Kptfile
+++ b/manifests/function/rook-cluster/dashboard/base/upstream/Kptfile
@ -5,7 +5,7 @@ metadata:
 upstream:
  type: git
  git:
-    commit: 69591248f69e23964734f0192944ef2442bc7885
+    commit: 16d2ffbb2d1ff8c36f3735c96f309c1cca0d9308
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph
-    ref: v1.6.3
+    ref: v1.7.5
--- a/manifests/function/rook-cluster/pools/base/upstream/Kptfile
+++ b/manifests/function/rook-cluster/pools/base/upstream/Kptfile
@ -5,7 +5,7 @@ metadata:
 upstream:
  type: git
  git:
-    commit: 69591248f69e23964734f0192944ef2442bc7885
+    commit: 16d2ffbb2d1ff8c36f3735c96f309c1cca0d9308
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph
-    ref: v1.6.3
+    ref: v1.7.5
--- a/manifests/function/rook-cluster/pools/base/upstream/pool.yaml
+++ b/manifests/function/rook-cluster/pools/base/upstream/pool.yaml
@ -18,15 +18,18 @@ spec:
    # Disallow setting pool with replica 1, this could lead to data loss without recovery.
    # Make sure you're *ABSOLUTELY CERTAIN* that is what you want
    requireSafeReplicaSize: true
+    # hybridStorage:
+    #   primaryDeviceClass: ssd
+    #   secondaryDeviceClass: hdd
    # The number for replicas per failure domain, the value must be a divisor of the replica count. If specified, the most common value is 2 for stretch clusters, where the replica count would be 4.
    # replicasPerFailureDomain: 2
    # The name of the failure domain to place further down replicas
    # subFailureDomain: host
  # Ceph CRUSH root location of the rule
-  # For reference: https://docs.ceph.com/docs/nautilus/rados/operations/crush-map/#types-and-buckets
+  # For reference: https://docs.ceph.com/docs/master/rados/operations/crush-map/#types-and-buckets
  #crushRoot: my-root
  # The Ceph CRUSH device class associated with the CRUSH replicated rule
-  # For reference: https://docs.ceph.com/docs/nautilus/rados/operations/crush-map/#device-classes
+  # For reference: https://docs.ceph.com/docs/master/rados/operations/crush-map/#device-classes
  #deviceClass: my-class
  # Enables collecting RBD per-image IO statistics by enabling dynamic OSD performance counters. Defaults to false.
  # For reference: https://docs.ceph.com/docs/master/mgr/prometheus/#rbd-io-statistics
@ -35,7 +38,7 @@ spec:
  # see https://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values
  parameters:
    # Inline compression mode for the data pool
-    # Further reference: https://docs.ceph.com/docs/nautilus/rados/configuration/bluestore-config-ref/#inline-compression
+    # Further reference: https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#inline-compression
    compression_mode: none
    # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
    # for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
--- a/manifests/function/rook-cluster/storageclasses/block/upstream/Kptfile
+++ b/manifests/function/rook-cluster/storageclasses/block/upstream/Kptfile
@ -5,7 +5,7 @@ metadata:
 upstream:
  type: git
  git:
-    commit: 69591248f69e23964734f0192944ef2442bc7885
+    commit: 16d2ffbb2d1ff8c36f3735c96f309c1cca0d9308
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph/csi/rbd
-    ref: v1.6.3
+    ref: v1.7.5
--- a/manifests/function/rook-cluster/storageclasses/block/upstream/storageclass.yaml
+++ b/manifests/function/rook-cluster/storageclasses/block/upstream/storageclass.yaml
@ -66,9 +66,9 @@ parameters:
  # in hyperconverged settings where the volume is mounted on the same node as the osds.
  csi.storage.k8s.io/fstype: ext4
 # uncomment the following to use rbd-nbd as mounter on supported nodes
-# **IMPORTANT**: If you are using rbd-nbd as the mounter, during upgrade you will be hit a ceph-csi
-# issue that causes the mount to be disconnected. You will need to follow special upgrade steps
-# to restart your application pods. Therefore, this option is not recommended.
+# **IMPORTANT**: CephCSI v3.4.0 onwards a volume healer functionality is added to reattach
+# the PVC to application pod if nodeplugin pod restart.
+# Its still in Alpha support. Therefore, this option is not recommended for production use.
 #mounter: rbd-nbd
 allowVolumeExpansion: true
 reclaimPolicy: Delete
--- a/manifests/function/rook-cluster/storageclasses/file/upstream/Kptfile
+++ b/manifests/function/rook-cluster/storageclasses/file/upstream/Kptfile
@ -5,7 +5,7 @@ metadata:
 upstream:
  type: git
  git:
-    commit: 69591248f69e23964734f0192944ef2442bc7885
+    commit: 16d2ffbb2d1ff8c36f3735c96f309c1cca0d9308
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph/csi/cephfs
-    ref: v1.6.3
+    ref: v1.7.5
--- a/manifests/function/rook-operator/Kptfile
+++ b/manifests/function/rook-operator/Kptfile
@ -5,5 +5,5 @@ dependencies:
  git:
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph
-    ref: v1.6.3
+    ref: v1.7.5
  updateStrategy: force-delete-replace
--- a/manifests/function/rook-operator/kustomization.yaml
+++ b/manifests/function/rook-operator/kustomization.yaml
@ -12,9 +12,9 @@ patches:
    namespace: rook-ceph
  data:
    CSI_ENABLE_HOST_NETWORK: "true"
-    ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.3.1"
-    ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1"
-    ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1"
-    ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4"
-    ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0"
-    ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2"
+    ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.4.0"
+    ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0"
+    ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
+    ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0"
+    ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0"
+    ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.3.0"
--- a/manifests/function/rook-operator/upstream/Kptfile
+++ b/manifests/function/rook-operator/upstream/Kptfile
@ -5,7 +5,7 @@ metadata:
 upstream:
  type: git
  git:
-    commit: 69591248f69e23964734f0192944ef2442bc7885
+    commit: 16d2ffbb2d1ff8c36f3735c96f309c1cca0d9308
    repo: https://github.com/rook/rook
    directory: cluster/examples/kubernetes/ceph
-    ref: v1.6.3
+    ref: v1.7.5
--- a/manifests/function/rook-operator/upstream/common.yaml
+++ b/manifests/function/rook-operator/upstream/common.yaml
@ -91,6 +91,25 @@ rules:
      - update
      - delete
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: rook-ceph-system
+  labels:
+    operator: rook
+    storage-backend: ceph
+rules:
+    # Most resources are represented by a string representation of their name, such as “pods”, just as it appears in the URL for the relevant API endpoint.
+    # However, some Kubernetes APIs involve a “subresource”, such as the logs for a pod. [...]
+    # To represent this in an RBAC role, use a slash to delimit the resource and subresource.
+    # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources
+  - apiGroups: [""]
+    resources: ["pods", "pods/log"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
+---
 # The role for the operator to manage resources in its own namespace
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@ -271,6 +290,7 @@ rules:
      - configmaps
      - nodes
      - nodes/proxy
+      - persistentvolumes
    verbs:
      - get
      - list
@ -285,6 +305,14 @@ rules:
      - list
      - get
      - watch
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - storageclasses
+    verbs:
+      - get
+      - list
+      - watch
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@ -351,6 +379,22 @@ subjects:
    name: rook-ceph-system
    namespace: rook-ceph # namespace:operator
 ---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rook-ceph-system
+  labels:
+    operator: rook
+    storage-backend: ceph
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: rook-ceph-system
+subjects:
+  - kind: ServiceAccount
+    name: rook-ceph-system
+    namespace: rook-ceph # namespace:operator
+---
 # Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@ -485,6 +529,20 @@ rules:
      - "*"
    verbs:
      - "*"
+  - apiGroups:
+      - apps
+    resources:
+      - deployments/scale
+      - deployments
+    verbs:
+      - patch
+      - delete
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumeclaims
+    verbs:
+      - delete
 # OLM: END CLUSTER ROLE
 # OLM: BEGIN CMD REPORTER ROLE
 ---
@ -1028,6 +1086,9 @@ rules:
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["serviceaccounts"]
+    verbs: ["get"]
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@ -1094,6 +1155,9 @@ rules:
  - apiGroups: ["replication.storage.openshift.io"]
    resources: ["volumereplicationclasses/status"]
    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["serviceaccounts"]
+    verbs: ["get"]
 # OLM: END CSI RBD CLUSTER ROLE
 # OLM: BEGIN CSI RBD CLUSTER ROLEBINDING
 ---
@ -1149,3 +1213,44 @@ roleRef:
  name: rbd-external-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
 # OLM: END CSI RBD CLUSTER ROLEBINDING
+---
+# Aspects of ceph osd purge job that require access to the operator/cluster namespace
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rook-ceph-purge-osd
+  namespace: rook-ceph # namespace:operator
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get", "delete"]
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "update", "delete"]
+---
+# Allow the osd purge job to run in this namespace
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rook-ceph-purge-osd
+  namespace: rook-ceph # namespace:operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: rook-ceph-purge-osd
+subjects:
+  - kind: ServiceAccount
+    name: rook-ceph-purge-osd
+    namespace: rook-ceph # namespace:operator
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rook-ceph-purge-osd
+  namespace: rook-ceph # namespace:operator
--- a/manifests/function/rook-operator/upstream/crds.yaml
+++ b/manifests/function/rook-operator/upstream/crds.yaml
--- a/manifests/function/rook-operator/upstream/operator.yaml
+++ b/manifests/function/rook-operator/upstream/operator.yaml
@ -22,6 +22,9 @@ metadata:
  # should be in the namespace of the operator
  namespace: rook-ceph # namespace:operator
 data:
+  # The logging level for the operator: INFO | DEBUG
+  ROOK_LOG_LEVEL: "INFO"
+
  # Enable the CSI driver.
  # To run the non-default version of the CSI driver, see the override-able image properties in operator.yaml
  ROOK_CSI_ENABLE_CEPHFS: "true"
@ -38,6 +41,9 @@ data:
  # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
  # CSI_LOG_LEVEL: "0"

+  # Set replicas for csi provisioner deployment.
+  CSI_PROVISIONER_REPLICAS: "2"
+
  # OMAP generator will generate the omap mapping between the PV name and the RBD image.
  # CSI_ENABLE_OMAP_GENERATOR need to be enabled when we are using rbd mirroring feature.
  # By default OMAP generator sidecar is deployed with CSI provisioner pod, to disable
@ -62,19 +68,19 @@ data:

  # (Optional) policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted.
  # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
-  CSI_CEPHFS_FSGROUPPOLICY: "ReadWriteOnceWithFSType"
+  CSI_CEPHFS_FSGROUPPOLICY: "None"

  # (Optional) Allow starting unsupported ceph-csi image
  ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false"
  # The default version of CSI supported by Rook will be started. To change the version
  # of the CSI driver to something other than what is officially supported, change
  # these images to the desired release of the CSI driver.
-  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.3.1"
-  # ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1"
-  # ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1"
-  # ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4"
-  # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0"
-  # ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2"
+  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.4.0"
+  # ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0"
+  # ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
+  # ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0"
+  # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0"
+  # ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.3.0"

  # (Optional) set user created priorityclassName for csi plugin pods.
  # CSI_PLUGIN_PRIORITY_CLASSNAME: "system-node-critical"
@ -97,9 +103,10 @@ data:
  # Labels to add to the CSI RBD Deployments and DaemonSets Pods.
  # ROOK_CSI_RBD_POD_LABELS: "key1=value1,key2=value2"

-  # (Optional) Ceph Provisioner NodeAffinity.
+  # (Optional) CephCSI provisioner NodeAffinity(applied to both CephFS and RBD provisioner).
  # CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
-  # (Optional) CEPH CSI provisioner tolerations list. Put here list of taints you want to tolerate in YAML format.
+  # (Optional) CephCSI provisioner tolerations list(applied to both CephFS and RBD provisioner).
+  # Put here list of taints you want to tolerate in YAML format.
  # CSI provisioner would be best to start on the same nodes as other ceph daemons.
  # CSI_PROVISIONER_TOLERATIONS: |
  #   - effect: NoSchedule
@ -108,9 +115,10 @@ data:
  #   - effect: NoExecute
  #     key: node-role.kubernetes.io/etcd
  #     operator: Exists
-  # (Optional) Ceph CSI plugin NodeAffinity.
+  # (Optional) CephCSI plugin NodeAffinity(applied to both CephFS and RBD plugin).
  # CSI_PLUGIN_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
-  # (Optional) CEPH CSI plugin tolerations list. Put here list of taints you want to tolerate in YAML format.
+  # (Optional) CephCSI plugin tolerations list(applied to both CephFS and RBD plugin).
+  # Put here list of taints you want to tolerate in YAML format.
  # CSI plugins need to be started on all the nodes where the clients need to mount the storage.
  # CSI_PLUGIN_TOLERATIONS: |
  #   - effect: NoSchedule
@ -120,6 +128,40 @@ data:
  #     key: node-role.kubernetes.io/etcd
  #     operator: Exists

+  # (Optional) CephCSI RBD provisioner NodeAffinity(if specified, overrides CSI_PROVISIONER_NODE_AFFINITY).
+  # CSI_RBD_PROVISIONER_NODE_AFFINITY: "role=rbd-node"
+  # (Optional) CephCSI RBD provisioner tolerations list(if specified, overrides CSI_PROVISIONER_TOLERATIONS).
+  # Put here list of taints you want to tolerate in YAML format.
+  # CSI provisioner would be best to start on the same nodes as other ceph daemons.
+  # CSI_RBD_PROVISIONER_TOLERATIONS: |
+  #   - key: node.rook.io/rbd
+  #     operator: Exists
+  # (Optional) CephCSI RBD plugin NodeAffinity(if specified, overrides CSI_PLUGIN_NODE_AFFINITY).
+  # CSI_RBD_PLUGIN_NODE_AFFINITY: "role=rbd-node"
+  # (Optional) CephCSI RBD plugin tolerations list(if specified, overrides CSI_PLUGIN_TOLERATIONS).
+  # Put here list of taints you want to tolerate in YAML format.
+  # CSI plugins need to be started on all the nodes where the clients need to mount the storage.
+  # CSI_RBD_PLUGIN_TOLERATIONS: |
+  #   - key: node.rook.io/rbd
+  #     operator: Exists
+
+  # (Optional) CephCSI CephFS provisioner NodeAffinity(if specified, overrides CSI_PROVISIONER_NODE_AFFINITY).
+  # CSI_CEPHFS_PROVISIONER_NODE_AFFINITY: "role=cephfs-node"
+  # (Optional) CephCSI CephFS provisioner tolerations list(if specified, overrides CSI_PROVISIONER_TOLERATIONS).
+  # Put here list of taints you want to tolerate in YAML format.
+  # CSI provisioner would be best to start on the same nodes as other ceph daemons.
+  # CSI_CEPHFS_PROVISIONER_TOLERATIONS: |
+  #   - key: node.rook.io/cephfs
+  #     operator: Exists
+  # (Optional) CephCSI CephFS plugin NodeAffinity(if specified, overrides CSI_PLUGIN_NODE_AFFINITY).
+  # CSI_CEPHFS_PLUGIN_NODE_AFFINITY: "role=cephfs-node"
+  # (Optional) CephCSI CephFS plugin tolerations list(if specified, overrides CSI_PLUGIN_TOLERATIONS).
+  # Put here list of taints you want to tolerate in YAML format.
+  # CSI plugins need to be started on all the nodes where the clients need to mount the storage.
+  # CSI_CEPHFS_PLUGIN_TOLERATIONS: |
+  #   - key: node.rook.io/cephfs
+  #     operator: Exists
+
  # (Optional) CEPH CSI RBD provisioner resource requirement list, Put here list of resource
  # requests and limits you want to apply for provisioner pod
  # CSI_RBD_PROVISIONER_RESOURCE: |
@ -285,7 +327,11 @@ data:
  # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
  # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
  ROOK_ENABLE_DISCOVERY_DAEMON: "false"
-  # Enable volume replication controller
+  # The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15.
+  ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15"
+  # Enable the volume replication controller.
+  # Before enabling, ensure the Volume Replication CRDs are created.
+  # See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring
  CSI_ENABLE_VOLUME_REPLICATION: "false"
  # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"

@ -323,7 +369,7 @@ spec:
      serviceAccountName: rook-ceph-system
      containers:
        - name: rook-ceph-operator
-          image: rook/ceph:v1.6.3
+          image: rook/ceph:v1.7.5
          args: ["ceph", "operator"]
          volumeMounts:
            - mountPath: /var/lib/rook
@ -335,9 +381,6 @@ spec:
            # If this is not set to true, the operator will watch for cluster CRDs in all namespaces.
            - name: ROOK_CURRENT_NAMESPACE_ONLY
              value: "false"
-            # To disable RBAC, uncomment the following:
-            # - name: RBAC_ENABLED
-            #   value: "false"
            # Rook Agent toleration. Will tolerate all taints with all keys.
            # Choose between NoSchedule, PreferNoSchedule and NoExecute:
            # - name: AGENT_TOLERATION
@ -402,10 +445,6 @@ spec:
            # - name: DISCOVER_AGENT_POD_LABELS
            #   value: "key1=value1,key2=value2"

-            # The logging level for the operator: INFO | DEBUG
-            - name: ROOK_LOG_LEVEL
-              value: "INFO"
-
            # The duration between discovering devices in the rook-discover daemonset.
            - name: ROOK_DISCOVER_DEVICES_INTERVAL
              value: "60m"
@ -468,6 +507,14 @@ spec:
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
+          # Recommended resource requests and limits, if desired
+          #resources:
+          #  limits:
+          #    cpu: 500m
+          #    memory: 256Mi
+          #  requests:
+          #    cpu: 100m
+          #    memory: 128Mi

          #  Uncomment it to run lib bucket provisioner in multithreaded mode
          #- name: LIB_BUCKET_PROVISIONER_THREADS
--- a/manifests/function/treasuremap-base-catalogues/versions-treasuremap.yaml
+++ b/manifests/function/treasuremap-base-catalogues/versions-treasuremap.yaml
@ -120,28 +120,28 @@ spec:
    rook-operator:
      rook-ceph-operator:
        rook-ceph-operator:
-          image: rook/ceph:v1.6.3
+          image: rook/ceph:v1.7.5
      rook-ceph-operator-config:
        ceph_daemon:
-          image: ceph/ceph:v15.2.11
+          image: quay.io/ceph/ceph:v16.2.6
        rook_csi_ceph_image:
-          image: quay.io/cephcsi/cephcsi:v3.3.1
+          image: quay.io/cephcsi/cephcsi:v3.4.0
        rook_csi_registrar_image:
-          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1
+          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0
        rook_csi_resizer_image:
-          image: k8s.gcr.io/sig-storage/csi-resizer:v1.0.1
+          image: k8s.gcr.io/sig-storage/csi-resizer:v1.3.0
        rook_csi_provisioner_image:
-          image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4
+          image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
        rook_csi_snapshotter_image:
-          image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
+          image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
        rook_csi_attacher_image:
-          image: k8s.gcr.io/sig-storage/csi-attacher:v3.0.2
+          image: k8s.gcr.io/sig-storage/csi-attacher:v3.3.0
    storage-rook:
      ceph:
        ceph-version:
-          image: ceph/ceph:v15.2.11
+          image: quay.io/ceph/ceph:v16.2.6
        rook-ceph-tools:
-          image: rook/ceph:v1.6.3
+          image: rook/ceph:v1.7.5
    localstorage:
      provisioner:
        provisioner: