openstack/openstack-helm-infra
Code Issues Proposed changes

Enable Ceph CSI Provisioner to Stand Alone

Browse Source 
The current implementation of the Ceph CSI provisioner is tied too
closely with the older Ceph RBD provisioner, which doesn't let the
deployer deploy Ceph CSI provisioner without the old RBD provisioner.

This patchset will decouple them such that they can be deployed
independently from one another.

A few other changes are needed as well:
1) The deployment/gate scripts are updated so that the old RBD and
   CSI RBD provisioners are separately enabled/disabled as needed.
   The original RBD provisioner is now deprecated.
2) Ceph-mon chart is updated because it had some RBD storageclass
   data in values.yaml that is not needed for ceph-mon deployment.
3) Fixed a couple of bugs in job-cephfs-client-key.yaml where RBD
   parameters were being used instead of cephfs parameters.

Change-Id: Icb5f78dcefa51990baf1b6d92411eb641c2ea9e2
This commit is contained in:
Parsons, Cliff (cp769u) 2021-06-10 22:34:01 +00:00
parent bd17d4b849
commit d9404f89c2
32 changed files with 126 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ceph-mon/Chart.yaml
View File

@ -15,6 +15,6 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Ceph Mon
name: ceph-mon
version: 0.1.8
version: 0.1.9
home: https://github.com/ceph/ceph
...

13
ceph-mon/values.yaml
View File

@ -305,21 +305,8 @@ bootstrap:
# and derive the manifest.
storageclass:
rbd:
provision_storage_class: true
provisioner: ceph.com/rbd
ceph_configmap_name: ceph-etc
metadata:
default_storage_class: true
name: general
parameters:
pool: rbd
adminId: admin
adminSecretName: pvc-ceph-conf-combined-storageclass
adminSecretNamespace: ceph
userId: admin
userSecretName: pvc-ceph-client-key
imageFormat: "2"
imageFeatures: layering
cephfs:
provision_storage_class: true
provisioner: ceph.com/cephfs

2
ceph-provisioners/Chart.yaml
View File

@ -15,6 +15,6 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Ceph Provisioner
name: ceph-provisioners
version: 0.1.7
version: 0.1.8
home: https://github.com/ceph/ceph
...

4
ceph-provisioners/templates/configmap-etc-client.yaml
View File

@ -46,5 +46,9 @@ data:
{{- end }}
{{- end }}
{{- if .Values.manifests.configmap_etc }}
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
{{- list .Values.storageclass.csi_rbd.ceph_configmap_name . | include "ceph.configmap.etc" }}
{{- else }}
{{- list .Values.storageclass.rbd.ceph_configmap_name . | include "ceph.configmap.etc" }}
{{- end }}
{{- end }}

4
ceph-provisioners/templates/configmap-etc-csi.yaml
View File

@ -17,7 +17,7 @@ limitations under the License.
{{- $envAll := index . 1 }}
{{- with $envAll }}
{{- if and (.Values.deployment.ceph) (.Values.deployment.csi) }}
{{- if and (.Values.deployment.ceph) (.Values.deployment.csi_rbd_provisioner) }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
@ -44,5 +44,5 @@ metadata:
{{- end }}
{{- if .Values.manifests.configmap_etc }}
{{- list .Values.storageclass.rbd.ceph_configmap_name . | include "ceph.configmap.etc.csi" }}
{{- list .Values.storageclass.csi_rbd.ceph_configmap_name . | include "ceph.configmap.etc.csi" }}
{{- end }}

2
ceph-provisioners/templates/daemonset-csi-rbd-plugin.yaml
View File

@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.rbd_provisioner }}
{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.csi_rbd_provisioner }}
{{- $envAll := . }}
{{- $serviceAccountName := printf "%s-%s" .Release.Name "ceph-rbd-csi-nodeplugin" }}

8
ceph-provisioners/templates/deployment-csi-rbd-provisioner.yaml
View File

@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.rbd_provisioner }}
{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.csi_rbd_provisioner }}
{{- $envAll := . }}
{{- $serviceAccountName := printf "%s-%s" .Release.Name "ceph-rbd-csi-provisioner" }}
@ -112,7 +112,7 @@ metadata:
labels:
{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
replicas: {{ .Values.pod.replicas.rbd_provisioner }}
replicas: {{ .Values.pod.replicas.csi_rbd_provisioner }}
selector:
matchLabels:
{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
@ -129,7 +129,7 @@ spec:
serviceAccountName: {{ $serviceAccountName }}
affinity:
{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
{{ tuple $envAll "rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
{{ tuple $envAll "csi_rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
nodeSelector:
{{ .Values.labels.provisioner.node_selector_key }}: {{ .Values.labels.provisioner.node_selector_value }}
initContainers:
@ -137,7 +137,7 @@ spec:
containers:
- name: ceph-rbd-provisioner
{{ tuple $envAll "csi_provisioner" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.rbd_provisioner | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.csi_rbd_provisioner | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: DEPLOYMENT_NAMESPACE

4
ceph-provisioners/templates/job-bootstrap.yaml
View File

@ -72,7 +72,11 @@ spec:
defaultMode: 0555
- name: ceph-etc
configMap:
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
name: {{ .Values.storageclass.csi_rbd.ceph_configmap_name }}
{{- else }}
name: {{ .Values.storageclass.rbd.ceph_configmap_name }}
{{- end }}
defaultMode: 0444
- name: ceph-client-admin-keyring
secret:

4
ceph-provisioners/templates/job-cephfs-client-key.yaml
View File

@ -52,7 +52,7 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
namespace: {{ .Values.storageclass.cephfs.parameters.adminSecretNamespace }}
rules:
- apiGroups:
- ""
@ -66,7 +66,7 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
namespace: {{ .Values.storageclass.cephfs.parameters.adminSecretNamespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role

16
ceph-provisioners/templates/job-namespace-client-ceph-config.yaml
View File

@ -52,7 +52,11 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
namespace: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
rules:
- apiGroups:
- ""
@ -66,7 +70,11 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
namespace: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
@ -102,13 +110,21 @@ spec:
{{ dict "envAll" $envAll "application" "client_ceph_config_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: CEPH_CONF_ETC
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
value: {{ .Values.storageclass.csi_rbd.ceph_configmap_name }}
{{- else }}
value: {{ .Values.storageclass.rbd.ceph_configmap_name }}
{{- end }}
- name: DEPLOYMENT_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
value: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
value: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
- name: MON_PORT
value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MON_PORT_V2

4
ceph-provisioners/templates/job-namespace-client-key-cleaner.yaml
View File

@ -77,7 +77,11 @@ spec:
fieldRef:
fieldPath: metadata.namespace
- name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
value: {{ .Values.storageclass.csi_rbd.parameters.userSecretName }}
{{- else }}
value: {{ .Values.storageclass.rbd.parameters.userSecretName }}
{{- end }}
command:
- /tmp/provisioner-rbd-namespace-client-key-cleaner.sh
volumeMounts:

17
ceph-provisioners/templates/job-namespace-client-key.yaml
View File

@ -52,7 +52,11 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
namespace: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
rules:
- apiGroups:
- ""
@ -66,7 +70,11 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
namespace: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
@ -105,12 +113,21 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
- name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
value: {{ .Values.storageclass.csi_rbd.parameters.userSecretName }}
- name: PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME
value: {{ .Values.storageclass.csi_rbd.parameters.adminSecretName }}
- name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
value: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
- name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
value: {{ .Values.storageclass.rbd.parameters.userSecretName }}
- name: PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME
value: {{ .Values.storageclass.rbd.parameters.adminSecretName }}
- name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
value: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
command:
- /tmp/provisioner-rbd-namespace-client-key-manager.sh
volumeMounts:

36
ceph-provisioners/values.yaml
View File

@ -19,8 +19,10 @@
deployment:
ceph: true
client_secrets: false
# Original rbd_provisioner is now DEPRECATED. It will be removed in the
# next release; CSI RBD provisioner should be used instead.
rbd_provisioner: true
csi: true
csi_rbd_provisioner: true
cephfs_provisioner: true
release_group: null
@ -144,6 +146,7 @@ pod:
replicas:
cephfs_provisioner: 2
rbd_provisioner: 2
csi_rbd_provisioner: 2
lifecycle:
upgrades:
deployments:
@ -171,6 +174,13 @@ pod:
limits:
memory: "50Mi"
cpu: "500m"
csi_rbd_provisioner:
requests:
memory: "5Mi"
cpu: "250m"
limits:
memory: "50Mi"
cpu: "500m"
cephfs_provisioner:
requests:
memory: "5Mi"
@ -239,6 +249,16 @@ pod:
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 60
csi_rbd_provisioner:
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 60
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 60
cephfs_provisioner:
tolerations:
- effect: NoExecute
@ -312,6 +332,12 @@ dependencies:
services:
- endpoint: internal
service: ceph_mon
csi_rbd_provisioner:
jobs:
- ceph-rbd-pool
services:
- endpoint: internal
service: ceph_mon
image_repo_sync:
services:
- endpoint: internal
@ -353,6 +379,7 @@ storageclass:
csi_rbd:
provision_storage_class: true
provisioner: ceph.rbd.csi.ceph.com
ceph_configmap_name: ceph-etc
metadata:
default_storage_class: true
name: general
@ -368,6 +395,11 @@ storageclass:
imageFeatures: layering
imageFormat: "2"
pool: rbd
adminId: admin
adminSecretName: pvc-ceph-conf-combined-storageclass
adminSecretNamespace: ceph
userId: admin
userSecretName: pvc-ceph-client-key
cephfs:
provision_storage_class: true
provisioner: ceph.com/cephfs
@ -411,6 +443,8 @@ manifests:
configmap_bin_common: true
configmap_etc: true
deployment_rbd_provisioner: true
# Original rbd_provisioner is now DEPRECATED. It will be removed in the
# next release; CSI RBD provisioner should be used instead.
deployment_csi_rbd_provisioner: true
deployment_cephfs_provisioner: true
job_bootstrap: false

1
doc/source/testing/ceph-resiliency/failure-domain.rst
View File

@ -696,6 +696,7 @@ An example of a lab enviroment had the following paramters set for the ceph yaml
storage_secrets: true
ceph: true
rbd_provisioner: true
csi_rbd_provisioner: true
cephfs_provisioner: true
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
releasenotes/notes/ceph-mon.yaml
View File

@ -9,4 +9,5 @@ ceph-mon:
- 0.1.6 Fix python3 issue for util scripts
- 0.1.7 remove deprecated svc annotation tolerate-unready-endpoints
- 0.1.8 Use full image ref for docker official images
- 0.1.9 Remove unnecessary parameters for ceph-mon
...

1
releasenotes/notes/ceph-provisioners.yaml
View File

@ -8,4 +8,5 @@ ceph-provisioners:
- 0.1.5 Fix Helm tests for the Ceph provisioners
- 0.1.6 Update ceph_mon config as per new ceph clients
- 0.1.7 Use full image ref for docker official images
- 0.1.8 Enable Ceph CSI Provisioner to Stand Alone
...

1
tools/deployment/armada/manifests/armada-ceph.yaml
View File

@ -293,6 +293,7 @@ data:
deployment:
ceph: true
rbd_provisioner: true
csi_rbd_provisioner: true
cephfs_provisioner: false
client_secrets: false
storageclass:

2
tools/deployment/armada/manifests/armada-lma.yaml
View File

@ -93,6 +93,7 @@ data:
deployment:
ceph: False
rbd_provisioner: False
csi_rbd_provisioner: False
cephfs_provisioner: False
client_secrets: True
storageclass:
@ -166,6 +167,7 @@ data:
deployment:
ceph: True
rbd_provisioner: False
csi_rbd_provisioner: False
cephfs_provisioner: False
client_secrets: False
rgw_keystone_user_and_endpoints: False

1
tools/deployment/multinode/030-ceph.sh
View File

@ -56,6 +56,7 @@ deployment:
storage_secrets: true
ceph: true
rbd_provisioner: true
csi_rbd_provisioner: true
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
tools/deployment/multinode/035-ceph-ns-activate.sh
View File

@ -28,6 +28,7 @@ deployment:
storage_secrets: false
ceph: false
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: true
rgw_keystone_user_and_endpoints: false

1
tools/deployment/multinode/115-radosgw-osh-infra.sh
View File

@ -33,6 +33,7 @@ deployment:
storage_secrets: false
ceph: true
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
tools/deployment/openstack-support/025-ceph-ns-activate.sh
View File

@ -30,6 +30,7 @@ deployment:
storage_secrets: false
ceph: false
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: true
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging-tls/020-ceph.sh
View File

@ -62,6 +62,7 @@ deployment:
storage_secrets: true
ceph: true
rbd_provisioner: true
csi_rbd_provisioner: true
cephfs_provisioner: true
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging-tls/025-ceph-ns-activate.sh
View File

@ -30,6 +30,7 @@ deployment:
storage_secrets: false
ceph: false
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: true
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging-tls/030-radosgw-osh-infra.sh
View File

@ -34,6 +34,7 @@ deployment:
storage_secrets: false
ceph: true
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging/020-ceph.sh
View File

@ -62,6 +62,7 @@ deployment:
storage_secrets: true
ceph: true
rbd_provisioner: true
csi_rbd_provisioner: true
cephfs_provisioner: true
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
View File

@ -30,6 +30,7 @@ deployment:
storage_secrets: false
ceph: false
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: true
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh
View File

@ -31,6 +31,7 @@ deployment:
storage_secrets: false
ceph: true
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false

7
tools/deployment/tenant-ceph/030-ceph.sh
View File

@ -56,7 +56,8 @@ network:
deployment:
storage_secrets: true
ceph: true
rbd_provisioner: true
rbd_provisioner: false
csi_rbd_provisioner: true
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false
@ -107,8 +108,10 @@ conf:
location: ${CEPH_OSD_DB_WAL_DEVICE}
size: "2GB"
storageclass:
rbd:
csi_rbd:
ceph_configmap_name: ceph-etc
rbd:
provision_storage_class: false
cephfs:
provision_storage_class: false
ceph_mgr_modules_config:

1
tools/deployment/tenant-ceph/040-tenant-ceph.sh
View File

@ -65,6 +65,7 @@ deployment:
storage_secrets: true
ceph: true
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false

15
tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh
View File

@ -35,6 +35,7 @@ deployment:
storage_secrets: false
ceph: false
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: true
rgw_keystone_user_and_endpoints: false
@ -45,16 +46,24 @@ conf:
enabled: true
storageclass:
rbd:
ceph_configmap_name: tenant-ceph-etc
provision_storage_class: false
provision_storage_class: true
metadata:
name: tenant-rbd
parameters:
adminSecretName: pvc-tenant-ceph-conf-combined-storageclass
adminSecretNamespace: tenant-ceph
userSecretName: pvc-tenant-ceph-client-key
csi_rbd:
ceph_configmap_name: tenant-ceph-etc
provision_storage_class: true
metadata:
name: tenant-csi-rbd
parameters:
adminSecretName: pvc-tenant-ceph-conf-combined-storageclass
adminSecretNamespace: tenant-ceph
userSecretName: pvc-tenant-ceph-client-key
cephfs:
provision_storage_class: false
provision_storage_class: true
metadata:
name: cephfs
parameters:

1
tools/deployment/tenant-ceph/060-radosgw-openstack.sh
View File

@ -38,6 +38,7 @@ deployment:
storage_secrets: false
ceph: true
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false