[CEPH] Extend RadosGW S3 configuration support

- Support using custom client params for S3 configurations
- Move common tuning for S3 and Keystone into there own
configuration option
- Cleanup the rgw helm tests, since copying the ceph admin key is
no longer required
- Cleanup duplicate portions of the code for configuring the RGW
backend and frontend port
- Add an rgw helm test check for the osh-infra-logging gates

Change-Id: I46dbb4c45b0b96f5cf555077e49d2e09a1171424

ceph-rgw/templates/bin/rgw/_init.sh.tpl

@@ -18,12 +18,20 @@ limitations under the License.
 
 set -ex
 
-cp -va /tmp/ceph.conf /etc/ceph/ceph.conf
+cp -va /etc/ceph/ceph.conf.template /etc/ceph/ceph.conf
 
 cat >> /etc/ceph/ceph.conf <<EOF
 
 [client.rgw.$(hostname -s)]
 rgw_frontends = "beast port=${RGW_FRONTEND_PORT}"
+{{ range $key, $value := .Values.conf.rgw.config -}}
+{{- if kindIs "slice" $value -}}
+{{ $key }} = {{ include "helm-toolkit.joinListWithComma" $value | quote }}
+{{ else -}}
+{{ $key }} = {{ $value | quote  }}
+{{ end -}}
+{{- end -}}
+{{ if .Values.conf.rgw_ks.enabled }}
 rgw_keystone_url = "${KEYSTONE_URL}"
 rgw_keystone_admin_user = "${OS_USERNAME}"
 rgw_keystone_admin_password = "${OS_PASSWORD}"
@@ -36,4 +44,13 @@ rgw_keystone_admin_domain = "${OS_USER_DOMAIN_NAME}"
 {{ $key }} = {{ $value | quote  }}
 {{ end -}}
 {{- end -}}
+{{ end }}
+{{ range $key, $value := .Values.conf.rgw_s3.config -}}
+{{- if kindIs "slice" $value -}}
+{{ $key }} = {{ include "helm-toolkit.joinListWithComma" $value | quote }}
+{{ else -}}
+{{ $key }} = {{ $value | quote  }}
+{{ end -}}
+{{- end -}}
+
 EOF

ceph-rgw/templates/bin/rgw/_start.sh.tpl

@@ -54,12 +54,6 @@ if [ ! -e "${RGW_KEYRING}" ]; then
   chmod 0600 "${RGW_KEYRING}"
 fi
 
-if [[ $(ceph -v | egrep -c "luminous" | xargs echo) -gt 0 ]]; then
-  RGW_FRONTENDS="civitweb port=${RGW_FRONTEND_PORT}"
-else
-  RGW_FRONTENDS="beast port=${RGW_FRONTEND_PORT}"
-fi
-
 /usr/bin/radosgw \
   --cluster "${CLUSTER}" \
   --setuser "ceph" \
@@ -69,5 +63,4 @@ fi
   -k "${RGW_KEYRING}" \
   --rgw-socket-path="" \
   --rgw-zonegroup="${RGW_ZONEGROUP}" \
-  --rgw-zone="${RGW_ZONE}" \
+  --rgw-zone="${RGW_ZONE}"
-  --rgw-frontends="${RGW_FRONTENDS}"

ceph-rgw/templates/configmap-bin.yaml

@@ -32,8 +32,8 @@ data:
 
   rgw-start.sh: |
 {{ tuple "bin/rgw/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  rgw-init-keystone.sh: |
+  rgw-init.sh: |
-{{ tuple "bin/rgw/_init_keystone.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ tuple "bin/rgw/_init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   storage-init.sh: |
 {{ tuple "bin/_ceph-rgw-storage-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   ceph-admin-keyring.sh: |

ceph-rgw/templates/deployment-rgw.yaml

@@ -66,8 +66,7 @@ spec:
             - name: pod-run
               mountPath: /run
               readOnly: false
-{{ if .Values.conf.rgw_ks.enabled }}
+        - name: ceph-rgw-init
-        - name: ceph-rgw-ks-init
 {{ tuple $envAll "ceph_rgw" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.rgw | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           env:
@@ -78,27 +77,28 @@ spec:
                 fieldRef:
                   apiVersion: v1
                   fieldPath: metadata.name
+{{ if .Values.conf.rgw_ks.enabled }}
 {{- with $env := dict "ksUserSecret" .Values.secrets.identity.user_rgw }}
 {{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
 {{- end }}
             - name: KEYSTONE_URL
               value: {{ tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.identity.path.default | quote }}
+{{ end }}
             - name: RGW_FRONTEND_PORT
               value: "{{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}"
           command:
-            - /tmp/rgw-init-keystone.sh
+            - /tmp/rgw-init.sh
           volumeMounts:
             - name: pod-etc-ceph
               mountPath: /etc/ceph
             - name: ceph-rgw-bin
-              mountPath: /tmp/rgw-init-keystone.sh
+              mountPath: /tmp/rgw-init.sh
-              subPath: rgw-init-keystone.sh
+              subPath: rgw-init.sh
               readOnly: true
             - name: ceph-rgw-etc
-              mountPath: /tmp/ceph.conf
+              mountPath: /etc/ceph/ceph.conf.template
               subPath: ceph.conf
               readOnly: true
-{{ end }}
       containers:
         - name: ceph-rgw
 {{ tuple $envAll "ceph_rgw" | include "helm-toolkit.snippets.image" | indent 10 }}
@@ -130,12 +130,10 @@ spec:
               readOnly: true
             - name: pod-etc-ceph
               mountPath: /etc/ceph
-{{- if not .Values.conf.rgw_ks.enabled }}
             - name: ceph-rgw-etc
-              mountPath: /etc/ceph/ceph.conf
+              mountPath: /etc/ceph/ceph.conf.template
               subPath: ceph.conf
               readOnly: true
-{{- end }}
             - name: ceph-bootstrap-rgw-keyring
               mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
               subPath: ceph.keyring

ceph-rgw/templates/pod-helm-tests.yaml

@@ -31,25 +31,6 @@ metadata:
 spec:
   restartPolicy: Never
   serviceAccountName: {{ $serviceAccountName }}
-  initContainers:
-{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }}
-    - name: ceph-keyring-placement
-{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 6 }}
-      securityContext:
-        runAsUser: 0
-      command:
-        - /tmp/ceph-admin-keyring.sh
-      volumeMounts:
-        - name: etcceph
-          mountPath: /etc/ceph
-        - name: ceph-rgw-bin
-          mountPath: /tmp/ceph-admin-keyring.sh
-          subPath: ceph-admin-keyring.sh
-          readOnly: true
-        - name: ceph-keyring
-          mountPath: /tmp/client-keyring
-          subPath: key
-          readOnly: true
   containers:
 {{ if .Values.conf.rgw_ks.enabled }}
     - name: ceph-rgw-ks-validation

ceph-rgw/values.yaml

@@ -273,16 +273,8 @@ conf:
         application: rgw
         replication: 3
         percent_total_data: 34.8
-  rgw_ks:
+  rgw:
-    enabled: false
     config:
-      rgw_keystone_api_version: 3
-      rgw_keystone_accepted_roles: "admin, member"
-      rgw_keystone_implicit_tenants: true
-      rgw_keystone_make_new_tenants: true
-      rgw_s3_auth_use_keystone: true
-      rgw_swift_account_in_url: true
-      rgw_swift_url: null
       #NOTE (portdirect): See http://tracker.ceph.com/issues/21226
       rgw_keystone_token_cache_size: 0
       #NOTE (JCL): See http://tracker.ceph.com/issues/7073
@@ -294,11 +286,22 @@ conf:
       rgw_thread_pool_size: 512
       rgw_num_rados_handles: 4
       rgw_override_bucket_index_max_shards: 8
-      #NOTE (supamatt): Unfortunately we do not conform to S3 compliant names with some of our charts
+  rgw_ks:
-      rgw_relaxed_s3_bucket_names: true
+    enabled: false
+    config:
+      rgw_keystone_api_version: 3
+      rgw_keystone_accepted_roles: "admin, member"
+      rgw_keystone_implicit_tenants: true
+      rgw_keystone_make_new_tenants: true
+      rgw_s3_auth_use_keystone: true
+      rgw_swift_account_in_url: true
+      rgw_swift_url: null
   rgw_s3:
     enabled: false
     admin_caps: "users=*;buckets=*;zone=*"
+    config:
+      #NOTE (supamatt): Unfortunately we do not conform to S3 compliant names with some of our charts
+      rgw_relaxed_s3_bucket_names: true
   ceph:
     global:
       # auth

tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh

@@ -56,3 +56,6 @@ helm upgrade --install radosgw-osh-infra ./ceph-rgw \
 
 #NOTE: Validate Deployment info
 helm status radosgw-osh-infra
+
+#NOTE: Test Deployment
+helm test radosgw-osh-infra --timeout 900