Ceph: RadosGW

This PS implements the ceph radosgw and also provides keystone
intergration, allowing ceph to provide a swift like service if
desired for object storage.

In addtion it updates the endpoint lookups to use valid yaml when
dealing with keystone services with a '-' in their name.

Change-Id: I9162ad657df2f77c1bc1afa93a8b999894b1b470

barbican/templates/configmap-etc.yaml

@@ -63,9 +63,9 @@ limitations under the License.
 {{- tuple "oslo_messaging" "internal" "user" "amqp" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.barbican.default.oslo.messaging "transport_url" | quote | trunc 0 -}}
 {{- end -}}
 
-{{- $barbicanPath := index .Values "endpoints" "key-manager" "path" "default" }}
+{{- $barbicanPath := index .Values "endpoints" "key_manager" "path" "default" }}
 {{- if empty .Values.conf.barbican.default.barbican.common.config.host_href -}}
-{{- tuple "key-manager" "public" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix $barbicanPath | set .Values.conf.barbican.default.barbican.common.config "host_href" | quote | trunc 0 -}}
+{{- tuple "key_manager" "public" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix $barbicanPath | set .Values.conf.barbican.default.barbican.common.config "host_href" | quote | trunc 0 -}}
 {{- end -}}
 
 ---

barbican/values.yaml

@@ -337,7 +337,7 @@ endpoints:
         default: 35357
       api:
         default: 80
-  key-manager:
+  key_manager:
     name: barbican
     hosts:
       default: barbican-api

ceph/templates/bin/_init_rgw_ks.sh.tpl

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+cp -va /tmp/ceph.conf /etc/ceph/ceph.conf
+
+cat >> /etc/ceph/ceph.conf <<EOF
+
+[client.rgw.${POD_NAME}]
+rgw_frontends = "civetweb port=${RGW_CIVETWEB_PORT}"
+rgw_keystone_url = "${KEYSTONE_URL}"
+rgw_keystone_admin_user = "${OS_USERNAME}"
+rgw_keystone_admin_password = "${OS_PASSWORD}"
+rgw_keystone_admin_project = "${OS_PROJECT_NAME}"
+rgw_keystone_admin_domain = "${OS_USER_DOMAIN_NAME}"
+{{ range $key, $value := .Values.conf.rgw_ks.config -}}
+{{- if kindIs "slice" $value -}}
+{{ $key }} = {{ include "helm-toolkit.joinListWithComma" $value | quote }}
+{{ else -}}
+{{ $key }} = {{ $value | quote  }}
+{{ end -}}
+{{- end -}}
+EOF

ceph/templates/configmap-bin-clients.yaml

@@ -0,0 +1,31 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_bin_clients }}
+{{- $envAll := . }}
+{{- if .Values.deployment.client_secrets }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-bin-clients
+data:
+  ceph-namespace-client-key.sh: |+
+{{ tuple "bin/_ceph-namespace-client-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ceph-namespace-client-key-cleaner.sh: |+
+{{ tuple "bin/_ceph-namespace-client-key-cleaner.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
+{{- end }}

ceph/templates/configmap-bin-ks.yaml

@@ -0,0 +1,33 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_bin_ks }}
+{{- $envAll := . }}
+{{- if .Values.deployment.rgw_keystone_user_and_endpoints }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-bin-ks
+data:
+  ks-service.sh: |+
+{{- include "helm-toolkit.scripts.keystone_service" . | indent 4 }}
+  ks-endpoints.sh: |+
+{{- include "helm-toolkit.scripts.keystone_endpoints" . | indent 4 }}
+  ks-user.sh: |+
+{{- include "helm-toolkit.scripts.keystone_user" . | indent 4 }}
+{{- end }}
+{{- end }}

ceph/templates/configmap-bin.yaml

@@ -14,7 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- if .Values.manifests.configmap_bin }}
 {{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -25,20 +27,12 @@ data:
   bootstrap.sh: |+
 {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
-{{- if .Values.manifests_enabled.storage_secrets }}
   ceph-key.py: |+
 {{ tuple "bin/_ceph-key.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   ceph-key.sh: |+
 {{ tuple "bin/_ceph-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   ceph-storage-key.sh: |+
 {{ tuple "bin/_ceph-storage-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{- end }}
-{{- if .Values.manifests_enabled.client_secrets }}
-  ceph-namespace-client-key.sh: |+
-{{ tuple "bin/_ceph-namespace-client-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  ceph-namespace-client-key-cleaner.sh: |+
-{{ tuple "bin/_ceph-namespace-client-key-cleaner.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{- end }}
   init_dirs.sh: |+
 {{ tuple "bin/_init_dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   common_functions.sh: |+
@@ -67,6 +61,8 @@ data:
 {{ tuple "bin/_start_mds.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   start_rgw.sh: |+
 {{ tuple "bin/_start_rgw.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  init_rgw_ks.sh: |+
+{{ tuple "bin/_init_rgw_ks.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   watch_mon_health.sh: |+
 {{ tuple "bin/_watch_mon_health.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   variables_entrypoint.sh: |
@@ -75,3 +71,5 @@ data:
 {{ tuple "bin/_check_zombie_mons.py.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
   rbd-provisioner.sh: |
 {{ tuple "bin/_rbd-provisioner.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
+{{- end }}

ceph/templates/configmap-etc.yaml

@@ -14,13 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- if .Values.manifests.configmap_etc }}
 {{- $envAll := . }}
+{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
 
 {{- if empty .Values.conf.ceph.config.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
-{{- $monHostDomain := default .Release.Namespace .Values.ceph.namespace }}
-{{- $monHostURI := cat $monHost "." $monHostDomain | nospace -}}
-{{- $monHostURI | set .Values.conf.ceph.config.global "mon_host" | quote | trunc 0 -}}
+{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $monHost | set .Values.conf.ceph.config.global "mon_host" | quote | trunc 0 -}}
 {{- end -}}
 
 {{- if empty .Values.conf.ceph.config.global.fsid -}}
@@ -43,3 +43,5 @@ metadata:
 data:
   ceph.conf: |+
 {{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph.config | indent 4 }}
+{{- end }}
+{{- end }}

ceph/templates/configmap-templates.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.storage_secrets }}
+{{- if .Values.manifests.configmap_templates }}
 {{- $envAll := . }}
+{{- if .Values.deployment.storage_secrets }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -33,3 +34,4 @@ data:
   mon.keyring: |+
 {{ tuple "templates/_mon.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
+{{- end }}

ceph/templates/daemonset-mon.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.deployment }}
+{{- if .Values.manifests.daemonset_mon }}
 {{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
 {{- $dependencies := .Values.dependencies.mon }}
 ---
 kind: DaemonSet
@@ -82,6 +83,15 @@ spec:
                     - /remove-mon.sh
           ports:
             - containerPort: 6789
+          livenessProbe:
+              tcpSocket:
+                port: 6789
+              initialDelaySeconds: 60
+              timeoutSeconds: 5
+          readinessProbe:
+              tcpSocket:
+                port: 6789
+              timeoutSeconds: 5
           volumeMounts:
             - name: ceph-bin
               mountPath: /start_mon.sh
@@ -129,15 +139,6 @@ spec:
             - name: pod-run
               mountPath: /run
               readOnly: false
-          livenessProbe:
-              tcpSocket:
-                port: 6789
-              initialDelaySeconds: 60
-              timeoutSeconds: 5
-          readinessProbe:
-              tcpSocket:
-                port: 6789
-              timeoutSeconds: 5
       volumes:
         - name: ceph-bin
           configMap:
@@ -168,3 +169,4 @@ spec:
           secret:
             secretName: {{ .Values.secrets.keyrings.rgw }}
 {{- end }}
+{{- end }}

ceph/templates/daemonset-osd.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.deployment }}
+{{- if .Values.manifests.daemonset_osd }}
 {{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
 {{- $dependencies := .Values.dependencies.osd }}
 ---
 kind: DaemonSet
@@ -167,3 +168,4 @@ spec:
           hostPath:
             path: {{ .Values.ceph.storage.osd_directory }}
 {{- end }}
+{{- end }}

ceph/templates/deployment-mds.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.deployment }}
+{{- if .Values.manifests.deployment_mds }}
 {{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
 {{- if .Values.ceph.enabled.mds }}
 {{- $dependencies := .Values.dependencies.mds }}
 ---
@@ -70,6 +71,15 @@ spec:
             - /start_mds.sh
           ports:
             - containerPort: 6800
+          livenessProbe:
+              tcpSocket:
+                port: 6800
+              initialDelaySeconds: 60
+              timeoutSeconds: 5
+          readinessProbe:
+              tcpSocket:
+                port: 6800
+              timeoutSeconds: 5
           volumeMounts:
             - name: ceph-bin
               mountPath: /start_mds.sh
@@ -109,15 +119,6 @@ spec:
             - name: pod-run
               mountPath: /run
               readOnly: false
-          livenessProbe:
-              tcpSocket:
-                port: 6800
-              initialDelaySeconds: 60
-              timeoutSeconds: 5
-          readinessProbe:
-              tcpSocket:
-                port: 6800
-              timeoutSeconds: 5
       volumes:
         - name: ceph-etc
           configMap:
@@ -149,3 +150,4 @@ spec:
             secretName: {{ .Values.secrets.keyrings.rgw }}
 {{- end }}
 {{- end }}
+{{- end }}

ceph/templates/deployment-moncheck.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.deployment }}
+{{- if .Values.manifests.deployment_moncheck }}
 {{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
 {{- $dependencies := .Values.dependencies.moncheck }}
 ---
 kind: Deployment
@@ -23,7 +24,7 @@ apiVersion: apps/v1beta1
 metadata:
   name: ceph-mon-check
 spec:
-  replicas: {{ .Values.replicas.mon_check }}
+  replicas: {{ .Values.pod.replicas.mon_check }}
   template:
     metadata:
       labels:
@@ -143,3 +144,4 @@ spec:
           secret:
             secretName: {{ .Values.secrets.keyrings.rgw }}
 {{- end }}
+{{- end }}

ceph/templates/deployment-rbd-provisioner.yaml

@@ -1,19 +1,22 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
 
-{{- if .Values.manifests_enabled.rbd_provisioner }}
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.deployment_rbd_provisioner }}
 {{- $envAll := . }}
+{{- if .Values.deployment.rbd_provisioner }}
 {{- $dependencies := .Values.dependencies.rbd_provisioner }}
 ---
 kind: Deployment
@@ -21,7 +24,7 @@ apiVersion: extensions/v1beta1
 metadata:
   name: ceph-rbd-provisioner
 spec:
-  replicas: {{ .Values.replicas.rbd_provisioner }}
+  replicas: {{ .Values.pod.replicas.rbd_provisioner }}
   strategy:
     type: Recreate
   template:
@@ -56,3 +59,4 @@ spec:
             name: ceph-bin
             defaultMode: 0555
 {{- end }}
+{{- end }}

ceph/templates/deployment-rgw.yaml

@@ -14,9 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.deployment }}
-{{- if .Values.ceph.enabled.rgw }}
+{{- if .Values.manifests.deployment_rgw }}
 {{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
+{{- if .Values.ceph.enabled.rgw }}
 {{- $dependencies := .Values.dependencies.rgw }}
 ---
 kind: Deployment
@@ -24,7 +25,7 @@ apiVersion: apps/v1beta1
 metadata:
   name: ceph-rgw
 spec:
-  replicas: {{ .Values.replicas.rgw }}
+  replicas: {{ .Values.pod.replicas.rgw }}
   template:
     metadata:
       labels:
@@ -57,6 +58,38 @@ spec:
             - name: pod-run
               mountPath: /run
               readOnly: false
+{{ if .Values.ceph.rgw_keystone_auth }}
+        - name: ceph-rgw-ks-init
+          image: {{ .Values.images.daemon }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.rgw | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+{{- with $env := dict "ksUserSecret" .Values.secrets.identity.user_rgw }}
+{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
+{{- end }}
+            - name: KEYSTONE_URL
+              value: {{ tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.identity.path.default | quote }}
+            - name: RGW_CIVETWEB_PORT
+              value: "{{ .Values.network.port.rgw }}"
+          command:
+            - /tmp/init_rgw_ks.sh
+          volumeMounts:
+            - name: pod-etc-ceph
+              mountPath: /etc/ceph
+            - name: ceph-bin
+              mountPath: /tmp/init_rgw_ks.sh
+              subPath: init_rgw_ks.sh
+              readOnly: true
+            - name: ceph-etc
+              mountPath: /tmp/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+{{ end }}
       containers:
         - name: ceph-rgw
           image: {{ .Values.images.daemon }}
@@ -64,23 +97,31 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.rgw | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           env:
             - name: RGW_CIVETWEB_PORT
-              value: "{{ .Values.network.port.rgw_target }}"
+              value: "{{ .Values.network.port.rgw }}"
           command:
             - /start_rgw.sh
           ports:
-            - containerPort: {{ .Values.network.port.rgw_target }}
+            - containerPort: {{ .Values.network.port.rgw }}
           livenessProbe:
               httpGet:
                 path: /
-                port: {{ .Values.network.port.rgw_target }}
+                port: {{ .Values.network.port.rgw }}
               initialDelaySeconds: 120
               timeoutSeconds: 5
           readinessProbe:
               httpGet:
                 path: /
-                port: {{ .Values.network.port.rgw_target }}
+                port: {{ .Values.network.port.rgw }}
               timeoutSeconds: 5
           volumeMounts:
+            - name: pod-etc-ceph
+              mountPath: /etc/ceph
+{{- if not .Values.ceph.rgw_keystone_auth }}
+            - name: ceph-etc
+              mountPath: /ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+{{- end }}
             - name: ceph-bin
               mountPath: /start_rgw.sh
               subPath: start_rgw.sh
@@ -89,10 +130,6 @@ spec:
               mountPath: /common_functions.sh
               subPath: common_functions.sh
               readOnly: true
-            - name: ceph-etc
-              mountPath: /etc/ceph/ceph.conf
-              subPath: ceph.conf
-              readOnly: true
             - name: ceph-client-admin-keyring
               mountPath: /etc/ceph/ceph.client.admin.keyring
               subPath: ceph.client.admin.keyring
@@ -120,6 +157,8 @@ spec:
               mountPath: /run
               readOnly: false
       volumes:
+        - name: pod-etc-ceph
+          emptyDir: {}
         - name: ceph-bin
           configMap:
             name: ceph-bin
@@ -150,3 +189,4 @@ spec:
             secretName: {{ .Values.secrets.keyrings.rgw }}
 {{- end }}
 {{- end }}
+{{- end }}

ceph/templates/job-bootstrap.yaml

@@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- if .Values.manifests.job_bootstrap }}
 {{- $envAll := . }}
 {{- if .Values.bootstrap.enabled }}
 {{- $dependencies := .Values.dependencies.bootstrap }}
@@ -30,7 +31,7 @@ spec:
     spec:
       restartPolicy: OnFailure
       nodeSelector:
-        {{ .Values.labels.bootstrap.node_selector_key }}: {{ .Values.labels.bootstrap.node_selector_value }}
+        {{ .Values.labels.jobs.node_selector_key }}: {{ .Values.labels.jobs.node_selector_value }}
       initContainers:
 {{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
       containers:
@@ -66,3 +67,4 @@ spec:
           secret:
             secretName: {{ .Values.secrets.keyrings.admin }}
 {{- end }}
+{{- end }}

ceph/templates/job-keyring.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.storage_secrets }}
+{{- if .Values.manifests.job_keyring }}
 {{- $envAll := . }}
+{{- if .Values.deployment.storage_secrets }}
 {{- range $key1, $cephBootstrapKey := tuple "mds" "osd" "rgw" "mon" }}
 {{- $jobName := print $cephBootstrapKey "-keyring-generator" }}
 ---
@@ -30,19 +31,13 @@ spec:
 {{ tuple $envAll "ceph" $jobName | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
     spec:
       restartPolicy: OnFailure
+      nodeSelector:
+        {{ $envAll.Values.labels.jobs.node_selector_key }}: {{ $envAll.Values.labels.jobs.node_selector_value }}
       containers:
         - name:  ceph-{{ $jobName }}
           image: {{ $envAll.Values.images.ceph_config_helper }}
           imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
-          {{- if $envAll.Values.pod.resources.enabled }}
-          resources:
-            requests:
-              memory: {{ .Values.pod.resources.jobs.secret_provisioning.requests.memory | quote }}
-              cpu: {{ .Values.pod.resources.jobs.secret_provisioning.requests.cpu | quote }}
-            limits:
-              memory: {{ .Values.pod.resources.jobs.secret_provisioning.limits.memory | quote }}
-              cpu: {{ .Values.pod.resources.jobs.secret_provisioning.limits.cpu | quote }}
-          {{- end }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           env:
             - name: DEPLOYMENT_NAMESPACE
               valueFrom:
@@ -88,5 +83,6 @@ spec:
           configMap:
             name: ceph-templates
             defaultMode: 0444
-{{ end }}
-{{ end }}
+{{- end }}
+{{- end }}
+{{- end }}

ceph/templates/job-ks-endpoints.yaml

@@ -0,0 +1,71 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_ks_endpoints }}
+{{- $envAll := . }}
+{{- if .Values.deployment.rgw_keystone_user_and_endpoints }}
+{{- $dependencies := .Values.dependencies.ks_endpoints }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-ks-endpoints
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph" "ks-endpoints" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ $envAll.Values.labels.jobs.node_selector_key }}: {{ $envAll.Values.labels.jobs.node_selector_value }}
+      initContainers:
+{{ tuple $envAll $dependencies "[]" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+{{- range $key1, $osServiceType := tuple "object-store" }}
+{{- range $key2, $osServiceEndPoint := tuple "admin" "internal" "public" }}
+        - name: {{ $osServiceType }}-ks-endpoints-{{ $osServiceEndPoint }}
+          image: {{ $envAll.Values.images.ks_endpoints }}
+          imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_endpoints | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/ks-endpoints.sh
+          volumeMounts:
+            - name: ks-endpoints-sh
+              mountPath: /tmp/ks-endpoints.sh
+              subPath: ks-endpoints.sh
+              readOnly: true
+          env:
+{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin }}
+{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
+{{- end }}
+            - name: OS_SVC_ENDPOINT
+              value: {{ $osServiceEndPoint }}
+            - name: OS_SERVICE_NAME
+              value: {{ tuple $osServiceType $envAll | include "helm-toolkit.endpoints.keystone_endpoint_name_lookup" }}
+            - name: OS_SERVICE_TYPE
+              value: {{ $osServiceType }}
+            - name: OS_SERVICE_ENDPOINT
+              value: {{ tuple $osServiceType $osServiceEndPoint "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+{{- end }}
+{{- end }}
+      volumes:
+        - name: ks-endpoints-sh
+          configMap:
+            name: ceph-bin-ks
+            defaultMode: 0555
+{{- end }}
+{{- end }}

ceph/templates/job-ks-service.yaml

@@ -0,0 +1,65 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_ks_service }}
+{{- $envAll := . }}
+{{- if .Values.deployment.rgw_keystone_user_and_endpoints }}
+{{- $dependencies := .Values.dependencies.ks_service }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-ks-service
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph" "ks-service" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ $envAll.Values.labels.jobs.node_selector_key }}: {{ $envAll.Values.labels.jobs.node_selector_value }}
+      initContainers:
+{{ tuple $envAll $dependencies "[]" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+{{- range $key1, $osServiceType := tuple "object-store" }}
+        - name: {{ $osServiceType }}-ks-service-registration
+          image: {{ $envAll.Values.images.ks_service }}
+          imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_service | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/ks-service.sh
+          volumeMounts:
+            - name: ks-service-sh
+              mountPath: /tmp/ks-service.sh
+              subPath: ks-service.sh
+              readOnly: true
+          env:
+{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin }}
+{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
+{{- end }}
+            - name: OS_SERVICE_NAME
+              value: {{ tuple $osServiceType $envAll | include "helm-toolkit.endpoints.keystone_endpoint_name_lookup" }}
+            - name: OS_SERVICE_TYPE
+              value: {{ $osServiceType }}
+{{- end }}
+      volumes:
+        - name: ks-service-sh
+          configMap:
+            name: ceph-bin-ks
+            defaultMode: 0555
+{{- end }}
+{{- end }}

ceph/templates/job-ks-user.yaml

@@ -0,0 +1,66 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_ks_user }}
+{{- $envAll := . }}
+{{- if .Values.deployment.rgw_keystone_user_and_endpoints }}
+{{- $dependencies := .Values.dependencies.ks_user }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-ks-user
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph" "ks-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ $envAll.Values.labels.jobs.node_selector_key }}: {{ $envAll.Values.labels.jobs.node_selector_value }}
+      initContainers:
+{{ tuple $envAll $dependencies "[]" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: ceph-ks-user
+          image: {{ .Values.images.ks_user }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/ks-user.sh
+          volumeMounts:
+            - name: ks-user-sh
+              mountPath: /tmp/ks-user.sh
+              subPath: ks-user.sh
+              readOnly: true
+          env:
+{{- with $env := dict "ksUserSecret" .Values.secrets.identity.admin }}
+{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
+{{- end }}
+            - name: SERVICE_OS_SERVICE_NAME
+              value: "ceph"
+{{- with $env := dict "ksUserSecret" .Values.secrets.identity.user }}
+{{- include "helm-toolkit.snippets.keystone_user_create_env_vars" $env | indent 12 }}
+{{- end }}
+            - name: SERVICE_OS_ROLE
+              value: {{ .Values.endpoints.identity.auth.user.role | quote }}
+      volumes:
+        - name: ks-user-sh
+          configMap:
+            name: ceph-bin-ks
+            defaultMode: 0555
+{{- end }}
+{{- end }}

ceph/templates/job-namespace-client-key-cleaner.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.client_secrets }}
+{{- if .Values.manifests.job_namespace_client_key_cleaner }}
 {{- $envAll := . }}
+{{- if .Values.deployment.client_secrets }}
 ---
 apiVersion: batch/v1
 kind: Job
@@ -30,19 +31,13 @@ spec:
 {{ tuple $envAll "ceph" "client-key-cleaner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
     spec:
       restartPolicy: OnFailure
+      nodeSelector:
+        {{ $envAll.Values.labels.jobs.node_selector_key }}: {{ $envAll.Values.labels.jobs.node_selector_value }}
       containers:
         - name:  ceph-namespace-client-keys-cleaner
           image: {{ .Values.images.ceph_config_helper }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
-          {{- if .Values.pod.resources.enabled }}
-          resources:
-            requests:
-              memory: {{ .Values.pod.resources.jobs.secret_provisioning.requests.memory | quote }}
-              cpu: {{ .Values.pod.resources.jobs.secret_provisioning.requests.cpu | quote }}
-            limits:
-              memory: {{ .Values.pod.resources.jobs.secret_provisioning.limits.memory | quote }}
-              cpu: {{ .Values.pod.resources.jobs.secret_provisioning.limits.cpu | quote }}
-          {{- end }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           env:
             - name: DEPLOYMENT_NAMESPACE
               valueFrom:
@@ -60,6 +55,7 @@ spec:
       volumes:
         - name: ceph-bin
           configMap:
-            name: ceph-bin
+            name: ceph-bin-clients
             defaultMode: 0555
 {{- end }}
+{{- end }}

ceph/templates/job-namespace-client-key.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.client_secrets }}
+{{- if .Values.manifests.job_namespace_client_key }}
 {{- $envAll := . }}
+{{- if .Values.deployment.client_secrets }}
 ---
 apiVersion: batch/v1
 kind: Job
@@ -28,19 +29,13 @@ spec:
 {{ tuple $envAll "ceph" "client-key-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
     spec:
       restartPolicy: OnFailure
+      nodeSelector:
+        {{ $envAll.Values.labels.jobs.node_selector_key }}: {{ $envAll.Values.labels.jobs.node_selector_value }}
       containers:
         - name:  ceph-storage-keys-generator
           image: {{ .Values.images.ceph_config_helper }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
-          {{- if .Values.pod.resources.enabled }}
-          resources:
-            requests:
-              memory: {{ .Values.pod.resources.jobs.secret_provisioning.requests.memory | quote }}
-              cpu: {{ .Values.pod.resources.jobs.secret_provisioning.requests.cpu | quote }}
-            limits:
-              memory: {{ .Values.pod.resources.jobs.secret_provisioning.limits.memory | quote }}
-              cpu: {{ .Values.pod.resources.jobs.secret_provisioning.limits.cpu | quote }}
-          {{- end }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           env:
             - name: DEPLOYMENT_NAMESPACE
               valueFrom:
@@ -62,6 +57,7 @@ spec:
       volumes:
         - name: ceph-bin
           configMap:
-            name: ceph-bin
+            name: ceph-bin-clients
             defaultMode: 0555
 {{- end }}
+{{- end }}

ceph/templates/job-storage-admin-keys.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.storage_secrets }}
+{{- if .Values.manifests.job_storage_admin_keys }}
 {{- $envAll := . }}
+{{- if .Values.deployment.storage_secrets }}
 ---
 apiVersion: batch/v1
 kind: Job
@@ -28,19 +29,13 @@ spec:
 {{ tuple $envAll "ceph" "storage-keys-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
     spec:
       restartPolicy: OnFailure
+      nodeSelector:
+        {{ $envAll.Values.labels.jobs.node_selector_key }}: {{ $envAll.Values.labels.jobs.node_selector_value }}
       containers:
         - name:  ceph-storage-keys-generator
           image: {{ .Values.images.ceph_config_helper }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
-          {{- if .Values.pod.resources.enabled }}
-          resources:
-            requests:
-              memory: {{ .Values.pod.resources.jobs.secret_provisioning.requests.memory | quote }}
-              cpu: {{ .Values.pod.resources.jobs.secret_provisioning.requests.cpu | quote }}
-            limits:
-              memory: {{ .Values.pod.resources.jobs.secret_provisioning.limits.memory | quote }}
-              cpu: {{ .Values.pod.resources.jobs.secret_provisioning.limits.cpu | quote }}
-          {{- end }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           env:
             - name: DEPLOYMENT_NAMESPACE
               valueFrom:
@@ -82,3 +77,4 @@ spec:
             name: ceph-templates
             defaultMode: 0444
 {{- end }}
+{{- end }}

ceph/templates/secret-keystone-rgw.yaml

@@ -0,0 +1,32 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_keystone_rgw }}
+{{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
+{{- range $key1, $userClass := tuple "user" }}
+{{- $secretName := index $envAll.Values.secrets.identity "user_rgw" }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+{{- tuple $userClass "internal" $envAll | include "helm-toolkit.snippets.keystone_secret_openrc" | indent 2 -}}
+{{- end }}
+{{- end }}
+{{- end }}

ceph/templates/secret-keystone.yaml

@@ -0,0 +1,32 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_keystone }}
+{{- $envAll := . }}
+{{- if .Values.deployment.rgw_keystone_user_and_endpoints }}
+{{- range $key1, $userClass := tuple "admin" "user" }}
+{{- $secretName := index $envAll.Values.secrets.identity $userClass }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+{{- tuple $userClass "internal" $envAll | include "helm-toolkit.snippets.keystone_secret_openrc" | indent 2 -}}
+{{- end }}
+{{- end }}
+{{- end }}

ceph/templates/service-mon.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.deployment }}
+{{- if .Values.manifests.service_mon }}
 {{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
 ---
 kind: Service
 apiVersion: v1
@@ -38,3 +39,4 @@ spec:
 {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
   clusterIP: None
 {{- end }}
+{{- end }}

ceph/templates/service-rgw.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.deployment }}
+{{- if .Values.manifests.service_rgw }}
 {{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
 {{- if .Values.ceph.enabled.rgw }}
 ---
 apiVersion: v1
@@ -24,10 +25,11 @@ metadata:
   name: ceph-rgw
 spec:
   ports:
-  - port: {{ .Values.network.port.rgw_ingress }}
+  - port: {{ .Values.network.port.rgw }}
     protocol: TCP
-    targetPort: {{ .Values.network.port.rgw_target }}
+    targetPort: {{ .Values.network.port.rgw }}
   selector:
 {{ tuple $envAll "ceph" "rgw" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 {{- end }}
 {{- end }}
+{{- end }}

ceph/templates/storageclass.yaml

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests_enabled.deployment }}
+{{- if .Values.manifests.storageclass }}
 {{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
@@ -31,3 +32,4 @@ parameters:
     userId: {{ .Values.storageclass.user_id }}
     userSecretName: {{ .Values.storageclass.user_secret_name }}
 {{- end }}
+{{- end }}

ceph/values.yaml

@@ -12,22 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-manifests_enabled:
+deployment:
+  ceph: true
   storage_secrets: true
   client_secrets: true
-  deployment: true
   rbd_provisioner: true
-
-replicas:
-  rgw: 1
-  mon_check: 1
-  rbd_provisioner: 2
-
-service:
-  mon:
-    name: ceph-mon
+  rgw_keystone_user_and_endpoints: false
 
 images:
+  ks_user: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
+  ks_service: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
+  ks_endpoints: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
   bootstrap: quay.io/attcomdev/ceph-daemon:tag-build-master-jewel-ubuntu-16.04
   dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0
   daemon: quay.io/attcomdev/ceph-daemon:tag-build-master-jewel-ubuntu-16.04
@@ -36,6 +31,9 @@ images:
   pull_policy: "IfNotPresent"
 
 labels:
+  jobs:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
   mon:
     node_selector_key: ceph-mon
     node_selector_value: enabled
@@ -48,12 +46,13 @@ labels:
   rgw:
     node_selector_key: ceph-rgw
     node_selector_value: enabled
-  bootstrap:
-    node_selector_key: ceph-mon
-    node_selector_value: enabled
 
 pod:
   dns_policy: "ClusterFirstWithHostNet"
+  replicas:
+    rgw: 1
+    mon_check: 1
+    rbd_provisioner: 2
   affinity:
       anti:
         type:
@@ -119,6 +118,27 @@ pod:
         requests:
           memory: "128Mi"
           cpu: "500m"
+      ks_endpoints:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+      ks_service:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+      ks_user:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
 
 secrets:
   keyrings:
@@ -127,18 +147,25 @@ secrets:
     osd: ceph-bootstrap-osd-keyring
     rgw: ceph-bootstrap-rgw-keyring
     admin: ceph-client-admin-keyring
+  identity:
+    admin: ceph-keystone-admin
+    user: ceph-keystone-user
+    user_rgw: ceph-keystone-user-rgw
 
 network:
-#  public: "192.168.0.0/16"
-#  cluster: "192.168.0.0/16"
-  public: "10.1.10.0/24"
-  cluster: "10.1.10.0/24"
+  public: 192.168.0.0/16
+  cluster: 192.168.0.0/16
   port:
     mon: 6789
-    rgw_ingress: 80
-    rgw_target: 8088
+    rgw: 8088
 
 conf:
+  rgw_ks:
+    config:
+      rgw_keystone_api_version: 3
+      rgw_keystone_accepted_roles: "admin, _member_"
+      rgw_keystone_implicit_tenants: true
+      rgw_s3_auth_use_keystone: true
   ceph:
     override:
     append:
@@ -233,20 +260,31 @@ dependencies:
     jobs:
     - service: ceph_mon
       endpoint: internal
+  ks_user:
+    services:
+    - service: identity
+      endpoint: internal
+  ks_service:
+    services:
+    - service: identity
+      endpoint: internal
+  ks_endpoints:
+    jobs:
+    - ceph-ks-service
+    services:
+    - service: identity
+      endpoint: internal
 
 ceph:
+  rgw_keystone_auth: false
   enabled:
     mds: true
-    rgw: false
+    rgw: true
   storage:
     osd_directory: /var/lib/openstack-helm/ceph/osd
     var_directory: /var/lib/openstack-helm/ceph/ceph
     mon_directory: /var/lib/openstack-helm/ceph/mon
 
-# rgw is optionally disabled
-rgw:
-  enabled: false
-
 bootstrap:
   enabled: false
   script: |
@@ -256,7 +294,6 @@ bootstrap:
     }
     ensure_pool volumes 8
     ensure_pool images 8
-    ensure_pool vms 8
 
 # if you change provision_storage_class to false
 # it is presumed you manage your own storage
@@ -275,7 +312,55 @@ storageclass:
 
 endpoints:
   cluster_domain_suffix: cluster.local
+  identity:
+    name: keystone
+    namespace: null
+    auth:
+      admin:
+        region_name: RegionOne
+        username: admin
+        password: password
+        project_name: admin
+        user_domain_name: default
+        project_domain_name: default
+      user:
+        role: admin
+        region_name: RegionOne
+        username: swift
+        password: password
+        project_name: service
+        user_domain_name: default
+        project_domain_name: default
+    hosts:
+      default: keystone-api
+      public: keystone
+    host_fqdn_override:
+      default: null
+    path:
+      default: /v3
+    scheme:
+      default: http
+    port:
+      admin:
+        default: 35357
+      api:
+        default: 80
+  object_store:
+    name: swift
+    namespace: null
+    hosts:
+      default: ceph-rgw
+    host_fqdn_override:
+      default: null
+    path:
+      default: /swift/v1
+    scheme:
+      default: http
+    port:
+      api:
+        default: 8088
   ceph_mon:
+    namespace: null
     hosts:
       default: ceph-mon
     host_fqdn_override:
@@ -283,3 +368,30 @@ endpoints:
     port:
       mon:
         default: 6789
+
+
+manifests:
+  configmap_bin_clients: true
+  configmap_bin_ks: true
+  configmap_bin: true
+  configmap_etc: true
+  configmap_templates: true
+  daemonset_mon: true
+  daemonset_osd: true
+  deployment_mds: true
+  deployment_moncheck: true
+  deployment_rbd_provisioner: true
+  deployment_rgw: true
+  job_bootstrap: true
+  job_keyring: true
+  job_ks_endpoints: true
+  job_ks_service: true
+  job_ks_user: true
+  job_namespace_client_key_cleaner: true
+  job_namespace_client_key: true
+  job_storage_admin_keys: true
+  secret_keystone_rgw: true
+  secret_keystone: true
+  service_mon: true
+  service_rgw: true
+  storageclass: true

doc/source/install/multinode.rst

@@ -233,6 +233,7 @@ Nodes are labeled according to their Openstack roles:
 * **Ceph MON Nodes:** ``ceph-mon``
 * **Ceph OSD Nodes:** ``ceph-osd``
 * **Ceph MDS Nodes:** ``ceph-mds``
+* **Ceph RGW Nodes:** ``ceph-rgw``
 * **Control Plane:** ``openstack-control-plane``
 * **Compute Nodes:** ``openvswitch``, ``openstack-compute-node``
 
@@ -242,6 +243,7 @@ Nodes are labeled according to their Openstack roles:
     kubectl label nodes ceph-mon=enabled --all
     kubectl label nodes ceph-osd=enabled --all
     kubectl label nodes ceph-mds=enabled --all
+    kubectl label nodes ceph-rgw=enabled --all
     kubectl label nodes openvswitch=enabled --all
     kubectl label nodes openstack-compute-node=enabled --all
 
@@ -267,8 +269,8 @@ by issuing the following commands:
 
 ::
 
-    export osd_cluster_network=10.26.0.0/26
-    export osd_public_network=10.26.0.0/26
+    export OSD_CLUSTER_NETWORK=10.26.0.0/26
+    export OSD_PUBLIC_NETWORK=10.26.0.0/26
 
 Helm Preparation
 ----------------
@@ -325,19 +327,42 @@ Ceph Installation and Verification
 ----------------------------------
 
 Install the first service, which is Ceph. If all instructions have been
-followed as mentioned above, this installation should go smoothly. Use
-the following command to install Ceph in the ``openstack-helm`` project folder:
+followed as mentioned above, this installation should go smoothly. It is at this
+point you can also decide to enable keystone authentication for the RadosGW if
+you wish to use ceph for tenant facing object storage. If you do not wish to do
+this then you should set the value of ``CEPH_RGW_KEYSTONE_ENABLED=false`` before
+running the following commands in the ``openstack-helm`` project folder:
 
 ::
 
-  helm install --namespace=ceph ./ceph --name=ceph \
-    --set manifests_enabled.client_secrets=false \
-    --set network.public=$osd_public_network \
-    --set network.cluster=$osd_cluster_network \
+  : ${CEPH_RGW_KEYSTONE_ENABLED:="true"}
+  helm install --namespace=ceph ${WORK_DIR}/ceph --name=ceph \
+    --set endpoints.identity.namespace=openstack \
+    --set endpoints.object_store.namespace=ceph \
+    --set endpoints.ceph_mon.namespace=ceph \
+    --set ceph.rgw_keystone_auth=${CEPH_RGW_KEYSTONE_ENABLED} \
+    --set network.public=${OSD_PUBLIC_NETWORK} \
+    --set network.cluster=${OSD_CLUSTER_NETWORK} \
+    --set deployment.storage_secrets=true \
+    --set deployment.ceph=true \
+    --set deployment.rbd_provisioner=true \
+    --set deployment.client_secrets=false \
+    --set deployment.rgw_keystone_user_and_endpoints=false \
     --set bootstrap.enabled=true
 
-You may want to validate that Ceph is deployed successfully. For more
-information on this, please see the section entitled `Ceph
+After Ceph has deployed and all the pods are running, you can check the health
+of your cluster by running:
+
+::
+
+  MON_POD=$(kubectl get pods \
+    --namespace=ceph \
+    --selector="application=ceph" \
+    --selector="component=mon" \
+    --no-headers | awk '{ print $1; exit }')
+  kubectl exec -n ceph ${MON_POD} -- ceph -s
+
+For more information on this, please see the section entitled `Ceph
 Troubleshooting <../../operator/troubleshooting/persistent-storage.html>`__.
 
 Activating Control-Plane Namespace for Ceph
@@ -351,12 +376,19 @@ deploy the client keyring and ``ceph.conf`` to the ``openstack`` namespace:
 
 ::
 
-    helm install --namespace=openstack ./ceph --name=ceph-openstack-config \
-      --set manifests_enabled.storage_secrets=false \
-      --set manifests_enabled.deployment=false \
-      --set ceph.namespace=ceph \
-      --set network.public=$osd_public_network \
-      --set network.cluster=$osd_cluster_network
+    : ${CEPH_RGW_KEYSTONE_ENABLED:="true"}
+    helm install --namespace=openstack ${WORK_DIR}/ceph --name=ceph-openstack-config \
+      --set endpoints.identity.namespace=openstack \
+      --set endpoints.object_store.namespace=ceph \
+      --set endpoints.ceph_mon.namespace=ceph \
+      --set ceph.rgw_keystone_auth=${CEPH_RGW_KEYSTONE_ENABLED} \
+      --set network.public=${OSD_PUBLIC_NETWORK} \
+      --set network.cluster=${OSD_CLUSTER_NETWORK} \
+      --set deployment.storage_secrets=false \
+      --set deployment.ceph=false \
+      --set deployment.rbd_provisioner=false \
+      --set deployment.client_secrets=true \
+      --set deployment.rgw_keystone_user_and_endpoints=false
 
 MariaDB Installation and Verification
 -------------------------------------
@@ -390,6 +422,26 @@ Now you can easily install the other services simply by going in order:
     helm install --namespace=openstack --name=keystone ./keystone \
       --set pod.replicas.api=2
 
+**Install RadosGW Object Storage:**
+
+If you elected to install Ceph with Keystone support for the RadosGW you can
+now create endpoints in the Keystone service catalog:
+
+::
+
+    helm install --namespace=openstack ${WORK_DIR}/ceph --name=radosgw-openstack \
+      --set endpoints.identity.namespace=openstack \
+      --set endpoints.object_store.namespace=ceph \
+      --set endpoints.ceph_mon.namespace=ceph \
+      --set ceph.rgw_keystone_auth=${CEPH_RGW_KEYSTONE_ENABLED} \
+      --set network.public=${OSD_PUBLIC_NETWORK} \
+      --set network.cluster=${OSD_CLUSTER_NETWORK} \
+      --set deployment.storage_secrets=false \
+      --set deployment.ceph=false \
+      --set deployment.rbd_provisioner=false \
+      --set deployment.client_secrets=false \
+      --set deployment.rgw_keystone_user_and_endpoints=true
+
 **Install Horizon:**
 
 ::

helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl

@@ -28,7 +28,8 @@ limitations under the License.
 {{- $userclass := index . 2 -}}
 {{- $port := index . 3 -}}
 {{- $context := index . 4 -}}
-{{- $endpointMap := index $context.Values.endpoints $type }}
+{{- $typeYamlSafe := $type | replace "-" "_" }}
+{{- $endpointMap := index $context.Values.endpoints $typeYamlSafe }}
 {{- $userMap := index $endpointMap.auth $userclass }}
 {{- $clusterSuffix := printf "%s.%s" "svc" $context.Values.endpoints.cluster_domain_suffix }}
 {{- with $endpointMap -}}

helm-toolkit/templates/endpoints/_endpoint_port_lookup.tpl

@@ -27,7 +27,8 @@ limitations under the License.
 {{- $endpoint := index . 1 -}}
 {{- $port := index . 2 -}}
 {{- $context := index . 3 -}}
-{{- $endpointMap := index $context.Values.endpoints $type }}
+{{- $typeYamlSafe := $type | replace "-" "_" }}
+{{- $endpointMap := index $context.Values.endpoints $typeYamlSafe }}
 {{- with $endpointMap -}}
 {{- $endpointPortMAP := index .port $port }}
 {{- $endpointPort := index $endpointPortMAP $endpoint | default (index $endpointPortMAP "default") }}

helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl

@@ -27,8 +27,9 @@ limitations under the License.
 {{- $endpoint := index . 1 -}}
 {{- $port := index . 2 -}}
 {{- $context := index . 3 -}}
+{{- $typeYamlSafe := $type | replace "-" "_" }}
 {{- $clusterSuffix := printf "%s.%s" "svc" $context.Values.endpoints.cluster_domain_suffix }}
-{{- $endpointMap := index $context.Values.endpoints $type }}
+{{- $endpointMap := index $context.Values.endpoints $typeYamlSafe }}
 {{- with $endpointMap -}}
 {{- $namespace := .namespace | default $context.Release.Namespace }}
 {{- $endpointScheme := .scheme }}

helm-toolkit/templates/endpoints/_hostname_fqdn_endpoint_lookup.tpl

@@ -24,8 +24,9 @@ limitations under the License.
 {{- $type := index . 0 -}}
 {{- $endpoint := index . 1 -}}
 {{- $context := index . 2 -}}
+{{- $typeYamlSafe := $type | replace "-" "_" }}
 {{- $clusterSuffix := printf "%s.%s" "svc" $context.Values.endpoints.cluster_domain_suffix }}
-{{- $endpointMap := index $context.Values.endpoints $type }}
+{{- $endpointMap := index $context.Values.endpoints $typeYamlSafe }}
 {{- with $endpointMap -}}
 {{- $namespace := .namespace | default $context.Release.Namespace }}
 {{- $endpointScheme := .scheme }}

helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_lookup.tpl

@@ -24,7 +24,8 @@ limitations under the License.
 {{- $type := index . 0 -}}
 {{- $endpoint := index . 1 -}}
 {{- $context := index . 2 -}}
-{{- $endpointMap := index $context.Values.endpoints $type }}
+{{- $typeYamlSafe := $type | replace "-" "_" }}
+{{- $endpointMap := index $context.Values.endpoints $typeYamlSafe }}
 {{- with $endpointMap -}}
 {{- $namespace := .namespace | default $context.Release.Namespace }}
 {{- $endpointScheme := .scheme }}

helm-toolkit/templates/endpoints/_hostname_short_endpoint_lookup.tpl

@@ -24,7 +24,8 @@ limitations under the License.
 {{- $type := index . 0 -}}
 {{- $endpoint := index . 1 -}}
 {{- $context := index . 2 -}}
-{{- $endpointMap := index $context.Values.endpoints $type }}
+{{- $typeYamlSafe := $type | replace "-" "_" }}
+{{- $endpointMap := index $context.Values.endpoints $typeYamlSafe }}
 {{- with $endpointMap -}}
 {{- $endpointScheme := .scheme }}
 {{- $endpointHost := index .hosts $endpoint | default .hosts.default}}

helm-toolkit/templates/endpoints/_keystone_endpoint_name_lookup.tpl

@@ -22,7 +22,8 @@ limitations under the License.
 {{- define "helm-toolkit.endpoints.keystone_endpoint_name_lookup" -}}
 {{- $type := index . 0 -}}
 {{- $context := index . 1 -}}
-{{- $endpointMap := index $context.Values.endpoints $type }}
+{{- $typeYamlSafe := $type | replace "-" "_" }}
+{{- $endpointMap := index $context.Values.endpoints $typeYamlSafe }}
 {{- $endpointName := index $endpointMap "name" }}
 {{- $endpointName | quote -}}
 {{- end -}}

helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl

@@ -24,7 +24,8 @@ limitations under the License.
 {{- $endpoint := index . 1 -}}
 {{- $port := index . 2 -}}
 {{- $context := index . 3 -}}
-{{- $endpointMap := index $context.Values.endpoints $type }}
+{{- $typeYamlSafe := $type | replace "-" "_" }}
+{{- $endpointMap := index $context.Values.endpoints $typeYamlSafe }}
 {{- with $endpointMap -}}
 {{- $endpointPath := index .path $endpoint | default .path.default | default "/" }}
 {{- printf "%s" $endpointPath -}}

helm-toolkit/templates/endpoints/_keystone_endpoint_uri_lookup.tpl

@@ -24,8 +24,9 @@ limitations under the License.
 {{- $endpoint := index . 1 -}}
 {{- $port := index . 2 -}}
 {{- $context := index . 3 -}}
+{{- $typeYamlSafe := $type | replace "-" "_" }}
 {{- $clusterSuffix := printf "%s.%s" "svc" $context.Values.endpoints.cluster_domain_suffix }}
-{{- $endpointMap := index $context.Values.endpoints $type }}
+{{- $endpointMap := index $context.Values.endpoints $typeYamlSafe }}
 {{- with $endpointMap -}}
 {{- $namespace := $endpointMap.namespace | default $context.Release.Namespace }}
 {{- $endpointScheme :=  index .scheme $endpoint | default .scheme.default }}

magnum/values.yaml

@@ -201,7 +201,7 @@ endpoints:
         default: 35357
       api:
         default: 80
-  container-infra:
+  container_infra:
     name: magnum
     hosts:
       default: magnum-api

tools/deployment/armada/openstack-master-aio.yaml

@@ -68,13 +68,26 @@ data:
             - component: osd-keyring-generator
             - release_group: osh-ceph
   values:
-    manifests_enabled:
-      client_secrets: False
-    bootstrap:
-      enabled: True
+    endpoints:
+      identity:
+        namespace: openstack
+      object_store:
+        namespace: ceph
+      ceph_mon:
+        namespace: ceph
+    ceph:
+      rgw_keystone_auth: true
     network:
       public: 192.168.0.0/16
       cluster: 192.168.0.0/16
+    deployment:
+      storage_secrets: true
+      ceph: true
+      rbd_provisioner: true
+      client_secrets: false
+      rgw_keystone_user_and_endpoints: false
+    bootstrap:
+      enabled: true
     conf:
       ceph:
         config:
@@ -112,14 +125,31 @@ data:
             - component: namespace-client-key-generator
             - release_group: osh-ceph
   values:
+    endpoints:
+      identity:
+        namespace: openstack
+      object_store:
+        namespace: ceph
+      ceph_mon:
+        namespace: ceph
     ceph:
-      namespace: ceph
-    manifests_enabled:
-      deployment: False
-      storage_secrets: False
+      rgw_keystone_auth: true
     network:
       public: 192.168.0.0/16
       cluster: 192.168.0.0/16
+    deployment:
+      storage_secrets: false
+      ceph: false
+      rbd_provisioner: false
+      client_secrets: true
+      rgw_keystone_user_and_endpoints: false
+    conf:
+      ceph:
+        config:
+          global:
+            osd_pool_default_size: 1
+          osd:
+            osd_crush_chooseleaf_type: 0
   source:
     type: local
     location: /opt/openstack-helm/charts
@@ -337,6 +367,54 @@ data:
     - helm-toolkit
 ---
 schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ceph-radosgw-keystone
+data:
+  chart_name: ceph-radosgw-keystone
+  release: ceph-radosgw-keystone
+  namespace: openstack
+  timeout: 3600
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+  values:
+    endpoints:
+      identity:
+        namespace: openstack
+      object_store:
+        namespace: ceph
+      endpoints:
+        ceph_mon:
+          namespace: ceph
+    ceph:
+      rgw_keystone_auth: true
+    network:
+      public: 192.168.0.0/16
+      cluster: 192.168.0.0/16
+    deployment:
+      storage_secrets: false
+      ceph: false
+      rbd_provisioner: false
+      client_secrets: false
+      rgw_keystone_user_and_endpoints: true
+    conf:
+      ceph:
+        config:
+          global:
+            osd_pool_default_size: 1
+          osd:
+            osd_crush_chooseleaf_type: 0
+  source:
+    type: local
+    location: /opt/openstack-helm/charts
+    subpath: ceph
+    reference: master
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
 metadata:
   schema: metadata/Document/v1
   name: horizon
@@ -760,6 +838,7 @@ data:
   sequenced: False
   chart_group:
     - keystone
+    - ceph-radosgw-keystone
     - horizon
     - glance
     - neutron

tools/deployment/armada/openstack-master.yaml

@@ -68,13 +68,26 @@ data:
             - component: osd-keyring-generator
             - release_group: osh-ceph
   values:
-    manifests_enabled:
-      client_secrets: False
-    bootstrap:
-      enabled: True
+    endpoints:
+      identity:
+        namespace: openstack
+      object_store:
+        namespace: ceph
+      ceph_mon:
+        namespace: ceph
+    ceph:
+      rgw_keystone_auth: true
     network:
       public: 192.168.0.0/16
       cluster: 192.168.0.0/16
+    deployment:
+      storage_secrets: true
+      ceph: true
+      rbd_provisioner: true
+      client_secrets: false
+      rgw_keystone_user_and_endpoints: false
+    bootstrap:
+      enabled: true
   source:
     type: local
     location: /opt/openstack-helm/charts
@@ -105,14 +118,24 @@ data:
             - component: namespace-client-key-generator
             - release_group: osh-ceph
   values:
+    endpoints:
+      identity:
+        namespace: openstack
+      object_store:
+        namespace: ceph
+      ceph_mon:
+        namespace: ceph
     ceph:
-      namespace: ceph
-    manifests_enabled:
-      deployment: False
-      storage_secrets: False
+      rgw_keystone_auth: true
     network:
       public: 192.168.0.0/16
       cluster: 192.168.0.0/16
+    deployment:
+      storage_secrets: false
+      ceph: false
+      rbd_provisioner: false
+      client_secrets: true
+      rgw_keystone_user_and_endpoints: false
   source:
     type: local
     location: /opt/openstack-helm/charts
@@ -330,6 +353,47 @@ data:
     - helm-toolkit
 ---
 schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ceph-radosgw-keystone
+data:
+  chart_name: ceph-radosgw-keystone
+  release: ceph-radosgw-keystone
+  namespace: openstack
+  timeout: 3600
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+  values:
+    endpoints:
+      identity:
+        namespace: openstack
+      object_store:
+        namespace: ceph
+      endpoints:
+        ceph_mon:
+          namespace: ceph
+    ceph:
+      rgw_keystone_auth: true
+    network:
+      public: 192.168.0.0/16
+      cluster: 192.168.0.0/16
+    deployment:
+      storage_secrets: false
+      ceph: false
+      rbd_provisioner: false
+      client_secrets: false
+      rgw_keystone_user_and_endpoints: true
+  source:
+    type: local
+    location: /opt/openstack-helm/charts
+    subpath: ceph
+    reference: master
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
 metadata:
   schema: metadata/Document/v1
   name: horizon
@@ -753,6 +817,7 @@ data:
   sequenced: False
   chart_group:
     - keystone
+    - ceph-radosgw-keystone
     - horizon
     - glance
     - neutron

tools/gate/README.rst

@@ -70,3 +70,10 @@ It is also possible to customise the CNI used in the deployment:
 
     export KUBE_CNI=calico # or "canal" "weave" "flannel"
     export CNI_POD_CIDR=192.168.0.0/16
+
+If you wish to deploy using Armada then you just need to export the following
+variable:
+
+.. code:: bash
+
+    export INTEGRATION_TYPE=armada

tools/gate/launch-osh/armada.sh

@@ -39,6 +39,7 @@ sudo docker run -d \
 
 sudo docker exec armada armada tiller --status
 sudo docker exec armada armada apply ${ARMADA_MANIFEST}
+sudo docker rm -f armada
 kube_wait_for_pods ceph 600
 kube_wait_for_pods openstack 1200
 

tools/gate/launch-osh/basic.sh

@@ -29,37 +29,59 @@ if [ "x$PVC_BACKEND" == "xceph" ]; then
     SUBNET_RANGE=$(find_subnet_range)
   fi
 
-  export osd_cluster_network=${SUBNET_RANGE}
-  export osd_public_network=${SUBNET_RANGE}
-
   if [ "x$INTEGRATION" == "xaio" ]; then
     helm install --namespace=ceph ${WORK_DIR}/ceph --name=ceph \
-      --set manifests_enabled.client_secrets=false \
-      --set network.public=$osd_public_network \
-      --set network.cluster=$osd_cluster_network \
+      --set endpoints.identity.namespace=openstack \
+      --set endpoints.object_store.namespace=ceph \
+      --set endpoints.ceph_mon.namespace=ceph \
+      --set ceph.rgw_keystone_auth=${CEPH_RGW_KEYSTONE_ENABLED} \
+      --set network.public=${SUBNET_RANGE} \
+      --set network.cluster=${SUBNET_RANGE} \
+      --set deployment.storage_secrets=true \
+      --set deployment.ceph=true \
+      --set deployment.rbd_provisioner=true \
+      --set deployment.client_secrets=false \
+      --set deployment.rgw_keystone_user_and_endpoints=false \
       --set bootstrap.enabled=true \
       --values=${WORK_DIR}/tools/overrides/mvp/ceph.yaml
   else
     helm install --namespace=ceph ${WORK_DIR}/ceph --name=ceph \
-      --set manifests_enabled.client_secrets=false \
-      --set network.public=$osd_public_network \
-      --set network.cluster=$osd_cluster_network \
+      --set endpoints.identity.namespace=openstack \
+      --set endpoints.object_store.namespace=ceph \
+      --set endpoints.ceph_mon.namespace=ceph \
+      --set ceph.rgw_keystone_auth=${CEPH_RGW_KEYSTONE_ENABLED} \
+      --set network.public=${SUBNET_RANGE} \
+      --set network.cluster=${SUBNET_RANGE} \
+      --set deployment.storage_secrets=true \
+      --set deployment.ceph=true \
+      --set deployment.rbd_provisioner=true \
+      --set deployment.client_secrets=false \
+      --set deployment.rgw_keystone_user_and_endpoints=false \
       --set bootstrap.enabled=true
   fi
 
   kube_wait_for_pods ceph ${SERVICE_LAUNCH_TIMEOUT}
 
-  MON_POD=$(kubectl get pods -l application=ceph -l component=mon -n ceph --no-headers | awk '{ print $1; exit }')
+  MON_POD=$(kubectl get pods \
+    --namespace=ceph \
+    --selector="application=ceph" \
+    --selector="component=mon" \
+    --no-headers | awk '{ print $1; exit }')
 
   kubectl exec -n ceph ${MON_POD} -- ceph -s
 
   helm install --namespace=openstack ${WORK_DIR}/ceph --name=ceph-openstack-config \
-    --set manifests_enabled.storage_secrets=false \
-    --set manifests_enabled.deployment=false \
-    --set manifests_enabled.rbd_provisioner=false \
-    --set ceph.namespace=ceph \
-    --set network.public=$osd_public_network \
-    --set network.cluster=$osd_cluster_network
+    --set endpoints.identity.namespace=openstack \
+    --set endpoints.object_store.namespace=ceph \
+    --set endpoints.ceph_mon.namespace=ceph \
+    --set ceph.rgw_keystone_auth=${CEPH_RGW_KEYSTONE_ENABLED} \
+    --set network.public=${SUBNET_RANGE} \
+    --set network.cluster=${SUBNET_RANGE} \
+    --set deployment.storage_secrets=false \
+    --set deployment.ceph=false \
+    --set deployment.rbd_provisioner=false \
+    --set deployment.client_secrets=true \
+    --set deployment.rgw_keystone_user_and_endpoints=false
 
   kube_wait_for_pods openstack ${SERVICE_LAUNCH_TIMEOUT}
 fi
@@ -69,32 +91,53 @@ if [ "x$INTEGRATION" == "xmulti" ]; then
   helm install --namespace=openstack ${WORK_DIR}/mariadb --name=mariadb
 else
   helm install --namespace=openstack ${WORK_DIR}/mariadb --name=mariadb \
-      --set=pod.replicas.server=1
+    --set=pod.replicas.server=1
 fi
 helm install --namespace=openstack ${WORK_DIR}/memcached --name=memcached
+kube_wait_for_pods openstack ${SERVICE_LAUNCH_TIMEOUT}
+
+helm install --namespace=openstack ${WORK_DIR}/keystone --name=keystone
+kube_wait_for_pods openstack ${SERVICE_LAUNCH_TIMEOUT}
+
+if [ "x$OPENSTACK_OBJECT_STORAGE" == "xradosgw" ]; then
+  helm install --namespace=openstack ${WORK_DIR}/ceph --name=radosgw-openstack \
+    --set endpoints.identity.namespace=openstack \
+    --set endpoints.object_store.namespace=ceph \
+    --set endpoints.ceph_mon.namespace=ceph \
+    --set ceph.rgw_keystone_auth=${CEPH_RGW_KEYSTONE_ENABLED} \
+    --set network.public=${SUBNET_RANGE} \
+    --set network.cluster=${SUBNET_RANGE} \
+    --set deployment.storage_secrets=false \
+    --set deployment.ceph=false \
+    --set deployment.rbd_provisioner=false \
+    --set deployment.client_secrets=false \
+    --set deployment.rgw_keystone_user_and_endpoints=true
+  kube_wait_for_pods openstack ${SERVICE_LAUNCH_TIMEOUT}
+fi
+
 helm install --namespace=openstack ${WORK_DIR}/etcd --name=etcd-rabbitmq
 helm install --namespace=openstack ${WORK_DIR}/rabbitmq --name=rabbitmq
 helm install --namespace=openstack ${WORK_DIR}/libvirt --name=libvirt
 helm install --namespace=openstack ${WORK_DIR}/openvswitch --name=openvswitch
 kube_wait_for_pods openstack ${SERVICE_LAUNCH_TIMEOUT}
-helm install --namespace=openstack ${WORK_DIR}/keystone --name=keystone
+
 if [ "x$PVC_BACKEND" == "xceph" ]; then
   helm install --namespace=openstack ${WORK_DIR}/glance --name=glance
 else
   helm install --namespace=openstack ${WORK_DIR}/glance --name=glance \
-      --values=${WORK_DIR}/tools/overrides/mvp/glance.yaml
+    --values=${WORK_DIR}/tools/overrides/mvp/glance.yaml
 fi
 kube_wait_for_pods openstack ${SERVICE_LAUNCH_TIMEOUT}
 if [ "x$PVC_BACKEND" == "xceph" ]; then
   helm install --namespace=openstack ${WORK_DIR}/nova --name=nova \
-      --set=conf.nova.libvirt.nova.conf.virt_type=qemu
+    --set=conf.nova.libvirt.nova.conf.virt_type=qemu
 else
   helm install --namespace=openstack ${WORK_DIR}/nova --name=nova \
-      --values=${WORK_DIR}/tools/overrides/mvp/nova.yaml \
-      --set=conf.nova.libvirt.nova.conf.virt_type=qemu
+    --values=${WORK_DIR}/tools/overrides/mvp/nova.yaml \
+    --set=conf.nova.libvirt.nova.conf.virt_type=qemu
 fi
 helm install --namespace=openstack ${WORK_DIR}/neutron --name=neutron \
-    --values=${WORK_DIR}/tools/overrides/mvp/neutron.yaml
+  --values=${WORK_DIR}/tools/overrides/mvp/neutron.yaml
 kube_wait_for_pods openstack ${SERVICE_LAUNCH_TIMEOUT}
 
 helm install --namespace=openstack ${WORK_DIR}/heat --name=heat
@@ -105,7 +148,7 @@ if [ "x$INTEGRATION" == "xmulti" ]; then
     helm install --namespace=openstack ${WORK_DIR}/cinder --name=cinder
   else
     helm install --namespace=openstack ${WORK_DIR}/cinder --name=cinder \
-        --values=${WORK_DIR}/tools/overrides/mvp/cinder.yaml
+      --values=${WORK_DIR}/tools/overrides/mvp/cinder.yaml
   fi
   helm install --namespace=openstack ${WORK_DIR}/horizon --name=horizon
   kube_wait_for_pods openstack ${SERVICE_LAUNCH_TIMEOUT}

tools/gate/launch-osh/common.sh

@@ -31,6 +31,7 @@ if [ "x$PVC_BACKEND" == "xceph" ]; then
   kubectl label nodes ceph-mon=enabled --all
   kubectl label nodes ceph-osd=enabled --all
   kubectl label nodes ceph-mds=enabled --all
+  kubectl label nodes ceph-rgw=enabled --all
 fi
 
 helm install --namespace=openstack ${WORK_DIR}/dns-helper --name=dns-helper

tools/gate/vars.sh

@@ -37,6 +37,10 @@ export KUBE_CNI=${KUBE_CNI:="calico"}
 # Set PVC Backend
 export PVC_BACKEND=${PVC_BACKEND:-"ceph"}
 
+# Set Object Storage options
+export CEPH_RGW_KEYSTONE_ENABLED=${CEPH_RGW_KEYSTONE_ENABLED:-"true"}
+export OPENSTACK_OBJECT_STORAGE=${OPENSTACK_OBJECT_STORAGE:-"radosgw"}
+
 # Set Upstream DNS
 export UPSTREAM_DNS=${UPSTREAM_DNS:-"8.8.8.8"}