Ceph: use ingress controller for RGW

This PS updates the ceph chart to use the ingress controller for
the radosgw.

Change-Id: Ie0ff4d9dd91d8228170e0ed4f2793599132077a7
Signed-off-by: Pete Birley <pete@port.direct>

ceph-client/templates/configmap-etc-client.yaml

@@ -34,6 +34,10 @@ limitations under the License.
 {{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
 {{- end -}}
 
+{{- if empty .Values.conf.rgw_ks.config.rgw_swift_url -}}
+{{- $_ := tuple "object_store" "public" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.object_store.path.default | set .Values.conf.rgw_ks.config "rgw_swift_url" -}}
+{{- end -}}
+
 ---
 apiVersion: v1
 kind: ConfigMap

ceph-client/templates/deployment-rgw.yaml

@@ -79,7 +79,7 @@ spec:
             - name: KEYSTONE_URL
               value: {{ tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.identity.path.default | quote }}
             - name: RGW_CIVETWEB_PORT
-              value: "{{ .Values.network.port.rgw }}"
+              value: "{{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}"
           command:
             - /tmp/rgw-init-keystone.sh
           volumeMounts:
@@ -102,21 +102,21 @@ spec:
             - name: CLUSTER
               value: "ceph"
             - name: RGW_CIVETWEB_PORT
-              value: "{{ .Values.network.port.rgw }}"
+              value: "{{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}"
           command:
             - /tmp/rgw-start.sh
           ports:
-            - containerPort: {{ .Values.network.port.rgw }}
+            - containerPort: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           livenessProbe:
               httpGet:
                 path: /
-                port: {{ .Values.network.port.rgw }}
+                port: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
               initialDelaySeconds: 120
               timeoutSeconds: 5
           readinessProbe:
               httpGet:
                 path: /
-                port: {{ .Values.network.port.rgw }}
+                port: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
               timeoutSeconds: 5
           volumeMounts:
             - name: ceph-client-bin

ceph-client/templates/ingress-rgw.yaml

@@ -0,0 +1,20 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.ingress_rgw ( and .Values.deployment.ceph (and .Values.network.api.ingress.public .Values.conf.features.rgw ) ) }}
+{{- $ingressOpts := dict "envAll" . "backendServiceType" "object_store" "backendPort" "ceph-rgw" -}}
+{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
+{{- end }}

ceph-client/templates/service-ingress-rgw.yaml

@@ -0,0 +1,20 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.service_ingress_rgw ( and .Values.deployment.ceph (and .Values.network.api.ingress.public .Values.conf.features.rgw ) ) }}
+{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "object_store" -}}
+{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }}
+{{- end }}

ceph-client/templates/service-rgw.yaml

@@ -23,9 +23,19 @@ metadata:
   name: ceph-rgw
 spec:
   ports:
-  - port: {{ .Values.network.port.rgw }}
+  - name: ceph-rgw
+    port: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
     protocol: TCP
-    targetPort: {{ .Values.network.port.rgw }}
+    targetPort: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  {{ if .Values.network.api.node_port.enabled }}
+    nodePort: {{ .Values.network.api.node_port.port }}
+  {{ end }}
   selector:
 {{ tuple $envAll "ceph" "rgw" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+  {{ if .Values.network.api.node_port.enabled }}
+  type: NodePort
+  {{ if .Values.network.api.external_policy_local }}
+  externalTrafficPolicy: Local
+  {{ end }}
+  {{ end }}
 {{- end }}

ceph-client/values.yaml

@@ -162,11 +162,24 @@ secrets:
     user_rgw: ceph-keystone-user-rgw
 
 network:
+  api:
+    ingress:
+      public: true
+      classes:
+        namespace: "nginx"
+        cluster: "nginx-cluster"
+      annotations:
+        nginx.ingress.kubernetes.io/rewrite-target: /
+        nginx.ingress.kubernetes.io/proxy-body-size: "0"
+        nginx.org/proxy-max-temp-file-size: "0"
+    external_policy_local: false
+    node_port:
+      enabled: false
+      port: 30004
   public: 192.168.0.0/16
   cluster: 192.168.0.0/16
   port:
     mon: 6789
-    rgw: 8088
     mgr: 7000
 
 conf:
@@ -280,6 +293,7 @@ conf:
       rgw_keystone_make_new_tenants: true
       rgw_s3_auth_use_keystone: true
       rgw_swift_account_in_url: true
+      rgw_swift_url: null
   ceph:
     global:
       # auth
@@ -488,6 +502,7 @@ endpoints:
     namespace: null
     hosts:
       default: ceph-rgw
+      public: radosgw
     host_fqdn_override:
       default: null
     path:
@@ -497,6 +512,7 @@ endpoints:
     port:
       api:
         default: 8088
+        public: 80
   ceph_mon:
     namespace: null
     hosts:
@@ -538,6 +554,7 @@ manifests:
   deployment_cephfs_provisioner: true
   deployment_rgw: true
   deployment_mgr: true
+  ingress_rgw: true
   job_bootstrap: false
   job_cephfs_client_key: true
   job_image_repo_sync: true
@@ -549,6 +566,7 @@ manifests:
   job_rbd_pool: true
   secret_keystone_rgw: true
   secret_keystone: true
+  service_ingress_rgw: true
   service_mgr: true
   service_rgw: true
   storageclass_cephfs: true

glance/values.yaml

@@ -239,6 +239,7 @@ conf:
       swift_store_container: glance
       swift_store_create_container_on_put: true
       swift_store_config_file: /etc/glance/swift-store.conf
+      swift_store_endpoint_type: internalURL
     paste_deploy:
       flavor: keystone
     database:
@@ -588,6 +589,7 @@ endpoints:
         tmpurlkey: supersecret
     hosts:
       default: ceph-rgw
+      public: radosgw
     host_fqdn_override:
       default: null
     path:
@@ -597,6 +599,7 @@ endpoints:
     port:
       api:
         default: 8088
+        public: 80
   ceph_object_store:
     name: radosgw
     namespace: ceph
@@ -607,6 +610,7 @@ endpoints:
         tmpurlkey: supersecret
     hosts:
       default: ceph-rgw
+      public: radosgw
     host_fqdn_override:
       default: null
     path:
@@ -616,6 +620,7 @@ endpoints:
     port:
       api:
         default: 8088
+        public: 80
 
 pod:
   user:

tools/deployment/armada/multinode/armada-ceph.yaml

@@ -16,6 +16,42 @@ data:
   dependencies: []
 ---
 schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ceph-ingress-controller
+data:
+  chart_name: ceph-ingress-controller
+  release: ceph-ingress-controller
+  namespace: ceph
+  wait:
+    timeout: 1800
+    labels:
+      release_group: osh-ceph-ingress-controller
+  install:
+    no_hooks: False
+  upgrade:
+    no_hooks: False
+  values:
+    labels:
+      server:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+      error_server:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+    pod:
+      replicas:
+        error_page: 2
+        ingress: 2
+  source:
+    type: local
+    location: ${OSH_PATH}
+    subpath: ingress
+    reference: master
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
 metadata:
   schema: metadata/Document/v1
   name: ceph-mon
@@ -270,6 +306,7 @@ data:
   description: "Ceph Storage"
   sequenced: True
   chart_group:
+    - ceph-ingress-controller
     - ceph-mon
     - ceph-osd
     - ceph-client

tools/deployment/developer/common/030-ingress.sh

@@ -34,16 +34,22 @@ helm upgrade --install ingress-kube-system ./ingress \
   ${OSH_EXTRA_HELM_ARGS} \
   ${OSH_EXTRA_HELM_ARGS_INGRESS_KUBE_SYSTEM}
 
-#NOTE: Deploy namespace ingress
-helm upgrade --install ingress-openstack ./ingress \
-  --namespace=openstack \
-  ${OSH_EXTRA_HELM_ARGS} \
-  ${OSH_EXTRA_HELM_ARGS_INGRESS_OPENSTACK}
-
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh kube-system
-./tools/deployment/common/wait-for-pods.sh openstack
 
 #NOTE: Display info
 helm status ingress-kube-system
-helm status ingress-openstack
+
+#NOTE: Deploy namespace ingress
+for NAMESPACE in openstack ceph; do
+  helm upgrade --install ingress-${NAMESPACE} ./ingress \
+    --namespace=${NAMESPACE} \
+    ${OSH_EXTRA_HELM_ARGS} \
+    ${OSH_EXTRA_HELM_ARGS_INGRESS_OPENSTACK}
+
+  #NOTE: Wait for deploy
+  ./tools/deployment/common/wait-for-pods.sh ${NAMESPACE}
+
+  #NOTE: Display info
+  helm status ingress-openstack
+done

tools/deployment/multinode/020-ingress.sh

@@ -33,21 +33,28 @@ helm upgrade --install ingress-kube-system ./ingress \
   ${OSH_EXTRA_HELM_ARGS} \
   ${OSH_EXTRA_HELM_ARGS_INGRESS_KUBE_SYSTEM}
 
-#NOTE: Deploy namespace ingress
-tee /tmp/ingress-openstack.yaml << EOF
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh kube-system
+
+#NOTE: Display info
+helm status ingress-kube-system
+
+#NOTE: Deploy namespaced ingress controllers
+for NAMESPACE in openstack ceph; do
+  #NOTE: Deploy namespace ingress
+  tee /tmp/ingress-${NAMESPACE}.yaml << EOF
 pod:
   replicas:
     ingress: 2
     error_page: 2
 EOF
-helm upgrade --install ingress-openstack ./ingress \
-  --namespace=openstack \
-  --values=/tmp/ingress-openstack.yaml
+  helm upgrade --install ingress-${NAMESPACE} ./ingress \
+    --namespace=${NAMESPACE} \
+    --values=/tmp/ingress-${NAMESPACE}.yaml
 
-#NOTE: Wait for deploy
-./tools/deployment/common/wait-for-pods.sh kube-system
-./tools/deployment/common/wait-for-pods.sh openstack
+  #NOTE: Wait for deploy
+  ./tools/deployment/common/wait-for-pods.sh ${NAMESPACE}
 
-#NOTE: Display info
-helm status ingress-kube-system
-helm status ingress-openstack
+  #NOTE: Display info
+  helm status ingress-${NAMESPACE}
+done