From 0f107f1b813ce452fdc4a189842c154821219dc2 Mon Sep 17 00:00:00 2001 From: portdirect Date: Wed, 27 Dec 2017 17:19:49 -0500 Subject: [PATCH] Ceph: update dependency logic to be more strict This PS updates the dependency tree in ceph to take into account the keyring jobs and also the tollerance for unready endpoints introduced in the k8s 1.6 era. Change-Id: If76efeafdbcbe88ee699e7553f0effd5da7ce624 --- ceph/templates/configmap-etc.yaml | 2 +- ceph/templates/service-mon-discovery.yaml | 42 +++++++++++++++++++++++ ceph/templates/service-mon.yaml | 10 +----- ceph/values.yaml | 16 ++++++++- 4 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 ceph/templates/service-mon-discovery.yaml diff --git a/ceph/templates/configmap-etc.yaml b/ceph/templates/configmap-etc.yaml index b6223f0c78..dc325292c2 100644 --- a/ceph/templates/configmap-etc.yaml +++ b/ceph/templates/configmap-etc.yaml @@ -19,7 +19,7 @@ limitations under the License. {{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }} {{- if empty .Values.conf.ceph.config.global.mon_host -}} -{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $monHost := tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} {{- $monHost | set .Values.conf.ceph.config.global "mon_host" | quote | trunc 0 -}} {{- end -}} diff --git a/ceph/templates/service-mon-discovery.yaml b/ceph/templates/service-mon-discovery.yaml new file mode 100644 index 0000000000..a9e96c393c --- /dev/null +++ b/ceph/templates/service-mon-discovery.yaml @@ -0,0 +1,42 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_mon_discovery }} +{{- $envAll := . }} +{{- if .Values.deployment.ceph }} +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + annotations: + # In kubernetes 1.6 and beyond, it seems there was a change in behavior + # requiring us to tolerate unready endpoints to form a quorum. I can only + # guess at some small timing change causing statefulset+2 to not see the + # now ready statefulset+1, and because we do not tolerate unready endpoints + # a newly provisioned ceph-mon will most certainly never see itself in the + # peer list. This change allows us to form a quorum reliably everytime + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + ports: + - port: {{ .Values.network.port.mon }} + protocol: TCP + targetPort: {{ .Values.network.port.mon }} + selector: +{{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + clusterIP: None +{{- end }} +{{- end }} diff --git a/ceph/templates/service-mon.yaml b/ceph/templates/service-mon.yaml index acb8375c9f..37e7ad18b7 100644 --- a/ceph/templates/service-mon.yaml +++ b/ceph/templates/service-mon.yaml @@ -21,15 +21,7 @@ limitations under the License. kind: Service apiVersion: v1 metadata: - name: {{ .Values.endpoints.ceph_mon.hosts.default }} - annotations: - # In kubernetes 1.6 and beyond, it seems there was a change in behavior - # requiring us to tolerate unready endpoints to form a quorum. I can only - # guess at some small timing change causing statefulset+2 to not see the - # now ready statefulset+1, and because we do not tolerate unready endpoints - # a newly provisioned ceph-mon will most certainly never see itself in the - # peer list. This change allows us to form a quorum reliably everytime - service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + name: {{ tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: ports: - port: {{ .Values.network.port.mon }} diff --git a/ceph/values.yaml b/ceph/values.yaml index 63c0c00124..a0017cbd3b 100644 --- a/ceph/values.yaml +++ b/ceph/values.yaml @@ -212,23 +212,33 @@ dependencies: jobs: mon: jobs: + - ceph-storage-keys-generator + - ceph-mon-keyring-generator osd: jobs: + - ceph-storage-keys-generator + - ceph-osd-keyring-generator services: - service: ceph_mon endpoint: internal moncheck: jobs: + - ceph-storage-keys-generator + - ceph-mon-keyring-generator services: - service: ceph_mon - endpoint: internal + endpoint: discovery rgw: jobs: + - ceph-storage-keys-generator + - ceph-rgw-keyring-generator services: - service: ceph_mon endpoint: internal mds: jobs: + - ceph-storage-keys-generator + - ceph-mds-keyring-generator services: - service: ceph_mon endpoint: internal @@ -258,6 +268,8 @@ dependencies: endpoint: internal mgr: jobs: + - ceph-storage-keys-generator + - ceph-mgr-keyring-generator services: - service: ceph_mon endpoint: internal @@ -356,6 +368,7 @@ endpoints: namespace: null hosts: default: ceph-mon + discovery: ceph-mon-discovery host_fqdn_override: default: null port: @@ -388,5 +401,6 @@ manifests: secret_keystone: true service_mgr: true service_mon: true + service_mon_discovery: true service_rgw: true storageclass: true