From fc68439414816b2384aed1e88120713f645db8d8 Mon Sep 17 00:00:00 2001 From: Mihnea Saracin Date: Fri, 11 Sep 2020 18:46:48 +0300 Subject: [PATCH] Fix apply of stx-openstack when host is locked Currently, all of the stx-openstack services have the replica count set to the number of the controllers. If one of the controllers is locked their replicas number will still be 2 which is incorrect. We solve this by changing the number of replicas to be equal to the number of the active controllers. The rabbitmq and mariadb services cannot use this approach because they are unable to work properly if their replica number is decreased from 2 to 1. So a kubernetes toleration is used here to allow the rabbitmq and mariadb pods to be deployed on the locked controller. Change-Id: I15cf2a3f62525751435ddbe66760935f3ab21d2b Closes-Bug: 1879018 Signed-off-by: Mihnea Saracin --- .../centos/openstack-helm-infra.spec | 4 ++ ...14-Add-tolerations-to-rabbitmq-chart.patch | 26 ++++++++++++ ...015-Add-tolerations-to-mariadb-chart.patch | 40 +++++++++++++++++++ .../k8sapp_openstack/helm/aodh.py | 8 ++-- .../k8sapp_openstack/helm/barbican.py | 2 +- .../k8sapp_openstack/helm/ceilometer.py | 4 +- .../k8sapp_openstack/helm/cinder.py | 8 ++-- .../k8sapp_openstack/helm/fm_rest_api.py | 2 +- .../k8sapp_openstack/helm/glance.py | 2 +- .../k8sapp_openstack/helm/gnocchi.py | 2 +- .../k8sapp_openstack/helm/heat.py | 8 ++-- .../k8sapp_openstack/helm/ingress.py | 4 +- .../k8sapp_openstack/helm/ironic.py | 4 +- .../k8sapp_openstack/helm/keystone.py | 2 +- .../k8sapp_openstack/helm/magnum.py | 4 +- .../k8sapp_openstack/helm/neutron.py | 2 +- .../k8sapp_openstack/helm/nova.py | 14 +++---- .../k8sapp_openstack/helm/nova_api_proxy.py | 2 +- .../k8sapp_openstack/helm/panko.py | 2 +- .../k8sapp_openstack/helm/placement.py | 2 +- .../manifests/manifest.yaml | 6 +++ 21 files changed, 112 insertions(+), 36 deletions(-) create mode 100644 openstack-helm-infra/files/0014-Add-tolerations-to-rabbitmq-chart.patch create mode 100644 openstack-helm-infra/files/0015-Add-tolerations-to-mariadb-chart.patch diff --git a/openstack-helm-infra/centos/openstack-helm-infra.spec b/openstack-helm-infra/centos/openstack-helm-infra.spec index 8569c9f2..9428208c 100644 --- a/openstack-helm-infra/centos/openstack-helm-infra.spec +++ b/openstack-helm-infra/centos/openstack-helm-infra.spec @@ -28,6 +28,8 @@ Patch10: 0010-Mariadb-use-utf8_general_ci-collation-as-default.patch Patch11: 0011-Add-mariadb-database-config-override-to-support-ipv6.patch Patch12: 0012-enable-Values.conf.database.config_override-for-mari.patch Patch13: 0013-Allow-set-public-endpoint-url-for-all-openstack-types.patch +Patch14: 0014-Add-tolerations-to-rabbitmq-chart.patch +Patch15: 0015-Add-tolerations-to-mariadb-chart.patch BuildRequires: helm BuildRequires: chartmuseum @@ -50,6 +52,8 @@ Openstack Helm Infra charts %patch11 -p1 %patch12 -p1 %patch13 -p1 +%patch14 -p1 +%patch15 -p1 %build # Host a server for the charts diff --git a/openstack-helm-infra/files/0014-Add-tolerations-to-rabbitmq-chart.patch b/openstack-helm-infra/files/0014-Add-tolerations-to-rabbitmq-chart.patch new file mode 100644 index 00000000..3a7fdc55 --- /dev/null +++ b/openstack-helm-infra/files/0014-Add-tolerations-to-rabbitmq-chart.patch @@ -0,0 +1,26 @@ +From c8f3a96fad3344cfdb058c7c0fee77431f77a001 Mon Sep 17 00:00:00 2001 +From: Mihnea Saracin +Date: Wed, 22 Jul 2020 15:53:25 +0300 +Subject: [PATCH] Add tolerations to rabbitmq chart + +Signed-off-by: Mihnea Saracin +--- + rabbitmq/templates/statefulset.yaml | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/rabbitmq/templates/statefulset.yaml b/rabbitmq/templates/statefulset.yaml +index 9e40a103..cf5d1811 100644 +--- a/rabbitmq/templates/statefulset.yaml ++++ b/rabbitmq/templates/statefulset.yaml +@@ -85,6 +85,8 @@ spec: + {{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ $envAll.Values.labels.server.node_selector_key }}: {{ $envAll.Values.labels.server.node_selector_value | quote }} ++ tolerations: ++{{ toYaml .Values.tolerations | indent 8 }} + initContainers: + {{ tuple $envAll "rabbitmq" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: rabbitmq-password +-- +2.17.1 + diff --git a/openstack-helm-infra/files/0015-Add-tolerations-to-mariadb-chart.patch b/openstack-helm-infra/files/0015-Add-tolerations-to-mariadb-chart.patch new file mode 100644 index 00000000..1ae38564 --- /dev/null +++ b/openstack-helm-infra/files/0015-Add-tolerations-to-mariadb-chart.patch @@ -0,0 +1,40 @@ +From 7394d591310bf5342f3b66e4ee0a6a3cbd38c558 Mon Sep 17 00:00:00 2001 +From: Mihnea Saracin +Date: Tue, 11 Aug 2020 10:52:15 +0300 +Subject: [PATCH] Add tolerations to mariadb chart + +Signed-off-by: Mihnea Saracin +--- + mariadb/templates/deployment-ingress.yaml | 2 ++ + mariadb/templates/statefulset.yaml | 2 ++ + 2 files changed, 4 insertions(+) + +diff --git a/mariadb/templates/deployment-ingress.yaml b/mariadb/templates/deployment-ingress.yaml +index 72bea94a..9fb02c8d 100644 +--- a/mariadb/templates/deployment-ingress.yaml ++++ b/mariadb/templates/deployment-ingress.yaml +@@ -147,6 +147,8 @@ spec: + nodeSelector: + {{ .Values.labels.ingress.node_selector_key }}: {{ .Values.labels.ingress.node_selector_value }} + terminationGracePeriodSeconds: 60 ++ tolerations: ++{{ toYaml .Values.tolerations | indent 8 }} + initContainers: + {{ tuple $envAll "ingress" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: +diff --git a/mariadb/templates/statefulset.yaml b/mariadb/templates/statefulset.yaml +index 7ccc219b..120427ae 100644 +--- a/mariadb/templates/statefulset.yaml ++++ b/mariadb/templates/statefulset.yaml +@@ -108,6 +108,8 @@ spec: + {{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value }} ++ tolerations: ++{{ toYaml .Values.tolerations | indent 8 }} + initContainers: + {{ tuple $envAll "mariadb" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + {{- if .Values.volume.chown_on_start }} +-- +2.17.1 + diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/aodh.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/aodh.py index c55af195..f4f5ed15 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/aodh.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/aodh.py @@ -39,10 +39,10 @@ class AodhHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): overrides = { 'replicas': { - 'api': self._num_controllers(), - 'evaluator': self._num_controllers(), - 'listener': self._num_controllers(), - 'notifier': self._num_controllers() + 'api': self._num_provisioned_controllers(), + 'evaluator': self._num_provisioned_controllers(), + 'listener': self._num_provisioned_controllers(), + 'notifier': self._num_provisioned_controllers() } } return overrides diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/barbican.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/barbican.py index edd98c64..941f722c 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/barbican.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/barbican.py @@ -23,7 +23,7 @@ class BarbicanHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() } }, 'endpoints': self._get_endpoints_overrides(), diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ceilometer.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ceilometer.py index cab4c820..a4f3bf6b 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ceilometer.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ceilometer.py @@ -41,8 +41,8 @@ class CeilometerHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): return { 'replicas': { - 'central': self._num_controllers(), - 'notification': self._num_controllers() + 'central': self._num_provisioned_controllers(), + 'notification': self._num_provisioned_controllers() } } diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/cinder.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/cinder.py index 71afbce9..603ddf0b 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/cinder.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/cinder.py @@ -49,10 +49,10 @@ class CinderHelm(openstack.OpenstackBaseHelm): } }, 'replicas': { - 'api': self._num_controllers(), - 'volume': self._num_controllers(), - 'scheduler': self._num_controllers(), - 'backup': self._num_controllers() + 'api': self._num_provisioned_controllers(), + 'volume': self._num_provisioned_controllers(), + 'scheduler': self._num_provisioned_controllers(), + 'backup': self._num_provisioned_controllers() } }, 'conf': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/fm_rest_api.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/fm_rest_api.py index 360e0ff5..38b2cfd9 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/fm_rest_api.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/fm_rest_api.py @@ -27,7 +27,7 @@ class FmRestApiHelm(openstack.OpenstackBaseHelm): 'endpoints': self._get_endpoints_overrides(), 'pod': { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() }, }, } diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/glance.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/glance.py index dc954fe3..e84c8895 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/glance.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/glance.py @@ -51,7 +51,7 @@ class GlanceHelm(openstack.OpenstackBaseHelm): replicas_count = 1 ceph_backend = self._get_primary_ceph_backend() if ceph_backend: - replicas_count = self._num_controllers() + replicas_count = self._num_provisioned_controllers() return { 'replicas': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/gnocchi.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/gnocchi.py index b9fe09bb..9e98d647 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/gnocchi.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/gnocchi.py @@ -39,7 +39,7 @@ class GnocchiHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): return { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() } } diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/heat.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/heat.py index eb75a9af..f356fbb0 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/heat.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/heat.py @@ -38,10 +38,10 @@ class HeatHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): return { 'replicas': { - 'api': self._num_controllers(), - 'cfn': self._num_controllers(), - 'cloudwatch': self._num_controllers(), - 'engine': self._num_controllers() + 'api': self._num_provisioned_controllers(), + 'cfn': self._num_provisioned_controllers(), + 'cloudwatch': self._num_provisioned_controllers(), + 'engine': self._num_provisioned_controllers() } } diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ingress.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ingress.py index 8df7229a..fd9878fe 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ingress.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ingress.py @@ -62,8 +62,8 @@ class IngressHelm(base.BaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'ingress': self._num_controllers(), - 'error_page': self._num_controllers() + 'ingress': self._num_provisioned_controllers(), + 'error_page': self._num_provisioned_controllers() }, 'resources': { 'enabled': limit_enabled, diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ironic.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ironic.py index a2068af4..8e40e1ec 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ironic.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ironic.py @@ -45,8 +45,8 @@ class IronicHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'api': self._num_controllers(), - 'conductor': self._num_controllers() + 'api': self._num_provisioned_controllers(), + 'conductor': self._num_provisioned_controllers() } }, 'network': self._get_network_overrides(), diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/keystone.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/keystone.py index d8116468..f41f0e58 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/keystone.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/keystone.py @@ -49,7 +49,7 @@ class KeystoneHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): overrides = { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() }, 'lifecycle': { 'termination_grace_period': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/magnum.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/magnum.py index 75e45a26..9ecd32b9 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/magnum.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/magnum.py @@ -24,8 +24,8 @@ class MagnumHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'api': self._num_controllers(), - 'conductor': self._num_controllers() + 'api': self._num_provisioned_controllers(), + 'conductor': self._num_provisioned_controllers() } } } diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/neutron.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/neutron.py index ac00a0b6..ec68c8be 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/neutron.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/neutron.py @@ -49,7 +49,7 @@ class NeutronHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'server': self._num_controllers() + 'server': self._num_provisioned_controllers() }, }, 'conf': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova.py index bd7f0f74..e8a54f1b 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova.py @@ -107,13 +107,13 @@ class NovaHelm(openstack.OpenstackBaseHelm): } }, 'replicas': { - 'api_metadata': self._num_controllers(), - 'placement': self._num_controllers(), - 'osapi': self._num_controllers(), - 'conductor': self._num_controllers(), - 'consoleauth': self._num_controllers(), - 'scheduler': self._num_controllers(), - 'novncproxy': self._num_controllers() + 'api_metadata': self._num_provisioned_controllers(), + 'placement': self._num_provisioned_controllers(), + 'osapi': self._num_provisioned_controllers(), + 'conductor': self._num_provisioned_controllers(), + 'consoleauth': self._num_provisioned_controllers(), + 'scheduler': self._num_provisioned_controllers(), + 'novncproxy': self._num_provisioned_controllers() } }, 'conf': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova_api_proxy.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova_api_proxy.py index 727254df..e2c37371 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova_api_proxy.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova_api_proxy.py @@ -30,7 +30,7 @@ class NovaApiProxyHelm(openstack.OpenstackBaseHelm): } }, 'replicas': { - 'proxy': self._num_controllers() + 'proxy': self._num_provisioned_controllers() } }, 'conf': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/panko.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/panko.py index 34bc457e..d01e3cf1 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/panko.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/panko.py @@ -38,7 +38,7 @@ class PankoHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): overrides = { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() } } return overrides diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/placement.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/placement.py index 3cd060a3..702e1114 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/placement.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/placement.py @@ -26,7 +26,7 @@ class PlacementHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() } }, 'endpoints': self._get_endpoints_overrides() diff --git a/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml b/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml index cfe3a113..480cce89 100644 --- a/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml +++ b/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml @@ -190,6 +190,9 @@ data: config_override: | [mysqld] wsrep_provider_options="gmcast.listen_addr=tcp://0.0.0.0:{{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + tolerations: + - effect: NoExecute + operator: Exists source: type: tar location: http://172.17.0.1/helm_charts/starlingx/mariadb-0.1.0.tgz @@ -340,6 +343,9 @@ data: # filesystem race as seen in the LP. volume: size: 1Gi + tolerations: + - effect: NoExecute + operator: Exists source: type: tar location: http://172.17.0.1/helm_charts/starlingx/rabbitmq-0.1.0.tgz