From 754a1d33de7e16b454052190a2496f1a1d59c707 Mon Sep 17 00:00:00 2001 From: Mihnea Saracin Date: Thu, 23 Jul 2020 16:39:05 +0300 Subject: [PATCH] Fix apply of stx-openstack when host is locked Currently, all of the stx-openstack services have the replica count set to the number of the controllers. If one of the controllers is locked their replicas number will still be 2 which is incorrect. We solve this by changing the number of replicas to be equal to the number of the active controllers. The rabbitmq service cannot use this approach because it is unable to work properly if its replicas number is decreasaed from 2 to 1. So a kubernetes toleration is used here to allow the second rabbitmq pod to be deployed on the locked controller. Change-Id: Ie979c7b5f2755ad673bd180e38b68e0d53c5f9b2 Closes-Bug: 1879018 Signed-off-by: Mihnea Saracin --- .../centos/openstack-helm-infra.spec | 3 +++ ...12-Add-tolerations-to-rabbitmq-chart.patch | 26 +++++++++++++++++++ .../k8sapp_openstack/helm/aodh.py | 8 +++--- .../k8sapp_openstack/helm/barbican.py | 2 +- .../k8sapp_openstack/helm/ceilometer.py | 4 +-- .../k8sapp_openstack/helm/cinder.py | 8 +++--- .../k8sapp_openstack/helm/fm_rest_api.py | 2 +- .../k8sapp_openstack/helm/glance.py | 2 +- .../k8sapp_openstack/helm/gnocchi.py | 2 +- .../k8sapp_openstack/helm/heat.py | 8 +++--- .../k8sapp_openstack/helm/ingress.py | 4 +-- .../k8sapp_openstack/helm/ironic.py | 4 +-- .../k8sapp_openstack/helm/keystone.py | 2 +- .../k8sapp_openstack/helm/magnum.py | 4 +-- .../k8sapp_openstack/helm/mariadb.py | 4 +-- .../k8sapp_openstack/helm/neutron.py | 2 +- .../k8sapp_openstack/helm/nova.py | 14 +++++----- .../k8sapp_openstack/helm/nova_api_proxy.py | 2 +- .../k8sapp_openstack/helm/panko.py | 2 +- .../k8sapp_openstack/helm/placement.py | 2 +- .../manifests/manifest.yaml | 3 +++ 21 files changed, 70 insertions(+), 38 deletions(-) create mode 100644 openstack-helm-infra/files/0012-Add-tolerations-to-rabbitmq-chart.patch diff --git a/openstack-helm-infra/centos/openstack-helm-infra.spec b/openstack-helm-infra/centos/openstack-helm-infra.spec index a327e405..228760e1 100644 --- a/openstack-helm-infra/centos/openstack-helm-infra.spec +++ b/openstack-helm-infra/centos/openstack-helm-infra.spec @@ -26,6 +26,7 @@ Patch08: 0008-Fix-rabbitmq-could-not-bind-port-to-ipv6-address-iss.patch Patch09: 0009-Enable-override-of-mariadb-server-probe-parameters.patch Patch10: 0010-Mariadb-use-utf8_general_ci-collation-as-default.patch Patch11: 0011-Add-mariadb-database-config-override-to-support-ipv6.patch +Patch12: 0012-Add-tolerations-to-rabbitmq-chart.patch BuildRequires: helm BuildRequires: chartmuseum @@ -46,6 +47,8 @@ Openstack Helm Infra charts %patch09 -p1 %patch10 -p1 %patch11 -p1 +%patch12 -p1 + %build # Host a server for the charts diff --git a/openstack-helm-infra/files/0012-Add-tolerations-to-rabbitmq-chart.patch b/openstack-helm-infra/files/0012-Add-tolerations-to-rabbitmq-chart.patch new file mode 100644 index 00000000..3a7fdc55 --- /dev/null +++ b/openstack-helm-infra/files/0012-Add-tolerations-to-rabbitmq-chart.patch @@ -0,0 +1,26 @@ +From c8f3a96fad3344cfdb058c7c0fee77431f77a001 Mon Sep 17 00:00:00 2001 +From: Mihnea Saracin +Date: Wed, 22 Jul 2020 15:53:25 +0300 +Subject: [PATCH] Add tolerations to rabbitmq chart + +Signed-off-by: Mihnea Saracin +--- + rabbitmq/templates/statefulset.yaml | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/rabbitmq/templates/statefulset.yaml b/rabbitmq/templates/statefulset.yaml +index 9e40a103..cf5d1811 100644 +--- a/rabbitmq/templates/statefulset.yaml ++++ b/rabbitmq/templates/statefulset.yaml +@@ -85,6 +85,8 @@ spec: + {{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ $envAll.Values.labels.server.node_selector_key }}: {{ $envAll.Values.labels.server.node_selector_value | quote }} ++ tolerations: ++{{ toYaml .Values.tolerations | indent 8 }} + initContainers: + {{ tuple $envAll "rabbitmq" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: rabbitmq-password +-- +2.17.1 + diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/aodh.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/aodh.py index c55af195..f4f5ed15 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/aodh.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/aodh.py @@ -39,10 +39,10 @@ class AodhHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): overrides = { 'replicas': { - 'api': self._num_controllers(), - 'evaluator': self._num_controllers(), - 'listener': self._num_controllers(), - 'notifier': self._num_controllers() + 'api': self._num_provisioned_controllers(), + 'evaluator': self._num_provisioned_controllers(), + 'listener': self._num_provisioned_controllers(), + 'notifier': self._num_provisioned_controllers() } } return overrides diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/barbican.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/barbican.py index edd98c64..941f722c 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/barbican.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/barbican.py @@ -23,7 +23,7 @@ class BarbicanHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() } }, 'endpoints': self._get_endpoints_overrides(), diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ceilometer.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ceilometer.py index cab4c820..a4f3bf6b 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ceilometer.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ceilometer.py @@ -41,8 +41,8 @@ class CeilometerHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): return { 'replicas': { - 'central': self._num_controllers(), - 'notification': self._num_controllers() + 'central': self._num_provisioned_controllers(), + 'notification': self._num_provisioned_controllers() } } diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/cinder.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/cinder.py index 71afbce9..603ddf0b 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/cinder.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/cinder.py @@ -49,10 +49,10 @@ class CinderHelm(openstack.OpenstackBaseHelm): } }, 'replicas': { - 'api': self._num_controllers(), - 'volume': self._num_controllers(), - 'scheduler': self._num_controllers(), - 'backup': self._num_controllers() + 'api': self._num_provisioned_controllers(), + 'volume': self._num_provisioned_controllers(), + 'scheduler': self._num_provisioned_controllers(), + 'backup': self._num_provisioned_controllers() } }, 'conf': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/fm_rest_api.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/fm_rest_api.py index 360e0ff5..38b2cfd9 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/fm_rest_api.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/fm_rest_api.py @@ -27,7 +27,7 @@ class FmRestApiHelm(openstack.OpenstackBaseHelm): 'endpoints': self._get_endpoints_overrides(), 'pod': { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() }, }, } diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/glance.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/glance.py index dc954fe3..e84c8895 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/glance.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/glance.py @@ -51,7 +51,7 @@ class GlanceHelm(openstack.OpenstackBaseHelm): replicas_count = 1 ceph_backend = self._get_primary_ceph_backend() if ceph_backend: - replicas_count = self._num_controllers() + replicas_count = self._num_provisioned_controllers() return { 'replicas': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/gnocchi.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/gnocchi.py index b9fe09bb..9e98d647 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/gnocchi.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/gnocchi.py @@ -39,7 +39,7 @@ class GnocchiHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): return { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() } } diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/heat.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/heat.py index eb75a9af..f356fbb0 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/heat.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/heat.py @@ -38,10 +38,10 @@ class HeatHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): return { 'replicas': { - 'api': self._num_controllers(), - 'cfn': self._num_controllers(), - 'cloudwatch': self._num_controllers(), - 'engine': self._num_controllers() + 'api': self._num_provisioned_controllers(), + 'cfn': self._num_provisioned_controllers(), + 'cloudwatch': self._num_provisioned_controllers(), + 'engine': self._num_provisioned_controllers() } } diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ingress.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ingress.py index 8df7229a..fd9878fe 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ingress.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ingress.py @@ -62,8 +62,8 @@ class IngressHelm(base.BaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'ingress': self._num_controllers(), - 'error_page': self._num_controllers() + 'ingress': self._num_provisioned_controllers(), + 'error_page': self._num_provisioned_controllers() }, 'resources': { 'enabled': limit_enabled, diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ironic.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ironic.py index a2068af4..8e40e1ec 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ironic.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/ironic.py @@ -45,8 +45,8 @@ class IronicHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'api': self._num_controllers(), - 'conductor': self._num_controllers() + 'api': self._num_provisioned_controllers(), + 'conductor': self._num_provisioned_controllers() } }, 'network': self._get_network_overrides(), diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/keystone.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/keystone.py index 45210054..5c0c98ee 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/keystone.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/keystone.py @@ -49,7 +49,7 @@ class KeystoneHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): overrides = { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() }, 'lifecycle': { 'termination_grace_period': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/magnum.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/magnum.py index 75e45a26..9ecd32b9 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/magnum.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/magnum.py @@ -24,8 +24,8 @@ class MagnumHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'api': self._num_controllers(), - 'conductor': self._num_controllers() + 'api': self._num_provisioned_controllers(), + 'conductor': self._num_provisioned_controllers() } } } diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/mariadb.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/mariadb.py index fd7f5932..99673ced 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/mariadb.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/mariadb.py @@ -24,8 +24,8 @@ class MariadbHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'server': self._num_server_replicas(), - 'ingress': self._num_controllers() + 'server': self._num_provisioned_controllers(), + 'ingress': self._num_provisioned_controllers() } }, 'endpoints': self._get_endpoints_overrides(), diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/neutron.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/neutron.py index 7b61e5dc..beafa063 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/neutron.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/neutron.py @@ -33,7 +33,7 @@ class NeutronHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'server': self._num_controllers() + 'server': self._num_provisioned_controllers() }, }, 'conf': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova.py index 2945869b..806b87af 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova.py @@ -67,13 +67,13 @@ class NovaHelm(openstack.OpenstackBaseHelm): } }, 'replicas': { - 'api_metadata': self._num_controllers(), - 'placement': self._num_controllers(), - 'osapi': self._num_controllers(), - 'conductor': self._num_controllers(), - 'consoleauth': self._num_controllers(), - 'scheduler': self._num_controllers(), - 'novncproxy': self._num_controllers() + 'api_metadata': self._num_provisioned_controllers(), + 'placement': self._num_provisioned_controllers(), + 'osapi': self._num_provisioned_controllers(), + 'conductor': self._num_provisioned_controllers(), + 'consoleauth': self._num_provisioned_controllers(), + 'scheduler': self._num_provisioned_controllers(), + 'novncproxy': self._num_provisioned_controllers() } }, 'conf': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova_api_proxy.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova_api_proxy.py index 727254df..e2c37371 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova_api_proxy.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/nova_api_proxy.py @@ -30,7 +30,7 @@ class NovaApiProxyHelm(openstack.OpenstackBaseHelm): } }, 'replicas': { - 'proxy': self._num_controllers() + 'proxy': self._num_provisioned_controllers() } }, 'conf': { diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/panko.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/panko.py index 34bc457e..d01e3cf1 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/panko.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/panko.py @@ -38,7 +38,7 @@ class PankoHelm(openstack.OpenstackBaseHelm): def _get_pod_overrides(self): overrides = { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() } } return overrides diff --git a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/placement.py b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/placement.py index 3cd060a3..702e1114 100644 --- a/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/placement.py +++ b/python-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/helm/placement.py @@ -26,7 +26,7 @@ class PlacementHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'api': self._num_controllers() + 'api': self._num_provisioned_controllers() } }, 'endpoints': self._get_endpoints_overrides() diff --git a/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml b/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml index f168c024..8054fcb4 100644 --- a/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml +++ b/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml @@ -336,6 +336,9 @@ data: # filesystem race as seen in the LP. volume: size: 1Gi + tolerations: + - effect: NoExecute + operator: Exists source: type: tar location: http://172.17.0.1/helm_charts/starlingx/rabbitmq-0.1.0.tgz