From 7e281afa397146321c3ee157aaa54ac13a65a823 Mon Sep 17 00:00:00 2001 From: Chris Friesen Date: Mon, 28 Jan 2019 15:46:09 -0600 Subject: [PATCH] Add support for galera based mariadb replication Building on top of the previous commit which added the helm chart for the galera arbitrator, this does essentially three things: 1) Add the galera arbitrator (aka "garbd") helm chart to the stx-openstack application manifests as part of the mariadb chart group. 2) Modify the mariadb system overrides to specify a number of replicas equal to the number of controller nodes. 3) Modify sysinv to remove the garbd helm chart from the mariadb chart group if there is only one controller node (for the AIO-SX use-case). The rationale for this is that galera requires three DB nodes to allow for a quorum if a Kubernetes node goes down. The mariadb replicas use StatefulSets and are therefore pinned to specific Kubernetes nodes which is problematic for the third DB node--if its host node goes down it won't be moved to the remaining Kubernetes node. By running a galera arbitrator as the third DB node, we avoid this problem (because the arbitrator is stateless and can be restarted on the remaining Kubernetes node). The third item was a bit tricky and required testing a few different approaches. Just setting the chart manifests to "disabled" should have worked, but Armada got stuck waiting for resources that didn't exist. Newer Armada supports explicitly setting an empty list of resources to wait for, but that would require an Armada upgrade and would still require a meta-override to specify the wait parameters. Ultimately the path chosen was to add the ability for each helm class in sysinv to specify meta-overrides as well as regular values overrides. This allows them to change non-values portions of the "chart" section of the manifest, as well as the "chartgroup" section or even the main "manifest" section. Change-Id: I1fe66985a4af18d624aa54d45e91db7c121333f5 Story: 2004712 Task: 28743 Task: 29055 Task: 29056 Signed-off-by: Chris Friesen --- .../manifests/manifest-no-tests.yaml | 34 ++++++++++++++++++ .../manifests/manifest.yaml | 34 ++++++++++++++++++ sysinv/sysinv/centos/build_srpm.data | 2 +- .../sysinv/sysinv/conductor/kube_app.py | 15 ++++++++ sysinv/sysinv/sysinv/sysinv/helm/base.py | 13 +++++++ sysinv/sysinv/sysinv/sysinv/helm/garbd.py | 36 +++++++++++++++++++ sysinv/sysinv/sysinv/sysinv/helm/helm.py | 23 ++++++++++++ sysinv/sysinv/sysinv/sysinv/helm/mariadb.py | 2 +- 8 files changed, 157 insertions(+), 2 deletions(-) diff --git a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml index d02eaa26e6..2e6287838a 100644 --- a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml +++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml @@ -166,6 +166,39 @@ data: - helm-toolkit --- schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: openstack-garbd +data: + chart_name: garbd + release: openstack-garbd + namespace: openstack + wait: + timeout: 1800 + labels: + release_group: osh-openstack-garbd + install: + no_hooks: false + upgrade: + no_hooks: false + pre: + delete: + - type: job + labels: + release_group: osh-openstack-garbd + values: + labels: + server: + node_selector_key: openstack-compute-node + node_selector_value: enabled + source: + type: tar + location: http://172.17.0.1/helm_charts/garbd-0.1.0.tgz + subpath: garbd + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: openstack-memcached @@ -2446,6 +2479,7 @@ data: sequenced: true chart_group: - openstack-mariadb + - openstack-garbd --- schema: armada/ChartGroup/v1 metadata: diff --git a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml index e4f4e5be26..119ba1d7f0 100644 --- a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml +++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml @@ -166,6 +166,39 @@ data: - helm-toolkit --- schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: openstack-garbd +data: + chart_name: garbd + release: openstack-garbd + namespace: openstack + wait: + timeout: 1800 + labels: + release_group: osh-openstack-garbd + install: + no_hooks: false + upgrade: + no_hooks: false + pre: + delete: + - type: job + labels: + release_group: osh-openstack-garbd + values: + labels: + server: + node_selector_key: openstack-compute-node + node_selector_value: enabled + source: + type: tar + location: http://172.17.0.1/helm_charts/garbd-0.1.0.tgz + subpath: garbd + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: openstack-memcached @@ -2446,6 +2479,7 @@ data: sequenced: true chart_group: - openstack-mariadb + - openstack-garbd --- schema: armada/ChartGroup/v1 metadata: diff --git a/sysinv/sysinv/centos/build_srpm.data b/sysinv/sysinv/centos/build_srpm.data index fb9c4ef5cd..27453fdecb 100644 --- a/sysinv/sysinv/centos/build_srpm.data +++ b/sysinv/sysinv/centos/build_srpm.data @@ -1,2 +1,2 @@ SRC_DIR="sysinv" -TIS_PATCH_VER=296 +TIS_PATCH_VER=297 diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py index a1177e756d..12f37034b2 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py @@ -683,6 +683,21 @@ class AppOperator(object): missing_overrides.append(overrides_file) else: available_overrides.append(overrides_file) + + # Now handle any meta-overrides files. These can affect + # sections of the chart schema other than "values, and can + # affect the chartgroup or even the manifest. + if self._helm.generate_meta_overrides( + chart.name, chart.namespace): + overrides = chart.namespace + '-' + chart.name + \ + '-meta' + '.yaml' + overrides_file = os.path.join(common.HELM_OVERRIDES_PATH, + overrides) + if not os.path.exists(overrides_file): + missing_overrides.append(overrides_file) + else: + available_overrides.append(overrides_file) + if missing_overrides: LOG.error("Missing the following overrides: %s" % missing_overrides) return None diff --git a/sysinv/sysinv/sysinv/sysinv/helm/base.py b/sysinv/sysinv/sysinv/sysinv/helm/base.py index 07d194b8c1..fefc65e71a 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/base.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/base.py @@ -188,3 +188,16 @@ class BaseHelm(object): address = self._get_address_by_name( constants.CONTROLLER_HOSTNAME, constants.NETWORK_TYPE_MGMT) return address.address + + def get_meta_overrides(self, namespace): + """ + Return Armada-formatted chart-specific meta-overrides + + This allows a helm chart class to specify overrides (in Armada format) + for things other than the "values" section of a chart. This includes + other sections of a chart, as well as chart groups or even the + overall manifest itself. + + May be left blank to indicate that there are no additional overrides. + """ + return {} diff --git a/sysinv/sysinv/sysinv/sysinv/helm/garbd.py b/sysinv/sysinv/sysinv/sysinv/helm/garbd.py index e638044f5a..55b77186f6 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/garbd.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/garbd.py @@ -29,6 +29,42 @@ class GarbdHelm(base.BaseHelm): def get_namespaces(self): return self.SUPPORTED_NAMESPACES + def get_meta_overrides(self, namespace): + + def _meta_overrides(): + if self._num_controllers() < 2: + # If there are fewer than 2 controllers we'll use a single + # mariadb server and so we don't want to run garbd. This + # will remove "openstack-garbd" from the charts in the + # openstack-mariadb chartgroup. + return { + 'schema': 'armada/ChartGroup/v1', + 'metadata': { + 'schema': 'metadata/Document/v1', + 'name': 'openstack-mariadb', + }, + 'data': { + 'description': 'Mariadb', + 'sequenced': True, + 'chart_group': [ + 'openstack-mariadb', + ] + } + } + else: + return {} + + overrides = { + common.HELM_NS_OPENSTACK: _meta_overrides() + } + if namespace in self.SUPPORTED_NAMESPACES: + return overrides[namespace] + elif namespace: + raise exception.InvalidHelmNamespace(chart=self.CHART, + namespace=namespace) + else: + return overrides + def get_overrides(self, namespace=None): overrides = { common.HELM_NS_OPENSTACK: { diff --git a/sysinv/sysinv/sysinv/sysinv/helm/helm.py b/sysinv/sysinv/sysinv/sysinv/helm/helm.py index 7eb7301915..e890d4788e 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/helm.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/helm.py @@ -340,6 +340,18 @@ class HelmOperator(object): else: LOG.exception("chart name is required") + @helm_context + def generate_meta_overrides(self, chart_name, chart_namespace): + overrides = {} + if chart_name in self.implemented_charts: + try: + overrides.update( + self.chart_operators[chart_name].get_meta_overrides( + chart_namespace)) + except exception.InvalidHelmNamespace: + raise + return overrides + @helm_context def generate_helm_application_overrides(self, app_name, cnamespace=None, armada_format=False, @@ -404,6 +416,17 @@ class HelmOperator(object): overrides[key] = new_overrides self._write_chart_overrides(chart_name, cnamespace, overrides) + + # Write any meta-overrides for this chart. These will be in + # armada format already. + if armada_format: + overrides = self.generate_meta_overrides(chart_name, + cnamespace) + if overrides: + chart_meta_name = chart_name + '-meta' + self._write_chart_overrides( + chart_meta_name, cnamespace, overrides) + elif app_name: LOG.exception("%s application is not supported" % app_name) else: diff --git a/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py b/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py index 6cff6133c8..265c7f8698 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py @@ -29,7 +29,7 @@ class MariadbHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': { 'replicas': { - 'server': 1 + 'server': self._num_controllers() } }, 'images': self._get_images_overrides(),