Add support for galera based mariadb replication

Building on top of the previous commit which added the helm chart
for the galera arbitrator, this does essentially three things:

1) Add the galera arbitrator (aka "garbd") helm chart to the
stx-openstack application manifests as part of the mariadb
chart group.

2) Modify the mariadb system overrides to specify a number of
replicas equal to the number of controller nodes.

3) Modify sysinv to remove the garbd helm chart from the mariadb
chart group if there is only one controller node (for the AIO-SX
use-case).

The rationale for this is that galera requires three DB nodes to
allow for a quorum if a Kubernetes node goes down.  The mariadb
replicas use StatefulSets and are therefore pinned to specific
Kubernetes nodes which is problematic for the third DB node--if
its host node goes down it won't be moved to the remaining
Kubernetes node. By running a galera arbitrator as the third DB
node, we avoid this problem (because the arbitrator is stateless
and can be restarted on the remaining Kubernetes node).

The third item was a bit tricky and required testing a few different
approaches.  Just setting the chart manifests to "disabled" should
have worked, but Armada got stuck waiting for resources that didn't
exist.  Newer Armada supports explicitly setting an empty list of
resources to wait for, but that would require an Armada upgrade and
would still require a meta-override to specify the wait parameters.
Ultimately the path chosen was to add the ability for each helm
class in sysinv to specify meta-overrides as well as regular
values overrides.  This allows them to change non-values portions
of the "chart" section of the manifest, as well as the "chartgroup"
section or even the main "manifest" section.

Change-Id: I1fe66985a4af18d624aa54d45e91db7c121333f5
Story: 2004712
Task: 28743
Task: 29055
Task: 29056
Signed-off-by: Chris Friesen <chris.friesen@windriver.com>

kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml

@@ -166,6 +166,39 @@ data:
     - helm-toolkit
 ---
 schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: openstack-garbd
+data:
+  chart_name: garbd
+  release: openstack-garbd
+  namespace: openstack
+  wait:
+    timeout: 1800
+    labels:
+      release_group: osh-openstack-garbd
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release_group: osh-openstack-garbd
+  values:
+    labels:
+      server:
+        node_selector_key: openstack-compute-node
+        node_selector_value: enabled
+  source:
+    type: tar
+    location: http://172.17.0.1/helm_charts/garbd-0.1.0.tgz
+    subpath: garbd
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
 metadata:
   schema: metadata/Document/v1
   name: openstack-memcached
@@ -2446,6 +2479,7 @@ data:
   sequenced: true
   chart_group:
     - openstack-mariadb
+    - openstack-garbd
 ---
 schema: armada/ChartGroup/v1
 metadata:

kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml

@@ -166,6 +166,39 @@ data:
     - helm-toolkit
 ---
 schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: openstack-garbd
+data:
+  chart_name: garbd
+  release: openstack-garbd
+  namespace: openstack
+  wait:
+    timeout: 1800
+    labels:
+      release_group: osh-openstack-garbd
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release_group: osh-openstack-garbd
+  values:
+    labels:
+      server:
+        node_selector_key: openstack-compute-node
+        node_selector_value: enabled
+  source:
+    type: tar
+    location: http://172.17.0.1/helm_charts/garbd-0.1.0.tgz
+    subpath: garbd
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
 metadata:
   schema: metadata/Document/v1
   name: openstack-memcached
@@ -2446,6 +2479,7 @@ data:
   sequenced: true
   chart_group:
     - openstack-mariadb
+    - openstack-garbd
 ---
 schema: armada/ChartGroup/v1
 metadata:

sysinv/sysinv/centos/build_srpm.data

@@ -1,2 +1,2 @@
 SRC_DIR="sysinv"
-TIS_PATCH_VER=296
+TIS_PATCH_VER=297

sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py

@@ -683,6 +683,21 @@ class AppOperator(object):
                 missing_overrides.append(overrides_file)
             else:
                 available_overrides.append(overrides_file)
+
+                # Now handle any meta-overrides files.  These can affect
+                # sections of the chart schema other than "values, and can
+                # affect the chartgroup or even the manifest.
+                if self._helm.generate_meta_overrides(
+                        chart.name, chart.namespace):
+                    overrides = chart.namespace + '-' + chart.name + \
+                                '-meta' + '.yaml'
+                    overrides_file = os.path.join(common.HELM_OVERRIDES_PATH,
+                                                  overrides)
+                    if not os.path.exists(overrides_file):
+                        missing_overrides.append(overrides_file)
+                    else:
+                        available_overrides.append(overrides_file)
+
         if missing_overrides:
             LOG.error("Missing the following overrides: %s" % missing_overrides)
             return None

sysinv/sysinv/sysinv/sysinv/helm/base.py

@@ -188,3 +188,16 @@ class BaseHelm(object):
         address = self._get_address_by_name(
             constants.CONTROLLER_HOSTNAME, constants.NETWORK_TYPE_MGMT)
         return address.address
+
+    def get_meta_overrides(self, namespace):
+        """
+        Return Armada-formatted chart-specific meta-overrides
+
+        This allows a helm chart class to specify overrides (in Armada format)
+        for things other than the "values" section of a chart.  This includes
+        other sections of a chart, as well as chart groups or even the
+        overall manifest itself.
+
+        May be left blank to indicate that there are no additional overrides.
+        """
+        return {}

sysinv/sysinv/sysinv/sysinv/helm/garbd.py

@@ -29,6 +29,42 @@ class GarbdHelm(base.BaseHelm):
     def get_namespaces(self):
         return self.SUPPORTED_NAMESPACES
 
+    def get_meta_overrides(self, namespace):
+
+        def _meta_overrides():
+            if self._num_controllers() < 2:
+                # If there are fewer than 2 controllers we'll use a single
+                # mariadb server and so we don't want to run garbd.  This
+                # will remove "openstack-garbd" from the charts in the
+                # openstack-mariadb chartgroup.
+                return {
+                    'schema': 'armada/ChartGroup/v1',
+                    'metadata': {
+                        'schema': 'metadata/Document/v1',
+                        'name': 'openstack-mariadb',
+                    },
+                    'data': {
+                        'description': 'Mariadb',
+                        'sequenced': True,
+                        'chart_group': [
+                            'openstack-mariadb',
+                        ]
+                    }
+                }
+            else:
+                return {}
+
+        overrides = {
+            common.HELM_NS_OPENSTACK: _meta_overrides()
+        }
+        if namespace in self.SUPPORTED_NAMESPACES:
+            return overrides[namespace]
+        elif namespace:
+            raise exception.InvalidHelmNamespace(chart=self.CHART,
+                                                 namespace=namespace)
+        else:
+            return overrides
+
     def get_overrides(self, namespace=None):
         overrides = {
             common.HELM_NS_OPENSTACK: {

sysinv/sysinv/sysinv/sysinv/helm/helm.py

@@ -340,6 +340,18 @@ class HelmOperator(object):
         else:
             LOG.exception("chart name is required")
 
+    @helm_context
+    def generate_meta_overrides(self, chart_name, chart_namespace):
+        overrides = {}
+        if chart_name in self.implemented_charts:
+            try:
+                overrides.update(
+                    self.chart_operators[chart_name].get_meta_overrides(
+                        chart_namespace))
+            except exception.InvalidHelmNamespace:
+                raise
+        return overrides
+
     @helm_context
     def generate_helm_application_overrides(self, app_name, cnamespace=None,
                                             armada_format=False,
@@ -404,6 +416,17 @@ class HelmOperator(object):
                         overrides[key] = new_overrides
 
                 self._write_chart_overrides(chart_name, cnamespace, overrides)
+
+                # Write any meta-overrides for this chart.  These will be in
+                # armada format already.
+                if armada_format:
+                    overrides = self.generate_meta_overrides(chart_name,
+                                                             cnamespace)
+                    if overrides:
+                        chart_meta_name = chart_name + '-meta'
+                        self._write_chart_overrides(
+                            chart_meta_name, cnamespace, overrides)
+
         elif app_name:
             LOG.exception("%s application is not supported" % app_name)
         else:

sysinv/sysinv/sysinv/sysinv/helm/mariadb.py

@@ -29,7 +29,7 @@ class MariadbHelm(openstack.OpenstackBaseHelm):
             common.HELM_NS_OPENSTACK: {
                 'pod': {
                     'replicas': {
-                        'server': 1
+                        'server': self._num_controllers()
                     }
                 },
                 'images': self._get_images_overrides(),