Support Helm user overrides for generic applications

This commit updates to support the user overrides for generic
apps. The list of charts for generic apps is retrieved from
manifest and stored to database when uploading the app. The
application-upload and apply are changed to generate overrides
files for generic apps as well.

This also fixes the helm-override-list namespace display for
garbd, memcached and ingress charts in stx-openstack app

Tested(generic apps, stx-openstack app):
-system application-upload
-system helm-override-list
-system helm-override-update(--set --values)
-system helm-override-show
-system application-apply
-system application-remove
-system helm-override-delete

Change-Id: I45c561400fb82a69d51118159fad7bf03881f263
Story: 2003908
Task: 28631
Signed-off-by: Angie Wang <angie.wang@windriver.com>

sysinv/cgts-client/cgts-client/cgtsclient/v1/helm_shell.py

@@ -101,10 +101,9 @@ def do_helm_override_delete(cc, args):
                 'Can specify multiple times.')
 @utils.arg('--set', metavar='<commandline_overrides>', action='append',
            default=[],
-           help='Set helm chart override values on the command line (can '
+           help='Set helm chart override values on the command line. Multiple '
-                'specify multiple times or separate values with commas: '
+                'override values can be specified with multiple --set arguments.'
-                'key1=val1,key2=val2). These are processed after "--values" '
+                'These are processed after "--values" files.')
-                'files.')
 def do_helm_override_update(cc, args):
     """Update helm chart user overrides."""
 

sysinv/sysinv/sysinv/sysinv/api/controllers/v1/helm_charts.py

@@ -28,13 +28,12 @@ class HelmChartsController(rest.RestController):
         """Provides information about the available charts to override."""
 
         try:
-            objects.kube_app.get_by_name(
-                pecan.request.context, app_name)
-        except exception.KubeAppNotFound:
-            raise wsme.exc.ClientSideError(_("Application %s not found." % app_name))
-
             namespaces = pecan.request.rpcapi.get_helm_application_namespaces(
                 pecan.request.context, app_name)
+        except Exception as e:
+            raise wsme.exc.ClientSideError(_("Unable to get the helm charts for "
+                                             "application %s: %s" % (app_name, str(e))))
+
         charts = [{'name': chart, 'namespaces': namespaces[chart]}
                   for chart in namespaces]
 
@@ -60,23 +59,36 @@ class HelmChartsController(rest.RestController):
         except exception.KubeAppNotFound:
             raise wsme.exc.ClientSideError(_("Application %s not found." % app_name))
         except exception.HelmOverrideNotFound:
-            user_overrides = ''
+            user_overrides = None
 
-        # Get any system overrides.
+        system_apps = pecan.request.rpcapi.get_helm_applications(
+                      pecan.request.context)
+        if app_name in system_apps:
+            # Get any system overrides for system app.
             try:
                 system_overrides = pecan.request.rpcapi.get_helm_chart_overrides(
                     pecan.request.context, name, namespace)
-            system_overrides = yaml.safe_dump(system_overrides)
+                system_overrides = yaml.safe_dump(system_overrides) \
-        except Exception:
+                    if system_overrides else None
-            # Unsupported/invalid namespace
+            except Exception as e:
-            raise wsme.exc.ClientSideError(_("Override not found."))
+                raise wsme.exc.ClientSideError(_("Unable to get the helm chart overrides "
+                                                 "for chart %s under Namespace %s: %s"
+                                                 % (name, namespace, str(e))))
+        else:
+            # No system overrides for generic app
+            system_overrides = None
 
         # Merge the system overrides with the saved user-specified overrides,
         # with user-specified overrides taking priority over the system
         # overrides.
-        file_overrides = [system_overrides, user_overrides] \
+        file_overrides = []
-            if user_overrides else [system_overrides]
+        if system_overrides:
+            file_overrides.append(system_overrides)
+        if user_overrides:
+            file_overrides.append(user_overrides)
 
+        combined_overrides = None
+        if file_overrides:
             combined_overrides = pecan.request.rpcapi.merge_overrides(
                 pecan.request.context, file_overrides=file_overrides)
 
@@ -111,6 +123,14 @@ class HelmChartsController(rest.RestController):
         file_overrides = values.get('files', [])
         set_overrides = values.get('set', [])
 
+        if set_overrides:
+            for overrides in set_overrides:
+                if ',' in overrides:
+                    raise wsme.exc.ClientSideError(
+                        _("Invalid input: One (or more) set overrides contains "
+                          "multiple values. Consider using --values option "
+                          "instead."))
+
         # Get any stored user overrides for this chart.  We'll need this
         # object later either way.
         try:
@@ -124,28 +144,21 @@ class HelmChartsController(rest.RestController):
             pecan.request.dbapi.helm_override_create({
                 'name': name,
                 'namespace': namespace,
-                'user_overrides': '',
                 'app_id': app.id})
             db_chart = objects.helm_overrides.get_by_appid_name(
                 pecan.request.context, app.id, name, namespace)
 
+        user_overrides = db_chart.user_overrides
         if flag == 'reuse':
-            if db_chart.user_overrides is not None:
+            if user_overrides is not None:
-                file_overrides.insert(0, db_chart.user_overrides)
+                file_overrides.insert(0, user_overrides)
         elif flag == 'reset':
             pass
         else:
             raise wsme.exc.ClientSideError(_("Invalid flag: %s must be either "
                                              "'reuse' or 'reset'.") % flag)
 
-        if set_overrides:
+        if file_overrides or set_overrides:
-            for overrides in set_overrides:
-                if ',' in overrides:
-                    raise wsme.exc.ClientSideError(
-                        _("Invalid input: One (or more) set overrides contains "
-                          "multiple values. Consider using --values option "
-                          "instead."))
-
             user_overrides = pecan.request.rpcapi.merge_overrides(
                 pecan.request.context, file_overrides=file_overrides,
                 set_overrides=set_overrides)

sysinv/sysinv/sysinv/sysinv/cmd/helm.py

@@ -12,6 +12,7 @@ System Inventory Helm Utility.
 import sys
 
 from oslo_config import cfg
+from oslo_log import log
 
 from sysinv.common import service
 from sysinv.db import api
@@ -19,10 +20,17 @@ from sysinv.helm import helm
 
 CONF = cfg.CONF
 
+LOG = log.getLogger(__name__)
+
 
 def create_app_overrides_action(path, app_name=None, namespace=None):
     dbapi = api.get_instance()
     operator = helm.HelmOperator(dbapi=dbapi)
+    system_apps = operator.get_helm_applications()
+    if app_name not in system_apps:
+        LOG.info("Overrides generation for application %s is "
+                 "not supported via this command." % app_name)
+    else:
         operator.generate_helm_application_overrides(path, app_name, mode=None,
                                                      cnamespace=namespace)
 
@@ -30,6 +38,11 @@ def create_app_overrides_action(path, app_name=None, namespace=None):
 def create_armada_app_overrides_action(path, app_name=None, namespace=None):
     dbapi = api.get_instance()
     operator = helm.HelmOperator(dbapi=dbapi)
+    system_apps = operator.get_helm_applications()
+    if app_name not in system_apps:
+        LOG.info("Overrides generation for application %s is "
+                 "not supported via this command." % app_name)
+    else:
         operator.generate_helm_application_overrides(path, app_name, mode=None,
                                                      cnamespace=namespace,
                                                      armada_format=True,
@@ -73,11 +86,20 @@ CONF.register_cli_opt(
 def main():
     service.prepare_service(sys.argv)
     if CONF.action.name == 'create-app-overrides':
-
+        if not CONF.action.path:
+            LOG.error("overrides path is required")
+        elif not CONF.action.app_name:
+            LOG.error("application name is required")
+        else:
             CONF.action.func(CONF.action.path,
                              CONF.action.app_name,
                              CONF.action.namespace)
     elif CONF.action.name == 'create-armada-app-overrides':
+        if not CONF.action.path:
+            LOG.error("overrides path is required")
+        elif not CONF.action.app_name:
+            LOG.error("application name is required")
+        else:
             CONF.action.func(CONF.action.path,
                              CONF.action.app_name,
                              CONF.action.namespace)

sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py

@@ -128,7 +128,7 @@ def get_local_docker_registry_auth():
                 password=registry_password)
 
 
-Chart = namedtuple('Chart', 'name namespace location release labels sequenced')
+Chart = namedtuple('Chart', 'metadata_name name namespace location release labels sequenced')
 
 
 class AppOperator(object):
@@ -462,11 +462,11 @@ class AppOperator(object):
         # applicable. Save the list to the same location as the armada manifest
         # so it can be sync'ed.
         app.charts = self._get_list_of_charts(app.armada_mfile_abs)
-        if app.system_app:
         LOG.info("Generating application overrides...")
         self._helm.generate_helm_application_overrides(
             app.overrides_dir, app.name, mode=None, cnamespace=None,
             armada_format=True, armada_chart_info=app.charts, combined=True)
+        if app.system_app:
             self._save_images_list_by_charts(app)
             # Get the list of images from the updated images overrides
             images_to_download = self._get_image_tags_by_charts(
@@ -863,6 +863,7 @@ class AppOperator(object):
 
         The following chart data for each chart in the manifest file
         are extracted and stored into a namedtuple Chart object:
+         - metadata_name
          - chart_name
          - namespace
          - location
@@ -929,6 +930,7 @@ class AppOperator(object):
             for c_group in chart_groups:
                 for chart in chart_group[c_group]['chart_group']:
                     charts.append(Chart(
+                        metadata_name=chart,
                         name=armada_charts[chart]['chart_name'],
                         namespace=armada_charts[chart]['namespace'],
                         location=armada_charts[chart]['location'],
@@ -944,6 +946,7 @@ class AppOperator(object):
                 for c_group in chart_group:
                     for chart in chart_group[c_group]['chart_group']:
                         charts.append(Chart(
+                            metadata_name=chart,
                             name=armada_charts[chart]['chart_name'],
                             namespace=armada_charts[chart]['namespace'],
                             location=armada_charts[chart]['location'],
@@ -955,6 +958,7 @@ class AppOperator(object):
             if armada_charts:
                 for chart in armada_charts:
                     charts.append(Chart(
+                        metadata_name=chart,
                         name=armada_charts[chart]['chart_name'],
                         namespace=armada_charts[chart]['namespace'],
                         location=armada_charts[chart]['location'],
@@ -1500,20 +1504,13 @@ class AppOperator(object):
                                                         app.name, mode)
             if overrides_files:
                 LOG.info("Application overrides generated.")
-                    overrides_str =\
+                overrides_str = self._generate_armada_overrides_str(
-                        self._generate_armada_overrides_str(app.name, app.version,
+                    app.name, app.version, overrides_files)
-                                                            overrides_files)
                 self._update_app_status(
                     app, new_progress=constants.APP_PROGRESS_DOWNLOAD_IMAGES)
                 self._download_images(app)
             else:
                 ready = False
-            else:
-                # No support for custom app overrides at this point, just
-                # download the needed images.
-                self._update_app_status(
-                    app, new_progress=constants.APP_PROGRESS_DOWNLOAD_IMAGES)
-                self._download_images(app)
         except exception.KubeAppApplyFailure as e:
             # ex:Image download failure
             LOG.exception(e)

sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py

@@ -7398,8 +7398,9 @@ class Connection(api.Connection):
         return self._helm_override_get(app_id, name, namespace)
 
     @objects.objectify(objects.helm_overrides)
-    def helm_override_get_all(self):
+    def helm_override_get_all(self, app_id):
         query = model_query(models.HelmOverrides, read_deleted="no")
+        query = query.filter_by(app_id=app_id)
         return query.all()
 
     @objects.objectify(objects.helm_overrides)

sysinv/sysinv/sysinv/sysinv/helm/base.py

@@ -48,12 +48,6 @@ class BaseHelm(object):
     def quoted_str(value):
         return quoted_str(value)
 
-    def get_chart_location(self, chart_name, repo_name):
-        if repo_name is None:
-            repo_name = common.HELM_REPO_FOR_APPS
-        return 'http://controller:{}/helm_charts/{}/{}-0.1.0.tgz'.format(
-            utils.get_http_port(self.dbapi), repo_name, chart_name)
-
     @staticmethod
     def _generate_random_password(length=16):
         suffix = "Ti0*"

sysinv/sysinv/sysinv/sysinv/helm/garbd.py

@@ -25,6 +25,10 @@ class GarbdHelm(base.BaseHelm):
     CHART = constants.HELM_CHART_GARBD
     SUPPORTED_NAMESPACES = \
         base.BaseHelm.SUPPORTED_NAMESPACES + [common.HELM_NS_OPENSTACK]
+    SUPPORTED_APP_NAMESPACES = {
+        constants.HELM_APP_OPENSTACK:
+            base.BaseHelm.SUPPORTED_NAMESPACES + [common.HELM_NS_OPENSTACK]
+    }
 
     def get_meta_overrides(self, namespace, app_name=None, mode=None):
 

sysinv/sysinv/sysinv/sysinv/helm/helm.py

@@ -19,6 +19,7 @@ from six import iteritems
 from stevedore import extension
 from sysinv.common import constants
 from sysinv.common import exception
+from sysinv.common import utils
 from sysinv.openstack.common import log as logging
 from sysinv.helm import common
 
@@ -62,7 +63,7 @@ class HelmOperator(object):
         self.chart_operators = {}
 
         # dict containing sequence of helm charts per app
-        self.helm_applications = self.get_helm_applications()
+        self.helm_system_applications = self.get_helm_applications()
 
     def get_helm_applications(self):
         """Build a dictionary of supported helm applications"""
@@ -189,9 +190,15 @@ class HelmOperator(object):
                   overrides may be provided.
         """
 
+        try:
+            app = self.dbapi.kube_app_get(app_name)
+        except exception.KubeAppNotFound:
+            LOG.exception("Application %s not found." % app_name)
+            raise
+
         app_namespaces = {}
-        if app_name in self.helm_applications:
+        if app_name in self.helm_system_applications:
-            for chart_name in self.helm_applications[app_name]:
+            for chart_name in self.helm_system_applications[app_name]:
                 try:
                     app_namespaces.update(
                         {chart_name:
@@ -199,6 +206,13 @@ class HelmOperator(object):
                              chart_name, app_name)})
                 except exception.InvalidHelmNamespace as e:
                     LOG.info(e)
+        else:
+            # Generic apps
+            db_namespaces = self.dbapi.helm_override_get_all(app.id)
+            for chart in db_namespaces:
+                app_namespaces.setdefault(
+                    chart.name, []).append(chart.namespace)
+
         return app_namespaces
 
     @helm_context
@@ -248,8 +262,8 @@ class HelmOperator(object):
         }
         """
         overrides = {}
-        if app_name in self.helm_applications:
+        if app_name in self.helm_system_applications:
-            for chart_name in self.helm_applications[app_name]:
+            for chart_name in self.helm_system_applications[app_name]:
                 try:
                     overrides.update({chart_name:
                                       self._get_helm_chart_overrides(
@@ -259,37 +273,40 @@ class HelmOperator(object):
                     LOG.info(e)
         return overrides
 
-    def _get_helm_chart_location(self, chart_name, repo_name):
+    def _get_helm_chart_location(self, chart_name, repo_name, chart_tarfile):
-        """Get supported chart location.
+        """Get the chart location.
 
         This method returns the download location for a given chart.
 
         :param chart_name: name of the chart
-        :returns: a URL as location or None if the chart is not supported
+        :param repo_name: name of the repo that chart uploaded to
+        :param chart_tarfile: name of the chart tarfile
+        :returns: a URL as location
         """
-        if chart_name in self.chart_operators:
+        if repo_name is None:
-            return self.chart_operators[chart_name].get_chart_location(
+            repo_name = common.HELM_REPO_FOR_APPS
-                chart_name, repo_name)
+        if chart_tarfile is None:
-        return None
+            # TODO: Clean up the assumption
+            chart_tarfile = chart_name + '-0.1.0'
+        return 'http://controller:{}/helm_charts/{}/{}.tgz'.format(
+            utils.get_http_port(self.dbapi), repo_name, chart_tarfile)
+
+    def _add_armada_override_header(self, chart_name, chart_metadata_name, repo_name,
+                                    chart_tarfile, namespace, overrides):
+        if chart_metadata_name is None:
+            chart_metadata_name = namespace + '-' + chart_name
 
-    def _add_armada_override_header(self, chart_name, repo_name, namespace,
-                                    overrides):
-        use_chart_name_only = [common.HELM_NS_HELM_TOOLKIT]
-        if namespace in use_chart_name_only:
-            name = chart_name
-        else:
-            name = namespace + '-' + chart_name
         new_overrides = {
             'schema': 'armada/Chart/v1',
             'metadata': {
                 'schema': 'metadata/Document/v1',
-                'name': name
+                'name': chart_metadata_name
             },
             'data': {
                 'values': overrides
             }
         }
-        location = self._get_helm_chart_location(chart_name, repo_name)
+        location = self._get_helm_chart_location(chart_name, repo_name, chart_tarfile)
         if location:
             new_overrides['data'].update({
                 'source': {
@@ -298,31 +315,42 @@ class HelmOperator(object):
             })
         return new_overrides
 
-    def _get_repo_from_armada_chart_info(self, chart_name, chart_info_list):
+    def _get_chart_info_from_armada_chart(self, chart_name, chart_namespace,
-        """ Extract the repo from the armada manifest chart location.
+                                          chart_info_list):
+        """ Extract the metadata name of the armada chart, repo and the name of
+            the chart tarfile from the armada manifest chart.
 
         :param chart_name: name of the chart from the (application list)
+        :param chart_namespace: namespace of the chart
         :param chart_info_list: a list of chart objects containing information
             extracted from the armada manifest
-        :returns: the supported StarlingX repository or None if not present
+        :returns: the metadata name of the chart, the supported StarlingX repository,
+                  the name of the chart tarfile or None,None,None if not present
         """
 
         # Could be called without any armada_manifest info. Returning 'None'
         # will enable helm defaults to point to common.HELM_REPO_FOR_APPS
+        metadata_name = None
         repo = None
+        chart_tarfile = None
         if chart_info_list is None:
-            return repo
+            return metadata_name, repo, chart_tarfile
 
-        location = next(
+        location = None
-            (c.location for c in chart_info_list if c.name == chart_name),
+        for c in chart_info_list:
-            None)
+            if (c.name == chart_name and
+                    c.namespace == chart_namespace):
+                location = c.location
+                metadata_name = c.metadata_name
+                break
 
         if location:
-            match = re.search('/helm_charts/(.*)/', location)
+            match = re.search('/helm_charts/(.*)/(.*).tgz', location)
             if match:
                 repo = match.group(1)
+                chart_tarfile = match.group(2)
         LOG.debug("Chart %s can be found in repo: %s" % (chart_name, repo))
-        return repo
+        return metadata_name, repo, chart_tarfile
 
     def merge_overrides(self, file_overrides=[], set_overrides=[]):
         """ Merge helm overrides together.
@@ -460,12 +488,13 @@ class HelmOperator(object):
             system overrides
         """
 
-        if app_name in self.helm_applications:
         try:
             app = self.dbapi.kube_app_get(app_name)
         except exception.KubeAppNotFound:
             LOG.exception("Application %s not found." % app_name)
+            raise
 
+        if app_name in self.helm_system_applications:
             app_overrides = self._get_helm_application_overrides(app_name,
                                                                  cnamespace)
             for (chart_name, overrides) in iteritems(app_overrides):
@@ -505,10 +534,11 @@ class HelmOperator(object):
                 # structure of the yaml file somewhat
                 if armada_format:
                     for key in overrides:
-                        armada_chart_repo_name = self._get_repo_from_armada_chart_info(
+                        metadata_name, repo_name, chart_tarfile = \
-                            chart_name, armada_chart_info)
+                            self._get_chart_info_from_armada_chart(chart_name, key,
+                                                                   armada_chart_info)
                         new_overrides = self._add_armada_override_header(
-                            chart_name, armada_chart_repo_name,
+                            chart_name, metadata_name, repo_name, chart_tarfile,
                             key, overrides[key])
                         overrides[key] = new_overrides
                 self._write_chart_overrides(path, chart_name, cnamespace, overrides)
@@ -524,11 +554,45 @@ class HelmOperator(object):
                         chart_meta_name = chart_name + '-meta'
                         self._write_chart_overrides(
                             path, chart_meta_name, cnamespace, overrides)
-
-        elif app_name:
-            LOG.exception("%s application is not supported" % app_name)
         else:
-            LOG.exception("application name is required")
+            # Generic applications
+            for chart in armada_chart_info:
+                try:
+                    db_chart = self.dbapi.helm_override_get(
+                        app.id, chart.name, chart.namespace)
+                except exception.HelmOverrideNotFound:
+                    # This routine is to create helm overrides entries
+                    # in database during application-upload so that user
+                    # can list the supported helm chart overrides of the
+                    # application via helm-override-list
+                    try:
+                        values = {
+                            'name': chart.name,
+                            'namespace': chart.namespace,
+                            'app_id': app.id,
+                        }
+                        db_chart = self.dbapi.helm_override_create(values=values)
+                    except Exception as e:
+                        LOG.exception(e)
+                        return
+
+                user_overrides = {chart.namespace: {}}
+                db_user_overrides = db_chart.user_overrides
+                if db_user_overrides:
+                    user_overrides = yaml.load(yaml.dump(
+                        {chart.namespace: yaml.load(db_user_overrides)}))
+
+                if armada_format:
+                    metadata_name, repo_name, chart_tarfile =\
+                        self._get_chart_info_from_armada_chart(chart.name, chart.namespace,
+                                                               armada_chart_info)
+                    new_overrides = self._add_armada_override_header(
+                        chart.name, metadata_name, repo_name, chart_tarfile,
+                        chart.namespace, user_overrides[chart.namespace])
+                    user_overrides[chart.namespace] = new_overrides
+
+                self._write_chart_overrides(path, chart.name,
+                                            cnamespace, user_overrides)
 
     def remove_helm_chart_overrides(self, path, chart_name, cnamespace=None):
         """Remove the overrides files for a chart"""

sysinv/sysinv/sysinv/sysinv/helm/ingress.py

@@ -22,6 +22,11 @@ class IngressHelm(base.BaseHelm):
         common.HELM_NS_KUBE_SYSTEM,
         common.HELM_NS_OPENSTACK
     ]
+    SUPPORTED_APP_NAMESPACES = {
+        constants.HELM_APP_OPENSTACK:
+            base.BaseHelm.SUPPORTED_NAMESPACES + [common.HELM_NS_KUBE_SYSTEM,
+                                                  common.HELM_NS_OPENSTACK]
+    }
 
     def get_overrides(self, namespace=None):
         overrides = {

sysinv/sysinv/sysinv/sysinv/helm/memcached.py

@@ -19,6 +19,10 @@ class MemcachedHelm(base.BaseHelm):
     CHART = constants.HELM_CHART_MEMCACHED
     SUPPORTED_NAMESPACES = \
         base.BaseHelm.SUPPORTED_NAMESPACES + [common.HELM_NS_OPENSTACK]
+    SUPPORTED_APP_NAMESPACES = {
+        constants.HELM_APP_OPENSTACK:
+            base.BaseHelm.SUPPORTED_NAMESPACES + [common.HELM_NS_OPENSTACK]
+    }
 
     def get_overrides(self, namespace=None):
         overrides = {